mirror of
https://github.com/kubernetes/kubernetes.git
synced 2026-08-08 11:40:35 +00:00
Merge pull request #132299 from aramase/aramase/f/authn_config_hash_metric
Add `last_config_info` metric for authn, authz and encryption config
This commit is contained in:
@@ -68,6 +68,9 @@ type Config struct {
|
||||
// AuthorizationConfiguration stores the configuration for the Authorizer chain
|
||||
// It will deprecate most of the above flags when GA
|
||||
AuthorizationConfiguration *authzconfig.AuthorizationConfiguration
|
||||
// InitialAuthorizationConfigurationData holds the initial authorization configuration data
|
||||
// that was read from the authorization configuration file.
|
||||
InitialAuthorizationConfigurationData string
|
||||
}
|
||||
|
||||
// New returns the right sort of union of multiple authorizer.Authorizer objects
|
||||
@@ -84,6 +87,7 @@ func (config Config) New(ctx context.Context, serverID string) (authorizer.Autho
|
||||
initialConfig: config,
|
||||
apiServerID: serverID,
|
||||
lastLoadedConfig: config.AuthorizationConfiguration,
|
||||
lastReadData: []byte(config.InitialAuthorizationConfigurationData),
|
||||
reloadInterval: time.Minute,
|
||||
compiler: authorizationcel.NewDefaultCompiler(),
|
||||
}
|
||||
@@ -160,12 +164,16 @@ func GetNameForAuthorizerMode(mode string) string {
|
||||
return strings.ToLower(mode)
|
||||
}
|
||||
|
||||
func LoadAndValidateFile(configFile string, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, error) {
|
||||
func LoadAndValidateFile(configFile string, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, string, error) {
|
||||
data, err := os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
return LoadAndValidateData(data, compiler, requireNonWebhookTypes)
|
||||
config, err := LoadAndValidateData(data, compiler, requireNonWebhookTypes)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return config, string(data), nil
|
||||
}
|
||||
|
||||
func LoadAndValidateData(data []byte, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, error) {
|
||||
@@ -198,7 +206,6 @@ func LoadAndValidateData(data []byte, compiler authorizationcel.Compiler, requir
|
||||
if expectedName != authorizer.Name {
|
||||
allErrors = append(allErrors, fmt.Errorf("expected name %s for authorizer %s instead of %s", expectedName, authorizer.Type, authorizer.Name))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if missingTypes := requireNonWebhookTypes.Difference(seenModes); missingTypes.Len() > 0 {
|
||||
|
||||
@@ -192,7 +192,7 @@ type kubeapiserverWebhookMetrics struct {
|
||||
// Blocks until ctx is complete.
|
||||
func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) {
|
||||
metrics.RegisterMetrics()
|
||||
metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID)
|
||||
metrics.RecordAuthorizationConfigLastConfigInfo(r.apiServerID, string(r.lastReadData))
|
||||
|
||||
filesystem.WatchUntil(
|
||||
ctx,
|
||||
@@ -250,5 +250,5 @@ func (r *reloadableAuthorizerResolver) checkFile(ctx context.Context) {
|
||||
ruleResolver: ruleResolver,
|
||||
})
|
||||
klog.InfoS("reloaded authz config")
|
||||
metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID)
|
||||
metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID, string(data))
|
||||
}
|
||||
|
||||
@@ -727,6 +727,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
|
||||
|
||||
if len(o.AuthenticationConfigFile) > 0 {
|
||||
authenticationconfigmetrics.RegisterMetrics()
|
||||
authenticationconfigmetrics.RecordAuthenticationConfigLastConfigInfo(apiServerID, authenticatorConfig.AuthenticationConfigData)
|
||||
trackedAuthenticationConfigData := authenticatorConfig.AuthenticationConfigData
|
||||
var mu sync.Mutex
|
||||
|
||||
@@ -789,7 +790,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
|
||||
|
||||
trackedAuthenticationConfigData = authConfigData
|
||||
klog.InfoS("reloaded authentication config")
|
||||
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID)
|
||||
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData)
|
||||
},
|
||||
func(err error) { klog.ErrorS(err, "watching authentication config file") },
|
||||
)
|
||||
|
||||
@@ -118,7 +118,7 @@ func (o *BuiltInAuthorizationOptions) Validate() []error {
|
||||
}
|
||||
|
||||
// load/validate kube-apiserver authz config with no opinion about required modes
|
||||
_, err := authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
|
||||
_, _, err := authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
|
||||
if err != nil {
|
||||
return append(allErrors, err)
|
||||
}
|
||||
@@ -219,6 +219,7 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac
|
||||
|
||||
var authorizationConfiguration *authzconfig.AuthorizationConfiguration
|
||||
var err error
|
||||
var authorizationConfigData string
|
||||
|
||||
// if --authorization-config is set, check if
|
||||
// - the feature flag is set
|
||||
@@ -236,7 +237,7 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac
|
||||
return nil, fmt.Errorf("--%s can not be specified when --%s or --authorization-webhook-* flags are defined", authorizationConfigFlag, authorizationModeFlag)
|
||||
}
|
||||
// load/validate kube-apiserver authz config with no opinion about required modes
|
||||
authorizationConfiguration, err = authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
|
||||
authorizationConfiguration, authorizationConfigData, err = authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -252,8 +253,9 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac
|
||||
VersionedInformerFactory: versionedInformerFactory,
|
||||
WebhookRetryBackoff: o.WebhookRetryBackoff,
|
||||
|
||||
ReloadFile: o.AuthorizationConfigurationFile,
|
||||
AuthorizationConfiguration: authorizationConfiguration,
|
||||
ReloadFile: o.AuthorizationConfigurationFile,
|
||||
AuthorizationConfiguration: authorizationConfiguration,
|
||||
InitialAuthorizationConfigurationData: authorizationConfigData,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"k8s.io/apiserver/pkg/util/configmetrics"
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
@@ -52,14 +53,25 @@ var (
|
||||
},
|
||||
[]string{"status", "apiserver_id_hash"},
|
||||
)
|
||||
|
||||
authenticationConfigLastConfigInfo = metrics.NewDesc(
|
||||
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
|
||||
"Information about the last applied authentication configuration with hash as label, split by apiserver identity.",
|
||||
[]string{"apiserver_id_hash", "hash"},
|
||||
nil,
|
||||
metrics.ALPHA,
|
||||
"",
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
var configHashProvider = configmetrics.NewAtomicHashProvider()
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal)
|
||||
legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds)
|
||||
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authenticationConfigLastConfigInfo, configHashProvider))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -74,10 +86,16 @@ func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) {
|
||||
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
|
||||
}
|
||||
|
||||
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) {
|
||||
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData string) {
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
|
||||
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
|
||||
|
||||
RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData)
|
||||
}
|
||||
|
||||
func RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData string) {
|
||||
configHashProvider.SetHashes(getHash(apiServerID), getHash(authConfigData))
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
|
||||
@@ -27,6 +27,18 @@ import (
|
||||
const (
|
||||
testAPIServerID = "testAPIServerID"
|
||||
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
|
||||
testConfigData = `
|
||||
apiVersion: apiserver.config.k8s.io/v1
|
||||
kind: AuthenticationConfiguration
|
||||
jwt:
|
||||
- issuer:
|
||||
url: https://test-issuer
|
||||
audiences: [ "aud" ]
|
||||
claimMappings:
|
||||
username:
|
||||
claim: sub
|
||||
prefix: ""
|
||||
`
|
||||
)
|
||||
|
||||
func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) {
|
||||
@@ -53,15 +65,19 @@ func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) {
|
||||
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
|
||||
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
|
||||
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
|
||||
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_authentication_config_controller_last_config_info gauge
|
||||
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:ccbcaf98557c273dfc779222d54a5bd3e785ea5330048f3bf4278cf3997b669c"} 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reloads_total",
|
||||
namespace + "_" + subsystem + "_last_config_info",
|
||||
}
|
||||
|
||||
authenticationConfigAutomaticReloadsTotal.Reset()
|
||||
ResetMetricsForTest()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID)
|
||||
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -107,3 +123,34 @@ func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordAuthenticationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
|
||||
ResetMetricsForTest()
|
||||
RegisterMetrics()
|
||||
|
||||
// Record initial success with first config
|
||||
firstConfig := "config1"
|
||||
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
|
||||
|
||||
// Record success with different config - should clean up old metric
|
||||
secondConfig := "config2"
|
||||
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
|
||||
|
||||
// Verify only the latest hash is present
|
||||
expectedValue := `
|
||||
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
|
||||
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
|
||||
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
|
||||
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_authentication_config_controller_last_config_info gauge
|
||||
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reloads_total",
|
||||
namespace + "_" + subsystem + "_last_config_info",
|
||||
}
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"hash"
|
||||
"sync"
|
||||
|
||||
"k8s.io/apiserver/pkg/util/configmetrics"
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
@@ -53,10 +54,20 @@ var (
|
||||
},
|
||||
[]string{"status", "apiserver_id_hash"},
|
||||
)
|
||||
|
||||
authorizationConfigLastConfigInfo = metrics.NewDesc(
|
||||
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
|
||||
"Information about the last applied authorization configuration with hash as label, split by apiserver identity.",
|
||||
[]string{"apiserver_id_hash", "hash"},
|
||||
nil,
|
||||
metrics.ALPHA,
|
||||
"",
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
var hashPool *sync.Pool
|
||||
var configHashProvider = configmetrics.NewAtomicHashProvider()
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
@@ -67,6 +78,7 @@ func RegisterMetrics() {
|
||||
}
|
||||
legacyregistry.MustRegister(authorizationConfigAutomaticReloadsTotal)
|
||||
legacyregistry.MustRegister(authorizationConfigAutomaticReloadLastTimestampSeconds)
|
||||
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authorizationConfigLastConfigInfo, configHashProvider))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -81,10 +93,16 @@ func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) {
|
||||
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
|
||||
}
|
||||
|
||||
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID string) {
|
||||
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID, authzConfigData string) {
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
authorizationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
|
||||
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
|
||||
|
||||
RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData)
|
||||
}
|
||||
|
||||
func RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData string) {
|
||||
configHashProvider.SetHashes(getHash(apiServerID), getHash(authzConfigData))
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
|
||||
@@ -27,6 +27,27 @@ import (
|
||||
const (
|
||||
testAPIServerID = "testAPIServerID"
|
||||
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
|
||||
testConfigData = `
|
||||
apiVersion: apiserver.config.k8s.io/v1
|
||||
kind: AuthorizationConfiguration
|
||||
authorizers:
|
||||
- type: Webhook
|
||||
name: error.example.com
|
||||
webhook:
|
||||
timeout: 5s
|
||||
failurePolicy: Deny
|
||||
subjectAccessReviewVersion: v1
|
||||
matchConditionSubjectAccessReviewVersion: v1
|
||||
authorizedTTL: 1ms
|
||||
unauthorizedTTL: 1ms
|
||||
connectionInfo:
|
||||
type: KubeConfigFile
|
||||
kubeConfigFile: /path/to/kubeconfig
|
||||
matchConditions:
|
||||
- expression: has(request.resourceAttributes)
|
||||
- expression: 'request.resourceAttributes.namespace == "fail"'
|
||||
- expression: 'request.resourceAttributes.name == "error"'
|
||||
`
|
||||
)
|
||||
|
||||
func TestRecordAuthorizationConfigAutomaticReloadFailure(t *testing.T) {
|
||||
@@ -53,15 +74,19 @@ func TestRecordAuthorizationConfigAutomaticReloadSuccess(t *testing.T) {
|
||||
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
|
||||
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
|
||||
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
|
||||
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_authorization_config_controller_last_config_info gauge
|
||||
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:2a66c95e5593fc74e6c3dbbb708741361f0690ed45d17e4d4ac0b9c282b6538f"} 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reloads_total",
|
||||
namespace + "_" + subsystem + "_last_config_info",
|
||||
}
|
||||
|
||||
authorizationConfigAutomaticReloadsTotal.Reset()
|
||||
ResetMetricsForTest()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID)
|
||||
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -107,3 +132,34 @@ func TestAuthorizationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordAuthorizationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
|
||||
ResetMetricsForTest()
|
||||
RegisterMetrics()
|
||||
|
||||
// Record initial success with first config
|
||||
firstConfig := "config1"
|
||||
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
|
||||
|
||||
// Record success with different config - should clean up old metric
|
||||
secondConfig := "config2"
|
||||
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
|
||||
|
||||
// Verify only the latest hash is present
|
||||
expectedValue := `
|
||||
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
|
||||
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
|
||||
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
|
||||
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_authorization_config_controller_last_config_info gauge
|
||||
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reloads_total",
|
||||
namespace + "_" + subsystem + "_last_config_info",
|
||||
}
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -907,9 +907,8 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte,
|
||||
|
||||
// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
|
||||
// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
|
||||
// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
|
||||
func computeEncryptionConfigHash(data []byte) string {
|
||||
return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data))
|
||||
return fmt.Sprintf("sha256:%x", sha256.Sum256(data))
|
||||
}
|
||||
|
||||
var _ storagevalue.ResourceTransformers = &DynamicTransformers{}
|
||||
|
||||
@@ -1843,7 +1843,7 @@ func errString(err error) string {
|
||||
|
||||
func TestComputeEncryptionConfigHash(t *testing.T) {
|
||||
// hash the empty string to be sure that sha256 is being used
|
||||
expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
expect := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
sum := computeEncryptionConfigHash([]byte(""))
|
||||
if expect != sum {
|
||||
t.Errorf("expected hash %q but got %q", expect, sum)
|
||||
@@ -2220,7 +2220,7 @@ func TestGetEncryptionConfigHash(t *testing.T) {
|
||||
{
|
||||
name: "valid file",
|
||||
filepath: "testdata/valid-configs/secret-box-first.yaml",
|
||||
wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
|
||||
wantHash: "sha256:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
|
||||
wantErr: ``,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -174,7 +174,7 @@ func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Conte
|
||||
}
|
||||
|
||||
if updatedEffectiveConfig && err == nil {
|
||||
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
|
||||
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID, encryptionConfiguration.EncryptionFileContentHash)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -47,6 +47,9 @@ func TestController(t *testing.T) {
|
||||
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
|
||||
apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1
|
||||
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_last_config_info gauge
|
||||
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",hash="sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef"} 1
|
||||
`
|
||||
const expectedFailureMetricValue = `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
|
||||
@@ -67,7 +70,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
}{
|
||||
{
|
||||
name: "when invalid config is provided previous config shouldn't be changed",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -82,7 +85,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
{
|
||||
name: "when new valid config is provided it should be updated",
|
||||
wantECFileHash: "some new config hash",
|
||||
wantECFileHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantMetrics: expectedSuccessMetricValue,
|
||||
@@ -98,13 +101,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
err: nil,
|
||||
},
|
||||
},
|
||||
EncryptionFileContentHash: "some new config hash",
|
||||
EncryptionFileContentHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "when same valid config is provided previous config shouldn't be changed",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -122,13 +125,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
},
|
||||
// hash of initial "testdata/ec_config.yaml" config file before reloading
|
||||
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "when transformer's health check fails previous config shouldn't be changed",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -152,7 +155,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
{
|
||||
name: "when multiple health checks are present previous config shouldn't be changed",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -179,7 +182,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
{
|
||||
name: "when invalid health check URL is provided previous config shouldn't be changed",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -202,7 +205,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
{
|
||||
name: "when config is not updated transformers are closed correctly",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 1,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -220,13 +223,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
},
|
||||
// hash of initial "testdata/ec_config.yaml" config file before reloading
|
||||
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "when config hash is not updated transformers are closed correctly",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 0,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -234,7 +237,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
wantAddRateLimitedCount: 0,
|
||||
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
||||
// hash of initial "testdata/ec_config.yaml" config file before reloading
|
||||
return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
|
||||
return "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
|
||||
},
|
||||
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
||||
return nil, fmt.Errorf("should not be called")
|
||||
@@ -242,7 +245,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
},
|
||||
{
|
||||
name: "when config hash errors transformers are closed correctly",
|
||||
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
||||
wantLoadCalls: 0,
|
||||
wantHashCalls: 1,
|
||||
wantTransformerClosed: true,
|
||||
@@ -332,7 +335,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
|
||||
}
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics),
|
||||
"apiserver_encryption_config_controller_automatic_reloads_total",
|
||||
"apiserver_encryption_config_controller_automatic_reloads_total", "apiserver_encryption_config_controller_automatic_reload_last_config_info",
|
||||
); err != nil {
|
||||
t.Errorf("failed to validate metrics: %v", err)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"hash"
|
||||
"sync"
|
||||
|
||||
"k8s.io/apiserver/pkg/util/configmetrics"
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
@@ -53,10 +54,20 @@ var (
|
||||
},
|
||||
[]string{"status", "apiserver_id_hash"},
|
||||
)
|
||||
|
||||
encryptionConfigLastConfigInfo = metrics.NewDesc(
|
||||
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
|
||||
"Information about the last applied encryption configuration with hash as label, split by apiserver identity.",
|
||||
[]string{"apiserver_id_hash", "hash"},
|
||||
nil,
|
||||
metrics.ALPHA,
|
||||
"",
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
var hashPool *sync.Pool
|
||||
var configHashProvider = configmetrics.NewAtomicHashProvider()
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
@@ -67,25 +78,37 @@ func RegisterMetrics() {
|
||||
}
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
|
||||
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(encryptionConfigLastConfigInfo, configHashProvider))
|
||||
})
|
||||
}
|
||||
|
||||
func ResetMetricsForTest() {
|
||||
encryptionConfigAutomaticReloadsTotal.Reset()
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
|
||||
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID, encryptionConfigDataHash string) {
|
||||
apiServerIDHash := getHash(apiServerID)
|
||||
encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
|
||||
|
||||
RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash)
|
||||
}
|
||||
|
||||
func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash string) {
|
||||
configHashProvider.SetHashes(getHash(apiServerID), encryptionConfigDataHash)
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
const (
|
||||
testAPIServerID = "testAPIServerID"
|
||||
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
|
||||
testConfigDataHash = "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"
|
||||
)
|
||||
|
||||
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
|
||||
@@ -53,16 +54,19 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
|
||||
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
|
||||
apiserver_encryption_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
|
||||
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_last_config_info gauge
|
||||
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"} 1
|
||||
`
|
||||
metricNames := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reload_success_total",
|
||||
namespace + "_" + subsystem + "_automatic_reloads_total",
|
||||
namespace + "_" + subsystem + "_last_config_info",
|
||||
}
|
||||
|
||||
encryptionConfigAutomaticReloadsTotal.Reset()
|
||||
ResetMetricsForTest()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID)
|
||||
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, testConfigDataHash)
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metricNames...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -108,3 +112,41 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordEncryptionConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
|
||||
ResetMetricsForTest()
|
||||
RegisterMetrics()
|
||||
|
||||
// Record first config
|
||||
firstConfigHash := "sha256:firsthash"
|
||||
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, firstConfigHash)
|
||||
|
||||
// Verify first config metric exists
|
||||
firstExpected := `
|
||||
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_last_config_info gauge
|
||||
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:firsthash"} 1
|
||||
`
|
||||
metricNames := []string{
|
||||
namespace + "_" + subsystem + "_last_config_info",
|
||||
}
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(firstExpected), metricNames...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Record second config - should clean up first config's metric
|
||||
secondConfigHash := "sha256:secondhash"
|
||||
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, secondConfigHash)
|
||||
|
||||
// Verify only second config metric exists
|
||||
secondExpected := `
|
||||
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
|
||||
# TYPE apiserver_encryption_config_controller_last_config_info gauge
|
||||
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:secondhash"} 1
|
||||
`
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(secondExpected), metricNames...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
|
||||
encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
|
||||
encryptionconfigmetrics "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
|
||||
serverstorage "k8s.io/apiserver/pkg/server/storage"
|
||||
"k8s.io/apiserver/pkg/storage/etcd3/metrics"
|
||||
"k8s.io/apiserver/pkg/storage/storagebackend"
|
||||
@@ -298,6 +299,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encryptionconfigmetrics.RecordEncryptionConfigLastConfigInfo(c.APIServerID, encryptionConfiguration.EncryptionFileContentHash)
|
||||
|
||||
if s.EncryptionProviderConfigAutomaticReload {
|
||||
// with reload=true we will always have 1 health check
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package configmetrics
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
// HashProvider is an interface for getting the current config hash values
|
||||
type HashProvider interface {
|
||||
GetCurrentHashes() []string
|
||||
SetHashes(hashes ...string)
|
||||
}
|
||||
|
||||
// AtomicHashProvider implements HashProvider using a single atomic pointer to a slice
|
||||
type AtomicHashProvider struct {
|
||||
hashes *atomic.Pointer[[]string]
|
||||
}
|
||||
|
||||
// NewAtomicHashProvider creates a new atomic hash provider
|
||||
func NewAtomicHashProvider() *AtomicHashProvider {
|
||||
p := &AtomicHashProvider{
|
||||
hashes: &atomic.Pointer[[]string]{},
|
||||
}
|
||||
// Initialize with empty slice
|
||||
p.hashes.Store(ptr.To([]string{}))
|
||||
return p
|
||||
}
|
||||
|
||||
func (h *AtomicHashProvider) GetCurrentHashes() []string {
|
||||
hashesPtr := h.hashes.Load()
|
||||
if hashesPtr == nil {
|
||||
// should never happen, but just in case
|
||||
return []string{}
|
||||
}
|
||||
// Return a copy to prevent external modification
|
||||
hashes := *hashesPtr
|
||||
return slices.Clone(hashes)
|
||||
}
|
||||
|
||||
func (h *AtomicHashProvider) SetHashes(hashes ...string) {
|
||||
hashCopy := slices.Clone(hashes)
|
||||
h.hashes.Store(&hashCopy)
|
||||
}
|
||||
|
||||
// NewConfigInfoCustomCollector creates a custom collector for config hash info metrics.
|
||||
// This eliminates the need for state management and locks by collecting metrics on demand.
|
||||
func NewConfigInfoCustomCollector(desc *metrics.Desc, hashProvider HashProvider) metrics.StableCollector {
|
||||
return &configInfoCustomCollector{
|
||||
desc: desc,
|
||||
hashProvider: hashProvider,
|
||||
}
|
||||
}
|
||||
|
||||
type configInfoCustomCollector struct {
|
||||
metrics.BaseStableCollector
|
||||
desc *metrics.Desc
|
||||
hashProvider HashProvider
|
||||
}
|
||||
|
||||
var _ metrics.StableCollector = &configInfoCustomCollector{}
|
||||
|
||||
func (c *configInfoCustomCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
|
||||
ch <- c.desc
|
||||
}
|
||||
|
||||
func (c *configInfoCustomCollector) CollectWithStability(ch chan<- metrics.Metric) {
|
||||
hashes := c.hashProvider.GetCurrentHashes()
|
||||
if len(hashes) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
ch <- metrics.NewLazyConstMetric(c.desc, metrics.GaugeValue, 1, hashes...)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package configmetrics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// assertHashesEqual is a test helper that compares expected and actual hash slices
|
||||
func assertHashesEqual(t *testing.T, expected, actual []string) {
|
||||
t.Helper()
|
||||
if len(actual) != len(expected) {
|
||||
t.Errorf("expected %d hashes, got %d", len(expected), len(actual))
|
||||
return
|
||||
}
|
||||
for i, hash := range actual {
|
||||
if hash != expected[i] {
|
||||
t.Errorf("expected hash %q at index %d, got %q", expected[i], i, hash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicHashProvider(t *testing.T) {
|
||||
provider := NewAtomicHashProvider()
|
||||
|
||||
apiServerIDHash := "sha256:abc123"
|
||||
configHash := "sha256:hash1"
|
||||
|
||||
// Test initial state
|
||||
hashes := provider.GetCurrentHashes()
|
||||
assertHashesEqual(t, []string{}, hashes)
|
||||
|
||||
// Test setting hashes
|
||||
provider.SetHashes(apiServerIDHash, configHash)
|
||||
hashes = provider.GetCurrentHashes()
|
||||
assertHashesEqual(t, []string{apiServerIDHash, configHash}, hashes)
|
||||
|
||||
// Test updating hashes
|
||||
newConfigHash := "sha256:hash2"
|
||||
provider.SetHashes(apiServerIDHash, newConfigHash)
|
||||
hashes = provider.GetCurrentHashes()
|
||||
assertHashesEqual(t, []string{apiServerIDHash, newConfigHash}, hashes)
|
||||
|
||||
// Test empty hashes
|
||||
provider.SetHashes()
|
||||
hashes = provider.GetCurrentHashes()
|
||||
assertHashesEqual(t, []string{}, hashes)
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
@@ -1146,6 +1147,7 @@ jwt:
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1229,6 +1231,7 @@ jwt:
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1277,6 +1280,7 @@ jwt:
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1332,6 +1336,7 @@ jwt:
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1390,6 +1395,7 @@ jwt:
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_old_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1433,6 +1439,7 @@ kind: AuthenticationConfiguration
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -1490,6 +1497,7 @@ jwt:
|
||||
wantMetricStrings: []string{
|
||||
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`,
|
||||
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`,
|
||||
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_old_config_hash"} 1`,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1529,8 +1537,9 @@ jwt:
|
||||
_ = tempFile.Close()
|
||||
}()
|
||||
|
||||
newAuthConfig := tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert))
|
||||
// Write the new content to the temporary file
|
||||
_, err = tempFile.Write([]byte(tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert))))
|
||||
_, err = tempFile.Write([]byte(newAuthConfig))
|
||||
require.NoError(t, err)
|
||||
|
||||
// Atomically replace the original file with the temporary file
|
||||
@@ -1565,6 +1574,16 @@ jwt:
|
||||
_, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{})
|
||||
tt.newAssertErrFn(t, err)
|
||||
|
||||
oldAuthConfigHash := getHash(tt.authConfigFn(t, oidcServer.URL(), string(caCert)))
|
||||
newAuthConfigHash := getHash(newAuthConfig)
|
||||
for i := range tt.wantMetricStrings {
|
||||
if strings.Contains(tt.wantMetricStrings[i], "replace_with_new_config_hash") {
|
||||
tt.wantMetricStrings[i] = strings.ReplaceAll(tt.wantMetricStrings[i], "replace_with_new_config_hash", newAuthConfigHash)
|
||||
} else if strings.Contains(tt.wantMetricStrings[i], "replace_with_old_config_hash") {
|
||||
tt.wantMetricStrings[i] = strings.ReplaceAll(tt.wantMetricStrings[i], "replace_with_old_config_hash", oldAuthConfigHash)
|
||||
}
|
||||
}
|
||||
|
||||
adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig)
|
||||
body, err := adminClient.RESTClient().Get().AbsPath("/metrics").DoRaw(ctx)
|
||||
require.NoError(t, err)
|
||||
@@ -2114,3 +2133,10 @@ func testContext(t *testing.T) context.Context {
|
||||
t.Cleanup(cancel)
|
||||
return ctx
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
|
||||
}
|
||||
|
||||
@@ -735,8 +735,8 @@ authorizers:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if initialMetrics.reloadSuccess == nil {
|
||||
t.Fatal("expected success timestamp, got none")
|
||||
if initialMetrics.reloadSuccess != nil {
|
||||
t.Fatal("expected no success timestamp, got one")
|
||||
}
|
||||
if initialMetrics.reloadFailure != nil {
|
||||
t.Fatal("expected no failure timestamp, got one")
|
||||
@@ -754,19 +754,13 @@ authorizers:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reload1Metrics.reloadSuccess == nil {
|
||||
t.Fatal("expected success timestamp, got none")
|
||||
}
|
||||
if !reload1Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
|
||||
t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialMetrics.reloadSuccess.String(), reload1Metrics.reloadSuccess.String())
|
||||
if reload1Metrics.reloadSuccess != nil {
|
||||
t.Fatal("expected no success timestamp, got one")
|
||||
}
|
||||
if reload1Metrics.reloadFailure == nil {
|
||||
t.Log("expected failure timestamp, got nil, retrying")
|
||||
return false, nil
|
||||
}
|
||||
if !reload1Metrics.reloadFailure.After(*reload1Metrics.reloadSuccess) {
|
||||
t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Metrics.reloadFailure.String(), reload1Metrics.reloadSuccess.String())
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -829,15 +823,19 @@ authorizers:
|
||||
t.Fatalf("failure timestamp changed from reload1Metrics.reloadFailure %s to %s unexpectedly", reload1Metrics.reloadFailure.String(), reload2Metrics.reloadFailure.String())
|
||||
}
|
||||
if reload2Metrics.reloadSuccess == nil {
|
||||
t.Fatal("expected success timestamp, got none")
|
||||
}
|
||||
if reload2Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
|
||||
t.Log("success timestamp hasn't updated from initial success, retrying")
|
||||
t.Log("expected success timestamp, got nil, retrying")
|
||||
return false, nil
|
||||
}
|
||||
if !reload2Metrics.reloadSuccess.After(*reload2Metrics.reloadFailure) {
|
||||
t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Metrics.reloadSuccess.String(), reload2Metrics.reloadFailure.String())
|
||||
}
|
||||
if len(reload2Metrics.configHash) == 0 {
|
||||
t.Fatal("expected config hash, got none")
|
||||
}
|
||||
if reload2Metrics.configHash == initialMetrics.configHash {
|
||||
t.Logf("config hash %s is the same as initial %s, retrying", reload2Metrics.configHash, initialMetrics.configHash)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -883,6 +881,12 @@ authorizers:
|
||||
if !reload3Metrics.reloadSuccess.Equal(*reload2Metrics.reloadSuccess) {
|
||||
t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Metrics.reloadSuccess.String(), reload3Metrics.reloadSuccess.String())
|
||||
}
|
||||
if len(reload3Metrics.configHash) == 0 {
|
||||
t.Fatal("expected config hash, got none")
|
||||
}
|
||||
if reload3Metrics.configHash != reload2Metrics.configHash {
|
||||
t.Fatalf("expected config hash to be the same as reload2Metrics %s, got %s", reload2Metrics.configHash, reload3Metrics.configHash)
|
||||
}
|
||||
if reload3Metrics.reloadFailure == nil {
|
||||
t.Log("expected failure timestamp, got nil, retrying")
|
||||
return false, nil
|
||||
@@ -925,6 +929,7 @@ authorizers:
|
||||
type metrics struct {
|
||||
reloadSuccess *time.Time
|
||||
reloadFailure *time.Time
|
||||
configHash string
|
||||
decisions map[authorizerKey]map[string]int
|
||||
exclusions int
|
||||
evalErrors int
|
||||
@@ -945,12 +950,14 @@ var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authori
|
||||
var whTotalMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_total{name="(.*?)",result="(.*?)"} (\d+)`)
|
||||
var webhookDurationMetric = regexp.MustCompile(`apiserver_authorization_webhook_duration_seconds_count{name="(.*?)",result="(.*?)"} (\d+)`)
|
||||
var webhookFailOpenMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_fail_open_total{name="(.*?)",result="(.*?)"} (\d+)`)
|
||||
var configInfoMetric = regexp.MustCompile(`apiserver_authorization_config_controller_last_config_info\{apiserver_id_hash="sha256:[^"]*",hash="([^"]*)"\} (\d+)`)
|
||||
|
||||
func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
|
||||
data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
|
||||
|
||||
// apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="success"} 1.7002567356895502e+09
|
||||
// apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="failure"} 1.7002567356895502e+09
|
||||
// apiserver_authorization_config_controller_automatic_reload_last_config_info{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
|
||||
// apiserver_authorization_decisions_total{decision="allowed",name="allow.example.com",type="Webhook"} 2
|
||||
// apiserver_authorization_decisions_total{decision="allowed",name="allowreloaded.example.com",type="Webhook"} 1
|
||||
// apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1
|
||||
@@ -1052,6 +1059,12 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
|
||||
t.Log("failure", m.reloadFailure.String())
|
||||
}
|
||||
}
|
||||
if matches := configInfoMetric.FindStringSubmatch(line); matches != nil {
|
||||
t.Logf("line: %s\n", line)
|
||||
t.Log(matches)
|
||||
m.configHash = matches[1]
|
||||
t.Logf("config hash: %s", m.configHash)
|
||||
}
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
@@ -352,13 +353,6 @@ resources:
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// assert that the metrics we collect during the test run match expectations
|
||||
// NOTE: 2 successful automatic reload resulted from 2 config file updates
|
||||
wantMetricStrings := []string{
|
||||
`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`,
|
||||
}
|
||||
|
||||
test.secret, err = test.createSecret(testSecret, testNamespace)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test secret, error: %v", err)
|
||||
@@ -582,6 +576,15 @@ resources:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// assert that the metrics we collect during the test run match expectations
|
||||
// NOTE: 2 successful automatic reload resulted from 2 config file updates
|
||||
wantMetricStrings := []string{
|
||||
`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
|
||||
`apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`,
|
||||
fmt.Sprintf(`apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="%s"} 1`, getHash(encryptionConfigWithoutOldProvider)),
|
||||
}
|
||||
|
||||
defer func() {
|
||||
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
|
||||
if err != nil {
|
||||
@@ -1205,3 +1208,10 @@ resources:
|
||||
// the healthz check should be OK.
|
||||
mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
|
||||
}
|
||||
|
||||
func getHash(data string) string {
|
||||
if len(data) == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user