From 1473836a1e3c528221439e3b0ce90e729b31adb3 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Fri, 11 Oct 2024 16:09:11 +0900 Subject: [PATCH 1/6] TEST-55-OOMD: set ManagedOOMMemoryPressure= and friends in a drop-in config Fedora and friends has a drop-in config for the settings in /usr/lib/systemd/user/slice.d/ . Hence, settings in the main .slice may be overridden. Let's set below in a drop-in with higher decimal prefix. Also, rename override.conf -> 99-managed-oom-preference.conf for the same reason. --- test/units/TEST-55-OOMD-workload.slice | 2 -- test/units/TEST-55-OOMD-workload.slice.d/99-oom.conf | 7 +++++++ test/units/TEST-55-OOMD.sh | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 test/units/TEST-55-OOMD-workload.slice.d/99-oom.conf diff --git a/test/units/TEST-55-OOMD-workload.slice b/test/units/TEST-55-OOMD-workload.slice index d117b754baa..15580736044 100644 --- a/test/units/TEST-55-OOMD-workload.slice +++ b/test/units/TEST-55-OOMD-workload.slice @@ -7,5 +7,3 @@ CPUAccounting=true MemoryAccounting=true IOAccounting=true TasksAccounting=true -ManagedOOMMemoryPressure=kill -ManagedOOMMemoryPressureLimit=20% diff --git a/test/units/TEST-55-OOMD-workload.slice.d/99-oom.conf b/test/units/TEST-55-OOMD-workload.slice.d/99-oom.conf new file mode 100644 index 00000000000..1bed89c5eec --- /dev/null +++ b/test/units/TEST-55-OOMD-workload.slice.d/99-oom.conf @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +[Slice] +# Fedora and friends has a drop-in config for the settings in +# /usr/lib/systemd/user/slice.d/ . Hence, settings in the main .slice may be +# overridden. Let's set below in a drop-in with higher decimal prefix. +ManagedOOMMemoryPressure=kill +ManagedOOMMemoryPressureLimit=20% diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 944067c5413..4a7c7f78aec 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -153,7 +153,7 @@ if cgroupfs_supports_user_xattrs; then sleep 120 # wait for systemd-oomd kill cool down and elevated memory pressure to come down mkdir -p /run/systemd/system/TEST-55-OOMD-testbloat.service.d/ - cat >/run/systemd/system/TEST-55-OOMD-testbloat.service.d/override.conf </run/systemd/system/TEST-55-OOMD-testbloat.service.d/99-managed-oom-preference.conf < Date: Wed, 16 Oct 2024 14:52:49 +0900 Subject: [PATCH 2/6] TEST-55-OOMD: check global config earlier 'Default Memory Pressure Duration' field in oomctl, which can be configured with DefaultMemoryPressureDurationSec= in oomd.conf, is a global config. Let's check it earlier. This also drops unnecessary cleanup at the beginning. --- test/units/TEST-55-OOMD.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 4a7c7f78aec..2b003c9f73d 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -19,8 +19,6 @@ if [[ -s /skipped ]]; then exit 77 fi -rm -rf /run/systemd/system/TEST-55-OOMD-testbloat.service.d - # Activate swap file if we are in a VM if systemd-detect-virt --vm --quiet; then swapoff --all @@ -73,6 +71,9 @@ if systemctl is-active systemd-oomd.service; then systemctl restart systemd-oomd.service fi +# Check if the oomd.conf drop-in config is loaded. +assert_in 'Default Memory Pressure Duration: 2s' "$(oomctl)" + if [[ -v ASAN_OPTIONS || -v UBSAN_OPTIONS ]]; then # If we're running with sanitizers, sd-executor might pull in quite a significant chunk of shared # libraries, which in turn causes a lot of pressure that can put us in the front when sd-oomd decides to @@ -99,7 +100,6 @@ systemctl start TEST-55-OOMD-testbloat.service timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-workload.slice"; do sleep 1; done' oomctl | grep "/TEST-55-OOMD-workload.slice" oomctl | grep "20.00%" -oomctl | grep "Default Memory Pressure Duration: 2s" systemctl status TEST-55-OOMD-testchill.service @@ -128,7 +128,6 @@ systemctl start --machine "testuser@.host" --user TEST-55-OOMD-testbloat.service timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-workload.slice"; do sleep 1; done' oomctl | grep -E "/user.slice.*/TEST-55-OOMD-workload.slice" oomctl | grep "20.00%" -oomctl | grep "Default Memory Pressure Duration: 2s" systemctl --machine "testuser@.host" --user status TEST-55-OOMD-testchill.service From 6bba058076cf4ce91fd470841d5ced47c8d32d6a Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Wed, 16 Oct 2024 15:04:07 +0900 Subject: [PATCH 3/6] TEST-55-OOMD: split into small testcases Then, we can run each small test cases separately. --- test/units/TEST-55-OOMD.sh | 100 +++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 2b003c9f73d..43a9919165a 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -3,8 +3,10 @@ set -eux set -o pipefail +# shellcheck source=test/units/test-control.sh +. "$(dirname "$0")"/test-control.sh # shellcheck source=test/units/util.sh - . "$(dirname "$0")"/util.sh +. "$(dirname "$0")"/util.sh systemd-analyze log-level debug @@ -93,62 +95,52 @@ else systemd-run -t -p MemoryMax=10M -p MemorySwapMax=0 -p MemoryZSwapMax=0 /bin/true fi -systemctl start TEST-55-OOMD-testchill.service -systemctl start TEST-55-OOMD-testbloat.service +test_basic() { + systemctl "$@" start TEST-55-OOMD-testchill.service + systemctl "$@" start TEST-55-OOMD-testbloat.service -# Verify systemd-oomd is monitoring the expected units -timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-workload.slice"; do sleep 1; done' -oomctl | grep "/TEST-55-OOMD-workload.slice" -oomctl | grep "20.00%" + # Verify systemd-oomd is monitoring the expected units. + timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-workload.slice"; do sleep 1; done' + oomctl | grep "/TEST-55-OOMD-workload.slice" + oomctl | grep "20.00%" -systemctl status TEST-55-OOMD-testchill.service + systemctl "$@" status TEST-55-OOMD-testchill.service -# systemd-oomd watches for elevated pressure for 2 seconds before acting. -# It can take time to build up pressure so either wait 2 minutes or for the service to fail. -for _ in {0..59}; do - if ! systemctl status TEST-55-OOMD-testbloat.service; then - break + # systemd-oomd watches for elevated pressure for 2 seconds before acting. + # It can take time to build up pressure so either wait 2 minutes or for the service to fail. + for _ in {0..59}; do + if ! systemctl "$@" status TEST-55-OOMD-testbloat.service; then + break + fi + oomctl + sleep 2 + done + + # testbloat should be killed and testchill should be fine + if systemctl "$@" status TEST-55-OOMD-testbloat.service; then exit 42; fi + if ! systemctl "$@" status TEST-55-OOMD-testchill.service; then exit 24; fi +} + +testcase_basic_system() { + test_basic +} + +testcase_basic_user() { + # Make sure we also work correctly on user units. + loginctl enable-linger testuser + + test_basic --machine "testuser@.host" --user + + loginctl disable-linger testuser +} + +testcase_preference_avoid() { + # only run this portion of the test if we can set xattrs + if ! cgroupfs_supports_user_xattrs; then + echo "cgroup does not support user xattrs, skipping test for ManagedOOMPreference=avoid" + return 0 fi - oomctl - sleep 2 -done -# testbloat should be killed and testchill should be fine -if systemctl status TEST-55-OOMD-testbloat.service; then exit 42; fi -if ! systemctl status TEST-55-OOMD-testchill.service; then exit 24; fi - -# Make sure we also work correctly on user units. -loginctl enable-linger testuser - -systemctl start --machine "testuser@.host" --user TEST-55-OOMD-testchill.service -systemctl start --machine "testuser@.host" --user TEST-55-OOMD-testbloat.service - -# Verify systemd-oomd is monitoring the expected units -# Try to avoid racing the oomctl output check by checking in a loop with a timeout -timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-workload.slice"; do sleep 1; done' -oomctl | grep -E "/user.slice.*/TEST-55-OOMD-workload.slice" -oomctl | grep "20.00%" - -systemctl --machine "testuser@.host" --user status TEST-55-OOMD-testchill.service - -# systemd-oomd watches for elevated pressure for 2 seconds before acting. -# It can take time to build up pressure so either wait 2 minutes or for the service to fail. -for _ in {0..59}; do - if ! systemctl --machine "testuser@.host" --user status TEST-55-OOMD-testbloat.service; then - break - fi - oomctl - sleep 2 -done - -# testbloat should be killed and testchill should be fine -if systemctl --machine "testuser@.host" --user status TEST-55-OOMD-testbloat.service; then exit 42; fi -if ! systemctl --machine "testuser@.host" --user status TEST-55-OOMD-testchill.service; then exit 24; fi - -loginctl disable-linger testuser - -# only run this portion of the test if we can set xattrs -if cgroupfs_supports_user_xattrs; then sleep 120 # wait for systemd-oomd kill cool down and elevated memory pressure to come down mkdir -p /run/systemd/system/TEST-55-OOMD-testbloat.service.d/ @@ -174,7 +166,9 @@ EOF if ! systemctl status TEST-55-OOMD-testbloat.service; then exit 25; fi if systemctl status TEST-55-OOMD-testmunch.service; then exit 43; fi if ! systemctl status TEST-55-OOMD-testchill.service; then exit 24; fi -fi +} + +run_testcases systemd-analyze log-level info From 665882f8f3d9dee4dac84d244e253db956fd94ab Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Wed, 16 Oct 2024 15:12:32 +0900 Subject: [PATCH 4/6] TEST-55-OOMD: check slice more in detail system and user slice has same name, hence we need to check full path. --- test/units/TEST-55-OOMD.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 43a9919165a..ac496430746 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -96,13 +96,16 @@ else fi test_basic() { + local cgroup_path="${1:?}" + shift + systemctl "$@" start TEST-55-OOMD-testchill.service systemctl "$@" start TEST-55-OOMD-testbloat.service # Verify systemd-oomd is monitoring the expected units. - timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-workload.slice"; do sleep 1; done' - oomctl | grep "/TEST-55-OOMD-workload.slice" - oomctl | grep "20.00%" + timeout 1m bash -xec "until oomctl | grep -q -F 'Path: $cgroup_path'; do sleep 1; done" + assert_in 'Memory Pressure Limit: 20.00%' \ + "$(oomctl | tac | sed -e '/Memory Pressure Monitored CGroups:/q' | tac | grep -A7 "Path: $cgroup_path")" systemctl "$@" status TEST-55-OOMD-testchill.service @@ -122,14 +125,15 @@ test_basic() { } testcase_basic_system() { - test_basic + test_basic /TEST.slice/TEST-55.slice/TEST-55-OOMD.slice/TEST-55-OOMD-workload.slice } testcase_basic_user() { # Make sure we also work correctly on user units. loginctl enable-linger testuser - test_basic --machine "testuser@.host" --user + test_basic "/user.slice/user-$(id -u testuser).slice/user@$(id -u testuser).service/TEST.slice/TEST-55.slice/TEST-55-OOMD.slice/TEST-55-OOMD-workload.slice" \ + --machine "testuser@.host" --user loginctl disable-linger testuser } From 98a0bb8a6487a1bd0767223f3738525c2b3eae52 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Wed, 16 Oct 2024 15:18:41 +0900 Subject: [PATCH 5/6] TEST-55-OOMD: stop test units when unnecessary Then, sleep becomes not necessary anymore. This greatly improve performance. --- test/units/TEST-55-OOMD.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index ac496430746..84f4283fbfc 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -122,6 +122,11 @@ test_basic() { # testbloat should be killed and testchill should be fine if systemctl "$@" status TEST-55-OOMD-testbloat.service; then exit 42; fi if ! systemctl "$@" status TEST-55-OOMD-testchill.service; then exit 24; fi + + systemctl "$@" kill --signal=KILL TEST-55-OOMD-testbloat.service || : + systemctl "$@" stop TEST-55-OOMD-testbloat.service + systemctl "$@" stop TEST-55-OOMD-testchill.service + systemctl "$@" stop TEST-55-OOMD-workload.slice } testcase_basic_system() { @@ -145,8 +150,6 @@ testcase_preference_avoid() { return 0 fi - sleep 120 # wait for systemd-oomd kill cool down and elevated memory pressure to come down - mkdir -p /run/systemd/system/TEST-55-OOMD-testbloat.service.d/ cat >/run/systemd/system/TEST-55-OOMD-testbloat.service.d/99-managed-oom-preference.conf < Date: Wed, 16 Oct 2024 15:20:08 +0900 Subject: [PATCH 6/6] TEST-55-OOMD: check slice property before stressing slice --- test/units/TEST-55-OOMD.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 84f4283fbfc..c615e7a4b22 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -100,14 +100,15 @@ test_basic() { shift systemctl "$@" start TEST-55-OOMD-testchill.service - systemctl "$@" start TEST-55-OOMD-testbloat.service + systemctl "$@" status TEST-55-OOMD-testchill.service + systemctl "$@" status TEST-55-OOMD-workload.slice # Verify systemd-oomd is monitoring the expected units. timeout 1m bash -xec "until oomctl | grep -q -F 'Path: $cgroup_path'; do sleep 1; done" assert_in 'Memory Pressure Limit: 20.00%' \ "$(oomctl | tac | sed -e '/Memory Pressure Monitored CGroups:/q' | tac | grep -A7 "Path: $cgroup_path")" - systemctl "$@" status TEST-55-OOMD-testchill.service + systemctl "$@" start TEST-55-OOMD-testbloat.service # systemd-oomd watches for elevated pressure for 2 seconds before acting. # It can take time to build up pressure so either wait 2 minutes or for the service to fail.