mirror of
https://github.com/containerd/containerd.git
synced 2026-08-09 01:21:15 +00:00
Merge pull request #13871 from samuelkarp/remove_deprecated_checkpoint_restore
cri: remove restore in CreateContainer
This commit is contained in:
Samuel Karp
17
.github/workflows/ci.yml
vendored
17
.github/workflows/ci.yml
vendored
@@ -555,23 +555,6 @@ jobs:
|
||||
env
|
||||
sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"
|
||||
|
||||
- name: Install tools
|
||||
# The GitHub Actions image already has buildah and podman included. Not the
|
||||
# ubuntu-24.04-arm images. buildah and podman is needed to convert the
|
||||
# Kubernetes checkpoint archive to an OCI image.
|
||||
# See contrib/checkpoint/checkpoint-restore-cri-test.sh
|
||||
if: matrix.os == 'ubuntu-24.04-arm'
|
||||
run: sudo apt-get install -y buildah podman
|
||||
|
||||
- if: matrix.os != 'ubuntu-24.04-arm'
|
||||
name: Checkpoint/Restore via CRI
|
||||
env:
|
||||
TEST_RUNTIME: ${{ matrix.runtime }}
|
||||
CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
|
||||
run: |
|
||||
env
|
||||
sudo -E PATH=$PATH ./contrib/checkpoint/checkpoint-restore-cri-test.sh
|
||||
|
||||
- if: matrix.os != 'ubuntu-24.04-arm'
|
||||
name: Checkpoint/Restore Disable via CRI
|
||||
env:
|
||||
|
||||
@@ -613,7 +613,7 @@ The deprecated features are shown in the following table:
|
||||
| Go-Plugin library (`*.so`) as containerd runtime plugin | containerd v2.0 | containerd v2.1 ✅ | Use external plugins (proxy or binary) |
|
||||
| NRI v0.1.0 plugin support | containerd v2.2 | containerd v2.3 | Use the v010-adapter NRI plugin, or update v0.1.0 plugins to use the current NRI API |
|
||||
| cgroup v1 support | containerd v2.2 | (May 2029) | Use cgroup v2 |
|
||||
| Restoring checkpoint data during CRI `CreateContainer` | containerd v2.3 | containerd v2.4 | Follow [KEP-5823](https://github.com/kubernetes/enhancements/issues/5823) for a replacement `RestorePod` API |
|
||||
| Restoring checkpoint data during CRI `CreateContainer` | containerd v2.3 | containerd v2.4 ✅ | Follow [KEP-5823](https://github.com/kubernetes/enhancements/issues/5823) for a replacement `RestorePod` API |
|
||||
|
||||
- Pulling Schema 1 images has been disabled in containerd v2.0, but it still can be enabled by setting an environment variable `CONTAINERD_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=1`
|
||||
until containerd v2.1. `ctr` users have to specify `--local` too (e.g., `ctr images pull --local`). Users of CRI clients (such as Kubernetes and `crictl`) have to specify this environment variable on the containerd daemon (usually in the systemd unit).
|
||||
|
||||
@@ -201,87 +201,33 @@ fi
|
||||
echo "PASS: Test 1: Checkpoint fails fast and source container remains running"
|
||||
cleanup_container "$pod_id" "$ctr_id"
|
||||
|
||||
# Test 2: Restore fails fast when enable_criu = false.
|
||||
POD_JSON=$(mktemp)
|
||||
jq ".log_directory=\"${TESTDIR}\"" "$TESTDATA"/sandbox_config.json >"$POD_JSON"
|
||||
pod_id=$(crictl runp "$POD_JSON")
|
||||
touch "$TESTDIR/dummy-checkpoint.tar"
|
||||
RESTORE_JSON=$(mktemp)
|
||||
jq ".image.image=\"$TESTDIR/dummy-checkpoint.tar\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON"
|
||||
set +e
|
||||
output=$(crictl create "$pod_id" "$RESTORE_JSON" "$POD_JSON" 2>&1)
|
||||
exit_code=$?
|
||||
set -e
|
||||
rm -f "$RESTORE_JSON" "$POD_JSON"
|
||||
if [ $exit_code -eq 0 ] || [[ ! "$output" =~ "criu support is disabled by configuration" ]]; then
|
||||
echo "ERROR: Test 2 failed (restore did not fail fast). Output: $output"
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: Test 2: Restore fails fast when enable_criu = false"
|
||||
crictl rmp -f "$pod_id" >/dev/null 2>&1 || true
|
||||
|
||||
stop_containerd
|
||||
|
||||
# ==============================================================================
|
||||
# Group 2 (Configuration: enable_criu = true, normal PATH)
|
||||
# ==============================================================================
|
||||
start_containerd "true" ""
|
||||
|
||||
# Test 3: Normal checkpoint and restore preserves container state.
|
||||
ids=$(setup_container)
|
||||
pod_id=$(echo "$ids" | cut -d: -f1)
|
||||
ctr_id=$(echo "$ids" | cut -d: -f2)
|
||||
crictl exec "$ctr_id" touch /root/state_file
|
||||
rm -f "$TESTDIR"/state_checkpoint.tar
|
||||
crictl checkpoint --export="$TESTDIR"/state_checkpoint.tar "${ctr_id}"
|
||||
cleanup_container "$pod_id" "$ctr_id"
|
||||
|
||||
POD_JSON=$(mktemp)
|
||||
jq ".log_directory=\"${TESTDIR}\"" "$TESTDATA"/sandbox_config.json >"$POD_JSON"
|
||||
pod_id=$(crictl runp "$POD_JSON")
|
||||
RESTORE_JSON=$(mktemp)
|
||||
jq ".image.image=\"$TESTDIR/state_checkpoint.tar\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON"
|
||||
restored_ctr_id=$(crictl create "$pod_id" "$RESTORE_JSON" "$POD_JSON")
|
||||
rm -f "$RESTORE_JSON" "$POD_JSON"
|
||||
crictl start "$restored_ctr_id"
|
||||
set +e
|
||||
crictl exec "$restored_ctr_id" ls /root/state_file >/dev/null 2>&1
|
||||
exit_code=$?
|
||||
set -e
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "ERROR: Test 3 failed (state_file not found in restored container)."
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: Test 3: Normal checkpoint and restore preserves container state"
|
||||
cleanup_container "$pod_id" "$restored_ctr_id"
|
||||
|
||||
stop_containerd
|
||||
|
||||
# ==============================================================================
|
||||
# Group 3 (Configuration: enable_criu omitted/defaults, normal PATH)
|
||||
# Group 2 (Configuration: enable_criu omitted/defaults, normal PATH)
|
||||
# ==============================================================================
|
||||
start_containerd "omit" ""
|
||||
|
||||
# Test 4: enable_criu omitted from configuration defaults to true (allowing checkpoint/restore).
|
||||
# Test 2: enable_criu omitted from configuration defaults to true (allowing checkpoint).
|
||||
ids=$(setup_container)
|
||||
pod_id=$(echo "$ids" | cut -d: -f1)
|
||||
ctr_id=$(echo "$ids" | cut -d: -f2)
|
||||
rm -f "$TESTDIR"/omitted_checkpoint.tar
|
||||
crictl checkpoint --export="$TESTDIR"/omitted_checkpoint.tar "${ctr_id}"
|
||||
echo "PASS: Test 4: enable_criu omitted from configuration defaults to true"
|
||||
echo "PASS: Test 2: enable_criu omitted from configuration defaults to true"
|
||||
cleanup_container "$pod_id" "$ctr_id"
|
||||
|
||||
stop_containerd
|
||||
|
||||
# ==============================================================================
|
||||
# Group 4 (Configuration: enable_criu = true, cleaned PATH without CRIU)
|
||||
# Group 3 (Configuration: enable_criu = true, cleaned PATH without CRIU)
|
||||
# ==============================================================================
|
||||
if [ -n "${CRIU_DIR}" ]; then
|
||||
CLEANED_PATH=$(get_cleaned_path)
|
||||
|
||||
start_containerd "true" "${CLEANED_PATH}"
|
||||
|
||||
# Test 5: CRIU missing from PATH, enable_criu = true.
|
||||
# Test 3: CRIU missing from PATH, enable_criu = true.
|
||||
# Verifies that when CRIU is enabled but missing, it fails with the binary missing error.
|
||||
ids=$(setup_container)
|
||||
pod_id=$(echo "$ids" | cut -d: -f1)
|
||||
@@ -291,10 +237,10 @@ if [ -n "${CRIU_DIR}" ]; then
|
||||
exit_code=$?
|
||||
set -e
|
||||
if [ $exit_code -eq 0 ] || [[ ! "$output" =~ "criu binary not found" ]]; then
|
||||
echo "ERROR: Test 5 failed. Output: $output"
|
||||
echo "ERROR: Test 3 failed. Output: $output"
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: Test 5: Fails with binary not found as expected when enable_criu = true"
|
||||
echo "PASS: Test 3: Fails with binary not found as expected when enable_criu = true"
|
||||
cleanup_container "$pod_id" "$ctr_id"
|
||||
|
||||
stop_containerd
|
||||
|
||||
@@ -1,265 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
DIR="$(dirname "${0}")"
|
||||
|
||||
cd "${DIR}"
|
||||
go build -o checkcriu
|
||||
|
||||
if ! "./checkcriu"; then
|
||||
echo >&2 "ERROR: CRIU check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e "$(command -v crictl)" ]; then
|
||||
echo >&2 "ERROR: crictl binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TESTDIR=$(mktemp -d -p "${PWD}")
|
||||
|
||||
SUCCESS=0
|
||||
|
||||
function cleanup() {
|
||||
rm -f ./checkcriu
|
||||
crictl ps -a || true
|
||||
pkill containerd || true
|
||||
if [ "${SUCCESS}" == "1" ]; then
|
||||
echo PASS
|
||||
else
|
||||
echo "--> containerd logs"
|
||||
sed 's/^/----> \t/' <"${TESTDIR}/containerd.log"
|
||||
echo FAIL
|
||||
fi
|
||||
umount "$(find "${TESTDIR}" -name shm -type d | head -1)" >/dev/null 2>&1 || true
|
||||
umount "$(find "${TESTDIR}" -name rootfs -type d | head -1)" >/dev/null 2>&1 || true
|
||||
rm -rf "${TESTDIR}" || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
TESTDATA=testdata
|
||||
# shellcheck disable=SC2034
|
||||
export CONTAINERD_ADDRESS="$TESTDIR/c.sock"
|
||||
export CONTAINER_RUNTIME_ENDPOINT="unix:///${CONTAINERD_ADDRESS}"
|
||||
|
||||
# Generate crictl config file with 30s timeout
|
||||
export CRI_CONFIG_FILE="${TESTDIR}/crictl.yaml"
|
||||
cat <<EOF > "${CRI_CONFIG_FILE}"
|
||||
runtime-endpoint: unix://${CONTAINERD_ADDRESS}
|
||||
image-endpoint: unix://${CONTAINERD_ADDRESS}
|
||||
timeout: 30
|
||||
EOF
|
||||
|
||||
TEST_IMAGE=ghcr.io/containerd/alpine
|
||||
|
||||
function test_from_archive() {
|
||||
echo "--> Deleting all pods: "
|
||||
crictl -t 5s rmp -fa | sed 's/^/----> \t/'
|
||||
echo -n "--> Pulling base image ${TEST_IMAGE}: "
|
||||
crictl pull "${TEST_IMAGE}"
|
||||
POD_JSON=$(mktemp)
|
||||
# adapt the log directory
|
||||
jq ".log_directory=\"${TESTDIR}\"" "$TESTDATA"/sandbox_config.json >"$POD_JSON"
|
||||
echo -n "--> Start pod: "
|
||||
pod_id=$(crictl runp "$POD_JSON")
|
||||
echo "$pod_id"
|
||||
CTR_JSON=$(mktemp)
|
||||
jq '.annotations = {"cdi.k8s.io/device":"gpu","safe.annotation":"true"}' "$TESTDATA"/container_sleep.json >"$CTR_JSON"
|
||||
echo -n "--> Create container: "
|
||||
ctr_id=$(crictl create "$pod_id" "$CTR_JSON" "$POD_JSON")
|
||||
echo "$ctr_id"
|
||||
rm -f "$CTR_JSON"
|
||||
echo -n "--> Start container: "
|
||||
crictl start "$ctr_id"
|
||||
lines_before=$(crictl logs "$ctr_id" | wc -l)
|
||||
# changes file system to see if changes are included in the checkpoint
|
||||
echo "--> Modifying container rootfs"
|
||||
crictl exec "$ctr_id" touch /root/testfile
|
||||
crictl exec "$ctr_id" rm /etc/motd
|
||||
echo -n "--> Checkpointing container: "
|
||||
crictl -t 10s checkpoint --export="$TESTDIR"/cp.tar "$ctr_id"
|
||||
echo "--> Cleanup container: "
|
||||
crictl rm -f "$ctr_id" | sed 's/^/----> \t/'
|
||||
echo "--> Cleanup pod: "
|
||||
crictl rmp -f "$pod_id" | sed 's/^/----> \t/'
|
||||
echo "--> Cleanup images: "
|
||||
crictl rmi "${TEST_IMAGE}" | sed 's/^/----> \t/'
|
||||
echo -n "--> Start pod: "
|
||||
pod_id=$(crictl runp "$POD_JSON")
|
||||
echo "$pod_id"
|
||||
# Replace original container with checkpoint image
|
||||
RESTORE_JSON=$(mktemp)
|
||||
jq ".image.image=\"$TESTDIR/cp.tar\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON"
|
||||
echo -n "--> Create container from checkpoint: "
|
||||
# This requires a larger timeout as we just deleted the image and
|
||||
# pulling can take some time.
|
||||
ctr_id=$(crictl -t 30s create "$pod_id" "$RESTORE_JSON" "$POD_JSON")
|
||||
echo "$ctr_id"
|
||||
rm -f "$RESTORE_JSON" "$POD_JSON"
|
||||
echo -n "--> Start container from checkpoint: "
|
||||
crictl start "$ctr_id"
|
||||
sleep 1
|
||||
lines_after=$(crictl logs "$ctr_id" | wc -l)
|
||||
if [ "$lines_before" -ge "$lines_after" ]; then
|
||||
echo "number of lines after checkpointing ($lines_after) " \
|
||||
"should be larger than before checkpointing ($lines_before)"
|
||||
false
|
||||
fi
|
||||
echo "--> Verifying CDI annotation filtering on restore: "
|
||||
actual_annots=$(crictl inspect "$ctr_id" | jq -c '.status.annotations')
|
||||
if jq -e 'has("cdi.k8s.io/device") or (has("safe.annotation") | not)' <<<"$actual_annots" >/dev/null; then
|
||||
echo "error: CDI annotation was not filtered or safe annotation missing: $actual_annots"
|
||||
exit 1
|
||||
fi
|
||||
echo "--> Verifying deprecation warning via API (archive): "
|
||||
archive_ts=$(../../bin/ctr deprecations list --format=json | jq -r '.[] | select(.id == "io.containerd.deprecation/cri-create-container-checkpoint-restore") | .lastOccurrence')
|
||||
if [ -z "$archive_ts" ] || [ "$archive_ts" = "null" ]; then
|
||||
echo "error: CRICreateContainerCheckpointRestore deprecation warning not found in API introspection (archive)"
|
||||
exit 1
|
||||
fi
|
||||
# Cleanup
|
||||
echo "--> Cleanup images: "
|
||||
(crictl rmi "${TEST_IMAGE}" || true) | sed 's/^/----> \t/'
|
||||
echo -n "--> Verifying container rootfs: "
|
||||
crictl exec "$ctr_id" ls -la /root/testfile
|
||||
if crictl exec "$ctr_id" ls -la /etc/motd >/dev/null 2>&1; then
|
||||
echo "error: file /etc/motd should not exist but it does"
|
||||
exit 1
|
||||
fi
|
||||
echo "--> Deleting all pods: "
|
||||
crictl -t 5s rmp -fa | sed 's/^/----> \t/'
|
||||
SUCCESS=1
|
||||
}
|
||||
|
||||
function test_from_oci() {
|
||||
echo "--> Deleting all pods: "
|
||||
crictl -t 5s rmp -fa | sed 's/^/----> \t/'
|
||||
echo -n "--> Pulling base image ${TEST_IMAGE}: "
|
||||
crictl pull "${TEST_IMAGE}"
|
||||
echo -n "--> Start pod: "
|
||||
pod_id=$(crictl runp "$TESTDATA"/sandbox_config.json)
|
||||
echo "$pod_id"
|
||||
echo -n "--> Create container: "
|
||||
ctr_id=$(crictl create "$pod_id" "$TESTDATA"/container_sleep.json "$TESTDATA"/sandbox_config.json)
|
||||
echo "$ctr_id"
|
||||
echo -n "--> Start container: "
|
||||
crictl start "$ctr_id"
|
||||
echo -n "--> Checkpointing container: "
|
||||
crictl -t 10s checkpoint --export="$TESTDIR"/cp.tar "$ctr_id"
|
||||
echo "--> Cleanup container: "
|
||||
crictl rm -f "$ctr_id" | sed 's/^/----> \t/'
|
||||
echo "--> Cleanup pod: "
|
||||
crictl rmp -f "$pod_id" | sed 's/^/----> \t/'
|
||||
echo "--> Cleanup pod: "
|
||||
crictl rmi "${TEST_IMAGE}" | sed 's/^/----> \t/'
|
||||
# Change cgroup of new sandbox
|
||||
RESTORE_POD_JSON=$(mktemp)
|
||||
jq ".linux.cgroup_parent=\"different_cgroup_789\"" "$TESTDATA"/sandbox_config.json >"$RESTORE_POD_JSON"
|
||||
echo -n "--> Start pod: "
|
||||
pod_id=$(crictl runp "$RESTORE_POD_JSON")
|
||||
echo "$pod_id"
|
||||
# Replace original container with checkpoint image
|
||||
RESTORE_JSON=$(mktemp)
|
||||
# Convert tar checkpoint archive to OCI image
|
||||
echo "--> Convert checkpoint archive $TESTDIR/cp.tar to OCI image 'checkpoint-image:latest': "
|
||||
newcontainer=$(buildah from scratch)
|
||||
echo -n "----> Add checkpoint archive to new OCI image: "
|
||||
buildah add "$newcontainer" "$TESTDIR"/cp.tar /
|
||||
echo "----> Add checkpoint annotation to new OCI image: "
|
||||
buildah config --annotation=org.criu.checkpoint.container.name=test "$newcontainer"
|
||||
echo "----> Save new OCI image: "
|
||||
buildah commit -q "$newcontainer" checkpoint-image:latest 2>&1 | sed 's/^/------> \t/'
|
||||
echo "----> Cleanup temporary images: "
|
||||
buildah rm "$newcontainer" | sed 's/^/------> \t/'
|
||||
# Export OCI image to disk
|
||||
echo "----> Export OCI image to disk: "
|
||||
podman image save -q --format oci-archive -o "$TESTDIR"/oci.tar localhost/checkpoint-image:latest | sed 's/^/------> \t/'
|
||||
echo "----> Cleanup temporary images: "
|
||||
buildah rmi localhost/checkpoint-image:latest | sed 's/^/------> \t/'
|
||||
# Remove potentially old version of the checkpoint image
|
||||
echo "----> Cleanup potential old copies: "
|
||||
../../bin/ctr -n k8s.io images rm --sync localhost/checkpoint-image:latest 2>&1 | sed 's/^/------> \t/'
|
||||
# Import image
|
||||
echo "----> Import new image: "
|
||||
../../bin/ctr -n k8s.io images import "$TESTDIR"/oci.tar 2>&1 | sed 's/^/------> \t/'
|
||||
jq ".image.image=\"localhost/checkpoint-image:latest\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON"
|
||||
echo -n "--> Create container from checkpoint: "
|
||||
ctr_id=$(crictl -t 30s create "$pod_id" "$RESTORE_JSON" "$RESTORE_POD_JSON")
|
||||
echo "$ctr_id"
|
||||
rm -f "$RESTORE_JSON" "$RESTORE_POD_JSON"
|
||||
echo -n "--> Start container from checkpoint: "
|
||||
crictl start "$ctr_id"
|
||||
echo "--> Verifying deprecation warning via API (oci): "
|
||||
oci_ts=$(../../bin/ctr deprecations list --format=json | jq -r '.[] | select(.id == "io.containerd.deprecation/cri-create-container-checkpoint-restore") | .lastOccurrence')
|
||||
if [ -z "$oci_ts" ] || [ "$oci_ts" = "null" ]; then
|
||||
echo "error: CRICreateContainerCheckpointRestore deprecation warning not found in API introspection (oci)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$archive_ts" = "$oci_ts" ]; then
|
||||
echo "error: expected lastOccurrence to update after OCI restore (was $archive_ts, now $oci_ts)"
|
||||
exit 1
|
||||
fi
|
||||
# Cleanup
|
||||
echo "--> Cleanup images: "
|
||||
../../bin/ctr -n k8s.io images rm localhost/checkpoint-image:latest | sed 's/^/----> \t/'
|
||||
echo "--> Cleanup images: "
|
||||
(crictl rmi "${TEST_IMAGE}" || true) | sed 's/^/----> \t/'
|
||||
echo "--> Deleting all pods: "
|
||||
crictl -t 5s rmp -fa | sed 's/^/----> \t/'
|
||||
SUCCESS=1
|
||||
}
|
||||
|
||||
cat >"${TESTDIR}/config.toml" <<EOF
|
||||
version = 3
|
||||
[plugins."io.containerd.cri.v1.runtime"]
|
||||
enable_cdi = false
|
||||
[plugins."io.containerd.cri.v1.runtime".containerd]
|
||||
default_runtime_name = "test-runtime"
|
||||
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.test-runtime]
|
||||
runtime_type = "${TEST_RUNTIME:-io.containerd.runc.v2}"
|
||||
EOF
|
||||
|
||||
mkdir -p "${TESTDIR}"/{root,state}
|
||||
|
||||
echo "--> Starting containerd: "
|
||||
../../bin/containerd \
|
||||
--address "${TESTDIR}/c.sock" \
|
||||
--config "${TESTDIR}/config.toml" \
|
||||
--root "${TESTDIR}/root" \
|
||||
--state "${TESTDIR}/state" \
|
||||
--log-level trace &>"${TESTDIR}/containerd.log" &
|
||||
# Make sure containerd is ready before calling critest.
|
||||
retry_counter=0
|
||||
max_retries=10
|
||||
while true; do
|
||||
((retry_counter += 1))
|
||||
if crictl info 2>&1 | sed 's/^/----> \t/'; then
|
||||
break
|
||||
else
|
||||
sleep 1.5
|
||||
fi
|
||||
if [ "${retry_counter}" -gt "${max_retries}" ]; then
|
||||
echo "--> Failed to start containerd"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
test_from_archive
|
||||
SUCCESS=0
|
||||
test_from_oci
|
||||
@@ -1,231 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
DIR="$(dirname "${0}")"
|
||||
|
||||
cd "${DIR}"
|
||||
go build -o checkcriu
|
||||
|
||||
if ! "./checkcriu"; then
|
||||
echo >&2 "ERROR: CRIU check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e "$(command -v crictl)" ]; then
|
||||
echo >&2 "ERROR: crictl binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e "$(command -v kubectl)" ]; then
|
||||
echo >&2 "ERROR: kubectl binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e "$(command -v ctr)" ]; then
|
||||
echo >&2 "ERROR: ctr binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TESTDIR=$(mktemp -d)
|
||||
OUTPUT=$(mktemp)
|
||||
SUCCESS=0
|
||||
|
||||
function cleanup() {
|
||||
# shellcheck disable=SC2317
|
||||
rm -f ./checkcriu
|
||||
# shellcheck disable=SC2317
|
||||
rm -rf "${TESTDIR}" "${OUTPUT}"
|
||||
# shellcheck disable=SC2317
|
||||
if [ "${SUCCESS}" == "1" ]; then
|
||||
# shellcheck disable=SC2317
|
||||
echo PASS
|
||||
else
|
||||
# shellcheck disable=SC2317
|
||||
echo FAIL
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
TESTDATA=testdata
|
||||
export CONTAINER_RUNTIME_ENDPOINT="unix:///run/containerd/containerd.sock"
|
||||
export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig
|
||||
|
||||
echo -n "--> Cleanup test pod/containers from previous run: "
|
||||
kubectl delete pod sleeper --grace-period=1 || true
|
||||
echo -n "--> Create new test pod/containers: "
|
||||
kubectl apply -f $TESTDATA/sleep.yaml
|
||||
|
||||
echo "--> Wait until test pod/containers are ready: "
|
||||
while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].started}")" == "false" ]; do
|
||||
echo "----> Waiting for pod/container sleeper/sleep to get ready"
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "--> Do curl request to the test container: "
|
||||
curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
COUNTER_BEFORE=$(cut -d\ -f2 <"${OUTPUT}")
|
||||
echo "--> Check logs of test container: "
|
||||
kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
LINES_BEFORE=$(wc -l <"${OUTPUT}")
|
||||
|
||||
if [ ! -e /var/run/kubernetes/client-admin.crt ]; then
|
||||
echo "Missing /var/run/kubernetes/client-admin.crt. Exiting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e /var/run/kubernetes/client-admin.key ]; then
|
||||
echo "Missing /var/run/kubernetes/client-admin.key. Exiting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -n "--> Creating checkpoint: "
|
||||
CP=$(curl -s --insecure --cert /var/run/kubernetes/client-admin.crt --key /var/run/kubernetes/client-admin.key -X POST "https://localhost:10250/checkpoint/default/sleeper/sleep" | jq -r ".items[0]")
|
||||
echo "$CP"
|
||||
|
||||
echo -n "--> Cleanup test pod/containers: "
|
||||
kubectl delete pod sleeper --grace-period=1
|
||||
|
||||
OCI="localhost/checkpoint-image:latest"
|
||||
|
||||
echo "--> Converting checkpoint archive $CP to OCI images $OCI"
|
||||
|
||||
newcontainer=$(buildah from scratch)
|
||||
echo -n "----> Add checkpoint archive to new OCI image: "
|
||||
buildah add "$newcontainer" "$CP" /
|
||||
echo "----> Add checkpoint annotation to new OCI image: "
|
||||
buildah config --annotation=org.criu.checkpoint.container.name=test "$newcontainer"
|
||||
echo "----> Save new OCI image: "
|
||||
buildah commit -q "$newcontainer" "$OCI" 2>&1 | sed 's/^/------> \t/'
|
||||
echo "----> Cleanup temporary images: "
|
||||
buildah rm "$newcontainer" | sed 's/^/------> \t/'
|
||||
# Export OCI image to disk
|
||||
echo "----> Export OCI image to disk: "
|
||||
podman image save -q --format oci-archive -o "$TESTDIR"/oci.tar "$OCI" | sed 's/^/------> \t/'
|
||||
echo "----> Cleanup temporary images: "
|
||||
buildah rmi "$OCI" | sed 's/^/------> \t/'
|
||||
# Remove potentially old version of the checkpoint image
|
||||
echo "----> Cleanup potential old copies: "
|
||||
ctr -n k8s.io images rm --sync localhost/checkpoint-image:latest 2>&1 | sed 's/^/------> \t/'
|
||||
# Import image
|
||||
echo "----> Import checkpoint image: "
|
||||
ctr -n k8s.io images import "$TESTDIR"/oci.tar 2>&1 | sed 's/^/------> \t/'
|
||||
|
||||
echo -n "--> Create pod/containers from checkpoint: "
|
||||
kubectl apply -f $TESTDATA/sleep-restore.yaml
|
||||
|
||||
echo "--> Wait until test pod/containers are ready: "
|
||||
while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].started}")" == "false" ]; do
|
||||
echo "----> Waiting for pod/container sleeper/sleep to get ready"
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "--> Do curl request to the test container: "
|
||||
curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
COUNTER_AFTER=$(cut -d\ -f2 <"${OUTPUT}")
|
||||
echo "--> Check logs of test container: "
|
||||
kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
LINES_AFTER=$(wc -l <"${OUTPUT}")
|
||||
|
||||
if [ "$LINES_BEFORE" -ge "$LINES_AFTER" ]; then
|
||||
echo "number of lines after checkpointing ($LINES_AFTER) " \
|
||||
"should be larger than before checkpointing ($LINES_BEFORE)"
|
||||
false
|
||||
fi
|
||||
|
||||
if [ "$COUNTER_BEFORE" -ge "$COUNTER_AFTER" ]; then
|
||||
echo "number of lines after checkpointing ($COUNTER_AFTER) " \
|
||||
"should be larger than before checkpointing ($COUNTER_BEFORE)"
|
||||
false
|
||||
fi
|
||||
|
||||
# Let's see if container restart also works correctly
|
||||
CONTAINER_ID=$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].containerID}" | cut -d '/' -f 3)
|
||||
CONTAINER_PID=$(crictl inspect "${CONTAINER_ID}" | jq .info.pid)
|
||||
|
||||
echo "--> Kill container $CONTAINER_ID by killing PID $CONTAINER_PID"
|
||||
# Kill PID in containner
|
||||
kill -9 "${CONTAINER_PID}"
|
||||
|
||||
# wait for the replacement container to come up
|
||||
echo "--> Wait until replacement pod/containers is started: "
|
||||
while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].restartCount}")" != "1" ]; do
|
||||
echo "----> Waiting for pod/container sleeper/sleep to get ready"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "--> Do curl request to the test container: "
|
||||
curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
COUNTER_AFTER=$(cut -d\ -f2 <"${OUTPUT}")
|
||||
echo "--> Check logs of test container: "
|
||||
kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
LINES_AFTER=$(wc -l <"${OUTPUT}")
|
||||
|
||||
if [ "$LINES_BEFORE" -ge "$LINES_AFTER" ]; then
|
||||
echo "number of lines after checkpointing ($LINES_AFTER) " \
|
||||
"should be larger than before checkpointing ($LINES_BEFORE)"
|
||||
false
|
||||
fi
|
||||
|
||||
if [ "$COUNTER_BEFORE" -ge "$COUNTER_AFTER" ]; then
|
||||
echo "number of lines after checkpointing ($COUNTER_AFTER) " \
|
||||
"should be larger than before checkpointing ($COUNTER_BEFORE)"
|
||||
false
|
||||
fi
|
||||
|
||||
# Let's see if container restart also works correctly a second time
|
||||
|
||||
CONTAINER_ID=$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].containerID}" | cut -d '/' -f 3)
|
||||
CONTAINER_PID=$(crictl inspect "${CONTAINER_ID}" | jq .info.pid)
|
||||
|
||||
# Kill PID in containner
|
||||
echo "--> Kill container $CONTAINER_ID by killing PID $CONTAINER_PID"
|
||||
kill -9 "${CONTAINER_PID}"
|
||||
|
||||
# wait for the replacement container to come up
|
||||
echo "--> Wait until replacement pod/containers is started: "
|
||||
while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].restartCount}")" != "2" ]; do
|
||||
echo "----> Waiting for pod/container sleeper/sleep to get ready"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "--> Do curl request to the test container: "
|
||||
curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
COUNTER_AFTER=$(cut -d\ -f2 <"${OUTPUT}")
|
||||
echo "--> Check logs of test container: "
|
||||
kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/'
|
||||
LINES_AFTER=$(wc -l <"${OUTPUT}")
|
||||
|
||||
if [ "$LINES_BEFORE" -ge "$LINES_AFTER" ]; then
|
||||
echo "number of lines after checkpointing ($LINES_AFTER) " \
|
||||
"should be larger than before checkpointing ($LINES_BEFORE)"
|
||||
false
|
||||
fi
|
||||
|
||||
if [ "$COUNTER_BEFORE" -ge "$COUNTER_AFTER" ]; then
|
||||
echo "number of lines after checkpointing ($COUNTER_AFTER) " \
|
||||
"should be larger than before checkpointing ($COUNTER_BEFORE)"
|
||||
false
|
||||
fi
|
||||
|
||||
echo -n "--> Creating checkpoint from restored container: "
|
||||
CP=$(curl -s --insecure --cert /var/run/kubernetes/client-admin.crt --key /var/run/kubernetes/client-admin.key -X POST "https://localhost:10250/checkpoint/default/sleeper/sleep" | jq -r ".items[0]")
|
||||
echo "$CP"
|
||||
|
||||
SUCCESS=1
|
||||
|
||||
exit 0
|
||||
10
contrib/checkpoint/testdata/sleep-restore.yaml
vendored
10
contrib/checkpoint/testdata/sleep-restore.yaml
vendored
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: sleeper
|
||||
spec:
|
||||
containers:
|
||||
- name: sleep
|
||||
image: localhost/checkpoint-image:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
restartPolicy: Always
|
||||
@@ -22,27 +22,11 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
containerstore "github.com/containerd/containerd/v2/internal/cri/store/container"
|
||||
"github.com/containerd/containerd/v2/internal/cri/store/sandbox"
|
||||
"github.com/containerd/errdefs"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
)
|
||||
|
||||
func (c *criService) checkIfCheckpointOCIImage(ctx context.Context, input string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (c *criService) CRImportCheckpoint(
|
||||
ctx context.Context,
|
||||
meta *containerstore.Metadata,
|
||||
sandbox *sandbox.Sandbox,
|
||||
sandboxConfig *runtime.PodSandboxConfig,
|
||||
) (ctrID string, retErr error) {
|
||||
return "", errdefs.ErrNotImplemented
|
||||
}
|
||||
|
||||
func (c *criService) CheckpointContainer(ctx context.Context, r *runtime.CheckpointContainerRequest) (res *runtime.CheckpointContainerResponse, err error) {
|
||||
// The next line is just needed to make the linter happy.
|
||||
containerCheckpointTimer.WithValues("no-runtime").UpdateSince(time.Now())
|
||||
|
||||
@@ -38,25 +38,13 @@ import (
|
||||
"github.com/containerd/containerd/v2/client"
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
"github.com/containerd/containerd/v2/core/mount"
|
||||
"github.com/containerd/containerd/v2/internal/cri/annotations"
|
||||
crilabels "github.com/containerd/containerd/v2/internal/cri/labels"
|
||||
containerstore "github.com/containerd/containerd/v2/internal/cri/store/container"
|
||||
imagestore "github.com/containerd/containerd/v2/internal/cri/store/image"
|
||||
"github.com/containerd/containerd/v2/internal/cri/store/sandbox"
|
||||
"github.com/containerd/containerd/v2/pkg/archive"
|
||||
"github.com/containerd/containerd/v2/pkg/protobuf/proto"
|
||||
ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types"
|
||||
"github.com/containerd/containerd/v2/plugins"
|
||||
"github.com/containerd/continuity/fs"
|
||||
"github.com/containerd/errdefs"
|
||||
"github.com/containerd/log"
|
||||
"github.com/containerd/platforms"
|
||||
|
||||
"github.com/distribution/reference"
|
||||
"github.com/opencontainers/image-spec/identity"
|
||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
spec "github.com/opencontainers/runtime-spec/specs-go"
|
||||
"golang.org/x/sys/unix"
|
||||
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
|
||||
@@ -102,41 +90,6 @@ func copyNoFollow(src, dst string, perm os.FileMode) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// checkpointArchiveEntryAllowed reports whether a tar entry from a checkpoint
|
||||
// archive may be unpacked. Legitimate checkpoint archives contain only regular
|
||||
// files and directories; other entry types (symlinks, hardlinks, device and fifo
|
||||
// nodes) are not produced by the checkpoint code and are rejected as a hardening
|
||||
// measure.
|
||||
func checkpointArchiveEntryAllowed(hdr *tar.Header) bool {
|
||||
switch hdr.Typeflag {
|
||||
//nolint:staticcheck // TypeRegA is deprecated but we may still receive an external tar with TypeRegA
|
||||
case tar.TypeReg, tar.TypeRegA, tar.TypeDir, tar.TypeXGlobalHeader:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// assertCheckpointDirSafe verifies that the populated restore directory contains
|
||||
// only regular files and directories.
|
||||
//
|
||||
// The OCI-image restore path copies checkpoint content into the restore dir with
|
||||
// fs.CopyDir, which (unlike the tar unpack filter) faithfully recreates any
|
||||
// symlinks and special files present in the image. Restore-time consumers open
|
||||
// paths under this directory, so non-regular entries are rejected before they run.
|
||||
func assertCheckpointDirSafe(root string) error {
|
||||
return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// d.Type() reports the entry type without following symlinks.
|
||||
if d.IsDir() || d.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("refusing to restore checkpoint: %s is not a regular file or directory", path)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *criService) checkCriu() error {
|
||||
c.checkCriuOnce.Do(func() {
|
||||
c.checkCriuErr = c.doCheckCriu()
|
||||
@@ -189,412 +142,6 @@ func resolveCriuPath(customPath string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// checkIfCheckpointOCIImage returns checks if the input refers to a checkpoint image.
|
||||
// It returns the StorageImageID of the image the input resolves to, nil otherwise.
|
||||
func (c *criService) checkIfCheckpointOCIImage(ctx context.Context, input string) (string, error) {
|
||||
if input == "" {
|
||||
return "", nil
|
||||
}
|
||||
if _, err := os.Stat(input); err == nil {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
image, err := c.LocalResolve(input)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve image %q: %w", input, err)
|
||||
}
|
||||
containerdImage, err := c.toContainerdImage(ctx, image)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get image from containerd %q: %w", input, err)
|
||||
}
|
||||
input = containerdImage.Name()
|
||||
images, err := c.client.ImageService().Get(ctx, input)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get image from containerd %q: %w", input, err)
|
||||
}
|
||||
|
||||
rawIndex, err := content.ReadBlob(ctx, c.client.ContentStore(), images.Target)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read image blob from containerd %q: %w", input, err)
|
||||
}
|
||||
|
||||
var index v1.Index
|
||||
if err = json.Unmarshal(rawIndex, &index); err != nil {
|
||||
return "", fmt.Errorf("failed to unmarshall blob into OCI index: %w", err)
|
||||
}
|
||||
|
||||
if index.Annotations == nil {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ann, ok := index.Annotations[crmetadata.CheckpointAnnotationName]
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
log.G(ctx).Infof("Found checkpoint of container %v in %v", ann, input)
|
||||
|
||||
return image.ID, nil
|
||||
}
|
||||
|
||||
func (c *criService) CRImportCheckpoint(
|
||||
ctx context.Context,
|
||||
meta *containerstore.Metadata,
|
||||
sandbox *sandbox.Sandbox,
|
||||
sandboxConfig *runtime.PodSandboxConfig,
|
||||
) (ctrID string, retErr error) {
|
||||
if err := c.checkCriu(); err != nil {
|
||||
return "", fmt.Errorf("checkpoint restore is not enabled: %w", err)
|
||||
}
|
||||
|
||||
var mountPoint string
|
||||
start := time.Now()
|
||||
// Ensure that the image to restore the checkpoint from has been provided.
|
||||
if meta.Config.Image == nil || meta.Config.Image.Image == "" {
|
||||
return "", errors.New(`attribute "image" missing from container definition`)
|
||||
}
|
||||
|
||||
inputImage := meta.Config.Image.Image
|
||||
createAnnotations := meta.Config.Annotations
|
||||
createLabels := meta.Config.Labels
|
||||
|
||||
restoreStorageImageID, err := c.checkIfCheckpointOCIImage(ctx, inputImage)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
mountPoint, err = os.MkdirTemp("", "checkpoint")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(mountPoint); err != nil {
|
||||
log.G(ctx).Errorf("Could not recursively remove %s: %q", mountPoint, err)
|
||||
}
|
||||
}()
|
||||
var archiveFile *os.File
|
||||
if restoreStorageImageID != "" {
|
||||
log.G(ctx).Debugf("Restoring from oci image %s", inputImage)
|
||||
platform, err := c.sandboxService.SandboxPlatform(ctx, sandbox.Sandboxer, sandbox.ID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to query sandbox platform: %w", err)
|
||||
}
|
||||
img, err := c.client.ImageService().Get(ctx, restoreStorageImageID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
i := client.NewImageWithPlatform(c.client, img, platforms.Only(platform))
|
||||
diffIDs, err := i.RootFS(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
chainID := identity.ChainID(diffIDs).String()
|
||||
ociRuntime, err := c.config.GetSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get sandbox runtime: %w", err)
|
||||
}
|
||||
s := c.client.SnapshotService(c.RuntimeSnapshotter(ctx, ociRuntime))
|
||||
|
||||
mounts, err := s.View(ctx, mountPoint, chainID)
|
||||
if err != nil {
|
||||
if errdefs.IsAlreadyExists(err) {
|
||||
mounts, err = s.Mounts(ctx, mountPoint)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := mount.All(mounts, mountPoint); err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
|
||||
archiveFile, err = os.Open(inputImage)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open checkpoint archive %s for import: %w", inputImage, err)
|
||||
}
|
||||
defer func(f *os.File) {
|
||||
if err := f.Close(); err != nil {
|
||||
log.G(ctx).Errorf("Unable to close file %s: %q", f.Name(), err)
|
||||
}
|
||||
}(archiveFile)
|
||||
|
||||
filter := archive.WithFilter(func(hdr *tar.Header) (bool, error) {
|
||||
// Reject entry types the checkpoint code never produces (symlinks,
|
||||
// hardlinks, device/fifo nodes) so they are not recreated on disk.
|
||||
if !checkpointArchiveEntryAllowed(hdr) {
|
||||
log.G(ctx).Warnf("Skipping unexpected checkpoint archive entry %q (type %d)", hdr.Name, hdr.Typeflag)
|
||||
return false, nil
|
||||
}
|
||||
// The checkpoint archive is unpacked twice if using a tar file directly.
|
||||
// The first time only the metadata files are relevant to prepare the
|
||||
// restore operation. This filter function ignores the large parts of
|
||||
// the checkpoint archive. This is usually the actual checkpoint
|
||||
// coming from CRIU as well as the rootfs diff tar file.
|
||||
excludePatterns := []string{
|
||||
"artifacts",
|
||||
"ctr.log",
|
||||
crmetadata.RootFsDiffTar,
|
||||
crmetadata.NetworkStatusFile,
|
||||
crmetadata.DeletedFilesFile,
|
||||
crmetadata.CheckpointDirectory,
|
||||
}
|
||||
for _, pattern := range excludePatterns {
|
||||
if strings.HasPrefix(hdr.Name, pattern) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
|
||||
_, err = archive.Apply(
|
||||
ctx,
|
||||
mountPoint,
|
||||
archiveFile,
|
||||
[]archive.ApplyOpt{filter}...,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unpacking of checkpoint archive %s failed: %w", mountPoint, err)
|
||||
}
|
||||
log.G(ctx).Debugf("Unpacked checkpoint in %s", mountPoint)
|
||||
}
|
||||
// Load spec.dump from temporary directory
|
||||
dumpSpec := new(spec.Spec)
|
||||
if _, err := crmetadata.ReadJSONFile(dumpSpec, mountPoint, crmetadata.SpecDumpFile); err != nil {
|
||||
return "", fmt.Errorf("failed to read %q: %w", crmetadata.SpecDumpFile, err)
|
||||
}
|
||||
|
||||
// Load config.dump from temporary directory
|
||||
config := new(crmetadata.ContainerConfig)
|
||||
if _, err := crmetadata.ReadJSONFile(config, mountPoint, crmetadata.ConfigDumpFile); err != nil {
|
||||
return "", fmt.Errorf("failed to read %q: %w", crmetadata.ConfigDumpFile, err)
|
||||
}
|
||||
|
||||
// Load status.dump from temporary directory
|
||||
containerStatus := new(runtime.ContainerStatus)
|
||||
if _, err := crmetadata.ReadJSONFile(containerStatus, mountPoint, crmetadata.StatusDumpFile); err != nil {
|
||||
return "", fmt.Errorf("failed to read %q: %w", crmetadata.StatusDumpFile, err)
|
||||
}
|
||||
|
||||
if meta.SandboxID == "" {
|
||||
// restore into previous sandbox
|
||||
meta.SandboxID = dumpSpec.Annotations[annotations.SandboxID]
|
||||
ctrID = config.ID
|
||||
} else {
|
||||
ctrID = ""
|
||||
}
|
||||
|
||||
ctrMetadata := runtime.ContainerMetadata{}
|
||||
|
||||
if meta.Config.Metadata != nil && meta.Config.Metadata.Name != "" {
|
||||
ctrMetadata.Name = containerStatus.GetMetadata().GetName()
|
||||
}
|
||||
|
||||
originalAnnotations := containerStatus.GetAnnotations()
|
||||
if originalAnnotations == nil {
|
||||
originalAnnotations = make(map[string]string)
|
||||
}
|
||||
originalLabels := containerStatus.GetLabels()
|
||||
|
||||
sandboxUID := sandboxConfig.GetMetadata().GetUid()
|
||||
|
||||
if sandboxUID != "" {
|
||||
if _, ok := originalLabels[crilabels.KubernetesPodUIDLabel]; ok {
|
||||
originalLabels[crilabels.KubernetesPodUIDLabel] = sandboxUID
|
||||
}
|
||||
if _, ok := originalAnnotations[crilabels.KubernetesPodUIDLabel]; ok {
|
||||
originalAnnotations[crilabels.KubernetesPodUIDLabel] = sandboxUID
|
||||
}
|
||||
}
|
||||
|
||||
if createLabels != nil {
|
||||
fixupLabels := []string{
|
||||
// Update the container name. It has already been update in metadata.Name.
|
||||
// It also needs to be updated in the container labels.
|
||||
crilabels.KubernetesContainerNameLabel,
|
||||
// Update pod name in the labels.
|
||||
crilabels.KubernetesPodNameLabel,
|
||||
// Also update namespace.
|
||||
crilabels.KubernetesPodNamespaceLabel,
|
||||
}
|
||||
|
||||
for _, annotation := range fixupLabels {
|
||||
_, ok1 := createLabels[annotation]
|
||||
_, ok2 := originalLabels[annotation]
|
||||
|
||||
// If the value is not set in the original container or
|
||||
// if it is not set in the new container, just skip
|
||||
// the step of updating metadata.
|
||||
if ok1 && ok2 {
|
||||
originalLabels[annotation] = createLabels[annotation]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
originalAnnotations = filterAndMergeAnnotations(
|
||||
ctx,
|
||||
originalAnnotations,
|
||||
createAnnotations,
|
||||
)
|
||||
|
||||
var containerdImage client.Image
|
||||
|
||||
containerdImage, err = c.client.GetImage(ctx, config.RootfsImageRef)
|
||||
if err != nil {
|
||||
if !errdefs.IsNotFound(err) {
|
||||
return "", fmt.Errorf("failed to get checkpoint base image %s: %w", config.RootfsImageRef, err)
|
||||
}
|
||||
// Pulling the image the checkpoint is based on. This is a bit different
|
||||
// than automatic image pulling. The checkpoint image is not automatically
|
||||
// pulled, but the image the checkpoint is based on.
|
||||
// During checkpointing the base image of the checkpoint is stored in the
|
||||
// checkpoint archive as NAME@DIGEST. The checkpoint archive also contains
|
||||
// the tag with which it was initially pulled.
|
||||
// First step is to pull NAME@DIGEST
|
||||
containerdImage, err = c.client.Pull(ctx, config.RootfsImageRef)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to pull checkpoint base image %s: %w", config.RootfsImageRef, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := reference.ParseAnyReference(config.RootfsImageName); err != nil {
|
||||
return "", fmt.Errorf("error parsing reference: %q is not a valid repository/tag %v", config.RootfsImageName, err)
|
||||
}
|
||||
|
||||
var image imagestore.Image
|
||||
for i := 1; i < 500; i++ {
|
||||
// This is probably wrong. Not sure how to wait for an image to appear in
|
||||
// the image (or content) store.
|
||||
log.G(ctx).Debugf("Trying to resolve %s:%d", containerdImage.Name(), i)
|
||||
image, err = c.LocalResolve(containerdImage.Name())
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Microsecond * time.Duration(i))
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve image %q during checkpoint import: %w", config.RootfsImageName, err)
|
||||
}
|
||||
imageConfig := image.ImageSpec.Config
|
||||
env := append([]string{}, imageConfig.Env...)
|
||||
for _, e := range meta.Config.GetEnvs() {
|
||||
env = append(env, e.GetKey()+"="+string(e.GetValue()))
|
||||
}
|
||||
imageConfig.Env = env
|
||||
|
||||
originalAnnotations["restored"] = "true"
|
||||
originalAnnotations["checkpointedAt"] = config.CheckpointedAt.Format(time.RFC3339Nano)
|
||||
originalAnnotations["checkpointImage"] = meta.Config.Image.GetUserSpecifiedImage()
|
||||
|
||||
meta.Config.Annotations = originalAnnotations
|
||||
|
||||
// Remove the checkpoint image name and show the base image name in the metadata.
|
||||
// The checkpoint image name is still available in the annotations.
|
||||
meta.Config.Image.Image = containerStatus.Image.GetImage()
|
||||
meta.Config.Image.UserSpecifiedImage = containerStatus.Image.GetUserSpecifiedImage()
|
||||
|
||||
cstatus, err := c.sandboxService.SandboxStatus(ctx, sandbox.Sandboxer, sandbox.ID, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get controller status: %w", err)
|
||||
}
|
||||
|
||||
containerRootDir, err := c.createContainer(
|
||||
&createContainerRequest{
|
||||
ctx: ctx,
|
||||
containerID: meta.ID,
|
||||
sandbox: sandbox,
|
||||
sandboxID: meta.SandboxID,
|
||||
imageID: image.ID,
|
||||
containerConfig: meta.Config,
|
||||
imageConfig: &imageConfig,
|
||||
podSandboxConfig: sandboxConfig,
|
||||
sandboxRuntimeHandler: sandbox.Metadata.RuntimeHandler,
|
||||
sandboxPid: cstatus.Pid,
|
||||
NetNSPath: sandbox.NetNSPath,
|
||||
containerName: containerName,
|
||||
containerdImage: &containerdImage,
|
||||
meta: meta,
|
||||
restore: true,
|
||||
start: start,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Confine all checkpoint content to a dedicated subdirectory of the container
|
||||
// state dir instead of unpacking it directly into the state dir, so it cannot
|
||||
// collide with containerd's own files there. Create it fresh; RemoveAll unlinks
|
||||
// any pre-existing entry without following it.
|
||||
restoreDir := filepath.Join(containerRootDir, checkpointRestoreDir)
|
||||
if err := os.RemoveAll(restoreDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Mkdir(restoreDir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if restoreStorageImageID != "" {
|
||||
if err := fs.CopyDir(restoreDir, mountPoint); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := mount.UnmountAll(mountPoint, 0); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// fs.CopyDir recreates any symlinks/special files from the image; reject
|
||||
// them here so restore-time consumers only ever open regular files.
|
||||
if err := assertCheckpointDirSafe(restoreDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
// unpack the checkpoint archive
|
||||
filter := archive.WithFilter(func(hdr *tar.Header) (bool, error) {
|
||||
// Reject entry types the checkpoint code never produces (symlinks,
|
||||
// hardlinks, device/fifo nodes) so they are not recreated on disk.
|
||||
if !checkpointArchiveEntryAllowed(hdr) {
|
||||
log.G(ctx).Warnf("Skipping unexpected checkpoint archive entry %q (type %d)", hdr.Name, hdr.Typeflag)
|
||||
return false, nil
|
||||
}
|
||||
excludePatterns := []string{
|
||||
crmetadata.ConfigDumpFile,
|
||||
crmetadata.SpecDumpFile,
|
||||
crmetadata.StatusDumpFile,
|
||||
}
|
||||
|
||||
for _, pattern := range excludePatterns {
|
||||
if strings.HasPrefix(hdr.Name, pattern) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
// Start from the beginning of the checkpoint archive
|
||||
archiveFile.Seek(0, 0)
|
||||
_, err = archive.Apply(ctx, restoreDir, archiveFile, []archive.ApplyOpt{filter}...)
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unpacking of checkpoint archive %s failed: %w", restoreDir, err)
|
||||
}
|
||||
}
|
||||
log.G(ctx).Debugf("Unpacked checkpoint in %s", restoreDir)
|
||||
|
||||
// Restore container log file (if it exists).
|
||||
//
|
||||
// container.log was unpacked from a checkpoint archive/OCI image, so it is
|
||||
// copied without following a final-component symlink.
|
||||
containerLog := filepath.Join(restoreDir, "container.log")
|
||||
if err := copyNoFollow(containerLog, meta.LogPath, 0600); err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return "", fmt.Errorf("restoring container log file %s failed: %w", containerLog, err)
|
||||
}
|
||||
}
|
||||
return meta.ID, nil
|
||||
}
|
||||
|
||||
func (c *criService) CheckpointContainer(ctx context.Context, r *runtime.CheckpointContainerRequest) (*runtime.CheckpointContainerResponse, error) {
|
||||
start := time.Now()
|
||||
if err := c.checkCriu(); err != nil {
|
||||
@@ -879,37 +426,3 @@ func writeSpecDumpFile(ctx context.Context, store content.Store, desc v1.Descrip
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func filterAndMergeAnnotations(
|
||||
ctx context.Context,
|
||||
checkpointAnnotations map[string]string,
|
||||
createAnnotations map[string]string,
|
||||
) map[string]string {
|
||||
result := make(map[string]string)
|
||||
|
||||
for k, v := range checkpointAnnotations {
|
||||
if strings.HasPrefix(k, "cdi.k8s.io/") || k == "cdi.k8s.io" {
|
||||
log.G(ctx).Warnf("Denying annotation %q in checkpoint restore", k)
|
||||
continue
|
||||
}
|
||||
result[k] = v
|
||||
}
|
||||
|
||||
// The hash also needs to be update or Kubernetes thinks the container needs to be restarted
|
||||
_, ok1 := createAnnotations["io.kubernetes.container.hash"]
|
||||
_, ok2 := result["io.kubernetes.container.hash"]
|
||||
|
||||
if ok1 && ok2 {
|
||||
result["io.kubernetes.container.hash"] = createAnnotations["io.kubernetes.container.hash"]
|
||||
}
|
||||
|
||||
// The restart count also needs to be correctly updated
|
||||
_, ok1 = createAnnotations["io.kubernetes.container.restartCount"]
|
||||
_, ok2 = result["io.kubernetes.container.restartCount"]
|
||||
|
||||
if ok1 && ok2 {
|
||||
result["io.kubernetes.container.restartCount"] = createAnnotations["io.kubernetes.container.restartCount"]
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -19,17 +19,13 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/containerd/log"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -107,137 +103,6 @@ func TestCopyNoFollowRejectsDirectory(t *testing.T) {
|
||||
assert.NoFileExists(t, dst)
|
||||
}
|
||||
|
||||
func TestAssertCheckpointDirSafe(t *testing.T) {
|
||||
t.Run("regular files and dirs allowed", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "checkpoint"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(root, "checkpoint", "img"), []byte("x"), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(root, "rootfs-diff.tar"), []byte("x"), 0o600))
|
||||
assert.NoError(t, assertCheckpointDirSafe(root))
|
||||
})
|
||||
|
||||
t.Run("symlink rejected", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.Symlink("/some/outside/path", filepath.Join(root, "rootfs-diff.tar")))
|
||||
assert.Error(t, assertCheckpointDirSafe(root))
|
||||
})
|
||||
|
||||
t.Run("symlink nested in subdir rejected", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "checkpoint"), 0o700))
|
||||
require.NoError(t, os.Symlink("/some/outside/path", filepath.Join(root, "checkpoint", "pages-1.img")))
|
||||
assert.Error(t, assertCheckpointDirSafe(root))
|
||||
})
|
||||
|
||||
t.Run("fifo rejected", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
require.NoError(t, unix.Mkfifo(filepath.Join(root, "fifo"), 0o600))
|
||||
assert.Error(t, assertCheckpointDirSafe(root))
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckpointArchiveEntryAllowed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
typ byte
|
||||
allowed bool
|
||||
}{
|
||||
{"regular", tar.TypeReg, true},
|
||||
//nolint:staticcheck // TypeRegA is deprecated but external tars may still use it
|
||||
{"regular-A", tar.TypeRegA, true},
|
||||
{"directory", tar.TypeDir, true},
|
||||
{"global-header", tar.TypeXGlobalHeader, true},
|
||||
{"symlink", tar.TypeSymlink, false},
|
||||
{"hardlink", tar.TypeLink, false},
|
||||
{"char-device", tar.TypeChar, false},
|
||||
{"block-device", tar.TypeBlock, false},
|
||||
{"fifo", tar.TypeFifo, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := checkpointArchiveEntryAllowed(&tar.Header{Typeflag: tc.typ, Name: tc.name})
|
||||
assert.Equal(t, tc.allowed, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type testLogHook struct {
|
||||
mu sync.Mutex
|
||||
entries []string
|
||||
}
|
||||
|
||||
func (h *testLogHook) Levels() []logrus.Level {
|
||||
return []logrus.Level{logrus.WarnLevel}
|
||||
}
|
||||
|
||||
func (h *testLogHook) Fire(entry *logrus.Entry) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.entries = append(h.entries, entry.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFilterAndMergeAnnotations(t *testing.T) {
|
||||
for desc, tc := range map[string]struct {
|
||||
checkpointAnnotations map[string]string
|
||||
createAnnotations map[string]string
|
||||
expectedAnnotations map[string]string
|
||||
expectedWarnings []string
|
||||
}{
|
||||
"cdi denied prefix boundaries": {
|
||||
checkpointAnnotations: map[string]string{
|
||||
"cdi.k8s.io/device": "gpu",
|
||||
"cdi.k8s.io/": "true",
|
||||
"cdi.k8s.io": "true",
|
||||
"safe.org/cdi.k8s.io": "ignored",
|
||||
"other": "val",
|
||||
},
|
||||
expectedAnnotations: map[string]string{
|
||||
"safe.org/cdi.k8s.io": "ignored",
|
||||
"other": "val",
|
||||
},
|
||||
expectedWarnings: []string{
|
||||
`Denying annotation "cdi.k8s.io/device" in checkpoint restore`,
|
||||
`Denying annotation "cdi.k8s.io/" in checkpoint restore`,
|
||||
`Denying annotation "cdi.k8s.io" in checkpoint restore`,
|
||||
},
|
||||
},
|
||||
|
||||
"createAnnotations update kubernetes metadata if present in both": {
|
||||
checkpointAnnotations: map[string]string{
|
||||
"io.kubernetes.container.hash": "old-hash",
|
||||
"io.kubernetes.container.restartCount": "1",
|
||||
"safe.annotation": "2",
|
||||
},
|
||||
createAnnotations: map[string]string{
|
||||
"io.kubernetes.container.hash": "new-hash",
|
||||
"io.kubernetes.container.restartCount": "2",
|
||||
},
|
||||
expectedAnnotations: map[string]string{
|
||||
"io.kubernetes.container.hash": "new-hash",
|
||||
"io.kubernetes.container.restartCount": "2",
|
||||
"safe.annotation": "2",
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(desc, func(t *testing.T) {
|
||||
logger := logrus.New()
|
||||
logger.SetLevel(logrus.WarnLevel)
|
||||
hook := &testLogHook{}
|
||||
logger.AddHook(hook)
|
||||
ctx := log.WithLogger(context.Background(), logrus.NewEntry(logger))
|
||||
|
||||
res := filterAndMergeAnnotations(
|
||||
ctx,
|
||||
tc.checkpointAnnotations,
|
||||
tc.createAnnotations,
|
||||
)
|
||||
|
||||
assert.Equal(t, tc.expectedAnnotations, res)
|
||||
assert.ElementsMatch(t, tc.expectedWarnings, hook.entries)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCriuPath(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "criu-test")
|
||||
if err != nil {
|
||||
@@ -357,15 +222,3 @@ func TestCheckpointContainerDisabled(t *testing.T) {
|
||||
t.Errorf("expected error containing 'criu support is disabled by configuration', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCRImportCheckpointDisabled(t *testing.T) {
|
||||
c := newTestCRIService()
|
||||
c.config.EnableCRIU = func() *bool { v := false; return &v }()
|
||||
_, err := c.CRImportCheckpoint(context.Background(), nil, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "criu support is disabled by configuration") {
|
||||
t.Errorf("expected error containing 'criu support is disabled by configuration', got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
|
||||
"github.com/containerd/containerd/v2/core/sandbox"
|
||||
sandboxstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox"
|
||||
"github.com/containerd/containerd/v2/pkg/deprecation"
|
||||
"github.com/containerd/containerd/v2/plugins/services/warning"
|
||||
)
|
||||
|
||||
type mockWarningService struct {
|
||||
emitted []deprecation.Warning
|
||||
}
|
||||
|
||||
func (m *mockWarningService) Emit(ctx context.Context, w deprecation.Warning) {
|
||||
m.emitted = append(m.emitted, w)
|
||||
}
|
||||
|
||||
func (m *mockWarningService) Warnings() []warning.Warning {
|
||||
return nil
|
||||
}
|
||||
|
||||
type testSandboxService struct {
|
||||
fakeSandboxService
|
||||
}
|
||||
|
||||
func (t *testSandboxService) SandboxStatus(ctx context.Context, sandboxer string, sandboxID string, verbose bool) (sandbox.ControllerStatus, error) {
|
||||
return sandbox.ControllerStatus{
|
||||
SandboxID: sandboxID,
|
||||
Pid: 1234,
|
||||
State: "READY",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestCreateContainerCheckpointWarning(t *testing.T) {
|
||||
c := newTestCRIService()
|
||||
mockWarn := &mockWarningService{}
|
||||
c.warningService = mockWarn
|
||||
c.sandboxService = &testSandboxService{}
|
||||
|
||||
sb := sandboxstore.NewSandbox(
|
||||
sandboxstore.Metadata{
|
||||
ID: "test-sandbox",
|
||||
Name: "test-sandbox",
|
||||
Config: &runtime.PodSandboxConfig{
|
||||
Metadata: &runtime.PodSandboxMetadata{Name: "test-sandbox", Namespace: "default"},
|
||||
},
|
||||
},
|
||||
sandboxstore.Status{
|
||||
State: sandboxstore.StateReady,
|
||||
},
|
||||
)
|
||||
require.NoError(t, c.sandboxStore.Add(sb))
|
||||
|
||||
// In newTestCRIService(), c.os is a FakeOS where Stat returns (nil, nil) (no error),
|
||||
// causing checkpointImage to evaluate as true when checked in CreateContainer.
|
||||
_, _ = c.CreateContainer(context.Background(), &runtime.CreateContainerRequest{
|
||||
PodSandboxId: "test-sandbox",
|
||||
Config: &runtime.ContainerConfig{
|
||||
Metadata: &runtime.ContainerMetadata{Name: "test-container"},
|
||||
Image: &runtime.ImageSpec{Image: "/path/to/checkpoint.tar"},
|
||||
},
|
||||
SandboxConfig: &runtime.PodSandboxConfig{
|
||||
Metadata: &runtime.PodSandboxMetadata{Name: "test-sandbox", Namespace: "default"},
|
||||
},
|
||||
})
|
||||
|
||||
assert.Contains(t, mockWarn.emitted, deprecation.CRICreateContainerCheckpointRestore, "expected CRICreateContainerCheckpointRestore deprecation warning to be emitted")
|
||||
}
|
||||
@@ -47,7 +47,6 @@ import (
|
||||
"github.com/containerd/containerd/v2/internal/cri/util"
|
||||
"github.com/containerd/containerd/v2/internal/registrar"
|
||||
"github.com/containerd/containerd/v2/pkg/blockio"
|
||||
"github.com/containerd/containerd/v2/pkg/deprecation"
|
||||
"github.com/containerd/containerd/v2/pkg/oci"
|
||||
"github.com/containerd/containerd/v2/pkg/tracing"
|
||||
)
|
||||
@@ -129,63 +128,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
|
||||
Config: config,
|
||||
}
|
||||
|
||||
// Check if image is a file. If it is a file it might be a checkpoint archive.
|
||||
checkpointImage, err := func() (bool, error) {
|
||||
if _, err := c.os.Stat(config.GetImage().GetImage()); err == nil {
|
||||
log.G(ctx).Infof(
|
||||
"%q is a file. Assuming it is a checkpoint archive",
|
||||
config.GetImage().GetImage(),
|
||||
)
|
||||
return true, nil
|
||||
}
|
||||
// Check if this is an OCI checkpoint image
|
||||
imageID, err := c.checkIfCheckpointOCIImage(ctx, config.GetImage().GetImage())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check if this is a checkpoint image: %w", err)
|
||||
}
|
||||
|
||||
return imageID != "", nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if checkpointImage {
|
||||
// This might be a checkpoint image. Let's pass
|
||||
// it to the checkpoint code.
|
||||
|
||||
if c.warningService != nil {
|
||||
c.warningService.Emit(ctx, deprecation.CRICreateContainerCheckpointRestore)
|
||||
if msg, ok := deprecation.Message(deprecation.CRICreateContainerCheckpointRestore); ok {
|
||||
log.G(ctx).WithFields(log.Fields{
|
||||
"podsandboxid": sandboxID,
|
||||
"containerid": id,
|
||||
"containername": name,
|
||||
}).Warn(msg)
|
||||
}
|
||||
}
|
||||
|
||||
if sandboxConfig.GetMetadata() == nil {
|
||||
return nil, fmt.Errorf("sandboxConfig must not be empty")
|
||||
}
|
||||
|
||||
ctrID, err := c.CRImportCheckpoint(
|
||||
ctx,
|
||||
&meta,
|
||||
&sandbox,
|
||||
sandboxConfig,
|
||||
)
|
||||
if err != nil {
|
||||
log.G(ctx).Errorf("failed to prepare %s for restore %q", ctrID, err)
|
||||
return nil, err
|
||||
}
|
||||
log.G(ctx).Infof("Prepared %s for restore", ctrID)
|
||||
|
||||
return &runtime.CreateContainerResponse{
|
||||
ContainerId: id,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Prepare container image snapshot. For container, the image should have
|
||||
// been pulled before creating the container, so do not ensure the image.
|
||||
image, err := c.LocalResolve(config.GetImage().GetImage())
|
||||
@@ -242,7 +184,6 @@ type createContainerRequest struct {
|
||||
containerName string
|
||||
containerdImage *containerd.Image
|
||||
meta *containerstore.Metadata
|
||||
restore bool
|
||||
start time.Time
|
||||
}
|
||||
|
||||
@@ -472,7 +413,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr
|
||||
}
|
||||
}()
|
||||
|
||||
status := containerstore.Status{CreatedAt: time.Now().UnixNano(), Restore: r.restore}
|
||||
status := containerstore.Status{CreatedAt: time.Now().UnixNano()}
|
||||
status = copyResourcesToStatus(spec, status)
|
||||
container, err := containerstore.NewContainer(*r.meta,
|
||||
containerstore.WithStatus(status, containerRootDir),
|
||||
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/v2/pkg/tracing"
|
||||
@@ -30,7 +28,6 @@ import (
|
||||
"github.com/containerd/log"
|
||||
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
|
||||
crmetadata "github.com/checkpoint-restore/checkpointctl/lib"
|
||||
containerd "github.com/containerd/containerd/v2/client"
|
||||
cio "github.com/containerd/containerd/v2/internal/cri/io"
|
||||
containerstore "github.com/containerd/containerd/v2/internal/cri/store/container"
|
||||
@@ -40,12 +37,6 @@ import (
|
||||
cioutil "github.com/containerd/containerd/v2/pkg/ioutil"
|
||||
)
|
||||
|
||||
// checkpointRestoreDir is the subdirectory under a container's persistent state
|
||||
// directory into which checkpoint content (CRIU images, container.log,
|
||||
// rootfs-diff.tar, ...) is unpacked during restore. Confining it here keeps
|
||||
// checkpoint content from colliding with containerd's own files in the state dir.
|
||||
const checkpointRestoreDir = "ctrd-restore"
|
||||
|
||||
// StartContainer starts the container.
|
||||
func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) {
|
||||
span := tracing.SpanFromContext(ctx)
|
||||
@@ -110,65 +101,6 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain
|
||||
return cntr.IO, nil
|
||||
}
|
||||
|
||||
if cntr.Status.Get().Restore {
|
||||
// If during start the container is detected as a checkpoint the container
|
||||
// will be marked with Restore() == true. In this case not the normal
|
||||
// start code is needed but this code which does a restore.
|
||||
pid, err := container.Restore(
|
||||
ctx,
|
||||
ioCreation,
|
||||
filepath.Join(c.getContainerRootDir(r.GetContainerId()), checkpointRestoreDir, crmetadata.CheckpointDirectory),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to restore containerd task: %w", err)
|
||||
}
|
||||
// Update container start timestamp.
|
||||
if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
|
||||
if pid < 0 {
|
||||
return status, fmt.Errorf("restore returned a PID < 0 (%d); that should not happen", pid)
|
||||
}
|
||||
status.Pid = uint32(pid)
|
||||
status.StartedAt = time.Now().UnixNano()
|
||||
return status, nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("failed to update container %q state: %w", id, err)
|
||||
}
|
||||
|
||||
c.generateAndSendContainerEvent(ctx, id, sandboxID, runtime.ContainerEventType_CONTAINER_STARTED_EVENT)
|
||||
|
||||
task, err := cntr.Container.Task(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get task for container %q: %w", id, err)
|
||||
}
|
||||
// wait is a long running background request, no timeout needed.
|
||||
exitCh, err := task.Wait(ctrdutil.NamespacedContext())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to wait for containerd task: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if retErr != nil {
|
||||
deferCtx, deferCancel := ctrdutil.DeferContext()
|
||||
defer deferCancel()
|
||||
err = c.nri.StopContainer(deferCtx, &sandbox, &cntr)
|
||||
if err != nil {
|
||||
log.G(ctx).WithError(err).Errorf("NRI stop failed for failed container %q", id)
|
||||
}
|
||||
}
|
||||
}()
|
||||
// It handles the TaskExit event and update container state after this.
|
||||
c.startContainerExitMonitor(context.Background(), id, task.Pid(), exitCh)
|
||||
|
||||
// cleanup checkpoint artifacts after restore.
|
||||
restoreDir := filepath.Join(c.getContainerRootDir(r.GetContainerId()), checkpointRestoreDir)
|
||||
if err := os.RemoveAll(restoreDir); err != nil {
|
||||
log.G(ctx).Warnf("Non-fatal: removal of checkpoint restore dir (%s) failed: %v", restoreDir, err)
|
||||
}
|
||||
|
||||
log.G(ctx).Infof("Restored container %s successfully", r.GetContainerId())
|
||||
return &runtime.StartContainerResponse{}, nil
|
||||
}
|
||||
// Recheck target container validity in Linux namespace options.
|
||||
if linux := config.GetLinux(); linux != nil {
|
||||
nsOpts := linux.GetSecurityContext().GetNamespaceOptions()
|
||||
|
||||
@@ -98,9 +98,6 @@ type Status struct {
|
||||
Unknown bool `json:"-"`
|
||||
// Resources has container runtime resource constraints
|
||||
Resources *runtime.ContainerResources
|
||||
// Restore marks this container as a container to be restored from a
|
||||
// checkpoint and not started.
|
||||
Restore bool
|
||||
}
|
||||
|
||||
// State returns current state of the container based on the container status.
|
||||
|
||||
@@ -43,8 +43,6 @@ const (
|
||||
RuncOptionsTaskAPIAddress Warning = Prefix + "runc-options-task-api-address"
|
||||
// RuncOptionsTaskAPIVersion is a warning for the use of `task_api_version` in runc options
|
||||
RuncOptionsTaskAPIVersion Warning = Prefix + "runc-options-task-api-version"
|
||||
// CRICreateContainerCheckpointRestore is a warning for restoring checkpoint data from an image or archive during CRI CreateContainer
|
||||
CRICreateContainerCheckpointRestore Warning = Prefix + "cri-create-container-checkpoint-restore"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -69,9 +67,8 @@ var messages = map[Warning]string{
|
||||
CgroupV1: "The support for cgroup v1 is deprecated since containerd v2.2 and will be removed by no later than May 2029. Upgrade the host to use cgroup v2.",
|
||||
CRIEnableCDI: "The `enable_cdi` property of `[plugins.\"io.containerd.cri.v1.runtime\"]` is deprecated, will be removed in containerd v2.3, and CDI support will always be enabled.",
|
||||
|
||||
RuncOptionsTaskAPIAddress: "The `task_api_address` field in runc options is deprecated since containerd v2.3. Set `task_api_address` on CreateTaskRequest instead.",
|
||||
RuncOptionsTaskAPIVersion: "The `task_api_version` field in runc options is deprecated since containerd v2.3. Set `task_api_version` on CreateTaskRequest instead.",
|
||||
CRICreateContainerCheckpointRestore: "Restoring checkpoint data from an image or archive during CRI CreateContainer is deprecated and will be removed in containerd v2.4.",
|
||||
RuncOptionsTaskAPIAddress: "The `task_api_address` field in runc options is deprecated since containerd v2.3. Set `task_api_address` on CreateTaskRequest instead.",
|
||||
RuncOptionsTaskAPIVersion: "The `task_api_version` field in runc options is deprecated since containerd v2.3. Set `task_api_version` on CreateTaskRequest instead.",
|
||||
}
|
||||
|
||||
// Valid checks whether a given Warning is valid
|
||||
|
||||
Reference in New Issue
Block a user