containerd/core/runtime/v2/shim.go

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package v2

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
	"google.golang.org/grpc"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/credentials/insecure"

	eventstypes "github.com/containerd/containerd/api/events"
	bootapi "github.com/containerd/containerd/api/runtime/bootstrap/v1"
	task "github.com/containerd/containerd/api/runtime/task/v3"
	"github.com/containerd/containerd/api/types"
	"github.com/containerd/errdefs"
	"github.com/containerd/errdefs/pkg/errgrpc"
	"github.com/containerd/log"
	"github.com/containerd/otelttrpc"
	"github.com/containerd/ttrpc"
	"github.com/containerd/typeurl/v2"

	"github.com/containerd/containerd/v2/core/events/exchange"
	"github.com/containerd/containerd/v2/core/runtime"
	"github.com/containerd/containerd/v2/pkg/archive"
	"github.com/containerd/containerd/v2/pkg/archive/compression"
	"github.com/containerd/containerd/v2/pkg/atomicfile"
	"github.com/containerd/containerd/v2/pkg/dialer"
	"github.com/containerd/containerd/v2/pkg/identifiers"
	"github.com/containerd/containerd/v2/pkg/protobuf"
	"github.com/containerd/containerd/v2/pkg/protobuf/proto"
	ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types"
	client "github.com/containerd/containerd/v2/pkg/shim"
	"github.com/containerd/containerd/v2/pkg/timeout"
)

const (
	loadTimeout     = "io.containerd.timeout.shim.load"
	cleanupTimeout  = "io.containerd.timeout.shim.cleanup"
	shutdownTimeout = "io.containerd.timeout.shim.shutdown"

	// rootFsDiffTar is the name of the rootfs diff archive written next to a
	// checkpoint. It is part of the checkpoint layout produced by CRIU tooling,
	// so it must stay in sync with github.com/checkpoint-restore/checkpointctl/lib.RootFsDiffTar.
	rootFsDiffTar = "rootfs-diff.tar"
)

func init() {
	timeout.Set(loadTimeout, 5*time.Second)
	timeout.Set(cleanupTimeout, 5*time.Second)
	timeout.Set(shutdownTimeout, 3*time.Second)
}

func loadShim(ctx context.Context, bundle *Bundle, onClose func()) (_ ShimInstance, retErr error) {
	shimCtx, cancelShimLog := context.WithCancel(ctx)
	defer func() {
		if retErr != nil {
			cancelShimLog()
		}
	}()
	f, err := openShimLog(shimCtx, bundle, client.AnonReconnectDialer)
	if err != nil {
		return nil, fmt.Errorf("open shim log pipe when reload: %w", err)
	}
	defer func() {
		if retErr != nil {
			f.Close()
		}
	}()
	// open the log pipe and block until the writer is ready
	// this helps with synchronization of the shim
	// copy the shim's logs to containerd's output
	go func() {
		defer f.Close()
		_, err := io.Copy(os.Stderr, f)
		// To prevent flood of error messages, the expected error
		// should be reset, like os.ErrClosed or os.ErrNotExist, which
		// depends on platform.
		err = checkCopyShimLogError(shimCtx, err)
		if err != nil {
			log.G(shimCtx).WithError(err).Error("copy shim log after reload")
		}
	}()
	onCloseWithShimLog := func() {
		onClose()
		cancelShimLog()
		f.Close()
	}

	params, err := restoreBootstrapParams(bundle.Path)
	if err != nil {
		return nil, fmt.Errorf("failed to read bootstrap.json when restoring bundle %q: %w", bundle.ID, err)
	}

	conn, err := makeConnection(ctx, bundle.ID, params, onCloseWithShimLog, client.AnonReconnectDialer)
	if err != nil {
		return nil, fmt.Errorf("unable to make connection: %w", err)
	}

	defer func() {
		if retErr != nil {
			conn.Close()
		}
	}()

	// The address is in the form like ttrpc+unix://<uds-path> or grpc+vsock://<cid>:<port>
	address := fmt.Sprintf("%s+%s", params.Protocol, params.Address)

	shim := &shim{
		bundle:    bundle,
		client:    conn,
		address:   address,
		version:   int(params.Version),
		bootstrap: params,
	}

	return shim, nil
}

func cleanupAfterDeadShim(ctx context.Context, id string, rt *runtime.NSMap[ShimInstance], events *exchange.Exchange, binaryCall *binary) {
	ctx, cancel := timeout.WithContext(ctx, cleanupTimeout)
	defer cancel()

	log.G(ctx).WithField("id", id).Info("cleaning up after shim disconnected")
	response, err := binaryCall.Delete(ctx)
	if err != nil {
		log.G(ctx).WithError(err).WithField("id", id).Warn("failed to clean up after shim disconnected")
	}

	s, err := rt.Get(ctx, id)
	if err != nil {
		// Task was never started, or its record has already been removed.
		// No need to publish events.
		return
	}

	// If the task delete already succeeded, the shim itself has delivered the
	// exit and delete events. No need to publish duplicates.
	if s.(taskDeleteState).taskDeleteResult() != nil {
		return
	}

	var (
		pid        uint32
		exitStatus uint32
		exitedAt   time.Time
	)
	if response != nil {
		pid = response.Pid
		exitStatus = response.Status
		exitedAt = response.Timestamp
	} else {
		exitStatus = 255
		exitedAt = time.Now()
	}
	events.Publish(ctx, runtime.TaskExitEventTopic, &eventstypes.TaskExit{
		ContainerID: id,
		ID:          id,
		Pid:         pid,
		ExitStatus:  exitStatus,
		ExitedAt:    protobuf.ToTimestamp(exitedAt),
	})

	events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
		ContainerID: id,
		Pid:         pid,
		ExitStatus:  exitStatus,
		ExitedAt:    protobuf.ToTimestamp(exitedAt),
	})
}

// cleanupShimTask reaps a shim task we have given up on, after a failed start or
// when loading a bundle left behind by a previous containerd. An unresponsive
// shim must not block the caller — on the load path that would stall containerd
// startup — so each call is bounded, and detached from the caller's context,
// which by then may be cancelled or out of budget.
//
// A failed delete returns before shutting the shim down and closing its client,
// so both are done here. It also leaves the bundle in place (only a successful
// delete removes it), so callers that own one must remove it on error. The shim
// map is untouched: callers reach this having already removed the task, or never
// added it.
func cleanupShimTask(ctx context.Context, st *shimTask) error {
	dctx, cancel := timeout.WithContext(context.WithoutCancel(ctx), cleanupTimeout)
	defer cancel()

	_, err := st.delete(dctx, func(context.Context, string) {})
	if err == nil {
		return nil
	}

	// Shutting down needs a context with time left on it. Check the deadline
	// rather than the error: a timeout only survives as context.DeadlineExceeded
	// over GRPC. Over TTRPC it arrives as the raw context error, which carries no
	// GRPC status, so errgrpc.ToNative flattens it into errdefs.ErrUnknown.
	if dctx.Err() != nil {
		dctx, cancel = timeout.WithContext(context.WithoutCancel(ctx), cleanupTimeout)
		defer cancel()
	}

	st.Shutdown(dctx)
	st.Close()

	return err
}

// CurrentShimVersion is the latest shim version supported by containerd (e.g. TaskService v3).
const CurrentShimVersion = 3

// ShimInstance represents running shim process managed by ShimManager.
type ShimInstance interface {
	io.Closer

	// ID of the shim.
	ID() string
	// Namespace of this shim.
	Namespace() string
	// Bundle is a file system path to shim's bundle.
	Bundle() string
	// Client returns the underlying TTRPC or GRPC client object for this shim.
	// The underlying object can be either *ttrpc.Client or grpc.ClientConnInterface.
	Client() any
	// Delete will close the client and remove bundle from disk.
	Delete(ctx context.Context) error
	// Endpoint returns shim's endpoint information,
	// including address and version.
	Endpoint() (string, int)
}

// shimCapabilities is implemented by shim instances that retain what the shim
// advertised when it started.
//
// This is a separate interface rather than a method on [ShimInstance] so that
// external implementations of ShimInstance keep compiling; a shim instance
// that does not implement it is treated as having advertised nothing.
type shimCapabilities interface {
	// BootstrapResult returns the result the shim returned when it started,
	// carrying the extensions it advertised. It may be nil.
	BootstrapResult() *bootapi.BootstrapResult
}

type clientVersionDowngrader interface {
	// Downgrade is to lower shim's client version.
	//
	// Assume there is a running pod created by containerd-shim-runc-v2 from v1.7.x.
	// After upgrading to v2.x, the containerd-shim-runc-v2 binary will support the
	// sandbox API, and calling `shim start` for the existing running pod will return
	// a version=3 address. However, that pod shim does not support the streaming IO API,
	// so we should downgrade the shim version.
	//
	// Additionally, if a container record was created with v1.7.x, it will not have
	// a SandboxID field in the metadata store. In the CRI case, this will cause
	// the new shim client to use the v3 protocol to send requests to a running shim
	// that still uses the v2 protocol, resulting in a failure to start.
	// In this case, we should also downgrade the shim version and retry.
	Downgrade() error
}

// taskDeleteState caches the result of a successful task delete on the shim
// instance, so that a Delete retried after a failed shutdown can still return
// the original exit.
type taskDeleteState interface {
	recordTaskDeleteResult(*runtime.Exit)
	taskDeleteResult() *runtime.Exit
}

func parseStartResponse(response []byte) (*bootapi.BootstrapResult, error) {
	var result bootapi.BootstrapResult

	if err := proto.Unmarshal(response, &result); err == nil {
		return &result, nil
	}

	// Fallback to legacy parsing for backward compatibility with legacy shims that return the address as a plain string or JSON.
	response = bytes.TrimSpace(response)

	// Decode into the whole message rather than a subset of its fields. A
	// bundle's bootstrap.json is written with encoding/json and read back
	// through here, so a field that is not decoded is silently lost on reload.
	params := &bootapi.BootstrapResult{}
	if err := json.Unmarshal(response, params); err != nil || params.Version < 2 {
		// Use TTRPC for legacy shims
		params = &bootapi.BootstrapResult{
			Address:  string(response),
			Protocol: "ttrpc",
			Version:  2,
		}
	}

	if params.Version > CurrentShimVersion {
		return nil, fmt.Errorf("unsupported shim version (%d): %w", params.Version, errdefs.ErrNotImplemented)
	}

	return params, nil
}

// writeBootstrapParams writes shim's bootstrap configuration (e.g. how to connect, version, etc).
func writeBootstrapParams(path string, params *bootapi.BootstrapResult) error {
	path, err := filepath.Abs(path)
	if err != nil {
		return err
	}

	data, err := json.Marshal(&params)
	if err != nil {
		return err
	}

	f, err := atomicfile.New(path, 0o644)
	if err != nil {
		return err
	}

	_, err = f.Write(data)
	if err != nil {
		f.Cancel()
		return err
	}

	return f.Close()
}

func readBootstrapParams(path string) (*bootapi.BootstrapResult, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	data, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}

	return parseStartResponse(data)
}

// makeConnection creates a new TTRPC or GRPC connection using the address and
// protocol from params. Legacy plain-string or JSON bootstrap responses are
// normalized by parseStartResponse before calling this function.
// The dialer parameter controls connection behavior: use AnonDialer for newly
// started shims (retries if pipe doesn't exist yet) or AnonReconnectDialer for
// reconnecting to already-running shims (fails fast if pipe is missing).
func makeConnection(ctx context.Context, id string, params *bootapi.BootstrapResult, onClose func(), dialer func(string, time.Duration) (net.Conn, error)) (_ io.Closer, retErr error) {
	log.G(ctx).WithFields(log.Fields{
		"address":  params.Address,
		"protocol": params.Protocol,
		"version":  params.Version,
	}).Infof("connecting to shim %s", id)

	switch strings.ToLower(params.Protocol) {
	case "ttrpc":
		conn, err := client.Connect(params.Address, dialer)
		if err != nil {
			return nil, fmt.Errorf("failed to create TTRPC connection: %w", err)
		}
		defer func() {
			if retErr != nil {
				conn.Close()
			}
		}()

		return ttrpc.NewClient(
			conn,
			ttrpc.WithOnClose(onClose),
			ttrpc.WithUnaryClientInterceptor(otelttrpc.UnaryClientInterceptor()),
		), nil
	case "grpc":
		gopts := []grpc.DialOption{
			grpc.WithTransportCredentials(insecure.NewCredentials()),
			grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
		}
		return grpcDialContext(params.Address, onClose, gopts...)
	default:
		return nil, fmt.Errorf("unexpected protocol: %q", params.Protocol)
	}
}

// grpcDialContext and the underlying grpcConn type exist solely
// so we can have something similar to ttrpc.WithOnClose to have
// a callback run when the connection is severed or explicitly closed.
func grpcDialContext(
	address string,
	onClose func(),
	gopts ...grpc.DialOption,
) (*grpcConn, error) {
	// If grpc.WithBlock is specified in gopts this causes the connection to block waiting for
	// a connection regardless of if the socket exists or has a listener when Dial begins. This
	// specific behavior of WithBlock is mostly undesirable for shims, as if the socket isn't
	// there when we go to load/connect there's likely an issue. However, getting rid of WithBlock is
	// also undesirable as we don't want the background connection behavior, we want to ensure
	// a connection before moving on. To bring this in line with the ttrpc connection behavior
	// lets do an initial dial to ensure the shims socket is actually available. stat wouldn't suffice
	// here as if the shim exited unexpectedly its socket may still be on the filesystem, but it'd return
	// ECONNREFUSED which grpc.DialContext will happily trudge along through for the full timeout.
	//
	// This is especially helpful on restart of containerd as if the shim died while containerd
	// was down, we end up waiting the full timeout.
	conn, err := net.DialTimeout("unix", address, time.Second*10)
	if err != nil {
		return nil, err
	}
	conn.Close()

	target := dialer.DialAddress(address)
	client, err := grpc.NewClient(target, gopts...)
	if err != nil {
		return nil, fmt.Errorf("failed to create GRPC connection: %w", err)
	}

	done := make(chan struct{})
	go func() {
		gctx := context.Background()
		sourceState := connectivity.Ready
		for {
			if client.WaitForStateChange(gctx, sourceState) {
				state := client.GetState()
				if state == connectivity.Idle || state == connectivity.Shutdown {
					break
				}
				// Could be transient failure. Lets see if we can get back to a working
				// state.
				log.G(gctx).WithFields(log.Fields{
					"state": state,
					"addr":  target,
				}).Warn("shim grpc connection unexpected state")
				sourceState = state
			}
		}
		onClose()
		close(done)
	}()

	return &grpcConn{
		ClientConn:  client,
		onCloseDone: done,
	}, nil
}

type grpcConn struct {
	*grpc.ClientConn
	onCloseDone chan struct{}
}

func (gc *grpcConn) UserOnCloseWait(ctx context.Context) error {
	select {
	case <-gc.onCloseDone:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

type shim struct {
	bundle  *Bundle
	client  any
	address string
	version int
	// bootstrap is what the shim advertised when it started. Retained whole
	// rather than decoded into fields here so that a new capability needs no
	// further plumbing through the shim instance.
	bootstrap *bootapi.BootstrapResult

	taskDeleteExit atomic.Pointer[runtime.Exit]
}

var _ ShimInstance = (*shim)(nil)
var _ clientVersionDowngrader = (*shim)(nil)
var _ shimCapabilities = (*shim)(nil)
var _ taskDeleteState = (*shim)(nil)

// BootstrapResult returns what the shim advertised when it started.
func (s *shim) BootstrapResult() *bootapi.BootstrapResult {
	return s.bootstrap
}

// ID of the shim/task
func (s *shim) ID() string {
	return s.bundle.ID
}

func (s *shim) Endpoint() (string, int) {
	return s.address, s.version
}

func (s *shim) Downgrade() error {
	if s.version >= CurrentShimVersion {
		s.version--
		return nil
	}
	return fmt.Errorf("unable to downgrade because shim version (%d) is lower than CurrentShimVersion (%d)",
		s.version, CurrentShimVersion)
}

func (s *shim) Namespace() string {
	return s.bundle.Namespace
}

func (s *shim) Bundle() string {
	return s.bundle.Path
}

func (s *shim) Client() any {
	return s.client
}

// Close closes the underlying client connection.
func (s *shim) Close() error {
	if ttrpcClient, ok := s.client.(*ttrpc.Client); ok {
		return ttrpcClient.Close()
	}

	if grpcClient, ok := s.client.(*grpcConn); ok {
		return grpcClient.Close()
	}

	return nil
}

func (s *shim) Delete(ctx context.Context) error {
	var result []error

	if ttrpcClient, ok := s.client.(*ttrpc.Client); ok {
		if err := ttrpcClient.Close(); err != nil {
			result = append(result, fmt.Errorf("failed to close ttrpc client: %w", err))
		}

		if err := ttrpcClient.UserOnCloseWait(ctx); err != nil {
			result = append(result, fmt.Errorf("close wait error: %w", err))
		}
	}

	if grpcClient, ok := s.client.(*grpcConn); ok {
		if err := grpcClient.Close(); err != nil {
			result = append(result, fmt.Errorf("failed to close grpc client: %w", err))
		}

		if err := grpcClient.UserOnCloseWait(ctx); err != nil {
			result = append(result, fmt.Errorf("close wait error: %w", err))
		}
	}

	if err := s.bundle.Delete(); err != nil {
		log.G(ctx).WithField("id", s.ID()).WithError(err).Error("failed to delete bundle")
		result = append(result, fmt.Errorf("failed to delete bundle: %w", err))
	}

	return errors.Join(result...)
}

// recordTaskDeleteResult caches the result of a successful task delete. The
// value is copied both in and out so that neither the caller that recorded it
// nor a later retry can mutate the cached result.
func (s *shim) recordTaskDeleteResult(exit *runtime.Exit) {
	cached := *exit
	s.taskDeleteExit.Store(&cached)
}

// taskDeleteResult returns a copy of the cached delete result, or nil if this
// containerd process has never deleted the task successfully.
func (s *shim) taskDeleteResult() *runtime.Exit {
	cached := s.taskDeleteExit.Load()
	if cached == nil {
		return nil
	}
	exit := *cached
	return &exit
}

var _ runtime.Task = &shimTask{}

// shimTask wraps shim process and adds task service client for compatibility with existing shim manager.
type shimTask struct {
	ShimInstance
	task TaskServiceClient
}

func newShimTask(shim ShimInstance) (*shimTask, error) {
	_, version := shim.Endpoint()
	taskClient, err := NewTaskClient(shim.Client(), version)
	if err != nil {
		return nil, err
	}

	return &shimTask{
		ShimInstance: shim,
		task:         taskClient,
	}, nil
}

func (s *shimTask) Shutdown(ctx context.Context) error {
	_, err := s.task.Shutdown(ctx, &task.ShutdownRequest{
		ID: s.ID(),
	})
	if err != nil && !errors.Is(err, ttrpc.ErrClosed) {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) waitShutdown(ctx context.Context) error {
	ctx, cancel := timeout.WithContext(ctx, shutdownTimeout)
	defer cancel()
	return s.Shutdown(ctx)
}

// PID of the task
func (s *shimTask) PID(ctx context.Context) (uint32, error) {
	response, err := s.task.Connect(ctx, &task.ConnectRequest{
		ID: s.ID(),
	})
	if err != nil {
		return 0, errgrpc.ToNative(err)
	}

	return response.TaskPid, nil
}

func (s *shimTask) delete(ctx context.Context, removeTask func(ctx context.Context, id string)) (*runtime.Exit, error) {
	response, shimErr := s.task.Delete(ctx, &task.DeleteRequest{
		ID: s.ID(),
	})
	if shimErr != nil {
		log.G(ctx).WithField("id", s.ID()).WithError(shimErr).Error("failed to delete task")
		if !errors.Is(shimErr, ttrpc.ErrClosed) {
			shimErr = errgrpc.ToNative(shimErr)
			if !errdefs.IsNotFound(shimErr) {
				return nil, shimErr
			}
		}
	}

	deleteState := s.ShimInstance.(taskDeleteState)

	// NOTE: If the shim has been killed and ttrpc connection has been
	// closed, the shimErr will not be nil. For this case, the event
	// subscriber, like moby/moby, might have received the exit or delete
	// events. Just in case, we should allow ttrpc-callback-on-close to
	// send the exit and delete events again. And the exit status will
	// depend on result of shimV2.Delete.
	//
	// If not, the shim has delivered the exit and delete events. Cache the
	// delete result so a retry can return it and prevent duplicate events from
	// ttrpc-callback-on-close.
	//
	// TODO: It's hard to guarantee that the event is unique and sent only
	// once. The moby/moby should not rely on that assumption that there is
	// only one exit event. The moby/moby should handle the duplicate events.
	//
	// REF: https://github.com/containerd/containerd/issues/4769
	var exit *runtime.Exit
	if shimErr == nil {
		exit = &runtime.Exit{
			Status:    response.ExitStatus,
			Timestamp: protobuf.FromTimestamp(response.ExitedAt),
			Pid:       response.Pid,
		}
		deleteState.recordTaskDeleteResult(exit)
	}

	// NOTE: Returning here deliberately leaves the shim record, the ttrpc
	// client and the bundle in place, so that the caller can retry Delete.
	// The result cached above lets that retry return the original exit, even
	// though the shim reports NotFound for the task by then.
	if err := s.waitShutdown(ctx); err != nil {
		return nil, fmt.Errorf("failed to invoke shutdown: %w", err)
	}

	if err := s.ShimInstance.Delete(ctx); err != nil {
		log.G(ctx).WithField("id", s.ID()).WithError(err).Error("failed to delete shim")
	}

	// remove self from the runtime task list
	// this seems dirty but it cleans up the API across runtimes, tasks, and the service
	removeTask(ctx, s.ID())

	if exit == nil {
		// An earlier attempt already deleted the task in the shim, so shimErr
		// is NotFound. Return the exit recorded by that attempt instead.
		exit = deleteState.taskDeleteResult()
	}
	if exit == nil {
		return nil, shimErr
	}
	return exit, nil
}

func (s *shimTask) Create(ctx context.Context, opts runtime.CreateOpts) (runtime.Task, error) {
	topts := opts.TaskOptions
	if topts == nil || topts.GetValue() == nil {
		topts = opts.RuntimeOptions
	}
	request := &task.CreateTaskRequest{
		ID:         s.ID(),
		Bundle:     s.Bundle(),
		Stdin:      opts.IO.Stdin,
		Stdout:     opts.IO.Stdout,
		Stderr:     opts.IO.Stderr,
		Terminal:   opts.IO.Terminal,
		Checkpoint: opts.Checkpoint,
		Options:    typeurl.MarshalProto(topts),
	}
	for _, m := range opts.Rootfs {
		request.Rootfs = append(request.Rootfs, &types.Mount{
			Type:    m.Type,
			Source:  m.Source,
			Target:  m.Target,
			Options: m.Options,
		})
	}

	_, err := s.task.Create(ctx, request)
	if err != nil {
		return nil, errgrpc.ToNative(err)
	}

	if opts.RestoreFromPath {
		// Unpack rootfs-diff.tar if it exists.
		// This needs to happen between the 'Create()' from above and before the 'Start()' from below.
		rootfsDiff := filepath.Join(opts.Checkpoint, "..", rootFsDiffTar)

		_, err = os.Stat(rootfsDiff)
		if err == nil {
			rootfsDiffTar, err := os.Open(rootfsDiff)
			if err != nil {
				return nil, fmt.Errorf("failed to open rootfs-diff archive %s for import: %w", rootfsDiffTar.Name(), err)
			}
			defer func(f *os.File) {
				if err := f.Close(); err != nil {
					log.G(ctx).Errorf("Unable to close file %s: %q", f.Name(), err)
				}
			}(rootfsDiffTar)

			decompressed, err := compression.DecompressStream(rootfsDiffTar)
			if err != nil {
				return nil, fmt.Errorf("failed to decompress archive %s for import: %w", rootfsDiffTar.Name(), err)
			}

			rootfs := filepath.Join(s.Bundle(), "rootfs")
			_, err = archive.Apply(
				ctx,
				rootfs,
				decompressed,
			)

			if err != nil {
				return nil, fmt.Errorf("unpacking of rootfs-diff archive %s into %s failed: %w", rootfsDiffTar.Name(), rootfs, err)
			}
			log.G(ctx).Debugf("Unpacked checkpoint in %s", rootfs)
		}
		// (adrianreber): This is unclear to me. But it works (and it is necessary).
		// This is probably connected to my misunderstanding why
		// restoring a container goes through Create().
		log.G(ctx).Infof("About to start with opts.Checkpoint %s", opts.Checkpoint)
		err = s.Start(ctx)
		if err != nil {
			return nil, err
		}
	}

	return s, nil
}

func (s *shimTask) Pause(ctx context.Context) error {
	if _, err := s.task.Pause(ctx, &task.PauseRequest{
		ID: s.ID(),
	}); err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Resume(ctx context.Context) error {
	if _, err := s.task.Resume(ctx, &task.ResumeRequest{
		ID: s.ID(),
	}); err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Start(ctx context.Context) error {
	_, err := s.task.Start(ctx, &task.StartRequest{
		ID: s.ID(),
	})
	if err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Kill(ctx context.Context, signal uint32, all bool) error {
	if _, err := s.task.Kill(ctx, &task.KillRequest{
		ID:     s.ID(),
		Signal: signal,
		All:    all,
	}); err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Exec(ctx context.Context, id string, opts runtime.ExecOpts) (runtime.ExecProcess, error) {
	if err := identifiers.Validate(id); err != nil {
		return nil, fmt.Errorf("invalid exec id %s: %w", id, err)
	}
	request := &task.ExecProcessRequest{
		ID:       s.ID(),
		ExecID:   id,
		Stdin:    opts.IO.Stdin,
		Stdout:   opts.IO.Stdout,
		Stderr:   opts.IO.Stderr,
		Terminal: opts.IO.Terminal,
		Spec:     opts.Spec,
	}
	if _, err := s.task.Exec(ctx, request); err != nil {
		return nil, errgrpc.ToNative(err)
	}
	return &process{
		id:   id,
		shim: s,
	}, nil
}

func (s *shimTask) Pids(ctx context.Context) ([]runtime.ProcessInfo, error) {
	resp, err := s.task.Pids(ctx, &task.PidsRequest{
		ID: s.ID(),
	})
	if err != nil {
		return nil, errgrpc.ToNative(err)
	}
	var processList []runtime.ProcessInfo
	for _, p := range resp.Processes {
		processList = append(processList, runtime.ProcessInfo{
			Pid:  p.Pid,
			Info: p.Info,
		})
	}
	return processList, nil
}

func (s *shimTask) ResizePty(ctx context.Context, size runtime.ConsoleSize) error {
	_, err := s.task.ResizePty(ctx, &task.ResizePtyRequest{
		ID:     s.ID(),
		Width:  size.Width,
		Height: size.Height,
	})
	if err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) CloseIO(ctx context.Context) error {
	_, err := s.task.CloseIO(ctx, &task.CloseIORequest{
		ID:    s.ID(),
		Stdin: true,
	})
	if err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Wait(ctx context.Context) (*runtime.Exit, error) {
	taskPid, err := s.PID(ctx)
	if err != nil {
		return nil, err
	}
	response, err := s.task.Wait(ctx, &task.WaitRequest{
		ID: s.ID(),
	})
	if err != nil {
		return nil, errgrpc.ToNative(err)
	}
	return &runtime.Exit{
		Pid:       taskPid,
		Timestamp: protobuf.FromTimestamp(response.ExitedAt),
		Status:    response.ExitStatus,
	}, nil
}

func (s *shimTask) Checkpoint(ctx context.Context, path string, options *ptypes.Any) error {
	request := &task.CheckpointTaskRequest{
		ID:      s.ID(),
		Path:    path,
		Options: options,
	}
	if _, err := s.task.Checkpoint(ctx, request); err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Update(ctx context.Context, resources *ptypes.Any, annotations map[string]string) error {
	if _, err := s.task.Update(ctx, &task.UpdateTaskRequest{
		ID:          s.ID(),
		Resources:   resources,
		Annotations: annotations,
	}); err != nil {
		return errgrpc.ToNative(err)
	}
	return nil
}

func (s *shimTask) Stats(ctx context.Context) (*ptypes.Any, error) {
	response, err := s.task.Stats(ctx, &task.StatsRequest{
		ID: s.ID(),
	})
	if err != nil {
		return nil, errgrpc.ToNative(err)
	}
	return response.Stats, nil
}

func (s *shimTask) Process(ctx context.Context, id string) (runtime.ExecProcess, error) {
	p := &process{
		id:   id,
		shim: s,
	}
	if _, err := p.State(ctx); err != nil {
		return nil, err
	}
	return p, nil
}

func (s *shimTask) State(ctx context.Context) (runtime.State, error) {
	response, err := s.task.State(ctx, &task.StateRequest{
		ID: s.ID(),
	})
	if err != nil {
		if errdefs.IsDeadlineExceeded(err) {
			return runtime.State{}, err
		}
		if !errors.Is(err, ttrpc.ErrClosed) {
			return runtime.State{}, errgrpc.ToNative(err)
		}
		return runtime.State{}, errdefs.ErrNotFound
	}
	return runtime.State{
		Pid:        response.Pid,
		Status:     statusFromProto(response.Status),
		Stdin:      response.Stdin,
		Stdout:     response.Stdout,
		Stderr:     response.Stderr,
		Terminal:   response.Terminal,
		ExitStatus: response.ExitStatus,
		ExitedAt:   protobuf.FromTimestamp(response.ExitedAt),
	}, nil
}