mirror of
https://github.com/containerd/containerd.git
synced 2026-08-13 17:07:26 +00:00
It's to ensure the data integrity during unexpected power failure. Background: Since release 1.3, in Linux system, containerD unpacks and writes files into overlayfs snapshot directly. It doesn’t involve any mount-umount operations so that the performance of pulling image has been improved. As we know, the umount syscall for overlayfs will force kernel to flush all the dirty pages into disk. Without umount syscall, the files’ data relies on kernel’s writeback threads or filesystem's commit setting (for instance, ext4 filesystem). The files in committed snapshot can be loss after unexpected power failure. However, the snapshot has been committed and the metadata also has been fsynced. There is data inconsistency between snapshot metadata and files in that snapshot. We, containerd, received several issues about data loss after unexpected power failure. * https://github.com/containerd/containerd/issues/5854 * https://github.com/containerd/containerd/issues/3369#issuecomment-1787334907 Solution: * Option 1: SyncFs after unpack Linux platform provides [syncfs][syncfs] syscall to synchronize just the filesystem containing a given file. * Option 2: Fsync directories recursively and fsync on regular file The fsync doesn't support symlink/block device/char device files. We need to use fsync the parent directory to ensure that entry is persisted. However, based on [xfstest-dev][xfstest-dev], there is no case to ensure fsync-on-parent can persist the special file's metadata, for example, uid/gid, access mode. Checkout [generic/690][generic/690]: Syncing parent dir can persist symlink. But for f2fs, it needs special mount option. And it doesn't say that uid/gid can be persisted. All the details are behind the implemetation. > NOTE: All the related test cases has `_flakey_drop_and_remount` in [xfstest-dev]. Based on discussion about [Documenting the crash-recovery guarantees of Linux file systems][kernel-crash-recovery-data-integrity], we can't rely on Fsync-on-parent. * Option 1 is winner This patch is using option 1. There is test result based on [test-tool][test-tool]. All the networking traffic created by pull is local. * Image: docker.io/library/golang:1.19.4 (992 MiB) * Current: 5.446738579s * WIOS=21081, WBytes=1329741824, RIOS=79, RBytes=1197056 * Option 1: 6.239686088s * WIOS=34804, WBytes=1454845952, RIOS=79, RBytes=1197056 * Option 2: 1m30.510934813s * WIOS=42143, WBytes=1471397888, RIOS=82, RBytes=1209344 * Image: docker.io/tensorflow/tensorflow:latest (1.78 GiB, ~32590 Inodes) * Current: 8.852718042s * WIOS=39417, WBytes=2412818432, RIOS=2673, RBytes=335987712 * Option 1: 9.683387174s * WIOS=42767, WBytes=2431750144, RIOS=89, RBytes=1238016 * Option 2: 1m54.302103719s * WIOS=54403, WBytes=2460528640, RIOS=1709, RBytes=208237568 The Option 1 will increase `wios`. So, the `image_pull_with_sync_fs` is option in CRI plugin. [syncfs]: <https://man7.org/linux/man-pages/man2/syncfs.2.html> [xfstest-dev]: <https://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git> [generic/690]: <https://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git/tree/tests/generic/690?h=v2023.11.19> [kernel-crash-recovery-data-integrity]: <https://lore.kernel.org/linux-fsdevel/1552418820-18102-1-git-send-email-jaya@cs.utexas.edu/> [test-tool]: <a17fb2010d/contrib/syncfs/containerd/main_test.go (L51)> Signed-off-by: Wei Fu <fuweid89@gmail.com> (cherry picked from commit23278c81fb) Signed-off-by: Wei Fu <fuweid89@gmail.com>
132 lines
3.3 KiB
Go
132 lines
3.3 KiB
Go
/*
|
|
Copyright The containerd Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package apply
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"github.com/containerd/containerd/content"
|
|
"github.com/containerd/containerd/diff"
|
|
"github.com/containerd/containerd/log"
|
|
"github.com/containerd/containerd/mount"
|
|
digest "github.com/opencontainers/go-digest"
|
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
)
|
|
|
|
// NewFileSystemApplier returns an applier which simply mounts
|
|
// and applies diff onto the mounted filesystem.
|
|
func NewFileSystemApplier(cs content.Provider) diff.Applier {
|
|
return &fsApplier{
|
|
store: cs,
|
|
}
|
|
}
|
|
|
|
type fsApplier struct {
|
|
store content.Provider
|
|
}
|
|
|
|
var emptyDesc = ocispec.Descriptor{}
|
|
|
|
// Apply applies the content associated with the provided digests onto the
|
|
// provided mounts. Archive content will be extracted and decompressed if
|
|
// necessary.
|
|
func (s *fsApplier) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispec.Descriptor, err error) {
|
|
t1 := time.Now()
|
|
defer func() {
|
|
if err == nil {
|
|
log.G(ctx).WithFields(log.Fields{
|
|
"d": time.Since(t1),
|
|
"digest": desc.Digest,
|
|
"size": desc.Size,
|
|
"media": desc.MediaType,
|
|
}).Debugf("diff applied")
|
|
}
|
|
}()
|
|
|
|
var config diff.ApplyConfig
|
|
for _, o := range opts {
|
|
if err := o(ctx, desc, &config); err != nil {
|
|
return emptyDesc, fmt.Errorf("failed to apply config opt: %w", err)
|
|
}
|
|
}
|
|
|
|
ra, err := s.store.ReaderAt(ctx, desc)
|
|
if err != nil {
|
|
return emptyDesc, fmt.Errorf("failed to get reader from content store: %w", err)
|
|
}
|
|
defer ra.Close()
|
|
|
|
var processors []diff.StreamProcessor
|
|
processor := diff.NewProcessorChain(desc.MediaType, content.NewReader(ra))
|
|
processors = append(processors, processor)
|
|
for {
|
|
if processor, err = diff.GetProcessor(ctx, processor, config.ProcessorPayloads); err != nil {
|
|
return emptyDesc, fmt.Errorf("failed to get stream processor for %s: %w", desc.MediaType, err)
|
|
}
|
|
processors = append(processors, processor)
|
|
if processor.MediaType() == ocispec.MediaTypeImageLayer {
|
|
break
|
|
}
|
|
}
|
|
defer processor.Close()
|
|
|
|
digester := digest.Canonical.Digester()
|
|
rc := &readCounter{
|
|
r: io.TeeReader(processor, digester.Hash()),
|
|
}
|
|
|
|
if err := apply(ctx, mounts, rc, config.SyncFs); err != nil {
|
|
return emptyDesc, err
|
|
}
|
|
|
|
// Read any trailing data
|
|
if _, err := io.Copy(io.Discard, rc); err != nil {
|
|
return emptyDesc, err
|
|
}
|
|
|
|
for _, p := range processors {
|
|
if ep, ok := p.(interface {
|
|
Err() error
|
|
}); ok {
|
|
if err := ep.Err(); err != nil {
|
|
return emptyDesc, err
|
|
}
|
|
}
|
|
}
|
|
return ocispec.Descriptor{
|
|
MediaType: ocispec.MediaTypeImageLayer,
|
|
Size: rc.c,
|
|
Digest: digester.Digest(),
|
|
}, nil
|
|
}
|
|
|
|
type readCounter struct {
|
|
r io.Reader
|
|
c int64
|
|
}
|
|
|
|
func (rc *readCounter) Read(p []byte) (n int, err error) {
|
|
n, err = rc.r.Read(p)
|
|
if n > 0 {
|
|
rc.c += int64(n)
|
|
}
|
|
return
|
|
}
|