erofs-snapshotter: protect layer blobs with FS_IMMUTABLE_FL

As documented in ioctl_iflags(2):
```
 FS_IMMUTABLE_FL
  The file is immutable: no changes are permitted to the file contents
  or metadata (permissions, timestamps, ownership, link count, and so
  on).  (This restriction applies even to the superuser.)
```

For example, any user cannot delete/move layer blobs when
FS_IMMUTABLE_FL is set:
``` sh
 # cd /var/lib/containerd/io.containerd.snapshotter.v1.erofs/snapshots/4
 # mv layer{,1}.erofs
 mv: cannot move 'layer.erofs' to 'layer1.erofs': Operation not permitted
 # rm layer.erofs
 rm: cannot remove 'layer.erofs': Operation not permitted
```

Note that it's a best-effort approach for data loss prevention.  IOWs,
just warn out if FS_IMMUTABLE_FL cannot be set anyway (e.g., due to lack
of support in the underlying filesystem.)

Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>

docs/snapshotters/erofs.md

@@ -28,9 +28,12 @@ on the backing filesystem, it applies OCI layers into EROFS blobs, therefore:
  - Improved image unpacking performance (~14% for WordPress image with the
    latest erofs-utils 1.8.2) due to reduced metadata overhead;
 
- - Full data protection for each snapshot using the S_IMMUTABLE file attribute
-   or fsverity. Currently, fsverity can only protect blob data in the content
-   store;
+ - Full data protection for each snapshot using the FS_IMMUTABLE_FL file
+   attribute and fsverity.  EROFS uses FS_IMMUTABLE_FL and fsverity to protect
+   each EROFS layer blob, ensuring the mounted tree remains immutable.  However,
+   since FS_IMMUTABLE_FL and fsverity protect individual files rather than a
+   sub-filesystem tree, other snapshotter implementations like the overlayfs
+   snapshotter are not quite applicable due to less efficiency at least;
 
  - Parallel unpacking can be supported in a more reliable way (fsync) compared
    to the overlayfs snapshotter (syncfs);

plugins/snapshots/erofs/erofs_linux.go

@@ -355,6 +355,31 @@ func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snap
 	return s.createSnapshot(ctx, snapshots.KindView, key, parent, opts)
 }
 
+func setImmutable(path string, enable bool) error {
+	//nolint:revive	// silence "don't use ALL_CAPS in Go names; use CamelCase"
+	const (
+		FS_IMMUTABLE_FL = 0x10
+	)
+	f, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to open: %w", err)
+	}
+	defer f.Close()
+
+	oldattr, err := unix.IoctlGetInt(int(f.Fd()), unix.FS_IOC_GETFLAGS)
+	if err != nil {
+		return fmt.Errorf("error getting inode flags: %w", err)
+	}
+	newattr := oldattr | FS_IMMUTABLE_FL
+	if !enable {
+		newattr ^= FS_IMMUTABLE_FL
+	}
+	if newattr == oldattr {
+		return nil
+	}
+	return unix.IoctlSetPointerInt(int(f.Fd()), unix.FS_IOC_SETFLAGS, newattr)
+}
+
 func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
 	var layerBlob, upperDir string
 
@@ -403,7 +428,10 @@ func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
 				return fmt.Errorf("failed to enable fsverity: %w", err)
 			}
 		}
-
+		// Set IMMUTABLE_FL on the EROFS layer to avoid artificial data loss
+		if err := setImmutable(layerBlob, true); err != nil {
+			log.G(ctx).WithError(err).Warnf("failed to set IMMUTABLE_FL for %s", layerBlob)
+		}
 		return nil
 	})
 
@@ -507,6 +535,11 @@ func (s *snapshotter) Remove(ctx context.Context, key string) (err error) {
 		if err != nil {
 			return fmt.Errorf("unable to get directories for removal: %w", err)
 		}
+		// Clear IMMUTABLE_FL before removal, since this flag avoids it.
+		err = setImmutable(s.layerBlobPath(id), false)
+		if err != nil {
+			return fmt.Errorf("failed to clear IMMUTABLE_FL: %w", err)
+		}
 		return nil
 	})
 }