diff --git a/solver/llbsolver/file/backend.go b/solver/llbsolver/file/backend.go
index 45d66d713..07044691f 100644
--- a/solver/llbsolver/file/backend.go
+++ b/solver/llbsolver/file/backend.go
@@ -27,13 +27,9 @@ func timestampToTime(ts int64) *time.Time {
 }
 
 func mapUser(user *copy.ChownOpt, idmap *idtools.IdentityMapping) (*copy.ChownOpt, error) {
-	if idmap == nil {
+	if idmap == nil || user == nil {
 		return user, nil
 	}
-	if user == nil {
-		identity := idmap.RootPair()
-		return &copy.ChownOpt{Uid: identity.UID, Gid: identity.GID}, nil
-	}
 	identity, err := idmap.ToHost(idtools.Identity{
 		UID: user.Uid,
 		GID: user.Gid,
@@ -138,7 +134,6 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *
 		return nil
 	}
 
-	// TODO(tonistiigi): this is wrong. fsutil.Copy can't handle non-forced user
 	u, err := mapUser(u, idmap)
 	if err != nil {
 		return err