diff --git a/contrib/dockerd-rootless.sh b/contrib/dockerd-rootless.sh
index a5d13d02e3..3bc12e72b0 100755
--- a/contrib/dockerd-rootless.sh
+++ b/contrib/dockerd-rootless.sh
@@ -190,24 +190,6 @@ if [ -z "$_DOCKERD_ROOTLESS_CHILD" ]; then
 else
 	[ "$_DOCKERD_ROOTLESS_CHILD" = 1 ]
 
-	# The Container Device Interface (CDI) specs can be found by default
-	# under {/etc,/var/run}/cdi. More information at:
-	# https://github.com/cncf-tags/container-device-interface
-	#
-	# In order to use the Container Device Interface (CDI) integration,
-	# the CDI paths need to exist before the Docker daemon is started in
-	# order for it to read the CDI specification files. Otherwise, a
-	# Docker daemon restart will be required for the daemon to discover
-	# them.
-	#
-	# If another set of CDI paths (other than the default /etc/cdi and
-	# /var/run/cdi) are configured through the Docker configuration file
-	# (using "cdi-spec-dirs"), they need to be bind mounted in rootless
-	# mode; otherwise the Docker daemon won't have access to the CDI
-	# specification files.
-	mount_directory /etc/cdi
-	mount_directory /var/run/cdi
-
 	# remove the symlinks for the existing files in the parent namespace if any,
 	# so that we can create our own files in our mount namespace.
 	rm -f /run/docker /run/containerd /run/xtables.lock
diff --git a/daemon/command/daemon.go b/daemon/command/daemon.go
index 67a11ad9f3..2fe591ed62 100644
--- a/daemon/command/daemon.go
+++ b/daemon/command/daemon.go
@@ -57,6 +57,7 @@ import (
 	"github.com/moby/moby/v2/pkg/homedir"
 	"github.com/moby/moby/v2/pkg/pidfile"
 	"github.com/moby/moby/v2/pkg/plugingetter"
+	"github.com/moby/sys/userns"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -668,7 +669,23 @@ func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) {
 			}
 			conf.CDISpecDirs = append(conf.CDISpecDirs, filepath.Join(xch, "cdi"), filepath.Join(xrd, "cdi"))
 		}
-	} else if len(conf.CDISpecDirs) == 1 && conf.CDISpecDirs[0] == "" {
+	}
+	// Filter out CDI spec directories that are not readable, and log appropriately
+	var cdiSpecDirs []string
+	for _, dir := range conf.CDISpecDirs {
+		// Non-existing directories are not filtered out here, as CDI spec directories are allowed to not exist.
+		if _, err := os.ReadDir(dir); err == nil || errors.Is(err, os.ErrNotExist) {
+			cdiSpecDirs = append(cdiSpecDirs, dir)
+		} else {
+			logLevel := log.ErrorLevel
+			if userns.RunningInUserNS() && errors.Is(err, os.ErrPermission) {
+				logLevel = log.DebugLevel
+			}
+			log.L.WithField("dir", dir).WithError(err).Log(logLevel, "CDI spec directory cannot be accessed, skipping")
+		}
+	}
+	conf.CDISpecDirs = cdiSpecDirs
+	if len(conf.CDISpecDirs) == 1 && conf.CDISpecDirs[0] == "" {
 		// If CDISpecDirs is set to an empty string, we clear it to ensure that CDI is disabled.
 		conf.CDISpecDirs = nil
 	}