daemon/command: add support for sd_notify "reload" notifications

commit f74b856e1a added an [ExecReload] to
the systemd unit, which allows users to signal the daemon to reload its
config through systemd (`systemctl reload docker.service`).

While reloading works, systemd expects the `ExecReload` command to be
synchronous, so that it knows when the reload completes, and can account
for this when managing dependent services.

> Note however that reloading a daemon by enqueuing a signal (...) is usually
> not a good choice, because this is an asynchronous operation and hence not
> suitable when ordering reloads of multiple services against each other.

Systemd 253 introduced a new Type (Type=notify-reload, see [systemd#25916]),
which allows setting a "ReloadSignal" instead, if the service supports
it by including a [MONOTONIC_USEC] value in its "RELOADING=1"
notifications.

This patch:

- adds reload notifications to the daemon to notify when the daemon got
  signaled to reload its configuration see [sd_notify(3)].
- appends a [MONOTONIC_USEC] value to the reload notification to support
  systemd 253's notify-reload protocol. This is backwards-compatible
  with older systemd releases as the systemd service manager ignores
  unknown assignments in notifications.
- adds a `notifyReady` callback to notify when the reload finished,
  which we currently send regardless if the reload was successful or
  failed (which could be due to an invalid config).

[ExecReload]: https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecReload=
[systemd#25916]: https://github.com/systemd/systemd/pull/25916
[MONOTONIC_USEC]: https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html#MONOTONIC_USEC=…
[sd_notify(3)]: https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html#RELOADING=1

Signed-off-by: Cory Snider <csnider@mirantis.com>
Co-authored-by: Sebastiaan van Stijn <github@gone.nl>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

daemon/command/daemon.go

@@ -521,6 +521,13 @@ func (cli *daemonCLI) reloadConfig() {
 		}
 	}
 
+	// On Linux, we use sd_notify to indicate we're reloading config. We send
+	// this signal as early as possible to let systemd know we're reloading,
+	// but must signal "ready" after this completes (even on failure), which
+	// is done by the reload function defined above.
+	done := notifyReloading()
+	defer done()
+
 	if err := config.Reload(*cli.configFile, cli.flags, reload); err != nil {
 		log.G(ctx).WithError(err).Error("Error reloading configuration")
 		return

daemon/command/daemon_freebsd.go

@@ -11,6 +11,10 @@ func preNotifyReady() error {
 func notifyReady() {
 }
 
+// notifyReloading sends a message to the host when the server got signaled to
+// reloading its configuration. It is a no-op on FreeBSD.
+func notifyReloading() func() { return func() {} }
+
 // notifyStopping sends a message to the host when the server is shutting down
 func notifyStopping() {
 }

daemon/command/daemon_linux.go

@@ -58,6 +58,25 @@ func notifyStopping() {
 	_, _ = systemdDaemon.SdNotify(false, systemdDaemon.SdNotifyStopping)
 }
 
+// notifyReloading sends a message to the host when the server got signaled to
+// reloading its configuration, see [sd_notify(3)]. The server should be running
+// as a systemd unit with "Type=notify" or "Type=notify-reload" (see
+// [systemd.service(5)]).
+//
+// notifyReloading returns a callback that must be called after reloading completes
+// (either successfully or unsuccessfully) to send [notifyReady].
+//
+// [sd_notify(3)]: https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html#RELOADING=1
+// [systemd.service(5)]: https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#Type=
+func notifyReloading() (done func()) {
+	sent, _ := systemdDaemon.SdNotify(false, systemdDaemon.SdNotifyReloading+"\n"+systemdDaemon.SdNotifyMonotonicUsec())
+	if !sent {
+		// Nothing to do if no reloading event was sent.
+		return func() {}
+	}
+	return notifyReady
+}
+
 func validateCPURealtimeOptions(cfg *config.Config) error {
 	if cfg.CPURealtimePeriod == 0 && cfg.CPURealtimeRuntime == 0 {
 		return nil

daemon/command/daemon_windows.go

@@ -56,6 +56,10 @@ func preNotifyReady() error {
 func notifyReady() {
 }
 
+// notifyReloading sends a message to the host when the server got signaled to
+// reloading its configuration. It is a no-op on Windows.
+func notifyReloading() func() { return func() {} }
+
 // notifyStopping sends a message to the host when the server is shutting down
 func notifyStopping() {
 }