Merge pull request #50908 from ndeloof/restarting

Add `restarting` attribute on `die` events

api/docs/CHANGELOG.md

@@ -15,6 +15,13 @@ keywords: "API, Docker, rcli, REST, documentation"
 
 ## v1.57 API changes
 
+* `GET /events`: `die` events for containers now include a `restarting`
+  attribute, set to `"true"` when the container is going to be restarted,
+  either by its restart policy or by a manual restart in progress. The
+  attribute is omitted when the container is not going to be restarted,
+  including when the daemon is shutting down. It announces the restart but
+  does not guarantee it: the restart can still be cancelled (for example by
+  stopping the container) or fail to start the container again.
 * The API specification now uses OpenAPI 3.2.0 instead of Swagger 2.0.
   A generated Swagger 2.0 version is still available for existing tooling.
 

daemon/monitor.go

@@ -120,6 +120,13 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine
 		"exitCode":     strconv.Itoa(ctrExitStatus.ExitCode),
 		"execDuration": strconv.Itoa(int(execDuration.Seconds())),
 	}
+	if restart || (c.HasBeenManuallyRestarted && !daemonShutdown) {
+		// The container will be restarted, either by the restart policy or
+		// by a manual "container restart" in progress (which doesn't
+		// complete if the daemon is shutting down).
+		attributes["restarting"] = "true"
+	}
+
 	daemon.Cleanup(context.TODO(), c)
 
 	if restart {

daemon/restart.go

@@ -32,7 +32,7 @@ func (daemon *Daemon) ContainerRestart(ctx context.Context, name string, options
 // container. When stopping, wait for the given duration in seconds to
 // gracefully stop, before forcefully terminating the container. If
 // given a negative duration, wait forever for a graceful stop.
-func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *configStore, container *container.Container, options backend.ContainerStopOptions) error {
+func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *configStore, container *container.Container, options backend.ContainerStopOptions) (retErr error) {
 	// Restarting is expected to be an atomic operation, and cancelling
 	// the request should not cancel the stop -> start sequence.
 	ctx = context.WithoutCancel(ctx)
@@ -60,6 +60,17 @@ func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *configSto
 		container.HasBeenManuallyRestarted = true
 		container.Unlock()
 
+		// The flag is only cleared once the container is started again. If
+		// the restart fails, make sure it doesn't linger: otherwise a later,
+		// unrelated exit would be reported as "going to restart".
+		defer func() {
+			if retErr != nil {
+				container.Lock()
+				container.HasBeenManuallyRestarted = false
+				container.Unlock()
+			}
+		}()
+
 		err := daemon.containerStop(ctx, container, options)
 		if err != nil {
 			return err

integration/container/restart_test.go

@@ -564,3 +564,139 @@ func testContainerRestartWithCancelledRequest(ctx context.Context, t *testing.T,
 	assert.NilError(t, err)
 	assert.Check(t, is.Equal(inspect.Container.State.Status, container.StateRunning))
 }
+
+// TestContainerDieEventRestartingAttribute verifies that "die" events tell
+// event consumers whether the container is going to be restarted, either by
+// its restart policy or by a manual restart, so that they don't have to
+// inspect the container to find out.
+func TestContainerDieEventRestartingAttribute(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows", "test requires a Linux shell")
+
+	ctx := setupTest(t)
+	apiClient := testEnv.APIClient()
+
+	// newContainer creates a container, and returns a function to wait for
+	// its events. Events are subscribed to before the container is started,
+	// so none of them can be missed.
+	newContainer := func(ctx context.Context, t *testing.T, ops ...func(*testContainer.TestContainerConfig)) (string, *containerEvents) {
+		t.Helper()
+		cID := testContainer.Create(ctx, t, apiClient, ops...)
+		t.Cleanup(func() {
+			apiClient.ContainerRemove(context.WithoutCancel(ctx), cID, client.ContainerRemoveOptions{Force: true})
+		})
+		evs := listenContainerEvents(ctx, t, apiClient, cID)
+		_, err := apiClient.ContainerStart(ctx, cID, client.ContainerStartOptions{})
+		assert.NilError(t, err)
+		return cID, evs
+	}
+	exitWithFailure := testContainer.WithCmd("sh", "-c", "exit 1")
+	stopTimeout := 1
+
+	t.Run("restart-policy", func(t *testing.T) {
+		ctx := testutil.StartSpan(ctx, t)
+		_, evs := newContainer(ctx, t, exitWithFailure, testContainer.WithRestartPolicy(container.RestartPolicyAlways))
+
+		die := evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], "true"))
+		evs.next(t, events.ActionStart)
+	})
+
+	t.Run("restart-policy-max-retries-reached", func(t *testing.T) {
+		ctx := testutil.StartSpan(ctx, t)
+		_, evs := newContainer(ctx, t, exitWithFailure, func(c *testContainer.TestContainerConfig) {
+			c.HostConfig.RestartPolicy = container.RestartPolicy{
+				Name:              container.RestartPolicyOnFailure,
+				MaximumRetryCount: 1,
+			}
+		})
+
+		// The first exit is followed by the one allowed retry.
+		die := evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], "true"))
+		evs.next(t, events.ActionStart)
+
+		// The second one isn't: no more retries.
+		die = evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], ""))
+	})
+
+	t.Run("no-restart-policy", func(t *testing.T) {
+		ctx := testutil.StartSpan(ctx, t)
+		_, evs := newContainer(ctx, t, exitWithFailure)
+
+		die := evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], ""))
+	})
+
+	t.Run("stopped-with-restart-policy", func(t *testing.T) {
+		ctx := testutil.StartSpan(ctx, t)
+		cID, evs := newContainer(ctx, t, testContainer.WithRestartPolicy(container.RestartPolicyAlways))
+		poll.WaitOn(t, testContainer.IsInState(ctx, apiClient, cID, container.StateRunning), poll.WithTimeout(30*time.Second))
+
+		_, err := apiClient.ContainerStop(ctx, cID, client.ContainerStopOptions{Timeout: &stopTimeout})
+		assert.NilError(t, err)
+
+		die := evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], ""))
+	})
+
+	t.Run("manual-restart", func(t *testing.T) {
+		ctx := testutil.StartSpan(ctx, t)
+		cID, evs := newContainer(ctx, t)
+		poll.WaitOn(t, testContainer.IsInState(ctx, apiClient, cID, container.StateRunning), poll.WithTimeout(30*time.Second))
+
+		_, err := apiClient.ContainerRestart(ctx, cID, client.ContainerRestartOptions{Timeout: &stopTimeout})
+		assert.NilError(t, err)
+
+		die := evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], "true"))
+		evs.next(t, events.ActionStart)
+		evs.next(t, events.ActionRestart)
+
+		// The container isn't flagged as restarting once it was restarted.
+		_, err = apiClient.ContainerStop(ctx, cID, client.ContainerStopOptions{Timeout: &stopTimeout})
+		assert.NilError(t, err)
+		die = evs.next(t, events.ActionDie)
+		assert.Check(t, is.Equal(die.Actor.Attributes["restarting"], ""))
+	})
+}
+
+// containerEvents delivers the events of a single container.
+type containerEvents struct {
+	messages <-chan events.Message
+	errs     <-chan error
+}
+
+// listenContainerEvents starts listening for the events of a container, until
+// the test completes.
+func listenContainerEvents(ctx context.Context, t *testing.T, apiClient client.APIClient, cID string) *containerEvents {
+	t.Helper()
+	ctx, cancel := context.WithCancel(ctx)
+	t.Cleanup(cancel)
+
+	// Events are replayed from now (the container is yet to be started), so
+	// that a container exiting right away doesn't beat the subscription.
+	result := apiClient.Events(ctx, client.EventsListOptions{
+		Since:   strconv.FormatInt(time.Now().Unix(), 10),
+		Filters: make(client.Filters).Add("container", cID),
+	})
+	return &containerEvents{messages: result.Messages, errs: result.Err}
+}
+
+// next waits for the next event with the given action, skipping any other one.
+func (e *containerEvents) next(t *testing.T, action events.Action) events.Message {
+	t.Helper()
+	timeout := time.After(30 * time.Second)
+	for {
+		select {
+		case m := <-e.messages:
+			if m.Action == action {
+				return m
+			}
+		case err := <-e.errs:
+			t.Fatalf("error while waiting for %q event: %v", action, err)
+		case <-timeout:
+			t.Fatalf("timeout waiting for %q event", action)
+		}
+	}
+}