From 56f77d5ade945b3b8816a6c8acb328b7c6dce9a7 Mon Sep 17 00:00:00 2001 From: "Erik St. Martin" Date: Tue, 7 Jun 2016 15:05:43 -0400 Subject: [PATCH] Implementing support for --cpu-rt-period and --cpu-rt-runtime so that containers may specify these cgroup values at runtime. This will allow processes to change their priority to real-time within the container when CONFIG_RT_GROUP_SCHED is enabled in the kernel. See #22380. Also added sanity checks for the new --cpu-rt-runtime and --cpu-rt-period flags to ensure that that the kernel supports these features and that runtime is not greater than period. Daemon will support a --cpu-rt-runtime flag to initialize the parent cgroup on startup, this prevents the administrator from alotting runtime to docker after each restart. There are additional checks that could be added but maybe too far? Check parent cgroups to ensure values are <= parent, inspecting rtprio ulimit and issuing a warning. Signed-off-by: Erik St. Martin --- api/types/container/host_config.go | 6 +- cli/command/container/update.go | 48 ++--- contrib/completion/bash/docker | 4 + contrib/completion/zsh/_docker | 2 + daemon/config_unix.go | 4 + daemon/daemon_unix.go | 44 ++++- daemon/oci_linux.go | 26 ++- docs/reference/api/docker_remote_api.md | 1 + docs/reference/api/docker_remote_api_v1.25.md | 8 + docs/reference/commandline/create.md | 2 + docs/reference/commandline/run.md | 2 + docs/reference/commandline/update.md | 2 + docs/reference/run.md | 2 + man/docker-create.1.md | 17 ++ man/docker-run.1.md | 15 ++ man/docker-update.1.md | 17 ++ pkg/sysinfo/sysinfo.go | 6 + pkg/sysinfo/sysinfo_linux.go | 19 +- pkg/sysinfo/sysinfo_solaris.go | 8 +- runconfig/config.go | 5 + runconfig/hostconfig_solaris.go | 6 + runconfig/hostconfig_test.go | 61 ++++++ runconfig/hostconfig_unix.go | 23 +++ runconfig/hostconfig_windows.go | 17 ++ runconfig/opts/parse.go | 174 +++++++++--------- 25 files changed, 404 insertions(+), 115 deletions(-) diff --git a/api/types/container/host_config.go b/api/types/container/host_config.go index caa1371371..6cc1dc7591 100644 --- a/api/types/container/host_config.go +++ b/api/types/container/host_config.go @@ -243,8 +243,10 @@ type Resources struct { BlkioDeviceWriteBps []*blkiodev.ThrottleDevice BlkioDeviceReadIOps []*blkiodev.ThrottleDevice BlkioDeviceWriteIOps []*blkiodev.ThrottleDevice - CPUPeriod int64 `json:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period - CPUQuota int64 `json:"CpuQuota"` // CPU CFS (Completely Fair Scheduler) quota + CPUPeriod int64 `json:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period + CPUQuota int64 `json:"CpuQuota"` // CPU CFS (Completely Fair Scheduler) quota + CPURealtimePeriod int64 `json:"CpuRealtimePeriod"` // CPU real-time period + CPURealtimeRuntime int64 `json:"CpuRealtimeRuntime"` // CPU real-time runtime CpusetCpus string // CpusetCpus 0-2, 0,1 CpusetMems string // CpusetMems 0-2, 0,1 Devices []DeviceMapping // List of devices to map inside the container diff --git a/cli/command/container/update.go b/cli/command/container/update.go index b5770c8997..5bacc9be75 100644 --- a/cli/command/container/update.go +++ b/cli/command/container/update.go @@ -15,17 +15,19 @@ import ( ) type updateOptions struct { - blkioWeight uint16 - cpuPeriod int64 - cpuQuota int64 - cpusetCpus string - cpusetMems string - cpuShares int64 - memoryString string - memoryReservation string - memorySwap string - kernelMemory string - restartPolicy string + blkioWeight uint16 + cpuPeriod int64 + cpuQuota int64 + cpuRealtimePeriod int64 + cpuRealtimeRuntime int64 + cpusetCpus string + cpusetMems string + cpuShares int64 + memoryString string + memoryReservation string + memorySwap string + kernelMemory string + restartPolicy string nFlag int @@ -51,6 +53,8 @@ func NewUpdateCommand(dockerCli *command.DockerCli) *cobra.Command { flags.Uint16Var(&opts.blkioWeight, "blkio-weight", 0, "Block IO (relative weight), between 10 and 1000") flags.Int64Var(&opts.cpuPeriod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period") flags.Int64Var(&opts.cpuQuota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota") + flags.Int64Var(&opts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds") + flags.Int64Var(&opts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit the CPU real-time runtime in microseconds") flags.StringVar(&opts.cpusetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)") flags.StringVar(&opts.cpusetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)") flags.Int64VarP(&opts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)") @@ -115,16 +119,18 @@ func runUpdate(dockerCli *command.DockerCli, opts *updateOptions) error { } resources := containertypes.Resources{ - BlkioWeight: opts.blkioWeight, - CpusetCpus: opts.cpusetCpus, - CpusetMems: opts.cpusetMems, - CPUShares: opts.cpuShares, - Memory: memory, - MemoryReservation: memoryReservation, - MemorySwap: memorySwap, - KernelMemory: kernelMemory, - CPUPeriod: opts.cpuPeriod, - CPUQuota: opts.cpuQuota, + BlkioWeight: opts.blkioWeight, + CpusetCpus: opts.cpusetCpus, + CpusetMems: opts.cpusetMems, + CPUShares: opts.cpuShares, + Memory: memory, + MemoryReservation: memoryReservation, + MemorySwap: memorySwap, + KernelMemory: kernelMemory, + CPUPeriod: opts.cpuPeriod, + CPUQuota: opts.cpuQuota, + CPURealtimePeriod: opts.cpuRealtimePeriod, + CPURealtimeRuntime: opts.cpuRealtimeRuntime, } updateConfig := containertypes.UpdateConfig{ diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index e76be9b785..129d5c5b14 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -1316,6 +1316,8 @@ _docker_container_run() { --cidfile --cpu-period --cpu-quota + --cpu-rt-period + --cpu-rt-runtime --cpuset-cpus --cpuset-mems --cpu-shares -c @@ -1667,6 +1669,8 @@ _docker_container_update() { --blkio-weight --cpu-period --cpu-quota + --cpu-rt-period + --cpu-rt-runtime --cpuset-cpus --cpuset-mems --cpu-shares -c diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 5385a37fd8..852a98633a 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -1433,6 +1433,8 @@ __docker_subcommand() { "($help -c --cpu-shares)"{-c=,--cpu-shares=}"[CPU shares (relative weight)]:CPU shares:(0 10 100 200 500 800 1000)" "($help)--cpu-period=[Limit the CPU CFS (Completely Fair Scheduler) period]:CPU period: " "($help)--cpu-quota=[Limit the CPU CFS (Completely Fair Scheduler) quota]:CPU quota: " + "($help)--cpu-rt-period=[Limit the CPU real-time period]:CPU real-time period in microseconds: " + "($help)--cpu-rt-runtime=[Limit the CPU real-time runtime]:CPU real-time runtime in microseconds: " "($help)--cpuset-cpus=[CPUs in which to allow execution]:CPUs: " "($help)--cpuset-mems=[MEMs in which to allow execution]:MEMs: " "($help -m --memory)"{-m=,--memory=}"[Memory limit]:Memory limit: " diff --git a/daemon/config_unix.go b/daemon/config_unix.go index e9201fa494..caca8f97d5 100644 --- a/daemon/config_unix.go +++ b/daemon/config_unix.go @@ -34,6 +34,8 @@ type Config struct { Ulimits map[string]*units.Ulimit `json:"default-ulimits,omitempty"` Runtimes map[string]types.Runtime `json:"runtimes,omitempty"` DefaultRuntime string `json:"default-runtime,omitempty"` + CPURealtimePeriod int64 `json:"cpu-rt-period,omitempty"` + CPURealtimeRuntime int64 `json:"cpu-rt-runtime,omitempty"` OOMScoreAdjust int `json:"oom-score-adjust,omitempty"` Init bool `json:"init,omitempty"` InitPath string `json:"init-path,omitempty"` @@ -97,6 +99,8 @@ func (config *Config) InstallFlags(flags *pflag.FlagSet) { flags.IntVar(&config.OOMScoreAdjust, "oom-score-adjust", -500, "Set the oom_score_adj for the daemon") flags.BoolVar(&config.Init, "init", false, "Run an init in the container to forward signals and reap processes") flags.StringVar(&config.InitPath, "init-path", "", "Path to the docker-init binary") + flags.Int64Var(&config.CPURealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds") + flags.Int64Var(&config.CPURealtimeRuntime, "cpu-rt-runtime", 0, "Limit the CPU real-time runtime in microseconds") config.attachExperimentalFlags(flags) } diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 7dcfa7051e..2ea6be25be 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -36,10 +36,11 @@ import ( "github.com/docker/libnetwork/options" lntypes "github.com/docker/libnetwork/types" "github.com/golang/protobuf/ptypes" + "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/label" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/runc/libcontainer/user" - "github.com/opencontainers/runtime-spec/specs-go" + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/vishvananda/netlink" ) @@ -118,6 +119,16 @@ func getCPUResources(config containertypes.Resources) *specs.CPU { cpu.Quota = "a } + if config.CPURealtimePeriod != 0 { + period := uint64(config.CPURealtimePeriod) + cpu.RealtimePeriod = &period + } + + if config.CPURealtimeRuntime != 0 { + runtime := uint64(config.CPURealtimeRuntime) + cpu.RealtimeRuntime = &runtime + } + return &cpu } @@ -1184,3 +1195,34 @@ func setupOOMScoreAdj(score int) error { f.Close() return err } + +func (daemon *Daemon) initCgroupsPath(path string) error { + if path == "/" || path == "." { + return nil + } + + daemon.initCgroupsPath(filepath.Dir(path)) + + _, root, err := cgroups.FindCgroupMountpointAndRoot("cpu") + if err != nil { + return err + } + + path = filepath.Join(root, path) + sysinfo := sysinfo.New(false) + if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { + return err + } + if sysinfo.CPURealtimePeriod && daemon.configStore.CPURealtimePeriod != 0 { + if err := ioutil.WriteFile(filepath.Join(path, "cpu.rt_period_us"), []byte(strconv.FormatInt(daemon.configStore.CPURealtimePeriod, 10)), 0700); err != nil { + return err + } + } + if sysinfo.CPURealtimeRuntime && daemon.configStore.CPURealtimeRuntime != 0 { + if err := ioutil.WriteFile(filepath.Join(path, "cpu.rt_runtime_us"), []byte(strconv.FormatInt(daemon.configStore.CPURealtimeRuntime, 10)), 0700); err != nil { + return err + } + } + + return nil +} diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 4aadc83b6f..4372f8ee94 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -21,9 +21,10 @@ import ( "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/volume" "github.com/opencontainers/runc/libcontainer/apparmor" + "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/user" - "github.com/opencontainers/runtime-spec/specs-go" + specs "github.com/opencontainers/runtime-spec/specs-go" ) func setResources(s *specs.Spec, r containertypes.Resources) error { @@ -655,6 +656,29 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } s.Linux.Resources.OOMScoreAdj = &c.HostConfig.OomScoreAdj s.Linux.Sysctl = c.HostConfig.Sysctls + + p := *s.Linux.CgroupsPath + if useSystemd { + initPath, err := cgroups.GetInitCgroupDir("cpu") + if err != nil { + return nil, err + } + p, _ = cgroups.GetThisCgroupDir("cpu") + if err != nil { + return nil, err + } + p = filepath.Join(initPath, p) + } + + // Clean path to guard against things like ../../../BAD + parentPath := filepath.Dir(p) + if !filepath.IsAbs(parentPath) { + parentPath = filepath.Clean("/" + parentPath) + } + + if err := daemon.initCgroupsPath(parentPath); err != nil { + return nil, fmt.Errorf("linux init cgroups path: %v", err) + } if err := setDevices(&s, c); err != nil { return nil, fmt.Errorf("linux runtime spec devices: %v", err) } diff --git a/docs/reference/api/docker_remote_api.md b/docs/reference/api/docker_remote_api.md index 9acc56e0c6..d3b6cb08a5 100644 --- a/docs/reference/api/docker_remote_api.md +++ b/docs/reference/api/docker_remote_api.md @@ -160,6 +160,7 @@ This section lists each version from latest to oldest. Each listing includes a * `POST /volumes/prune` prunes unused volumes. * `POST /networks/prune` prunes unused networks. * Every API response now includes a `Docker-Experimental` header specifying if experimental features are enabled (value can be `true` or `false`). +* The `hostConfig` option now accepts the fields `CpuRealtimePeriod` and `CpuRtRuntime` to allocate cpu runtime to rt tasks when `CONFIG_RT_GROUP_SCHED` is enabled in the kernel. ### v1.24 API changes diff --git a/docs/reference/api/docker_remote_api_v1.25.md b/docs/reference/api/docker_remote_api_v1.25.md index b3483d53a3..824a4c2b51 100644 --- a/docs/reference/api/docker_remote_api_v1.25.md +++ b/docs/reference/api/docker_remote_api_v1.25.md @@ -304,6 +304,8 @@ Create a container "CpuPercent": 80, "CpuShares": 512, "CpuPeriod": 100000, + "CpuRealtimePeriod": 1000000, + "CpuRealtimeRuntime": 10000, "CpuQuota": 50000, "CpusetCpus": "0,1", "CpusetMems": "0,1", @@ -426,6 +428,8 @@ Create a container - **CpuShares** - An integer value containing the container's CPU Shares (ie. the relative weight vs other containers). - **CpuPeriod** - The length of a CPU period in microseconds. + - **CpuRealtimePeriod** - The length of a CPU real-time period in microseconds (0=no time allocated for rt tasks) + - **CpuRealtimeRuntime** - The length of a CPU real-time runtime in microseconds (0=no time allocated for rt tasks) - **CpuQuota** - Microseconds of CPU time that the container can get in a CPU period. - **CpusetCpus** - String value containing the `cgroups CpusetCpus` to use. - **CpusetMems** - Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. @@ -615,6 +619,8 @@ Return low-level information on the container `id` "CpuPercent": 80, "CpuShares": 0, "CpuPeriod": 100000, + "CpuRealtimePeriod": 1000000, + "CpuRealtimeRuntime": 10000, "Devices": [], "Dns": null, "DnsOptions": null, @@ -1191,6 +1197,8 @@ Update configuration of one or more containers. "BlkioWeight": 300, "CpuShares": 512, "CpuPeriod": 100000, + "CpuRealtimePeriod": 1000000, + "CpuRealtimeRuntime": 10000, "CpuQuota": 50000, "CpusetCpus": "0,1", "CpusetMems": "0", diff --git a/docs/reference/commandline/create.md b/docs/reference/commandline/create.md index 91c5f30ff5..784b4940e1 100644 --- a/docs/reference/commandline/create.md +++ b/docs/reference/commandline/create.md @@ -35,6 +35,8 @@ Options: --cpu-period int Limit CPU CFS (Completely Fair Scheduler) period --cpu-quota int Limit CPU CFS (Completely Fair Scheduler) quota -c, --cpu-shares int CPU shares (relative weight) + --cpu-rt-period int Limit the CPU real-time period in microseconds + --cpu-rt-runtime int Limit the CPU real-time runtime in microseconds --cpuset-cpus string CPUs in which to allow execution (0-3, 0,1) --cpuset-mems string MEMs in which to allow execution (0-3, 0,1) --device value Add a host device to the container (default []) diff --git a/docs/reference/commandline/run.md b/docs/reference/commandline/run.md index 2a35ebc8c5..ec6a412009 100644 --- a/docs/reference/commandline/run.md +++ b/docs/reference/commandline/run.md @@ -33,6 +33,8 @@ Options: --cpu-period int Limit CPU CFS (Completely Fair Scheduler) period --cpu-quota int Limit CPU CFS (Completely Fair Scheduler) quota -c, --cpu-shares int CPU shares (relative weight) + --cpu-rt-period int Limit the CPU real-time period in microseconds + --cpu-rt-runtime int Limit the CPU real-time runtime in microseconds --cpuset-cpus string CPUs in which to allow execution (0-3, 0,1) --cpuset-mems string MEMs in which to allow execution (0-3, 0,1) -d, --detach Run container in background and print container ID diff --git a/docs/reference/commandline/update.md b/docs/reference/commandline/update.md index 1a6a2690ba..9ba7993bf7 100644 --- a/docs/reference/commandline/update.md +++ b/docs/reference/commandline/update.md @@ -25,6 +25,8 @@ Options: --cpu-period int Limit CPU CFS (Completely Fair Scheduler) period --cpu-quota int Limit CPU CFS (Completely Fair Scheduler) quota -c, --cpu-shares int CPU shares (relative weight) + --cpu-rt-period int Limit the CPU real-time period in microseconds + --cpu-rt-runtime int Limit the CPU real-time runtime in microseconds --cpuset-cpus string CPUs in which to allow execution (0-3, 0,1) --cpuset-mems string MEMs in which to allow execution (0-3, 0,1) --help Print usage diff --git a/docs/reference/run.md b/docs/reference/run.md index 24c695e24d..a48b44d4bf 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -690,6 +690,8 @@ container: | `--cpuset-cpus=""` | CPUs in which to allow execution (0-3, 0,1) | | `--cpuset-mems=""` | Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. | | `--cpu-quota=0` | Limit the CPU CFS (Completely Fair Scheduler) quota | +| `--cpu-rt-period=0` | Limit the CPU real-time period. In microseconds. Requires parent cgroups be set and cannot be higher than parent. Also check rtprio ulimits. | +| `--cpu-rt-runtime=0` | Limit the CPU real-time runtime. In microseconds. Requires parent cgroups be set and cannot be higher than parent. Also check rtprio ulimits. | | `--blkio-weight=0` | Block IO weight (relative weight) accepts a weight value between 10 and 1000. | | `--blkio-weight-device=""` | Block IO weight (relative device weight, format: `DEVICE_NAME:WEIGHT`) | | `--device-read-bps=""` | Limit read rate from a device (format: `:[]`). Number is a positive integer. Unit can be one of `kb`, `mb`, or `gb`. | diff --git a/man/docker-create.1.md b/man/docker-create.1.md index ece8a91f5f..08e8665903 100644 --- a/man/docker-create.1.md +++ b/man/docker-create.1.md @@ -17,6 +17,8 @@ docker-create - Create a new container [**--cidfile**[=*CIDFILE*]] [**--cpu-period**[=*0*]] [**--cpu-quota**[=*0*]] +[**--cpu-rt-period**[=*0*]] +[**--cpu-rt-runtime**[=*0*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] [**--cpuset-mems**[=*CPUSET-MEMS*]] [**--device**[=*[]*]] @@ -123,6 +125,8 @@ The initial status of the container created with **docker create** is 'created'. **--cpu-period**=*0* Limit the CPU CFS (Completely Fair Scheduler) period + Limit the container's CPU usage. This flag tell the kernel to restrict the container's CPU usage to the period you specify. + **--cpuset-cpus**="" CPUs in which to allow execution (0-3, 0,1) @@ -136,6 +140,19 @@ two memory nodes. **--cpu-quota**=*0* Limit the CPU CFS (Completely Fair Scheduler) quota +**--cpu-rt-period**=0 + Limit the CPU real-time period in microseconds + + Limit the container's Real Time CPU usage. This flag tell the kernel to restrict the container's Real Time CPU usage to the period you specify. + +**--cpu-rt-runtime**=0 + Limit the CPU real-time runtime in microseconds + + Limit the containers Real Time CPU usage. This flag tells the kernel to limit the amount of time in a given CPU period Real Time tasks may consume. Ex: + Period of 1,000,000us and Runtime of 950,000us means that this container could consume 95% of available CPU and leave the remaining 5% to normal priority tasks. + + The sum of all runtimes across containers cannot exceed the amount alotted to the parent cgroup. + **--device**=[] Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm) diff --git a/man/docker-run.1.md b/man/docker-run.1.md index a18bae469e..8cbd26242c 100644 --- a/man/docker-run.1.md +++ b/man/docker-run.1.md @@ -17,6 +17,8 @@ docker-run - Run a command in a new container [**--cidfile**[=*CIDFILE*]] [**--cpu-period**[=*0*]] [**--cpu-quota**[=*0*]] +[**--cpu-rt-period**[=*0*]] +[**--cpu-rt-runtime**[=*0*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] [**--cpuset-mems**[=*CPUSET-MEMS*]] [**-d**|**--detach**] @@ -192,6 +194,19 @@ two memory nodes. CPU resource. This flag tell the kernel to restrict the container's CPU usage to the quota you specify. +**--cpu-rt-period**=0 + Limit the CPU real-time period in microseconds + + Limit the container's Real Time CPU usage. This flag tell the kernel to restrict the container's Real Time CPU usage to the period you specify. + +**--cpu-rt-runtime**=0 + Limit the CPU real-time runtime in microseconds + + Limit the containers Real Time CPU usage. This flag tells the kernel to limit the amount of time in a given CPU period Real Time tasks may consume. Ex: + Period of 1,000,000us and Runtime of 950,000us means that this container could consume 95% of available CPU and leave the remaining 5% to normal priority tasks. + + The sum of all runtimes across containers cannot exceed the amount alotted to the parent cgroup. + **-d**, **--detach**=*true*|*false* Detached mode: run the container in the background and print the new container ID. The default is *false*. diff --git a/man/docker-update.1.md b/man/docker-update.1.md index 71fb6e476a..7ae7099344 100644 --- a/man/docker-update.1.md +++ b/man/docker-update.1.md @@ -10,6 +10,8 @@ docker-update - Update configuration of one or more containers [**--cpu-shares**[=*0*]] [**--cpu-period**[=*0*]] [**--cpu-quota**[=*0*]] +[**--cpu-rt-period**[=*0*]] +[**--cpu-rt-runtime**[=*0*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] [**--cpuset-mems**[=*CPUSET-MEMS*]] [**--help**] @@ -44,9 +46,24 @@ a running container with kernel memory initialized. **--cpu-period**=0 Limit the CPU CFS (Completely Fair Scheduler) period + Limit the container's CPU usage. This flag tell the kernel to restrict the container's CPU usage to the period you specify. + **--cpu-quota**=0 Limit the CPU CFS (Completely Fair Scheduler) quota +**--cpu-rt-period**=0 + Limit the CPU real-time period in microseconds + + Limit the container's Real Time CPU usage. This flag tell the kernel to restrict the container's Real Time CPU usage to the period you specify. + +**--cpu-rt-runtime**=0 + Limit the CPU real-time runtime in microseconds + + Limit the containers Real Time CPU usage. This flag tells the kernel to limit the amount of time in a given CPU period Real Time tasks may consume. Ex: + Period of 1,000,000us and Runtime of 950,000us means that this container could consume 95% of available CPU and leave the remaining 5% to normal priority tasks. + + The sum of all runtimes across containers cannot exceed the amount alotted to the parent cgroup. + **--cpuset-cpus**="" CPUs in which to allow execution (0-3, 0,1) diff --git a/pkg/sysinfo/sysinfo.go b/pkg/sysinfo/sysinfo.go index dc71dbfb80..f046de4b16 100644 --- a/pkg/sysinfo/sysinfo.go +++ b/pkg/sysinfo/sysinfo.go @@ -58,6 +58,12 @@ type cgroupCPUInfo struct { // Whether CPU CFS(Completely Fair Scheduler) quota is supported or not CPUCfsQuota bool + + // Whether CPU real-time period is supported or not + CPURealtimePeriod bool + + // Whether CPU real-time runtime is supported or not + CPURealtimeRuntime bool } type cgroupBlkioInfo struct { diff --git a/pkg/sysinfo/sysinfo_linux.go b/pkg/sysinfo/sysinfo_linux.go index 7dce3a3ec0..5823ff9e80 100644 --- a/pkg/sysinfo/sysinfo_linux.go +++ b/pkg/sysinfo/sysinfo_linux.go @@ -135,10 +135,23 @@ func checkCgroupCPU(cgMounts map[string]string, quiet bool) cgroupCPUInfo { if !quiet && !cpuCfsQuota { logrus.Warn("Your kernel does not support cgroup cfs quotas") } + + cpuRealtimePeriod := cgroupEnabled(mountPoint, "cpu.rt_period_us") + if !quiet && !cpuRealtimePeriod { + logrus.Warn("Your kernel does not support cgroup rt period") + } + + cpuRealtimeRuntime := cgroupEnabled(mountPoint, "cpu.rt_runtime_us") + if !quiet && !cpuRealtimeRuntime { + logrus.Warn("Your kernel does not support cgroup rt runtime") + } + return cgroupCPUInfo{ - CPUShares: cpuShares, - CPUCfsPeriod: cpuCfsPeriod, - CPUCfsQuota: cpuCfsQuota, + CPUShares: cpuShares, + CPUCfsPeriod: cpuCfsPeriod, + CPUCfsQuota: cpuCfsQuota, + CPURealtimePeriod: cpuRealtimePeriod, + CPURealtimeRuntime: cpuRealtimeRuntime, } } diff --git a/pkg/sysinfo/sysinfo_solaris.go b/pkg/sysinfo/sysinfo_solaris.go index 75a9c9bb2d..c858d57e08 100644 --- a/pkg/sysinfo/sysinfo_solaris.go +++ b/pkg/sysinfo/sysinfo_solaris.go @@ -77,9 +77,11 @@ func setCgroupMem(quiet bool) cgroupMemInfo { func setCgroupCPU(quiet bool) cgroupCPUInfo { return cgroupCPUInfo{ - CPUShares: true, - CPUCfsPeriod: false, - CPUCfsQuota: true, + CPUShares: true, + CPUCfsPeriod: false, + CPUCfsQuota: true, + CPURealtimePeriod: false, + CPURealtimeRuntime: false, } } diff --git a/runconfig/config.go b/runconfig/config.go index 4596f048ae..16e5e5c09f 100644 --- a/runconfig/config.go +++ b/runconfig/config.go @@ -7,6 +7,7 @@ import ( "github.com/docker/docker/api/types/container" networktypes "github.com/docker/docker/api/types/network" + "github.com/docker/docker/pkg/sysinfo" "github.com/docker/docker/volume" ) @@ -68,6 +69,10 @@ func DecodeContainerConfig(src io.Reader) (*container.Config, *container.HostCon return nil, nil, nil, err } + // Validate Resources + if err := ValidateResources(hc, sysinfo.New(true)); err != nil { + return nil, nil, nil, err + } return w.Config, hc, w.NetworkingConfig, nil } diff --git a/runconfig/hostconfig_solaris.go b/runconfig/hostconfig_solaris.go index 97b66480c7..399593a892 100644 --- a/runconfig/hostconfig_solaris.go +++ b/runconfig/hostconfig_solaris.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/sysinfo" ) // DefaultDaemonNetworkMode returns the default network stack the daemon should @@ -45,3 +46,8 @@ func ValidateIsolation(hc *container.HostConfig) error { func ValidateQoS(hc *container.HostConfig) error { return nil } + +// ValidateResources performs platform specific validation of the resource settings +func ValidateResources(hc *container.HostConfig, si *sysinfo.SysInfo) error { + return nil +} diff --git a/runconfig/hostconfig_test.go b/runconfig/hostconfig_test.go index 8fdc567636..a6a2b34fc1 100644 --- a/runconfig/hostconfig_test.go +++ b/runconfig/hostconfig_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/sysinfo" ) // TODO Windows: This will need addressing for a Windows daemon. @@ -220,3 +221,63 @@ func TestDecodeHostConfig(t *testing.T) { } } } + +func TestValidateResources(t *testing.T) { + type resourceTest struct { + ConfigCPURealtimePeriod int64 + ConfigCPURealtimeRuntime int64 + SysInfoCPURealtimePeriod bool + SysInfoCPURealtimeRuntime bool + ErrorExpected bool + FailureMsg string + } + + tests := []resourceTest{ + { + ConfigCPURealtimePeriod: 1000, + ConfigCPURealtimeRuntime: 1000, + SysInfoCPURealtimePeriod: true, + SysInfoCPURealtimeRuntime: true, + ErrorExpected: false, + FailureMsg: "Expected valid configuration", + }, + { + ConfigCPURealtimePeriod: 5000, + ConfigCPURealtimeRuntime: 5000, + SysInfoCPURealtimePeriod: false, + SysInfoCPURealtimeRuntime: true, + ErrorExpected: true, + FailureMsg: "Expected failure when cpu-rt-period is set but kernel doesn't support it", + }, + { + ConfigCPURealtimePeriod: 5000, + ConfigCPURealtimeRuntime: 5000, + SysInfoCPURealtimePeriod: true, + SysInfoCPURealtimeRuntime: false, + ErrorExpected: true, + FailureMsg: "Expected failure when cpu-rt-runtime is set but kernel doesn't support it", + }, + { + ConfigCPURealtimePeriod: 5000, + ConfigCPURealtimeRuntime: 10000, + SysInfoCPURealtimePeriod: true, + SysInfoCPURealtimeRuntime: false, + ErrorExpected: true, + FailureMsg: "Expected failure when cpu-rt-runtime is greater than cpu-rt-period", + }, + } + + for _, rt := range tests { + var hc container.HostConfig + hc.Resources.CPURealtimePeriod = rt.ConfigCPURealtimePeriod + hc.Resources.CPURealtimeRuntime = rt.ConfigCPURealtimeRuntime + + var si sysinfo.SysInfo + si.CPURealtimePeriod = rt.SysInfoCPURealtimePeriod + si.CPURealtimeRuntime = rt.SysInfoCPURealtimeRuntime + + if err := ValidateResources(&hc, &si); (err != nil) != rt.ErrorExpected { + t.Fatal(rt.FailureMsg, err) + } + } +} diff --git a/runconfig/hostconfig_unix.go b/runconfig/hostconfig_unix.go index c43718fc28..6e2b7f5ff7 100644 --- a/runconfig/hostconfig_unix.go +++ b/runconfig/hostconfig_unix.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/sysinfo" ) // DefaultDaemonNetworkMode returns the default network stack the daemon should @@ -104,3 +105,25 @@ func ValidateQoS(hc *container.HostConfig) error { } return nil } + +// ValidateResources performs platform specific validation of the resource settings +// cpu-rt-runtime and cpu-rt-period can not be greater than their parent, cpu-rt-runtime requires sys_nice +func ValidateResources(hc *container.HostConfig, si *sysinfo.SysInfo) error { + // We may not be passed a host config, such as in the case of docker commit + if hc == nil { + return nil + } + + if hc.Resources.CPURealtimePeriod > 0 && !si.CPURealtimePeriod { + return fmt.Errorf("invalid --cpu-rt-period: Your kernel does not support cgroup rt period") + } + + if hc.Resources.CPURealtimeRuntime > 0 && !si.CPURealtimeRuntime { + return fmt.Errorf("invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime") + } + + if hc.Resources.CPURealtimePeriod != 0 && hc.Resources.CPURealtimeRuntime != 0 && hc.Resources.CPURealtimeRuntime > hc.Resources.CPURealtimePeriod { + return fmt.Errorf("invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period") + } + return nil +} diff --git a/runconfig/hostconfig_windows.go b/runconfig/hostconfig_windows.go index edfd0b1e79..91bd6dcc3c 100644 --- a/runconfig/hostconfig_windows.go +++ b/runconfig/hostconfig_windows.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/sysinfo" ) // DefaultDaemonNetworkMode returns the default network stack the daemon should @@ -49,3 +50,19 @@ func ValidateIsolation(hc *container.HostConfig) error { func ValidateQoS(hc *container.HostConfig) error { return nil } + +// ValidateResources performs platform specific validation of the resource settings +func ValidateResources(hc *container.HostConfig, si *sysinfo.SysInfo) error { + // We may not be passed a host config, such as in the case of docker commit + if hc == nil { + return nil + } + + if hc.Resources.CPURealtimePeriod != 0 { + return fmt.Errorf("invalid --cpu-rt-period: Windows does not support this feature") + } + if hc.Resources.CPURealtimeRuntime != 0 { + return fmt.Errorf("invalid --cpu-rt-runtime: Windows does not support this feature") + } + return nil +} diff --git a/runconfig/opts/parse.go b/runconfig/opts/parse.go index e4221dee2d..e011800cce 100644 --- a/runconfig/opts/parse.go +++ b/runconfig/opts/parse.go @@ -23,90 +23,92 @@ import ( // ContainerOptions is a data object with all the options for creating a container type ContainerOptions struct { - attach opts.ListOpts - volumes opts.ListOpts - tmpfs opts.ListOpts - blkioWeightDevice WeightdeviceOpt - deviceReadBps ThrottledeviceOpt - deviceWriteBps ThrottledeviceOpt - links opts.ListOpts - aliases opts.ListOpts - linkLocalIPs opts.ListOpts - deviceReadIOps ThrottledeviceOpt - deviceWriteIOps ThrottledeviceOpt - env opts.ListOpts - labels opts.ListOpts - devices opts.ListOpts - ulimits *UlimitOpt - sysctls *opts.MapOpts - publish opts.ListOpts - expose opts.ListOpts - dns opts.ListOpts - dnsSearch opts.ListOpts - dnsOptions opts.ListOpts - extraHosts opts.ListOpts - volumesFrom opts.ListOpts - envFile opts.ListOpts - capAdd opts.ListOpts - capDrop opts.ListOpts - groupAdd opts.ListOpts - securityOpt opts.ListOpts - storageOpt opts.ListOpts - labelsFile opts.ListOpts - loggingOpts opts.ListOpts - privileged bool - pidMode string - utsMode string - usernsMode string - publishAll bool - stdin bool - tty bool - oomKillDisable bool - oomScoreAdj int - containerIDFile string - entrypoint string - hostname string - memoryString string - memoryReservation string - memorySwap string - kernelMemory string - user string - workingDir string - cpuShares int64 - cpuPercent int64 - cpuPeriod int64 - cpuQuota int64 - cpusetCpus string - cpusetMems string - blkioWeight uint16 - ioMaxBandwidth string - ioMaxIOps uint64 - swappiness int64 - netMode string - macAddress string - ipv4Address string - ipv6Address string - ipcMode string - pidsLimit int64 - restartPolicy string - readonlyRootfs bool - loggingDriver string - cgroupParent string - volumeDriver string - stopSignal string - stopTimeout int - isolation string - shmSize string - noHealthcheck bool - healthCmd string - healthInterval time.Duration - healthTimeout time.Duration - healthRetries int - runtime string - autoRemove bool - init bool - initPath string - credentialSpec string + attach opts.ListOpts + volumes opts.ListOpts + tmpfs opts.ListOpts + blkioWeightDevice WeightdeviceOpt + deviceReadBps ThrottledeviceOpt + deviceWriteBps ThrottledeviceOpt + links opts.ListOpts + aliases opts.ListOpts + linkLocalIPs opts.ListOpts + deviceReadIOps ThrottledeviceOpt + deviceWriteIOps ThrottledeviceOpt + env opts.ListOpts + labels opts.ListOpts + devices opts.ListOpts + ulimits *UlimitOpt + sysctls *opts.MapOpts + publish opts.ListOpts + expose opts.ListOpts + dns opts.ListOpts + dnsSearch opts.ListOpts + dnsOptions opts.ListOpts + extraHosts opts.ListOpts + volumesFrom opts.ListOpts + envFile opts.ListOpts + capAdd opts.ListOpts + capDrop opts.ListOpts + groupAdd opts.ListOpts + securityOpt opts.ListOpts + storageOpt opts.ListOpts + labelsFile opts.ListOpts + loggingOpts opts.ListOpts + privileged bool + pidMode string + utsMode string + usernsMode string + publishAll bool + stdin bool + tty bool + oomKillDisable bool + oomScoreAdj int + containerIDFile string + entrypoint string + hostname string + memoryString string + memoryReservation string + memorySwap string + kernelMemory string + user string + workingDir string + cpuShares int64 + cpuPercent int64 + cpuPeriod int64 + cpuRealtimePeriod int64 + cpuRealtimeRuntime int64 + cpuQuota int64 + cpusetCpus string + cpusetMems string + blkioWeight uint16 + ioMaxBandwidth string + ioMaxIOps uint64 + swappiness int64 + netMode string + macAddress string + ipv4Address string + ipv6Address string + ipcMode string + pidsLimit int64 + restartPolicy string + readonlyRootfs bool + loggingDriver string + cgroupParent string + volumeDriver string + stopSignal string + stopTimeout int + isolation string + shmSize string + noHealthcheck bool + healthCmd string + healthInterval time.Duration + healthTimeout time.Duration + healthRetries int + runtime string + autoRemove bool + init bool + initPath string + credentialSpec string Image string Args []string @@ -225,6 +227,8 @@ func AddFlags(flags *pflag.FlagSet) *ContainerOptions { flags.Int64Var(&copts.cpuPercent, "cpu-percent", 0, "CPU percent (Windows only)") flags.Int64Var(&copts.cpuPeriod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period") flags.Int64Var(&copts.cpuQuota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota") + flags.Int64Var(&copts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit CPU real-time period in microseconds") + flags.Int64Var(&copts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit CPU real-time runtime in microseconds") flags.Int64VarP(&copts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)") flags.Var(&copts.deviceReadBps, "device-read-bps", "Limit read rate (bytes per second) from a device") flags.Var(&copts.deviceReadIOps, "device-read-iops", "Limit read rate (IO per second) from a device") @@ -521,6 +525,8 @@ func Parse(flags *pflag.FlagSet, copts *ContainerOptions) (*container.Config, *c CpusetCpus: copts.cpusetCpus, CpusetMems: copts.cpusetMems, CPUQuota: copts.cpuQuota, + CPURealtimePeriod: copts.cpuRealtimePeriod, + CPURealtimeRuntime: copts.cpuRealtimeRuntime, PidsLimit: copts.pidsLimit, BlkioWeight: copts.blkioWeight, BlkioWeightDevice: copts.blkioWeightDevice.GetList(),