最後更新: 2016-06-21

介紹

 * system processes are called tasks in cgroup terminology.

 

---

準備

 

# Install

yum -y install libcgroup

# Auto start

service cgconfig restart

chkconfig cgconfig on

# checking cgroup subsystems working

ls /cgroup

blkio  cpu  cpuacct  cpuset  devices  freezer  memory  net_cls

比較特別的 cgroup

devices — this subsystem allows or denies access to devices by tasks in a cgroup

net_cls — this subsystem tags network packets with a class identifier (classid)

                 that allows the Linux traffic controller (tc) to identify packets originating from a particular cgroup task

net_prio — this subsystem provides a way to dynamically set the priority of network traffic per network interface

ns — this is the namespace subsystem

 

---

Configuration

 

cp -a /etc/cgconfig.conf /etc/cgconfig.conf.orig

建立 limitcpu, limitmem 這兩個 cgroups 去 limit resource

/etc/cgconfig.conf

mount {
        cpuset  = /cgroup/cpuset;
        cpu     = /cgroup/cpu;
        cpuacct = /cgroup/cpuacct;
        memory  = /cgroup/memory;
        devices = /cgroup/devices;
        freezer = /cgroup/freezer;
        net_cls = /cgroup/net_cls;
        blkio   = /cgroup/blkio;
}

group limitcpu{
        cpu {
                cpu.shares = 400;
        }
}

group limitmem{
        memory {
                # 可用 unit: m
                memory.limit_in_bytes = 512m;
        }
}

Remark

cgroup hierarchy is located at /cgroup on Centos 6

Centos 7 at /sys/fs/cgroup

每次修改後都要 restart

service cgconfig restart

Checking config

lscgroup

cpuset:/
cpu:/
cpu:/limitcpu
cpuacct:/
memory:/
memory:/limitmem
devices:/
freezer:/
net_cls:/
blkio:/

 

---

Starting a Process in a Control Group

 

* services that support sysconfig

/etc/sysconfig/servicename

[1] CGROUP_DAEMON="subsystem:control_group"

i.e.

CGROUP_DAEMON="cpu:/limitcpu"

[2] CGROUP_DAEMON="<space separated list of groups>"

i.e.

CGROUP_DAEMON="cpu:/limit_httpd memory:/limit_httpd"

[3] GROUP_DAEMON="subsystem1,subsystem2:/control_group"

i.e.

CGROUP_DAEMON="cpu,memory:/limit_httpd"

Details

The daemon() function in /etc/init.d/functions was modified to start the daemon in control group(s) specified by CGROUP_DAEMON environment variable.

 

---

Cgred (control group rules engine daemon)

 

cp /etc/cgrules.conf /etc/cgrules.conf.orig

/etc/cgrules.conf

#<user>                 <controllers>           <destination>
#<user>:<process name>  <controllers>           <destination>

<user>
# group: @group
# * for any user or group
# % — represents an item the same as the item in the line above.

<process name>
# a process name
# a full command path of a process

<controller>
# comma separated controller names

service cgred start

chkconfig cgred on

 

---

Example: Apache

 

/etc/sysconfig/httpd

CGROUP_DAEMON="cpuacct,cpu,cpuset,memory:/limit_httpd"

/etc/cgconfig.conf

mount {
        cpuset  = /cgroup/cpuset;
        cpu     = /cgroup/cpu;
        cpuacct = /cgroup/cpuacct;
        memory  = /cgroup/memory;
        devices = /cgroup/devices;
        freezer = /cgroup/freezer;
        net_cls = /cgroup/net_cls;
        blkio   = /cgroup/blkio;
}

group limit_httpd{
        cpu {
                cpu.shares = 700;
        }
        cpuset {
                cpuset.cpus = 0;
                cpuset.mems = 0;
        }
        cpuacct {
        }
        net_cls {
                net_cls.classid = 65568;
        }
        memory {
                memory.limit_in_bytes = 700m;
                memory.memsw.limit_in_bytes = 800m;
        }
}

Apply setting

service cgconfig restart

Check memory usage

cgget -r memory.memsw.usage_in_bytes /limit_httpd

cgget -r memory.stat  /limit_httpd

cgget -r memory.memsw.failcnt /limit_httpd

cgget -r memory.oom_control /limit_httpd

Put httpd inside cgroup

ps -e -o pid,cmd,cgroup | grep [h]ttpd

 2076 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2078 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2079 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2080 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2081 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2082 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2083 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2084 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2085 /usr/sbin/httpd             blkio:/;net_cls:/;freezer:/;devices:/;memory:/limit_httpd;cpuacct:/limit_httpd;cpu:/limit_httpd;cpuset:/limit_httpd
 2119 grep httpd                  blkio:/;net_cls:/;freezer:/;devices:/;memory:/;cpuacct:/;cpu:/;cpuset:/

Stress Testing

在 dmsg 出現以下 msg 後, apache 會 hang 了, 那時要 killall -9 httpd 才殺到

Memory cgroup out of memory: Kill process 21012 (httpd) score 142 or sacrifice child
Killed process 21012, UID 48, (httpd) total-vm:46664kB, anon-rss:4880kB, file-rss:5464kB

解決

/etc/httpd/conf/httpd.conf

<IfModule prefork.c>
StartServers       5
MinSpareServers    5
MaxSpareServers   10
MaxClients        25
ServerLimit      256
MaxRequestsPerChild 100
</IfModule>

ListenBacklog 100

# MaxRequestsPerChild

the limit on the number of requests that an individual child server process will handle

(memory leakage)

# 0=> the process will never expire.

# KeepAlive

only the first request is counted towards this limit.

(In effect, it changes the behavior to limit the number of connections per child.)

 

 

---