Linux containers in 500 lines of code (2016)

Listing 10: <<namespaces>> +=

	if (socketpair(AF_LOCAL, SOCK_SEQPACKET, 0, sockets)) {
		fprintf(stderr, "socketpair failed: %m\n");
		goto error;
	}
	if (fcntl(sockets[0], F_SETFD, FD_CLOEXEC)) {
		fprintf(stderr, "fcntl failed: %m\n");
		goto error;
	}
	config.fd = sockets[1];

Listing 13: <<namespaces>> +=

	#define STACK_SIZE (1024 * 1024)

	char *stack = 0;
	if (!(stack = malloc(STACK_SIZE))) {
		fprintf(stderr, "=> malloc failed, out of memory?\n");
		goto error;
	}

Listing 14: <<namespaces>> +=

	if (resources(&config)) {
		err = 1;
		goto clear_resources;
	}

Listing 15: <<namespaces>> +=

	int flags = CLONE_NEWNS
		| CLONE_NEWCGROUP
		| CLONE_NEWPID
		| CLONE_NEWIPC
		| CLONE_NEWNET
		| CLONE_NEWUTS;

Listing 16: <<namespaces>> +=

	if ((child_pid = clone(child, stack + STACK_SIZE, flags | SIGCHLD, &config)) == -1) {
		fprintf(stderr, "=> clone failed! %m\n");
		err = 1;
		goto clear_resources;
	}

Listing 17: <<namespaces>> +=

	close(sockets[1]);
	sockets[1] = 0;

Listing 21: <<child>> +=

#define USERNS_OFFSET 10000
#define USERNS_COUNT 2000

int handle_child_uid_map (pid_t child_pid, int fd)
{
	int uid_map = 0;
	int has_userns = -1;
	if (read(fd, &has_userns, sizeof(has_userns)) != sizeof(has_userns)) {
		fprintf(stderr, "couldn't read from child!\n");
		return -1;
	}
	if (has_userns) {
		char path[PATH_MAX] = {0};
		for (char **file = (char *[]) { "uid_map", "gid_map", 0 }; *file; file++) {
			if (snprintf(path, sizeof(path), "/proc/%d/%s", child_pid, *file)
			    > sizeof(path)) {
				fprintf(stderr, "snprintf too big? %m\n");
				return -1;
			}
			fprintf(stderr, "writing %s...", path);
			if ((uid_map = open(path, O_WRONLY)) == -1) {
				fprintf(stderr, "open failed: %m\n");
				return -1;
			}
			if (dprintf(uid_map, "0 %d %d\n", USERNS_OFFSET, USERNS_COUNT) == -1) {
				fprintf(stderr, "dprintf failed: %m\n");
				close(uid_map);
				return -1;
			}
			close(uid_map);
		}
	}
	if (write(fd, & (int) { 0 }, sizeof(int)) != sizeof(int)) {
		fprintf(stderr, "couldn't write: %m\n");
		return -1;
	}
	return 0;
}

Listing 23: <<child>> +=

int userns(struct child_config *config)
{
	fprintf(stderr, "=> trying a user namespace...");
	int has_userns = !unshare(CLONE_NEWUSER);
	if (write(config->fd, &has_userns, sizeof(has_userns)) != sizeof(has_userns)) {
		fprintf(stderr, "couldn't write: %m\n");
		return -1;
	}
	int result = 0;
	if (read(config->fd, &result, sizeof(result)) != sizeof(result)) {
		fprintf(stderr, "couldn't read: %m\n");
		return -1;
	}
	if (result) return -1;
	if (has_userns) {
		fprintf(stderr, "done.\n");
	} else {
		fprintf(stderr, "unsupported? continuing.\n");
	}
	fprintf(stderr, "=> switching to uid %d / gid %d...", config->uid, config->uid);
	if (setgroups(1, & (gid_t) { config->uid }) ||
	    setresgid(config->uid, config->uid, config->uid) ||
	    setresuid(config->uid, config->uid, config->uid)) {
		fprintf(stderr, "%m\n");
		return -1;
	}
	fprintf(stderr, "done.\n");
	return 0;
}

Listing 24: <<child>> +=

int child(void *arg)
{
	struct child_config *config = arg;
	if (sethostname(config->hostname, strlen(config->hostname))
	    || mounts(config)
	    || userns(config)
	    || capabilities()
	    || syscalls()) {
		close(config->fd);
		return -1;
	}
	if (close(config->fd)) {
		fprintf(stderr, "close failed: %m\n");
		return -1;
	}
	if (execve(config->argv[0], config->argv, NULL)) {
		fprintf(stderr, "execve failed! %m.\n");
		return -1;
	}
	return 0;
}

	During  an   execve(2),  the   kernel  calculates   the  new
	capabilities of the process using the following algorithm:

	    P'(ambient) = (file is privileged) ? 0 : P(ambient)

	    P'(permitted) = (P(inheritable) & F(inheritable)) |
					(F(permitted) & cap_bset) | P'(ambient)

	    P'(effective) = F(effective) ? P'(permitted) : P'(ambient)

	    P'(inheritable) = P(inheritable)    [i.e., unchanged]

	where:

	    P         denotes the  value of a thread  capability set
			    before the execve(2)

	    P'        denotes the  value of a thread  capability set
			    after the execve(2)

	    F         denotes a file capability set

	    cap_bset  is the  value of  the capability  bounding set
			    (described below).

int capabilities()
{
	fprintf(stderr, "=> dropping capabilities...");

CAP_AUDIT_CONTROL, _READ, and _WRITE allow access to the audit system of the kernel (i.e. functions like audit_set_enabled, usually used with auditctl). The kernel prevents messages that normally require CAP_AUDIT_CONTROL outside of the first pid namespace, but it does allow messages that would require CAP_AUDIT_READ and CAP_AUDIT_WRITE from any namespace.¹² So let's drop them all. We especially want to drop CAP_AUDIT_READ, since it isn't namespaced¹³ and may contain important information, but CAP_AUDIT_WRITE may also allow the contained process to falsify logs or DOS the audit system.

	int drop_caps[] = {
		CAP_AUDIT_CONTROL,
		CAP_AUDIT_READ,
		CAP_AUDIT_WRITE,

CAP_BLOCK_SUSPEND lets programs prevent the system from suspending, either with EPOLLWAKEUP or /proc/sys/wake_lock.¹⁴ Supend isn't namespaced, so we'd like to prevent this.

		CAP_BLOCK_SUSPEND,

CAP_DAC_READ_SEARCH lets programs call open_by_handle_at with an arbitrary struct file_handle *. struct file_handle is in theory an opaque type, but in practice it corresponds to inode numbers. So it's easy to brute-force them, and read arbitrary files. This was used by Sebastian Krahmer to write a program to read arbitrary system files from within Docker in 2014.¹⁵

		CAP_DAC_READ_SEARCH,

CAP_FSETID, without user namespacing, allows the process to modify a setuid executable without removing the setuid bit. This is pretty dangerous! It means that if we include a setuid binary in a container, it's easy for us to accidentally leave a dangerous setuid root binary on our disk, which any user can use to escalate privileges.¹⁶

		CAP_FSETID,

CAP_IPC_LOCK can be used to lock more of a process' own memory than would normally be allowed¹⁷, which could be a way to deny service.

		CAP_IPC_LOCK,

CAP_MAC_ADMIN and CAP_MAC_OVERRIDE are used by the mandatory acess control systems Apparmor, SELinux, and SMACK to restrict access to their settings. These aren't namespaced, so they could be used by the contained programs to circumvent system-wide access control.

		CAP_MAC_ADMIN,
		CAP_MAC_OVERRIDE,

CAP_MKNOD, without user namespacing, allows programs to create device files corresponding to real-world devices. This includes creating new device files for existing hardware. If this capability were not dropped, a contained process could re-create the hard disk device, remount it, and read or write to it.¹⁸

		CAP_MKNOD,

I was worried that CAP_SETFCAP could be used to add a capability to an executable and execve it, but it's not actually possible for a process to set capabilities it doesn't have¹⁹. But! An executable altered this way could be executed by any unsandboxed user, so I think it unacceptably undermines the security of the system.

		CAP_SETFCAP,

CAP_SYSLOG lets users perform destructive actions against the syslog. Importantly, it doesn't prevent contained processes from reading the syslog, which could be risky. It also exposes kernel addresses, which could be used to circumvent kernel address layout randomization²⁰.

		CAP_SYSLOG,

CAP_SYS_ADMIN allows many behaviors! We don't want most of them (mount, vm86, etc). Some would be nice to have (sethostname, mount for bind mounts…) but the extra complexity doesn't seem worth it.

		CAP_SYS_ADMIN,

CAP_SYS_BOOT allows programs to restart the system (the reboot syscall) and load new kernels (the kexec_load and kexec_file syscalls)²¹. We absolutely don't want this. reboot is user-namespaced, and the kexec* functions only work in the root user namespace, but neither of those help us.

		CAP_SYS_BOOT,

CAP_SYS_MODULE is used by the syscalls delete_module, init_module, finit_module ²², by the code for kmod ²³, and by the code for loading device modules with ioctl²⁴.

		CAP_SYS_MODULE,

CAP_SYS_NICE allows processes to set higher priority on given pids than the default²⁵. The default kernel scheduler doesn't know anything about pid namespaces, so it's possible for a contained process to deny service to the rest of the system²⁶.

		CAP_SYS_NICE,

CAP_SYS_RAWIO allows full access to the host systems memory with /proc/kcore, /dev/mem, and /dev/kmem ²⁷, but a contained process would need mknod to access these within the namespace.²⁸. But it also allows things like iopl and ioperm, which give raw access to the IO ports²⁹.

		CAP_SYS_RAWIO,

CAP_SYS_RESOURCE specifically allows circumventing kernel-wide limits, so we probably should drop it³⁰. But I don't think this can do more than DOS the kernel, in general³¹.

		CAP_SYS_RESOURCE,

CAP_SYS_TIME: setting the time isn't namespaced, so we should prevent contained processes from altering the system-wide time³².

		CAP_SYS_TIME,

CAP_WAKE_ALARM, like CAP_BLOCK_SUSPEND, lets the contained process interfere with suspend³³, and we'd like to prevent that.

		CAP_WAKE_ALARM
	};

	size_t num_caps = sizeof(drop_caps) / sizeof(*drop_caps);
	fprintf(stderr, "bounding...");
	for (size_t i = 0; i < num_caps; i++) {
		if (prctl(PR_CAPBSET_DROP, drop_caps[i], 0, 0, 0)) {
			fprintf(stderr, "prctl failed: %m\n");
			return 1;
		}
	}
	fprintf(stderr, "inheritable...");
	cap_t caps = NULL;
	if (!(caps = cap_get_proc())
	    || cap_set_flag(caps, CAP_INHERITABLE, num_caps, drop_caps, CAP_CLEAR)
	    || cap_set_proc(caps)) {
		fprintf(stderr, "failed: %m\n");
		if (caps) cap_free(caps);
		return 1;
	}
	cap_free(caps);
	fprintf(stderr, "done.\n");
	return 0;
}

It's important to keep track of the capabilities I'm not dropping, too.

I've heard multiple places³⁴ that CAP_DAC_OVERRIDE might expose the same functionality as CAP_DAC_READ_SEARCH (i.e. open_by_handle_at), but as far as I can tell that isn't true. shocker.c doesn't get anywhere with only CAP_DAC_OVERRIDE ³⁵, and the only usage in the kernel is in the Unix permission-checking code³⁶. So my understanding is that CAP_DAC_OVERRIDE on its own doesn't allow processes to read outside of their mount namespaces ("DAC" or "Discretionary Access Control" refers here to ordinary unix permissions).

CAP_FOWNER, CAP_LEASE, and CAP_LINUX_IMMUTABLE all operate on files inside of the mount namespace.

Likewise, CAP_SYS_PACCT allows processes to switch accounting on and off for itself. The acct system call takes a path to log to (which must be within the mount namespace), and only operates on the calling process. We're not using process accounting in our containerization, so turning it off should be harmless as well.³⁷

CAP_IPC_OWNER is only used by functions that respect IPC namespaces³⁸; since we're in a separate IPC namespace from the host, we can allow this.

CAP_NET_ADMIN lets processes create network devices; CAP_NET_BIND_SERVICE lets processes bind to low ports on those devices; CAP_NET_RAW lets processes send raw packets on those devices. Since we're going to isolate the networking with a virtual bridge, and the contained process is inside of a network namespace, these shouldn't be an issue³⁹. I was wondering whether we could recreate an existing device like mknod does, but I don't think it's possible ⁴⁰.

CAP_SYS_PTRACE doesn't allow ptrace across pid namespaces⁴¹. CAP_KILL doesn't allow signals across pid namespaces⁴².

CAP_SETUID and CAPSETGID have similar behaviors⁴³:

• Make arbitrary manipulations of process UIDS and GIDs and supplementary GID list, which will only apply to pids in the namespace.
• forge UID (GID) when passing socket credentials via UNIX domain sockets the mount namespace should prevent us from reading the host system's unix domain sockets.
• write a user(group ID) mapping in a user namespace (see user_namespaces(7)): this is /proc/self/uid_map, which will be hidden inside the container.

CAP_SETPCAP only lets processes add or drop capabilities they already effectively have; man 7 capabilities says

If file capabilities are supported: add any capability from the calling thread's bounding set to its inheritable set; drop capabilities from the bounding set (via prctl(2) PR_CAPBSET_DROP); make changes to the securebits flags.

We've dropped everything relevant from the bounding set, and dropping further capabilities should be harmless.

CAP_SYS_CHROOT is traditionally abused by changing root to a directory with a setuid root binary and tampered-with dynamic libraries⁴⁴. Additionally, it can be used to escape a chroot "jail"⁴⁵. Neither of those should be relevant in our setup so this should be harmless.

Brad Spengler, in "False Boundaries and Arbitrary Code Execution" says that CAP_SYS_TTYCONFIG can "temporarily change the keyboard mapping of an administrator's tty via the KDSETKEYCODE ioctl to cause a different command to be executed than intended", but again this is an ioctl against a device that should be impossible to access within the mount namespace.

Listing 108: <<mounts>> =

<<pivot-root>>

int mounts(struct child_config *config)
{
	fprintf(stderr, "=> remounting everything with MS_PRIVATE...");
	if (mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL)) {
		fprintf(stderr, "failed! %m\n");
		return -1;
	}
	fprintf(stderr, "remounted.\n");

	fprintf(stderr, "=> making a temp directory and a bind mount there...");
	char mount_dir[] = "/tmp/tmp.XXXXXX";
	if (!mkdtemp(mount_dir)) {
		fprintf(stderr, "failed making a directory!\n");
		return -1;
	}

	if (mount(config->mount_dir, mount_dir, NULL, MS_BIND | MS_PRIVATE, NULL)) {
		fprintf(stderr, "bind mount failed!\n");
		return -1;
	}

	char inner_mount_dir[] = "/tmp/tmp.XXXXXX/oldroot.XXXXXX";
	memcpy(inner_mount_dir, mount_dir, sizeof(mount_dir) - 1);
	if (!mkdtemp(inner_mount_dir)) {
		fprintf(stderr, "failed making the inner directory!\n");
		return -1;
	}
	fprintf(stderr, "done.\n");

	fprintf(stderr, "=> pivoting root...");
	if (pivot_root(mount_dir, inner_mount_dir)) {
		fprintf(stderr, "failed!\n");
		return -1;
	}
	fprintf(stderr, "done.\n");

	char *old_root_dir = basename(inner_mount_dir);
	char old_root[sizeof(inner_mount_dir) + 1] = { "/" };
	strcpy(&old_root[1], old_root_dir);

	fprintf(stderr, "=> unmounting %s...", old_root);
	if (chdir("/")) {
		fprintf(stderr, "chdir failed! %m\n");
		return -1;
	}
	if (umount2(old_root, MNT_DETACH)) {
		fprintf(stderr, "umount failed! %m\n");
		return -1;
	}
	if (rmdir(old_root)) {
		fprintf(stderr, "rmdir failed! %m\n");
		return -1;
	}
	fprintf(stderr, "done.\n");
	return 0;
}

Listing 109: <<pivot-root>> =

int pivot_root(const char *new_root, const char *put_old)
{
	return syscall(SYS_pivot_root, new_root, put_old);
}

#define SCMP_FAIL SCMP_ACT_ERRNO(EPERM)

int syscalls()
{
	scmp_filter_ctx ctx = NULL;
	fprintf(stderr, "=> filtering syscalls...");
	if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))

We want to prevent new setuid / setgid executables from being created, since in the absence of user namespaces the contained process could create a setuid binary that could be used by any user to get root.⁴⁸

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(chmod), 1,
				SCMP_A1(SCMP_CMP_MASKED_EQ, S_ISUID, S_ISUID))
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(chmod), 1,
				SCMP_A1(SCMP_CMP_MASKED_EQ, S_ISGID, S_ISGID))
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(fchmod), 1,
				SCMP_A1(SCMP_CMP_MASKED_EQ, S_ISUID, S_ISUID))
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(fchmod), 1,
				SCMP_A1(SCMP_CMP_MASKED_EQ, S_ISGID, S_ISGID))
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(fchmodat), 1,
				SCMP_A2(SCMP_CMP_MASKED_EQ, S_ISUID, S_ISUID))
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(fchmodat), 1,
				SCMP_A2(SCMP_CMP_MASKED_EQ, S_ISGID, S_ISGID))

Allowing contained processes to start new user namespaces can allow processes to gain new (albeit limited) capabilities, so we prevent it.

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(unshare), 1,
				SCMP_A0(SCMP_CMP_MASKED_EQ, CLONE_NEWUSER, CLONE_NEWUSER))
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(clone), 1,
				SCMP_A0(SCMP_CMP_MASKED_EQ, CLONE_NEWUSER, CLONE_NEWUSER))

TIOCSTI allows contained processes to write to the controlling terminal⁴⁹.

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(ioctl), 1,
				SCMP_A1(SCMP_CMP_MASKED_EQ, TIOCSTI, TIOCSTI))

The kernel keyring system isn't namespaced.⁵⁰

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(keyctl), 0)
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(add_key), 0)
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(request_key), 0)

Before Linux 4.8, ptrace totally breaks seccomp⁵¹.

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(ptrace), 0)

These system calls let processes assign NUMA nodes. I don't have anything specific in mind, but I could see these being used to deny service to some other NUMA-aware application on the host.

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(mbind), 0)
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(migrate_pages), 0)
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(move_pages), 0)
	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(set_mempolicy), 0)

userfaultd allows userspace to handle page faults⁵². It doesn't require any privileges, so in theory it should be safe to be called by an unprivileged user. But it can be used to pause execution in the kernel by triggering page faults in system calls. This is an important part in some kernel exploits⁵³. It's only rarely used legitimately, so I'll disable it.

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(userfaultfd), 0)

I was initially worried about perf_event_open because the Docker documentation says it "could leak a lot of information on the host", but it can't be used in our system to see information for out-of-namespace processes⁵⁴. But, if /proc/sys/kernel/perf_event_paranoid is less than 2, it can be used to discover kernel addresses and possibly uninitialized memory. 2 is the default since is the default since 4.6, but it can be changed, and relying on it seems like a bad idea⁵⁵.

	    || seccomp_rule_add(ctx, SCMP_FAIL, SCMP_SYS(perf_event_open), 0)

We'll set PR_SET_NO_NEW_PRIVS to 0. The name is a little vague: it specifically prevents setuid and setcap'd binaries from being executed with their additional privileges. This has some security benefits (it makes it harder for an unprivileged user in-container to exploit a vulnerability in a setuid or setcap executable to become in-container root, for example). But it's a little weird, and means that, for example, ping won't work in a container for an unprivileged user⁵⁶.

	    || seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, 0)

And we'll actually apply it to the process, and release the context.

	    || seccomp_load(ctx)) {
		if (ctx) seccomp_release(ctx);
		fprintf(stderr, "failed: %m\n");
		return 1;
	}
	seccomp_release(ctx);
	fprintf(stderr, "done.\n");
	return 0;
}

Here are the system calls that are disallowed by the default Docker policy but permitted by this code:

_sysctl is obsolete and disabled by default⁵⁷. alloc_hugepages and free_hugepages ⁵⁸, bdflush ⁵⁹, create_module ⁶⁰, nfsservctl ⁶¹, perfctr ⁶², get_kernel_syms ⁶³, and setup ⁶⁴ are not present on modern Linux.

clock_adjtime, clock_settime ⁶⁵, and adjtime ⁶⁶ depend on CAP_SYS_TIME.

pciconfig_read and pciconfig_write ⁶⁷ and all of the side-effecting operations of quotactl ⁶⁸ are prevented by CAP_SYS_ADMIN.

get_mempolicy and getpagesize reveal information about the memory layout of the system, but they can be made by unprivileged processes, and are probably harmless. pciconfig_iobase can be made by unprivileged processes, and reveals information about PCI decvices. ustat ⁶⁹ and sysfs ⁷⁰ leak some information about the filesystems, but are nothing that I see as critical. uselib is more-or-less obsolete, but is just used for loading a shared library in userspace ⁷¹

sync_file_range2 is sync_file_range with swapped argument order⁷².

readdir is mostly obsolete, but probably harmless⁷³.

kexec_file_load and kexec_load are prevented by CAP_SYS_BOOT ⁷⁴.

nice can only be used to lower priority without CAP_SYS_NICE ⁷⁵.

oldfstat, oldlstat, oldolduname, oldstat, and olduname are just older versions of their respective functions. I expect them to have the same security properties as the modern ones.

perfmonctl ⁷⁶ is only available on IA-64. ppc_rtas ⁷⁷, spu_create ⁷⁸ and spu_run ⁷⁹, and subpage_prot ⁸⁰ are only avaiable on PowerPC. utrap_install is only available on Sparc⁸¹. kern_features is only available on Sparc64, and should be harmless anyway⁸².

I don't believe pivot_root is a problem in our setup (but it could probably be used to circumvent path-based MAC).

preadv2 and pwritev2 are just extensions to preadv and pwritev / readv and writev, which are "scatter input" / "gather output" extensions to read and write ⁸³.

Listing 181: <<resources>> +=

#define MEMORY "1073741824"
#define SHARES "256"
#define PIDS "64"
#define WEIGHT "10"
#define FD_COUNT 64

struct cgrp_control {
	char control[256];
	struct cgrp_setting {
		char name[256];
		char value[256];
	} **settings;
};
struct cgrp_setting add_to_tasks = {
	.name = "tasks",
	.value = "0"
};

struct cgrp_control *cgrps[] = {
	& (struct cgrp_control) {
		.control = "memory",
		.settings = (struct cgrp_setting *[]) {
			& (struct cgrp_setting) {
				.name = "memory.limit_in_bytes",
				.value = MEMORY
			},
			& (struct cgrp_setting) {
				.name = "memory.kmem.limit_in_bytes",
				.value = MEMORY
			},
			&add_to_tasks,
			NULL
		}
	},
	& (struct cgrp_control) {
		.control = "cpu",
		.settings = (struct cgrp_setting *[]) {
			& (struct cgrp_setting) {
				.name = "cpu.shares",
				.value = SHARES
			},
			&add_to_tasks,
			NULL
		}
	},
	& (struct cgrp_control) {
		.control = "pids",
		.settings = (struct cgrp_setting *[]) {
			& (struct cgrp_setting) {
				.name = "pids.max",
				.value = PIDS
			},
			&add_to_tasks,
			NULL
		}
	},
	& (struct cgrp_control) {
		.control = "blkio",
		.settings = (struct cgrp_setting *[]) {
			& (struct cgrp_setting) {
				.name = "blkio.weight",
				.value = PIDS
			},
			&add_to_tasks,
			NULL
		}
	},
	NULL
};

Listing 183: <<resources>> +=

int resources(struct child_config *config)
{
	fprintf(stderr, "=> setting cgroups...");
	for (struct cgrp_control **cgrp = cgrps; *cgrp; cgrp++) {
		char dir[PATH_MAX] = {0};
		fprintf(stderr, "%s...", (*cgrp)->control);
		if (snprintf(dir, sizeof(dir), "/sys/fs/cgroup/%s/%s",
			     (*cgrp)->control, config->hostname) == -1) {
			return -1;
		}
		if (mkdir(dir, S_IRUSR | S_IWUSR | S_IXUSR)) {
			fprintf(stderr, "mkdir %s failed: %m\n", dir);
			return -1;
		}
		for (struct cgrp_setting **setting = (*cgrp)->settings; *setting; setting++) {
			char path[PATH_MAX] = {0};
			int fd = 0;
			if (snprintf(path, sizeof(path), "%s/%s", dir,
				     (*setting)->name) == -1) {
				fprintf(stderr, "snprintf failed: %m\n");
				return -1;
			}
			if ((fd = open(path, O_WRONLY)) == -1) {
				fprintf(stderr, "opening %s failed: %m\n", path);
				return -1;
			}
			if (write(fd, (*setting)->value, strlen((*setting)->value)) == -1) {
				fprintf(stderr, "writing to %s failed: %m\n", path);
				close(fd);
				return -1;
			}
			close(fd);
		}
	}
	fprintf(stderr, "done.\n");

Listing 185: <<resources>> +=

	fprintf(stderr, "=> setting rlimit...");
	if (setrlimit(RLIMIT_NOFILE,
		      & (struct rlimit) {
			.rlim_max = FD_COUNT,
			.rlim_cur = FD_COUNT,
		})) {
		fprintf(stderr, "failed: %m\n");
		return 1;
	}
	fprintf(stderr, "done.\n");
	return 0;
}

Listing 187: <<resources>> +=

int free_resources(struct child_config *config)
{
	fprintf(stderr, "=> cleaning cgroups...");
	for (struct cgrp_control **cgrp = cgrps; *cgrp; cgrp++) {
		char dir[PATH_MAX] = {0};
		char task[PATH_MAX] = {0};
		int task_fd = 0;
		if (snprintf(dir, sizeof(dir), "/sys/fs/cgroup/%s/%s",
			     (*cgrp)->control, config->hostname) == -1
		    || snprintf(task, sizeof(task), "/sys/fs/cgroup/%s/tasks",
				(*cgrp)->control) == -1) {
			fprintf(stderr, "snprintf failed: %m\n");
			return -1;
		}
		if ((task_fd = open(task, O_WRONLY)) == -1) {
			fprintf(stderr, "opening %s failed: %m\n", task);
			return -1;
		}
		if (write(task_fd, "0", 2) == -1) {
			fprintf(stderr, "writing to %s failed: %m\n", task);
			close(task_fd);
			return -1;
		}
		close(task_fd);
		if (rmdir(dir)) {
			fprintf(stderr, "rmdir %s failed: %m", dir);
			return -1;
		}
	}
	fprintf(stderr, "done.\n");
	return 0;
}