Skip to content

Fallback to bwrap in linux-sandbox when AppArmor denies mount in userns #32315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tripleyoung wants to merge 1 commit into
base: master
Choose a base branch
from
+287 −12
Open

Fallback to bwrap in linux-sandbox when AppArmor denies mount in userns #32315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 224 additions & 7 deletions src/main/tools/linux-sandbox-pid1.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ static int CreateTarget(const char* path, bool is_directory) {
return 0;
}

static void SetupSelfDestruction(int* pipe_to_parent) {
static void SetupSelfDestruction() {
// We could also poll() on the pipe fd to find out when the parent goes away,
// and rely on SIGCHLD interrupting that otherwise. That might require us to
// install some trivial handler for SIGCHLD. Using O_ASYNC to turn the pipe
Expand All @@ -226,17 +226,22 @@ static void SetupSelfDestruction(int* pipe_to_parent) {
if (setpgid(0, 0) < 0) {
DIE("setpgid");
}

// Verify that the parent still lives.
SignalPipe(pipe_to_parent);
}

static void SetupMountNamespace() {
static void SetupMountNamespace(int* pipe_to_parent) {
// Fully isolate our mount namespace private from outside events, so that
// mounts in the outside environment do not affect our sandbox.
if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) < 0) {
if (errno == EACCES && !opt.hermetic) {
close(pipe_to_parent[0]);
close(pipe_to_parent[1]);
_exit(kAppArmorBwrapFallbackExitCode);
}
DIE("mount");
}

// Verify that the parent still lives.
SignalPipe(pipe_to_parent);
}

static void SetupUserNamespace() {
Expand Down Expand Up @@ -923,10 +928,10 @@ int Pid1Main(void* args) {
// Start with default signal handlers and an empty signal mask.
ClearSignalMask();

SetupSelfDestruction(pid1Args.pipe_to_parent);
SetupSelfDestruction();

// Sandbox ourselves.
SetupMountNamespace();
SetupMountNamespace(pid1Args.pipe_to_parent);
SetupUserNamespace();
if (opt.fake_hostname) {
SetupUtsNamespace();
Expand Down Expand Up @@ -965,3 +970,215 @@ int Pid1Main(void* args) {
// automatically once we exit.
return WaitForChild();
}

pid_t SpawnBwrap(int argc, char* argv[]) {
for (const std::string& tmpfs_dir : opt.tmpfs_dirs) {
if (opt.working_dir.find(tmpfs_dir) == 0) {
DIE("The sandbox working directory cannot be below a path where we mount "
"tmpfs (you requested mounting %s in %s). Is your --output_base= "
"below one of your --sandbox_tmpfs_path values?",
opt.working_dir.c_str(), tmpfs_dir.c_str());
}
}

std::unordered_set<std::string> bind_mount_sources;
for (size_t i = 0; i < opt.bind_mount_sources.size(); i++) {
const std::string& source = opt.bind_mount_sources.at(i);
bind_mount_sources.insert(source);
const std::string& target = opt.bind_mount_targets.at(i);
PRINT_DEBUG("bind mount: %s -> %s", source.c_str(), target.c_str());
}
for (const std::string& tmpfs_dir : opt.tmpfs_dirs) {
PRINT_DEBUG("tmpfs: %s", tmpfs_dir.c_str());
}
for (const std::string& writable_file : opt.writable_files) {
PRINT_DEBUG("writable: %s", writable_file.c_str());
}
if (!opt.working_dir.empty()) {
PRINT_DEBUG("working dir: %s", opt.working_dir.c_str());
}
if (global_debug != nullptr) {
fflush(global_debug);
}

uid_t inner_uid = global_outer_uid;
gid_t inner_gid = global_outer_gid;
if (opt.fake_root) {
inner_uid = 0;
inner_gid = 0;
} else if (opt.fake_username) {
struct passwd pwd;
char buf[1024];
struct passwd* result = nullptr;
if (getpwnam_r("nobody", &pwd, buf, sizeof(buf), &result) != 0 ||
result == nullptr) {
DIE("getpwnam_r for nobody failed");
}
inner_uid = pwd.pw_uid;
inner_gid = pwd.pw_gid;
}
if (opt.enable_pty) {
struct group grp;
char buf[256];
struct group* result = nullptr;
getgrnam_r("tty", &grp, buf, sizeof(buf), &result);
if (result == nullptr) {
DIE("getgrnam_r");
}
inner_gid = grp.gr_gid;
}

char self_exe[4096];
ssize_t self_len = readlink("/proc/self/exe", self_exe, sizeof(self_exe) - 1);
if (self_len < 0) {
DIE("readlink(/proc/self/exe)");
}
self_exe[self_len] = '\0';

int pipe_from_child[2], pipe_to_child[2];
if (pipe(pipe_from_child) < 0 || pipe(pipe_to_child) < 0) {
DIE("pipe");
}

const pid_t child_pid = fork();
if (child_pid < 0) {
DIE("fork");
}

if (child_pid == 0) {
WaitPipe(pipe_to_child);
if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) {
DIE("prctl");
}
if (setpgid(0, 0) < 0) {
DIE("setpgid");
}
SignalPipe(pipe_from_child);

if (global_debug != nullptr) {
fclose(global_debug);
global_debug = nullptr;
}

// Ignore SIGTERM in bwrap itself so polite SIGTERM sent to the process
// group is caught and forwarded by the inner linux-sandbox process rather
// than terminating bwrap prematurely.
IgnoreSignal(SIGTERM);
if (setenv("BAZEL_LINUX_SANDBOX_BWRAP_PID1", "1", 1) < 0) {
DIE("setenv");
}

std::vector<std::string> args_storage;
auto add_arg = [&](const std::string& s) { args_storage.push_back(s); };
add_arg("/usr/bin/bwrap");
add_arg("--die-with-parent");
add_arg("--unshare-user");
add_arg("--unshare-pid");
add_arg("--unshare-ipc");
if (opt.create_netns != NO_NETNS) {
add_arg("--unshare-net");
}
if (opt.fake_hostname) {
add_arg("--unshare-uts");
add_arg("--hostname");
add_arg("localhost");
}
add_arg("--uid");
add_arg(std::to_string(inner_uid));
add_arg("--gid");
add_arg(std::to_string(inner_gid));
add_arg("--ro-bind");
add_arg("/");
add_arg("/");

const char* devs[] = {"/dev/null", "/dev/zero", "/dev/full",
"/dev/random", "/dev/urandom", nullptr};
for (int i = 0; devs[i] != nullptr; i++) {
add_arg("--dev-bind");
add_arg(devs[i]);
add_arg(devs[i]);
}
if (opt.enable_pty) {
add_arg("--dev-bind");
add_arg("/dev/pts");
add_arg("/dev/pts");
add_arg("--dev-bind");
add_arg("/dev/ptmx");
add_arg("/dev/ptmx");
}
add_arg("--proc");
add_arg("/proc");

for (size_t i = 0; i < opt.bind_mount_sources.size(); i++) {
const std::string& source = opt.bind_mount_sources.at(i);
const std::string& target = opt.bind_mount_targets.at(i);
bool writable = false;
for (const std::string& wf : opt.writable_files) {
if (wf == source) {
writable = true;
break;
}
}
add_arg(writable ? "--bind" : "--ro-bind");
add_arg(source);
add_arg(target);
}

for (const std::string& tmpfs_dir : opt.tmpfs_dirs) {
add_arg("--tmpfs");
add_arg(tmpfs_dir);
}

for (const std::string& writable_file : opt.writable_files) {
if (bind_mount_sources.find(writable_file) != bind_mount_sources.end()) {
continue;
}
add_arg("--bind");
add_arg(writable_file);
add_arg(writable_file);
}

if (!opt.working_dir.empty()) {
add_arg("--bind");
add_arg(opt.working_dir);
add_arg(opt.working_dir);
add_arg("--chdir");
add_arg(opt.working_dir);
}

add_arg("--");
add_arg(self_exe);
for (int i = 1; i < argc; i++) {
add_arg(argv[i]);
}

std::vector<char*> bwrap_argv;
bwrap_argv.reserve(args_storage.size() + 1);
for (std::string& s : args_storage) {
bwrap_argv.push_back(s.data());
}
bwrap_argv.push_back(nullptr);

execv("/usr/bin/bwrap", bwrap_argv.data());
DIE("execv(/usr/bin/bwrap)");
}

SignalPipe(pipe_to_child);
WaitPipe(pipe_from_child);
return child_pid;
}

int RunInBwrapPid1() {
if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) {
DIE("prctl");
}
ClearSignalMask();
if (!opt.working_dir.empty()) {
EnterWorkingDirectory();
}
IgnoreSignal(SIGTTIN);
IgnoreSignal(SIGTTOU);
SpawnChild();
InstallSignalHandler(SIGTERM, ForwardSignal);
return WaitForChild();
}
6 changes: 6 additions & 0 deletions src/main/tools/linux-sandbox-pid1.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@
#ifndef SRC_MAIN_TOOLS_LINUX_SANDBOX_PID1_H_
#define SRC_MAIN_TOOLS_LINUX_SANDBOX_PID1_H_

#include <sys/types.h>

constexpr int kAppArmorBwrapFallbackExitCode = 199;

struct Pid1Args {
int *pipe_to_parent;
int *pipe_from_parent;
};

int Pid1Main(void *pid1Args);
pid_t SpawnBwrap(int argc, char *argv[]);
int RunInBwrapPid1();

#endif
62 changes: 57 additions & 5 deletions src/main/tools/linux-sandbox.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ pid_t initial_ppid;
// Must we politely ask the child to exit before we send it a SIGKILL (once we
// want it to exit)? Holds only zero or one.
static std::atomic<int> global_need_polite_sigterm{false};
static std::atomic<int> global_using_bwrap_pgrp{false};

#if __cplusplus >= 201703L
static_assert(global_child_pid.is_always_lock_free);
Expand Down Expand Up @@ -192,6 +193,12 @@ static void OnTimeoutOrTerm(int) {
// Find the PID of the child, which main set up before installing us as a
// signal handler.
const pid_t child_pid = global_child_pid.load(std::memory_order_relaxed);
if (child_pid <= 0) {
return;
}
const pid_t target = global_using_bwrap_pgrp.load(std::memory_order_relaxed)
? -child_pid
: child_pid;

// Figure out whether we should send a SIGTERM here. If so, we won't want to
// next time we're called.
Expand All @@ -200,7 +207,7 @@ static void OnTimeoutOrTerm(int) {

// If we're not supposed to ask politely, simply forcibly kill the child.
if (!need_polite_sigterm) {
kill(child_pid, SIGKILL);
kill(target, SIGKILL);
return;
}

Expand All @@ -209,11 +216,11 @@ static void OnTimeoutOrTerm(int) {
//
// Note that main sets us up as the signal handler for SIGALRM, and arranges
// for this code path to be taken only if kill_delay_secs > 0.
kill(child_pid, SIGTERM);
kill(target, SIGTERM);
alarm(opt.kill_delay_secs);
}

static pid_t SpawnPid1() {
static pid_t SpawnPid1(int argc, char *argv[]) {
const int kStackSize = 1024 * 1024;
std::vector<char> child_stack(kStackSize);

Expand Down Expand Up @@ -262,7 +269,45 @@ static pid_t SpawnPid1() {
// the child process that we still existed after it ran
// prctl(PR_SET_PDEATHSIG, SIGKILL), thus preventing a race condition where
// the parent is killed before that call was made.
WaitPipe(pipe_from_child);
if (close(pipe_from_child[1]) < 0) {
DIE("close");
}
char buf = 0;
ssize_t n;
do {
n = read(pipe_from_child[0], &buf, 1);
} while (n < 0 && errno == EINTR);
if (n < 0) {
DIE("read");
}
if (close(pipe_from_child[0]) < 0) {
DIE("close");
}
if (n == 0) {
int status = 0;
while (waitpid(child_pid, &status, 0) < 0) {
if (errno != EINTR) {
DIE("waitpid");
}
}
if (WIFEXITED(status) &&
WEXITSTATUS(status) == kAppArmorBwrapFallbackExitCode) {
FILE* f =
fopen("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "r");
int restricted = 0;
if (f != nullptr) {
restricted = (fgetc(f) == '1');
fclose(f);
}
if (restricted && access("/usr/bin/bwrap", X_OK) == 0) {
global_using_bwrap_pgrp.store(1, std::memory_order_relaxed);
const pid_t bwrap_pid = SpawnBwrap(argc, argv);
MaybeAddChildProcessToCgroup(bwrap_pid);
return bwrap_pid;
}
}
DIE("child exited prematurely");
}

PRINT_DEBUG("done manipulating pipes");

Expand Down Expand Up @@ -333,6 +378,13 @@ static int WaitForPid1(const pid_t child_pid) {
}

int main(int argc, char *argv[]) {
if (getenv("BAZEL_LINUX_SANDBOX_BWRAP_PID1") != nullptr) {
unsetenv("BAZEL_LINUX_SANDBOX_BWRAP_PID1");
ParseOptions(argc, argv);
CloseFds();
return RunInBwrapPid1();
}

// Ask the kernel to kill us with SIGKILL if our parent dies.
if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0) {
DIE("prctl");
Expand Down Expand Up @@ -382,7 +434,7 @@ int main(int argc, char *argv[]) {

// Spawn the child that will fork the sandboxed program with fresh
// namespaces etc.
const pid_t child_pid = SpawnPid1();
const pid_t child_pid = SpawnPid1(argc, argv);

// Let the signal handlers installed below know the PID of the child.
global_child_pid.store(child_pid, std::memory_order_relaxed);
Expand Down