const LinuxThreadImpl = struct {
const linux = std.os.linux;
pub const ThreadHandle = i32;
threadlocal var tls_thread_id: ?Id = null;
fn getCurrentId() Id {
return tls_thread_id orelse {
const tid: u32 = @bitCast(linux.gettid());
tls_thread_id = tid;
return tid;
};
}
fn getCpuCount() !usize {
const cpu_set = try posix.sched_getaffinity(0);
return posix.CPU_COUNT(cpu_set);
}
thread: *ThreadCompletion,
const ThreadCompletion = struct {
completion: Completion = Completion.init(.running),
child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1),
parent_tid: i32 = undefined,
mapped: []align(std.heap.page_size_min) u8,
// On SPARC, the kernel needs to be able to restore the current register window from the
// stack when returning from a syscall. That presents a bit of a problem in `freeAndExit`
// since we're deallocating the stack! The good news is that, since we do not care about
// the contents of the incoming and local registers at that point, we can just tell the
// kernel that our stack is this undefined global buffer.
var sparc_exit_stack: [192]u8 align(16) = undefined;
/// Calls `munmap(mapped.ptr, mapped.len)` then `exit(1)` without touching the stack (which lives in `mapped.ptr`).
/// Ported over from musl libc's pthread detached implementation:
/// https://github.com/ifduyue/musl/search?q=__unmapself
fn freeAndExit(self: *ThreadCompletion) noreturn {
// If we do not reset the child_tidptr to null here, the kernel would later write the
// value zero to that address, which is inside the block we're unmapping below, after
// our thread exits. This can sometimes corrupt memory in other mmap blocks from
// unrelated concurrent threads.
_ = linux.set_tid_address(null);
// If a signal were delivered between SYS_munmap and SYS_exit, any installed signal
// handler would immediately segfault due to the stack being unmapped. To avoid this,
// we need to mask all signals before entering the inline asm.
posix.sigprocmask(std.posix.SIG.BLOCK, &std.os.linux.sigfillset(), null);
switch (target.cpu.arch) {
.x86 => asm volatile (
\\ movl $91, %%eax # SYS_munmap
\\ int $128
\\ movl $1, %%eax # SYS_exit
\\ movl $0, %%ebx
\\ int $128
:
: [ptr] "{ebx}" (@intFromPtr(self.mapped.ptr)),
[len] "{ecx}" (self.mapped.len),
),
.x86_64 => asm volatile (switch (target.abi) {
.gnux32, .muslx32, .x32 =>
\\ movl $0x4000000b, %%eax # SYS_munmap
\\ syscall
\\ movl $0x4000003c, %%eax # SYS_exit
\\ xor %%rdi, %%rdi
\\ syscall
,
else =>
\\ movl $11, %%eax # SYS_munmap
\\ syscall
\\ movl $60, %%eax # SYS_exit
\\ xor %%rdi, %%rdi
\\ syscall
,
}
:
: [ptr] "{rdi}" (@intFromPtr(self.mapped.ptr)),
[len] "{rsi}" (self.mapped.len),
),
.arm, .armeb, .thumb, .thumbeb => asm volatile (
\\ mov r7, #91 // SYS_munmap
\\ svc 0
\\ mov r7, #1 // SYS_exit
\\ mov r0, #0
\\ svc 0
:
: [ptr] "{r0}" (@intFromPtr(self.mapped.ptr)),
[len] "{r1}" (self.mapped.len),
),
.aarch64, .aarch64_be => asm volatile (
\\ mov x8, #215 // SYS_munmap
\\ svc 0
\\ mov x8, #93 // SYS_exit
\\ mov x0, #0
\\ svc 0
:
: [ptr] "{x0}" (@intFromPtr(self.mapped.ptr)),
[len] "{x1}" (self.mapped.len),
),
.alpha => asm volatile (
\\ ldi $0, 73 # SYS_munmap
\\ callsys
\\ ldi $0, 1 # SYS_exit
\\ ldi $16, 0
\\ callsys
:
: [ptr] "{$16}" (@intFromPtr(self.mapped.ptr)),
[len] "{$17}" (self.mapped.len),
),
.arc, .arceb => asm volatile (
\\ mov r8, 215 # SYS_munmap
\\ trap_s 0
\\ mov r8, 93 # SYS_exit
\\ mov r0, 0
\\ trap_s 0
:
: [ptr] "{r0}" (@intFromPtr(self.mapped.ptr)),
[len] "{r1}" (self.mapped.len),
),
.hexagon => asm volatile (
\\ r6 = #215 // SYS_munmap
\\ trap0(#1)
\\ r6 = #93 // SYS_exit
\\ r0 = #0
\\ trap0(#1)
:
: [ptr] "{r0}" (@intFromPtr(self.mapped.ptr)),
[len] "{r1}" (self.mapped.len),
),
.hppa => asm volatile (
\\ ble 0x100(%%sr2, %%r0)
\\ ldi 91, %%r20 /* SYS_munmap */
\\ ldi 0, %%r26
\\ ble 0x100(%%sr2, %%r0)
\\ ldi 1, %%r20 /* SYS_exit */
:
: [ptr] "{r26}" (@intFromPtr(self.mapped.ptr)),
[len] "{r25}" (self.mapped.len),
),
.m68k => asm volatile (
\\ move.l #91, %%d0 // SYS_munmap
\\ trap #0
\\ move.l #1, %%d0 // SYS_exit
\\ move.l #0, %%d1
\\ trap #0
:
: [ptr] "{d1}" (@intFromPtr(self.mapped.ptr)),
[len] "{d2}" (self.mapped.len),
),
.microblaze, .microblazeel => asm volatile (
\\ ori r12, r0, 91 # SYS_munmap
\\ brki r14, 0x8
\\ ori r12, r0, 1 # SYS_exit
\\ ori r5, r0, 0
\\ brki r14, 0x8
:
: [ptr] "{r5}" (@intFromPtr(self.mapped.ptr)),
[len] "{r6}" (self.mapped.len),
),
.mips, .mipsel => asm volatile (
\\ li $v0, 4091 # SYS_munmap
\\ syscall
\\ li $v0, 4001 # SYS_exit
\\ li $a0, 0
\\ syscall
:
: [ptr] "{$4}" (@intFromPtr(self.mapped.ptr)),
[len] "{$5}" (self.mapped.len),
),
.mips64, .mips64el => asm volatile (switch (target.abi) {
.gnuabin32, .muslabin32, .abin32 =>
\\ li $v0, 6011 # SYS_munmap
\\ syscall
\\ li $v0, 6058 # SYS_exit
\\ li $a0, 0
\\ syscall
,
else =>
\\ li $v0, 5011 # SYS_munmap
\\ syscall
\\ li $v0, 5058 # SYS_exit
\\ li $a0, 0
\\ syscall
,
}
:
: [ptr] "{$4}" (@intFromPtr(self.mapped.ptr)),
[len] "{$5}" (self.mapped.len),
),
.or1k => asm volatile (
\\ l.ori r11, r0, 215 # SYS_munmap
\\ l.sys 1
\\ l.ori r11, r0, 93 # SYS_exit
\\ l.ori r3, r0, 0
\\ l.sys 1
:
: [ptr] "{r3}" (@intFromPtr(self.mapped.ptr)),
[len] "{r4}" (self.mapped.len),
),
.powerpc, .powerpcle, .powerpc64, .powerpc64le => asm volatile (
\\ li 0, 91 # SYS_munmap
\\ sc
\\ li 0, 1 # SYS_exit
\\ li 3, 0
\\ sc
:
: [ptr] "{r3}" (@intFromPtr(self.mapped.ptr)),
[len] "{r4}" (self.mapped.len),
),
.riscv32, .riscv64 => asm volatile (
\\ li a7, 215 # SYS_munmap
\\ ecall
\\ li a7, 93 # SYS_exit
\\ mv a0, zero
\\ ecall
:
: [ptr] "{a0}" (@intFromPtr(self.mapped.ptr)),
[len] "{a1}" (self.mapped.len),
),
.s390x => asm volatile (
\\ svc 91 # SYS_munmap
\\ lghi %%r2, 0
\\ svc 1 # SYS_exit
:
: [ptr] "{r2}" (@intFromPtr(self.mapped.ptr)),
[len] "{r3}" (self.mapped.len),
),
.sh, .sheb => asm volatile (
\\ mov #91, r3 ! SYS_munmap
\\ trapa #31
\\ or r0, r0
\\ or r0, r0
\\ or r0, r0
\\ or r0, r0
\\ or r0, r0
\\ mov #1, r3 ! SYS_exit
\\ mov #0, r4
\\ trapa #31
\\ or r0, r0
\\ or r0, r0
\\ or r0, r0
\\ or r0, r0
\\ or r0, r0
:
: [ptr] "{r4}" (@intFromPtr(self.mapped.ptr)),
[len] "{r5}" (self.mapped.len),
),
.sparc => asm volatile (
\\ // See sparc64 comments below.
\\ t 0x3 // ST_FLUSH_WINDOWS
\\ mov %%g3, %%sp
\\ mov %%g1, %%o0
\\ mov %%g2, %%o1
\\ mov 73, %%g1 // SYS_munmap
\\ t 0x10
\\ mov 1, %%g1 // SYS_exit
\\ mov 0, %%o0
\\ t 0x10
:
: [ptr] "{g1}" (@intFromPtr(self.mapped.ptr)),
[len] "{g2}" (self.mapped.len),
[stack] "{g3}" (&sparc_exit_stack),
),
.sparc64 => asm volatile (
\\ // Ensure that the kernel only has to flush the current register window.
\\ flushw
\\ // Set up a fake stack for the syscall to restore l/i registers from. Local
\\ // and incoming registers must be treated as effectively garbage past this
\\ // instruction!
\\ sub %%g3, 2047, %%sp
\\ mov %%g1, %%o0
\\ mov %%g2, %%o1
\\ mov 73, %%g1 // SYS_munmap
\\ t 0x6d
\\ mov 1, %%g1 // SYS_exit
\\ mov 0, %%o0
\\ t 0x6d
:
: [ptr] "{g1}" (@intFromPtr(self.mapped.ptr)),
[len] "{g2}" (self.mapped.len),
[stack] "{g3}" (&sparc_exit_stack),
),
.loongarch32, .loongarch64 => asm volatile (
\\ ori $a7, $zero, 215 # SYS_munmap
\\ syscall 0 # call munmap
\\ ori $a0, $zero, 0
\\ ori $a7, $zero, 93 # SYS_exit
\\ syscall 0 # call exit
:
: [ptr] "{r4}" (@intFromPtr(self.mapped.ptr)),
[len] "{r5}" (self.mapped.len),
),
.csky => asm volatile (
\\ movi r7, 215 # SYS_munmap
\\ trap 0
\\ movi r7, 93 # SYS_exit
\\ movi r0, 0
\\ trap 0
:
: [ptr] "{r0}" (@intFromPtr(self.mapped.ptr)),
[len] "{r1}" (self.mapped.len),
),
.xtensa, .xtensaeb => asm volatile (
\\ movi a2, 81 // SYS_munmap
\\ syscall
\\ movi a6, 0
\\ movi a2, 118 // SYS_exit
\\ syscall
:
: [ptr] "{a6}" (@intFromPtr(self.mapped.ptr)),
[len] "{a3}" (self.mapped.len),
),
else => |cpu_arch| @compileError("Unsupported linux arch: " ++ @tagName(cpu_arch)),
}
unreachable;
}
};
fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl {
const page_size = std.heap.pageSize();
const Args = @TypeOf(args);
const Instance = struct {
fn_args: Args,
thread: ThreadCompletion,
fn entryFn(raw_arg: usize) callconv(.c) u8 {
const self = @as(*@This(), @ptrFromInt(raw_arg));
defer switch (self.thread.completion.swap(.completed, .seq_cst)) {
.running => {},
.completed => unreachable,
.detached => self.thread.freeAndExit(),
};
return callFn(f, self.fn_args);
}
};
var guard_offset: usize = undefined;
var stack_offset: usize = undefined;
var tls_offset: usize = undefined;
var instance_offset: usize = undefined;
const map_bytes = blk: {
var bytes: usize = page_size;
guard_offset = bytes;
bytes += @max(page_size, config.stack_size);
bytes = std.mem.alignForward(usize, bytes, page_size);
stack_offset = bytes;
bytes = std.mem.alignForward(usize, bytes, linux.tls.area_desc.alignment);
tls_offset = bytes;
bytes += linux.tls.area_desc.size;
bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance));
instance_offset = bytes;
bytes += @sizeOf(Instance);
bytes = std.mem.alignForward(usize, bytes, page_size);
break :blk bytes;
};
// map all memory needed without read/write permissions
// to avoid committing the whole region right away
// anonymous mapping ensures file descriptor limits are not exceeded
const mapped = posix.mmap(
null,
map_bytes,
.{},
.{ .TYPE = .PRIVATE, .ANONYMOUS = true },
-1,
0,
) catch |err| switch (err) {
error.MemoryMappingNotSupported => unreachable,
error.AccessDenied => unreachable,
error.PermissionDenied => unreachable,
error.ProcessFdQuotaExceeded => unreachable,
error.SystemFdQuotaExceeded => unreachable,
error.MappingAlreadyExists => unreachable,
else => |e| return e,
};
assert(mapped.len >= map_bytes);
errdefer posix.munmap(mapped);
// Map everything but the guard page as read/write.
const guarded: []align(std.heap.page_size_min) u8 = @alignCast(mapped[guard_offset..]);
const protection: posix.PROT = .{ .READ = true, .WRITE = true };
switch (posix.errno(posix.system.mprotect(guarded.ptr, guarded.len, protection))) {
.SUCCESS => {},
.NOMEM => return error.OutOfMemory,
else => |err| return posix.unexpectedErrno(err),
}
// Prepare the TLS segment and prepare a user_desc struct when needed on x86
var tls_ptr = linux.tls.prepareArea(mapped[tls_offset..][0..linux.tls.area_desc.size]);
var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined;
if (target.cpu.arch == .x86) {
defer tls_ptr = @intFromPtr(&user_desc);
user_desc = .{
.entry_number = linux.tls.area_desc.gdt_entry_number,
.base_addr = tls_ptr,
.limit = 0xfffff,
.flags = .{
.seg_32bit = 1,
.contents = 0, // Data
.read_exec_only = 0,
.limit_in_pages = 1,
.seg_not_present = 0,
.useable = 1,
},
};
}
const instance: *Instance = @ptrCast(@alignCast(&mapped[instance_offset]));
instance.* = .{
.fn_args = args,
.thread = .{ .mapped = mapped },
};
const flags: u32 = linux.CLONE.THREAD | linux.CLONE.DETACHED |
linux.CLONE.VM | linux.CLONE.FS | linux.CLONE.FILES |
linux.CLONE.PARENT_SETTID | linux.CLONE.CHILD_CLEARTID |
linux.CLONE.SIGHAND | linux.CLONE.SYSVSEM | linux.CLONE.SETTLS;
switch (linux.errno(linux.clone(
Instance.entryFn,
@intFromPtr(&mapped[stack_offset]),
flags,
@intFromPtr(instance),
&instance.thread.parent_tid,
tls_ptr,
&instance.thread.child_tid.raw,
))) {
.SUCCESS => return Impl{ .thread = &instance.thread },
.AGAIN => return error.ThreadQuotaExceeded,
.INVAL => unreachable,
.NOMEM => return error.SystemResources,
.NOSPC => unreachable,
.PERM => unreachable,
.USERS => unreachable,
else => |err| return posix.unexpectedErrno(err),
}
}
fn getHandle(self: Impl) ThreadHandle {
return self.thread.parent_tid;
}
fn detach(self: Impl) void {
switch (self.thread.completion.swap(.detached, .seq_cst)) {
.running => {},
.completed => self.join(),
.detached => unreachable,
}
}
fn join(self: Impl) void {
defer posix.munmap(self.thread.mapped);
while (true) {
const tid = self.thread.child_tid.load(.seq_cst);
if (tid == 0) break;
switch (linux.errno(linux.futex_4arg(
&self.thread.child_tid.raw,
.{ .cmd = .WAIT, .private = false },
@bitCast(tid),
null,
))) {
.SUCCESS => continue,
.INTR => continue,
.AGAIN => continue,
else => unreachable,
}
}
}
}
