fn free(
context: *anyopaque,
old_memory: []u8,
alignment: mem.Alignment,
return_address: usize,
) void {
const self: *Self = @ptrCast(@alignCast(context));
if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @backingInt(alignment));
if (size_class_index >= self.buckets.len) {
@branchHint(.unlikely);
self.freeLarge(old_memory, alignment, return_address);
return;
}
const slot_count = slot_counts[size_class_index];
const freed_addr = @intFromPtr(old_memory.ptr);
const page_addr = freed_addr & ~(page_size - 1);
const bucket: *BucketHeader = .fromPage(page_addr, slot_count);
if (bucket.canary != config.canary) @panic("Invalid free");
const page_offset = freed_addr - page_addr;
const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
const slot_index: SlotIndex = @intCast(page_offset / size_class);
const used_byte_index = slot_index / @bitSizeOf(usize);
const used_bit_index: Log2USize = @intCast(slot_index % @bitSizeOf(usize));
const used_byte = bucket.usedBits(used_byte_index);
const is_used = @as(u1, @truncate(used_byte.* >> used_bit_index)) != 0;
if (!is_used) {
if (config.safety) {
reportDoubleFree(
return_address,
bucketStackTrace(bucket, slot_count, slot_index, .alloc),
bucketStackTrace(bucket, slot_count, slot_index, .free),
);
// Recoverable since this is a free.
return;
} else {
unreachable;
}
}
// Definitely an in-use small alloc now.
if (config.safety) {
const requested_size = bucket.requestedSizes(slot_count)[slot_index];
if (requested_size == 0) @panic("Invalid free");
const slot_alignment = bucket.log2PtrAligns(slot_count)[slot_index];
if (old_memory.len != requested_size or alignment != slot_alignment) {
var addr_buf: [stack_n]usize = undefined;
const free_stack_trace = std.debug.captureCurrentStackTrace(.{ .first_address = return_address }, &addr_buf);
if (old_memory.len != requested_size) {
@branchHint(.cold);
log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {f} Free: {f}", .{
requested_size,
old_memory.len,
std.debug.FormatStackTrace{
.stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc),
.terminal_mode = std.log.terminalMode(),
},
std.debug.FormatStackTrace{
.stack_trace = free_stack_trace,
.terminal_mode = std.log.terminalMode(),
},
});
}
if (alignment != slot_alignment) {
@branchHint(.cold);
log.err("Allocation alignment {d} does not match free alignment {d}. Allocation: {f} Free: {f}", .{
slot_alignment.toByteUnits(),
alignment.toByteUnits(),
std.debug.FormatStackTrace{
.stack_trace = bucketStackTrace(bucket, slot_count, slot_index, .alloc),
.terminal_mode = std.log.terminalMode(),
},
std.debug.FormatStackTrace{
.stack_trace = free_stack_trace,
.terminal_mode = std.log.terminalMode(),
},
});
}
}
}
if (config.enable_memory_limit) {
self.total_requested_bytes -= old_memory.len;
}
if (config.stack_trace_frames > 0) {
// Capture stack trace to be the "first free", in case a double free happens.
bucket.captureStackTrace(return_address, slot_count, slot_index, .free);
}
used_byte.* &= ~(@as(usize, 1) << used_bit_index);
if (config.safety) {
bucket.requestedSizes(slot_count)[slot_index] = 0;
}
bucket.freed_count += 1;
if (bucket.freed_count == bucket.allocated_count) {
if (bucket.prev) |prev| {
prev.next = bucket.next;
}
if (bucket.next) |next| {
assert(self.buckets[size_class_index] != bucket);
next.prev = bucket.prev;
} else {
assert(self.buckets[size_class_index] == bucket);
self.buckets[size_class_index] = bucket.prev;
}
if (!config.never_unmap) {
const page: [*]align(page_size) u8 = @ptrFromInt(page_addr);
self.backing_allocator.rawFree(page[0..page_size], page_align, @returnAddress());
}
}
if (config.verbose_log) {
log.info("small free {d} bytes at {*}", .{ old_memory.len, old_memory.ptr });
}
}