904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
fn allocBucket(
s: *SafeAllocator,
t: *Thread,
b: *Bucket,
len: usize,
alignment: Alignment,
ra: usize,
) ?[*]u8 {
const fill = &b.fill;
const is_extended = AllocFooter.requiresExtended(len, alignment);
const alloc_len: u32 = @intCast(AllocFooter.allocLenBucket(s, len, is_extended));
const alloc_align = AllocFooter.allocAlign(alignment);
var prev_fill = @atomicLoad(Bucket.Fill, fill, .monotonic);
var start: u32 = undefined;
var end: u32 = undefined;
while (true) {
start = @intCast(alloc_align.forward(prev_fill.at));
end = start + alloc_len;
if (end > s.bucketSize()) {
@branchHint(.unlikely);
const prev_count = @atomicRmw(
Bucket.AllocCount,
&b.alloc_count,
.Sub,
.{ .filling = true, .n = 0 },
.acq_rel,
);
assert(prev_count.filling);
if (prev_count.n == 0) {
@branchHint(.unlikely);
freeAllocEntry(t, b.entry);
b.check(s);
s.backing.rawFree(b.bytes(s), @fromBackingInt(@intCast(s.bucket_size_log2)), ra);
}
return null;
}
prev_fill = @cmpxchgWeak(
Bucket.Fill,
fill,
prev_fill,
.{ .at = @intCast(end), .last_is_extended = is_extended },
.monotonic,
.monotonic,
) orelse {
@branchHint(.likely);
break;
};
// b.fill was changed during a resize (or a sporadic cmpxchgWeak failure)
}
const memory = b.bytes(s)[start..end];
_ = AllocFooter.populate(
@alignCast(memory),
len,
alignment,
ra,
is_extended,
prev_fill.last_is_extended,
.{ .bucket_prev = b.lastAlloc(s, prev_fill) },
s,
);
assert(@atomicRmw(
Bucket.AllocCount,
&b.alloc_count,
.Add,
.{ .filling = false, .n = 1 },
.acq_rel,
).filling);
return memory.ptr;
}
If this fails, future allocations to the bucket are illegal