diff --git a/druntime/src/core/internal/array/utils.d b/druntime/src/core/internal/array/utils.d index e7a661a0f2a0..2cda571ba719 100644 --- a/druntime/src/core/internal/array/utils.d +++ b/druntime/src/core/internal/array/utils.d @@ -159,7 +159,7 @@ void[] __arrayAlloc(T)(size_t arrSize) @trusted * so the GC can't finalize them. */ static if (typeInfoSize) - attr |= BlkAttr.STRUCTFINAL | BlkAttr.FINALIZE; + attr |= BlkAttr.FINALIZE; static if (!hasIndirections!T) attr |= BlkAttr.NO_SCAN; diff --git a/druntime/src/core/internal/gc/blockmeta.d b/druntime/src/core/internal/gc/blockmeta.d index 768dcd76ba30..75d11b79e67b 100644 --- a/druntime/src/core/internal/gc/blockmeta.d +++ b/druntime/src/core/internal/gc/blockmeta.d @@ -116,7 +116,7 @@ bool __setArrayAllocLengthImpl(ref BlkInfo info, size_t newlength, bool isshared *length = cast(ubyte)newlength; } } - else if (info.size < PAGESIZE) + else if (info.size <= PAGESIZE / 2) { if (newlength + MEDPAD + typeInfoSize > info.size) // new size does not fit inside block @@ -180,7 +180,7 @@ bool __setArrayAllocLengthImpl(ref BlkInfo info, size_t newlength, bool isshared */ void __setBlockFinalizerInfo(ref BlkInfo info, const TypeInfo ti) pure nothrow { - if ((info.attr & BlkAttr.APPENDABLE) && info.size >= PAGESIZE) + if ((info.attr & BlkAttr.APPENDABLE) && info.size > PAGESIZE / 2) { // if the structfinal bit is not set, we don't have a finalizer. But we // should still zero out the finalizer slot. @@ -207,14 +207,24 @@ void __setBlockFinalizerInfo(ref BlkInfo info, const TypeInfo ti) pure nothrow /** Get the finalizer info from the block (typeinfo). - Must be called on a block with STRUCTFINAL set. + If called on a block, without STRUCTFINAL set, returns null. */ const(TypeInfo) __getBlockFinalizerInfo(ref BlkInfo info) pure nothrow { - bool isLargeArray = (info.attr & BlkAttr.APPENDABLE) && info.size >= PAGESIZE; + return __getBlockFinalizerInfo(info.base, info.size, info.attr); +} + +/// ditto +const(TypeInfo) __getBlockFinalizerInfo(void* base, size_t size, uint attr) pure nothrow +{ + if (!(attr & BlkAttr.STRUCTFINAL)) + return null; + + bool isLargeArray = (attr & BlkAttr.APPENDABLE) && size > PAGESIZE / 2; auto typeInfo = isLargeArray ? - info.base + size_t.sizeof : - info.base + info.size - size_t.sizeof; + base + size_t.sizeof : + base + size - size_t.sizeof; + assert(*cast(size_t*)typeInfo != 0); return *cast(TypeInfo*)typeInfo; } @@ -228,7 +238,7 @@ size_t __arrayAllocLength(ref BlkInfo info) pure nothrow if (info.size <= 256) return *cast(ubyte *)(info.base + info.size - typeInfoSize - SMALLPAD); - if (info.size < PAGESIZE) + if (info.size <= PAGESIZE / 2) return *cast(ushort *)(info.base + info.size - typeInfoSize - MEDPAD); return *cast(size_t *)(info.base); @@ -245,7 +255,7 @@ size_t __arrayAllocLengthAtomic(ref BlkInfo info) pure nothrow if (info.size <= 256) return atomicLoad(*cast(shared(ubyte)*)(info.base + info.size - typeInfoSize - SMALLPAD)); - if (info.size < PAGESIZE) + if (info.size <= PAGESIZE / 2) return atomicLoad(*cast(shared(ushort)*)(info.base + info.size - typeInfoSize - MEDPAD)); return atomicLoad(*cast(shared(size_t)*)(info.base)); @@ -258,7 +268,7 @@ size_t __arrayAllocCapacity(ref BlkInfo info) pure nothrow in(info.attr & BlkAttr.APPENDABLE) { // Capacity is a calculation based solely on the block info. - if (info.size >= PAGESIZE) + if (info.size > PAGESIZE / 2) return info.size - LARGEPAD; auto typeInfoSize = (info.attr & BlkAttr.STRUCTFINAL) ? size_t.sizeof : 0; @@ -299,10 +309,52 @@ size_t __allocPad(size_t size, uint bits) nothrow pure @trusted * * Params: * info = array metadata + * base = pointer to base of memory block + * size = size of memory block. * Returns: * pointer to the start of the array */ void *__arrayStart()(return scope BlkInfo info) nothrow pure { - return info.base + ((info.size & BIGLENGTHMASK) ? LARGEPREFIX : 0); + return __arrayStart(info.base, info.size); +} +/// Ditto +void *__arrayStart()(return scope void* base, size_t size) nothrow pure +{ + return base + ((size & BIGLENGTHMASK) ? LARGEPREFIX : 0); +} + +/** + * Trim a block's extents to the known valid data that is not metadata. This + * takes into account the finalizer and array metadata. + */ +void __trimExtents(ref scope void* base, ref size_t blockSize, uint attr) nothrow pure @nogc +{ + if (attr & BlkAttr.APPENDABLE) + { + if (blockSize > PAGESIZE / 2) + { + // large block, it's always LARGEPREFIX bytes at the front. + blockSize = *(cast(size_t*)base); + base += LARGEPREFIX; + return; + } + + void *pend = base + blockSize; + if (attr & BlkAttr.STRUCTFINAL) + // skip the finalizer + pend -= (void*).sizeof; + + // get the actual length + if (blockSize <= 256) + blockSize = *(cast(ubyte*)pend - 1); + else // medium array block + blockSize = *(cast(ushort*)pend - 1); + } + else if (attr & BlkAttr.STRUCTFINAL) + { + // not appendable, but has a finalizer in the block. This is always + // stored at the end. + blockSize -= (void*).sizeof; + } } diff --git a/druntime/src/core/internal/gc/impl/conservative/gc.d b/druntime/src/core/internal/gc/impl/conservative/gc.d index 4cb7b57a6f4d..c9f41bb8a542 100644 --- a/druntime/src/core/internal/gc/impl/conservative/gc.d +++ b/druntime/src/core/internal/gc/impl/conservative/gc.d @@ -90,8 +90,8 @@ private { // to allow compilation of this module without access to the rt package, // make these functions available from rt.lifetime - void rt_finalizeFromGC(void* p, size_t size, uint attr) nothrow; - int rt_hasFinalizerInSegment(void* p, size_t size, uint attr, const scope void[] segment) nothrow; + void rt_finalizeFromGC(void* p, size_t size, uint attr, const(TypeInfo) typeInfo) nothrow; + int rt_hasFinalizerInSegment(void* p, size_t size, const(TypeInfo) typeInfo, const scope void[] segment) nothrow; // Declared as an extern instead of importing core.exception // to avoid inlining - see https://issues.dlang.org/show_bug.cgi?id=13725. @@ -2880,9 +2880,13 @@ struct Gcx if (pool.finals.nbits && pool.finals.clear(biti)) { + import core.internal.gc.blockmeta; size_t size = npages * PAGESIZE - SENTINEL_EXTRA; + size = sentinel_size(q, size); uint attr = pool.getBits(biti); - rt_finalizeFromGC(q, sentinel_size(q, size), attr); + auto ti = __getBlockFinalizerInfo(q, size, attr); + __trimExtents(q, size, attr); + rt_finalizeFromGC(q, size, attr, ti); } pool.clrBits(biti, ~BlkAttr.NONE ^ BlkAttr.FINALIZE); @@ -3003,7 +3007,14 @@ struct Gcx sentinel_Invariant(q); if (pool.finals.nbits && pool.finals.test(biti)) - rt_finalizeFromGC(q, sentinel_size(q, size), pool.getBits(biti)); + { + import core.internal.gc.blockmeta; + size_t ssize = sentinel_size(q, size); + uint attr = pool.getBits(biti); + auto ti = __getBlockFinalizerInfo(q, ssize, attr); + __trimExtents(q, ssize, attr); + rt_finalizeFromGC(q, ssize, attr, ti); + } assert(core.bitop.bt(toFree.ptr, i)); @@ -4523,10 +4534,13 @@ struct LargeObjectPool size_t size = sentinel_size(p, getSize(pn)); uint attr = getBits(biti); - if (!rt_hasFinalizerInSegment(p, size, attr, segment)) + import core.internal.gc.blockmeta; + auto ti = __getBlockFinalizerInfo(p, size, attr); + if (!rt_hasFinalizerInSegment(p, size, ti, segment)) continue; - rt_finalizeFromGC(p, size, attr); + __trimExtents(p, size, attr); + rt_finalizeFromGC(p, size, attr, ti); clrBits(biti, ~BlkAttr.NONE); @@ -4647,11 +4661,14 @@ struct SmallObjectPool auto q = sentinel_add(p); uint attr = getBits(biti); - const ssize = sentinel_size(q, size); - if (!rt_hasFinalizerInSegment(q, ssize, attr, segment)) + auto ssize = sentinel_size(q, size); + import core.internal.gc.blockmeta; + auto ti = __getBlockFinalizerInfo(q, ssize, attr); + if (!rt_hasFinalizerInSegment(q, ssize, ti, segment)) continue; - rt_finalizeFromGC(q, ssize, attr); + __trimExtents(q, ssize, attr); + rt_finalizeFromGC(q, ssize, attr, ti); freeBits = true; toFree.set(i); diff --git a/druntime/src/core/lifetime.d b/druntime/src/core/lifetime.d index 695f9bc84e74..a088bf6ca2d8 100644 --- a/druntime/src/core/lifetime.d +++ b/druntime/src/core/lifetime.d @@ -2828,21 +2828,11 @@ T* _d_newitemT(T)() @trusted import core.memory : GC; auto flags = !hasIndirections!T ? GC.BlkAttr.NO_SCAN : GC.BlkAttr.NONE; - immutable tiSize = TypeInfoSize!T; immutable itemSize = T.sizeof; - immutable totalSize = itemSize + tiSize; - if (tiSize) - flags |= GC.BlkAttr.STRUCTFINAL | GC.BlkAttr.FINALIZE; + if (TypeInfoSize!T) + flags |= GC.BlkAttr.FINALIZE; - auto blkInfo = GC.qalloc(totalSize, flags, null); - auto p = blkInfo.base; - - if (tiSize) - { - // The GC might not have cleared the padding area in the block. - *cast(TypeInfo*) (p + (itemSize & ~(size_t.sizeof - 1))) = null; - *cast(TypeInfo*) (p + blkInfo.size - tiSize) = cast() typeid(T); - } + auto p = GC.malloc(itemSize, flags, typeid(T)); emplaceInitializer(*(cast(T*) p)); @@ -2999,6 +2989,15 @@ version (D_ProfileGC) { version (D_TypeInfo) { + static if (is(T == struct)) + { + // prime the TypeInfo name, we don't want that affecting the allocated bytes + // Issue https://github.com/dlang/dmd/issues/20832 + static string typeName(TypeInfo_Struct ti) nothrow @trusted => ti.name; + auto tnPure = cast(string function(TypeInfo_Struct ti) nothrow pure @trusted)&typeName; + cast(void)tnPure(typeid(T)); + } + import core.internal.array.utils : TraceHook, gcStatsPure, accumulatePure; mixin(TraceHook!(T.stringof, "_d_newitemT")); diff --git a/druntime/src/rt/lifetime.d b/druntime/src/rt/lifetime.d index 355867448283..b63a5bf5266e 100644 --- a/druntime/src/rt/lifetime.d +++ b/druntime/src/rt/lifetime.d @@ -24,7 +24,6 @@ static import rt.tlsgc; debug (PRINTF) import core.stdc.stdio : printf; debug (VALGRIND) import etc.valgrind.valgrind; -alias BlkInfo = GC.BlkInfo; alias BlkAttr = GC.BlkAttr; // for now, all GC array functions are not exposed via core.memory. @@ -225,7 +224,7 @@ private uint __typeAttrs(const scope TypeInfo ti, void *copyAttrsFrom = null) pu if (typeid(ti) is typeid(TypeInfo_Struct)) { auto sti = cast(TypeInfo_Struct)cast(void*)ti; if (sti.xdtor) - attrs |= BlkAttr.STRUCTFINAL | BlkAttr.FINALIZE; + attrs |= BlkAttr.FINALIZE; } return attrs; } @@ -607,21 +606,16 @@ extern (C) CollectHandler rt_getCollectHandler() /** * */ -extern (C) int rt_hasFinalizerInSegment(void* p, size_t size, uint attr, scope const(void)[] segment) nothrow +extern (C) int rt_hasFinalizerInSegment(void* p, size_t size, TypeInfo typeInfo, scope const(void)[] segment) nothrow { if (!p) return false; - if (attr & BlkAttr.STRUCTFINAL) + if (typeInfo !is null) { - import core.internal.gc.blockmeta; - auto info = BlkInfo( - base: p, - size: size, - attr: attr - ); + assert(typeid(typeInfo) is typeid(TypeInfo_Struct)); - auto ti = cast(TypeInfo_Struct)cast(void*)__getBlockFinalizerInfo(info); + auto ti = cast(TypeInfo_Struct)cast(void*)typeInfo; return cast(size_t)(cast(void*)ti.xdtor - segment.ptr) < segment.length; } @@ -719,39 +713,31 @@ extern (C) void rt_finalize(void* p, bool det = true) nothrow rt_finalize2(p, det, true); } -extern (C) void rt_finalizeFromGC(void* p, size_t size, uint attr) nothrow +extern (C) void rt_finalizeFromGC(void* p, size_t size, uint attr, TypeInfo typeInfo) nothrow { // to verify: reset memory necessary? - if (!(attr & BlkAttr.STRUCTFINAL)) { + if (typeInfo is null) { rt_finalize2(p, false, false); // class return; } - // get the struct typeinfo from the block, and the used size. - import core.internal.gc.blockmeta; - auto info = BlkInfo( - base: p, - size: size, - attr: attr - ); + assert(typeid(typeInfo) is typeid(TypeInfo_Struct)); - auto si = cast(TypeInfo_Struct)cast(void*)__getBlockFinalizerInfo(info); + auto si = cast(TypeInfo_Struct)cast(void*)typeInfo; - if (attr & BlkAttr.APPENDABLE) + try { - auto usedsize = __arrayAllocLength(info); - auto arrptr = __arrayStart(info); - try + if (attr & BlkAttr.APPENDABLE) { - finalize_array(arrptr, usedsize, si); - } - catch (Exception e) - { - onFinalizeError(si, e); + finalize_array(p, size, si); } + else + finalize_struct(p, si); // struct + } + catch (Exception e) + { + onFinalizeError(si, e); } - else - finalize_struct(p, si); // struct } @@ -1583,7 +1569,7 @@ deprecated unittest dtorCount = 0; const(S1)[] carr1 = new const(S1)[5]; - BlkInfo blkinf1 = GC.query(carr1.ptr); + auto blkinf1 = GC.query(carr1.ptr); GC.runFinalizers((cast(char*)(typeid(S1).xdtor))[0..1]); assert(dtorCount == 5); GC.free(blkinf1.base); @@ -1595,7 +1581,7 @@ deprecated unittest assert(dtorCount == 4); // destructors run explicitely? dtorCount = 0; - BlkInfo blkinf = GC.query(arr2.ptr); + auto blkinf = GC.query(arr2.ptr); GC.runFinalizers((cast(char*)(typeid(S1).xdtor))[0..1]); assert(dtorCount == 6); GC.free(blkinf.base);