From 3a0a96e2d513d623153c4c2018e513fb28eb4a74 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 9 Dec 2022 21:29:08 -0500 Subject: [PATCH 1/5] First draft of minimal-mt --- cli/index.js | 1 + std/assembly/rt/index-incremental.ts | 5 +- std/assembly/rt/index-minimal-mt.ts | 3 + std/assembly/rt/index-minimal.ts | 5 +- std/assembly/rt/{tlsf.ts => tlsf-base.ts} | 1156 ++++++++++----------- std/assembly/rt/tlsf-mt.ts | 45 + std/assembly/rt/tlsf-mutex.ts | 30 + std/assembly/rt/tlsf-st.ts | 28 + 8 files changed, 680 insertions(+), 593 deletions(-) create mode 100644 std/assembly/rt/index-minimal-mt.ts rename std/assembly/rt/{tlsf.ts => tlsf-base.ts} (96%) create mode 100644 std/assembly/rt/tlsf-mt.ts create mode 100644 std/assembly/rt/tlsf-mutex.ts create mode 100644 std/assembly/rt/tlsf-st.ts diff --git a/cli/index.js b/cli/index.js index 36eeb4715e..ad0eb45af6 100644 --- a/cli/index.js +++ b/cli/index.js @@ -300,6 +300,7 @@ export async function main(argv, options) { switch (opts.runtime) { case "stub": runtime = 0; break; case "minimal": runtime = 1; break; + case "minimal-mt": runtime = 4; break; /* incremental */ default: runtime = 2; break; } diff --git a/std/assembly/rt/index-incremental.ts b/std/assembly/rt/index-incremental.ts index 4730344b4f..f18c8791db 100644 --- a/std/assembly/rt/index-incremental.ts +++ b/std/assembly/rt/index-incremental.ts @@ -1,2 +1,3 @@ -import "rt/tlsf"; -import "rt/itcms"; +import "rt/tlsf-base"; +import "rt/tlsf-st"; +import "rt/itcms"; diff --git a/std/assembly/rt/index-minimal-mt.ts b/std/assembly/rt/index-minimal-mt.ts new file mode 100644 index 0000000000..3595ba2dd6 --- /dev/null +++ b/std/assembly/rt/index-minimal-mt.ts @@ -0,0 +1,3 @@ +import "rt/tlsf-base"; +import "rt/tlsf-mt"; +import "rt/tcms"; diff --git a/std/assembly/rt/index-minimal.ts b/std/assembly/rt/index-minimal.ts index cf88ee158f..c020f7ed40 100644 --- a/std/assembly/rt/index-minimal.ts +++ b/std/assembly/rt/index-minimal.ts @@ -1,2 +1,3 @@ -import "rt/tlsf"; -import "rt/tcms"; +import "rt/tlsf-base"; +import "rt/tlsf-st"; +import "rt/tcms"; diff --git a/std/assembly/rt/tlsf.ts b/std/assembly/rt/tlsf-base.ts similarity index 96% rename from std/assembly/rt/tlsf.ts rename to std/assembly/rt/tlsf-base.ts index df437b82cb..c1caf483f6 100644 --- a/std/assembly/rt/tlsf.ts +++ b/std/assembly/rt/tlsf-base.ts @@ -1,589 +1,567 @@ -import { AL_BITS, AL_SIZE, AL_MASK, DEBUG, BLOCK, BLOCK_OVERHEAD, BLOCK_MAXSIZE } from "./common"; -import { oninit, onalloc, onresize, onmove, onfree } from "./rtrace"; -import { E_ALLOCATION_TOO_LARGE } from "../util/error"; - -// === The TLSF (Two-Level Segregate Fit) memory allocator === -// see: http://www.gii.upv.es/tlsf/ - -// - `ffs(x)` is equivalent to `ctz(x)` with x != 0 -// - `fls(x)` is equivalent to `sizeof(x) * 8 - clz(x) - 1` - -// ╒══════════════ Block size interpretation (32-bit) ═════════════╕ -// 3 2 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits -// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─╫─┴─┴─┴─┤ -// │ | FL │ SB = SL + AL │ ◄─ usize -// └───────────────────────────────────────────────┴───────╨───────┘ -// FL: first level, SL: second level, AL: alignment, SB: small block - -// @ts-ignore: decorator -@inline const SL_BITS: u32 = 4; -// @ts-ignore: decorator -@inline const SL_SIZE: u32 = 1 << SL_BITS; - -// @ts-ignore: decorator -@inline const SB_BITS: u32 = SL_BITS + AL_BITS; -// @ts-ignore: decorator -@inline const SB_SIZE: u32 = 1 << SB_BITS; - -// @ts-ignore: decorator -@inline const FL_BITS: u32 = 31 - SB_BITS; - -// [00]: < 256B (SB) [12]: < 1M -// [01]: < 512B [13]: < 2M -// [02]: < 1K [14]: < 4M -// [03]: < 2K [15]: < 8M -// [04]: < 4K [16]: < 16M -// [05]: < 8K [17]: < 32M -// [06]: < 16K [18]: < 64M -// [07]: < 32K [19]: < 128M -// [08]: < 64K [20]: < 256M -// [09]: < 128K [21]: < 512M -// [10]: < 256K [22]: <= 1G - OVERHEAD -// [11]: < 512K -// VMs limit to 2GB total (currently), making one 1G block max (or three 512M etc.) due to block overhead - -// Tags stored in otherwise unused alignment bits - -// @ts-ignore: decorator -@inline const FREE: usize = 1 << 0; -// @ts-ignore: decorator -@inline const LEFTFREE: usize = 1 << 1; -// @ts-ignore: decorator -@inline const TAGS_MASK: usize = FREE | LEFTFREE; // <= AL_MASK - -// ╒════════════════════ Block layout (32-bit) ════════════════════╕ -// 3 2 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits -// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┼─┤ ┐ -// │ size │L│F│ ◄─┐ info overhead -// ╞>ptr═══════════════════════════════════════════════════════╧═╧═╡ │ ┘ -// │ if free: ◄ prev │ ◄─┤ usize -// ├───────────────────────────────────────────────────────────────┤ │ -// │ if free: next ► │ ◄─┤ -// ├───────────────────────────────────────────────────────────────┤ │ -// │ ... │ │ >= 0 -// ├───────────────────────────────────────────────────────────────┤ │ -// │ if free: back ▲ │ ◄─┘ -// └───────────────────────────────────────────────────────────────┘ >= MIN SIZE -// F: FREE, L: LEFTFREE -@unmanaged export class Block extends BLOCK { - - /** Previous free block, if any. Only valid if free, otherwise part of payload. */ - prev: Block | null; - /** Next free block, if any. Only valid if free, otherwise part of payload. */ - next: Block | null; - - // If the block is free, there is a 'back'reference at its end pointing at its start. -} - -// Block constants. A block must have a minimum size of three pointers so it can hold `prev`, -// `next` and `back` if free. - -// @ts-ignore: decorator -@inline const BLOCK_MINSIZE: usize = ((3 * sizeof() + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; // prev + next + back -// @ts-ignore: decorator -// @inline const BLOCK_MAXSIZE: usize = 1 << (FL_BITS + SB_BITS - 1); // exclusive, lives in common.ts - -/** Gets the left block of a block. Only valid if the left block is free. */ -// @ts-ignore: decorator -@inline function GETFREELEFT(block: Block): Block { - return load(changetype(block) - sizeof()); -} - -/** Gets the right block of a block by advancing to the right by its size. */ -// @ts-ignore: decorator -@inline function GETRIGHT(block: Block): Block { - return changetype(changetype(block) + BLOCK_OVERHEAD + (block.mmInfo & ~TAGS_MASK)); -} - -// ╒═════════════════════ Root layout (32-bit) ════════════════════╕ -// 3 2 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits -// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐ -// │ 0 | flMap S│ ◄────┐ -// ╞═══════════════════════════════════════════════════════════════╡ │ -// │ slMap[0] S │ ◄─┐ │ -// ├───────────────────────────────────────────────────────────────┤ │ │ -// │ slMap[1] │ ◄─┤ │ -// ├───────────────────────────────────────────────────────────────┤ u32 │ -// │ slMap[22] │ ◄─┘ │ -// ╞═══════════════════════════════════════════════════════════════╡ usize -// │ head[0] │ ◄────┤ -// ├───────────────────────────────────────────────────────────────┤ │ -// │ ... │ ◄────┤ -// ├───────────────────────────────────────────────────────────────┤ │ -// │ head[367] │ ◄────┤ -// ╞═══════════════════════════════════════════════════════════════╡ │ -// │ tail │ ◄────┘ -// └───────────────────────────────────────────────────────────────┘ SIZE ┘ -// S: Small blocks map -@unmanaged class Root { - /** First level bitmap. */ - flMap: usize; -} - -// Root constants. Where stuff is stored inside of the root structure. - -// @ts-ignore: decorator -@inline const SL_START: usize = sizeof(); -// @ts-ignore: decorator -@inline const SL_END: usize = SL_START + (FL_BITS << alignof()); -// @ts-ignore: decorator -@inline const HL_START: usize = (SL_END + AL_MASK) & ~AL_MASK; -// @ts-ignore: decorator -@inline const HL_END: usize = HL_START + FL_BITS * SL_SIZE * sizeof(); -// @ts-ignore: decorator -@inline const ROOT_SIZE: usize = HL_END + sizeof(); - -// @ts-ignore: decorator -@lazy export let ROOT: Root = changetype(0); // unsafe initializion below - -/** Gets the second level map of the specified first level. */ -// @ts-ignore: decorator -@inline function GETSL(root: Root, fl: usize): u32 { - return load( - changetype(root) + (fl << alignof()), - SL_START - ); -} - -/** Sets the second level map of the specified first level. */ -// @ts-ignore: decorator -@inline function SETSL(root: Root, fl: usize, slMap: u32): void { - store( - changetype(root) + (fl << alignof()), - slMap, - SL_START - ); -} - -/** Gets the head of the free list for the specified combination of first and second level. */ -// @ts-ignore: decorator -@inline function GETHEAD(root: Root, fl: usize, sl: u32): Block | null { - return load( - changetype(root) + (((fl << SL_BITS) + sl) << alignof()), - HL_START - ); -} - -/** Sets the head of the free list for the specified combination of first and second level. */ -// @ts-ignore: decorator -@inline function SETHEAD(root: Root, fl: usize, sl: u32, head: Block | null): void { - store( - changetype(root) + (((fl << SL_BITS) + sl) << alignof()), - head, - HL_START - ); -} - -/** Gets the tail block.. */ -// @ts-ignore: decorator -@inline function GETTAIL(root: Root): Block { - return load( - changetype(root), - HL_END - ); -} - -/** Sets the tail block. */ -// @ts-ignore: decorator -@inline function SETTAIL(root: Root, tail: Block): void { - store( - changetype(root), - tail, - HL_END - ); -} - -/** Inserts a previously used block back into the free list. */ -function insertBlock(root: Root, block: Block): void { - if (DEBUG) assert(block); // cannot be null - let blockInfo = block.mmInfo; - if (DEBUG) assert(blockInfo & FREE); // must be free - - let right = GETRIGHT(block); - let rightInfo = right.mmInfo; - - // merge with right block if also free - if (rightInfo & FREE) { - removeBlock(root, right); - block.mmInfo = blockInfo = blockInfo + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); // keep block tags - right = GETRIGHT(block); - rightInfo = right.mmInfo; - // 'back' is set below - } - - // merge with left block if also free - if (blockInfo & LEFTFREE) { - let left = GETFREELEFT(block); - let leftInfo = left.mmInfo; - if (DEBUG) assert(leftInfo & FREE); // must be free according to right tags - removeBlock(root, left); - block = left; - block.mmInfo = blockInfo = leftInfo + BLOCK_OVERHEAD + (blockInfo & ~TAGS_MASK); // keep left tags - // 'back' is set below - } - - right.mmInfo = rightInfo | LEFTFREE; - // reference to right is no longer used now, hence rightInfo is not synced - - // we now know the size of the block - let size = blockInfo & ~TAGS_MASK; - if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be a valid size - if (DEBUG) assert(changetype(block) + BLOCK_OVERHEAD + size == changetype(right)); // must match - - // set 'back' to itself at the end of block - store(changetype(right) - sizeof(), block); - - // mapping_insert - let fl: usize, sl: u32; - if (size < SB_SIZE) { - fl = 0; - sl = (size >> AL_BITS); - } else { - const inv: usize = sizeof() * 8 - 1; - let boundedSize = min(size, BLOCK_MAXSIZE); - fl = inv - clz(boundedSize); - sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); - fl -= SB_BITS - 1; - } - if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range - - // perform insertion - let head = GETHEAD(root, fl, sl); - block.prev = null; - block.next = head; - if (head) head.prev = block; - SETHEAD(root, fl, sl, block); - - // update first and second level maps - root.flMap |= (1 << fl); - SETSL(root, fl, GETSL(root, fl) | (1 << sl)); -} - -/** Removes a free block from internal lists. */ -function removeBlock(root: Root, block: Block): void { - let blockInfo = block.mmInfo; - if (DEBUG) assert(blockInfo & FREE); // must be free - let size = blockInfo & ~TAGS_MASK; - if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be valid - - // mapping_insert - let fl: usize, sl: u32; - if (size < SB_SIZE) { - fl = 0; - sl = (size >> AL_BITS); - } else { - const inv: usize = sizeof() * 8 - 1; - let boundedSize = min(size, BLOCK_MAXSIZE); - fl = inv - clz(boundedSize); - sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); - fl -= SB_BITS - 1; - } - if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range - - // link previous and next free block - let prev = block.prev; - let next = block.next; - if (prev) prev.next = next; - if (next) next.prev = prev; - - // update head if we are removing it - if (block == GETHEAD(root, fl, sl)) { - SETHEAD(root, fl, sl, next); - - // clear second level map if head is empty now - if (!next) { - let slMap = GETSL(root, fl); - SETSL(root, fl, slMap &= ~(1 << sl)); - - // clear first level map if second level is empty now - if (!slMap) root.flMap &= ~(1 << fl); - } - } - // note: does not alter left/back because it is likely that splitting - // is performed afterwards, invalidating those changes. so, the caller - // must perform those updates. -} - -/** Searches for a free block of at least the specified size. */ -function searchBlock(root: Root, size: usize): Block | null { - // size was already asserted by caller - - // mapping_search - let fl: usize, sl: u32; - if (size < SB_SIZE) { - fl = 0; - sl = (size >> AL_BITS); - } else { - const halfMaxSize = BLOCK_MAXSIZE >> 1; // don't round last fl - const inv: usize = sizeof() * 8 - 1; - const invRound = inv - SL_BITS; - let requestSize = size < halfMaxSize - ? size + (1 << (invRound - clz(size))) - 1 - : size; - fl = inv - clz(requestSize); - sl = ((requestSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); - fl -= SB_BITS - 1; - } - if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range - - // search second level - let slMap = GETSL(root, fl) & (~0 << sl); - let head: Block | null = null; - if (!slMap) { - // search next larger first level - let flMap = root.flMap & (~0 << (fl + 1)); - if (!flMap) { - head = null; - } else { - fl = ctz(flMap); - slMap = GETSL(root, fl); - if (DEBUG) assert(slMap); // can't be zero if fl points here - head = GETHEAD(root, fl, ctz(slMap)); - } - } else { - head = GETHEAD(root, fl, ctz(slMap)); - } - return head; -} - -/** Prepares the specified block before (re-)use, possibly splitting it. */ -function prepareBlock(root: Root, block: Block, size: usize): void { - // size was already asserted by caller - - let blockInfo = block.mmInfo; - if (DEBUG) assert(!((size + BLOCK_OVERHEAD) & AL_MASK)); // size must be aligned so the new block is - - // split if the block can hold another MINSIZE block incl. overhead - let remaining = (blockInfo & ~TAGS_MASK) - size; - if (remaining >= BLOCK_OVERHEAD + BLOCK_MINSIZE) { - block.mmInfo = size | (blockInfo & LEFTFREE); // also discards FREE - - let spare = changetype(changetype(block) + BLOCK_OVERHEAD + size); - spare.mmInfo = (remaining - BLOCK_OVERHEAD) | FREE; // not LEFTFREE - insertBlock(root, spare); // also sets 'back' - - // otherwise tag block as no longer FREE and right as no longer LEFTFREE - } else { - block.mmInfo = blockInfo & ~FREE; - GETRIGHT(block).mmInfo &= ~LEFTFREE; - } -} - -/** Adds more memory to the pool. */ -function addMemory(root: Root, start: usize, end: usize): bool { - if (DEBUG) assert(start <= end); // must be valid - start = ((start + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; - end &= ~AL_MASK; - - let tail = GETTAIL(root); - let tailInfo: usize = 0; - if (tail) { // more memory - if (DEBUG) assert(start >= changetype(tail) + BLOCK_OVERHEAD); - - // merge with current tail if adjacent - const offsetToTail = AL_SIZE; - if (start - offsetToTail == changetype(tail)) { - start -= offsetToTail; - tailInfo = tail.mmInfo; - } else { - // We don't do this, but a user might `memory.grow` manually - // leading to non-adjacent pages managed by TLSF. - } - - } else if (DEBUG) { // first memory - assert(start >= changetype(root) + ROOT_SIZE); // starts after root - } - - // check if size is large enough for a free block and the tail block - let size = end - start; - if (size < BLOCK_OVERHEAD + BLOCK_MINSIZE + BLOCK_OVERHEAD) { - return false; - } - - // left size is total minus its own and the zero-length tail's header - let leftSize = size - 2 * BLOCK_OVERHEAD; - let left = changetype(start); - left.mmInfo = leftSize | FREE | (tailInfo & LEFTFREE); - left.prev = null; - left.next = null; - - // tail is a zero-length used block - tail = changetype(start + BLOCK_OVERHEAD + leftSize); - tail.mmInfo = 0 | LEFTFREE; - SETTAIL(root, tail); - - insertBlock(root, left); // also merges with free left before tail / sets 'back' - - return true; -} - -/** Grows memory to fit at least another block of the specified size. */ -function growMemory(root: Root, size: usize): void { - if (ASC_LOW_MEMORY_LIMIT) { - unreachable(); - return; - } - // Here, both rounding performed in searchBlock ... - const halfMaxSize = BLOCK_MAXSIZE >> 1; - if (size < halfMaxSize) { // don't round last fl - const invRound = (sizeof() * 8 - 1) - SL_BITS; - size += (1 << (invRound - clz(size))) - 1; - } - // and additional BLOCK_OVERHEAD must be taken into account. If we are going - // to merge with the tail block, that's one time, otherwise it's two times. - let pagesBefore = memory.size(); - size += BLOCK_OVERHEAD << usize((pagesBefore << 16) - BLOCK_OVERHEAD != changetype(GETTAIL(root))); - let pagesNeeded = (((size + 0xffff) & ~0xffff) >>> 16); - let pagesWanted = max(pagesBefore, pagesNeeded); // double memory - if (memory.grow(pagesWanted) < 0) { - if (memory.grow(pagesNeeded) < 0) unreachable(); - } - let pagesAfter = memory.size(); - addMemory(root, pagesBefore << 16, pagesAfter << 16); -} - -/** Computes the size (excl. header) of a block. */ -function computeSize(size: usize): usize { - // Size must be large enough and aligned minus preceeding overhead - return size <= BLOCK_MINSIZE - ? BLOCK_MINSIZE - : ((size + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; -} - -/** Prepares and checks an allocation size. */ -function prepareSize(size: usize): usize { - if (size > BLOCK_MAXSIZE) throw new Error(E_ALLOCATION_TOO_LARGE); - return computeSize(size); -} - -/** Initializes the root structure. */ -function initialize(): void { - if (isDefined(ASC_RTRACE)) oninit(__heap_base); - let rootOffset = (__heap_base + AL_MASK) & ~AL_MASK; - let pagesBefore = memory.size(); - let pagesNeeded = ((((rootOffset + ROOT_SIZE) + 0xffff) & ~0xffff) >>> 16); - if (pagesNeeded > pagesBefore && memory.grow(pagesNeeded - pagesBefore) < 0) unreachable(); - let root = changetype(rootOffset); - root.flMap = 0; - SETTAIL(root, changetype(0)); - for (let fl: usize = 0; fl < FL_BITS; ++fl) { - SETSL(root, fl, 0); - for (let sl: u32 = 0; sl < SL_SIZE; ++sl) { - SETHEAD(root, fl, sl, null); - } - } - let memStart = rootOffset + ROOT_SIZE; - if (ASC_LOW_MEMORY_LIMIT) { - const memEnd = ASC_LOW_MEMORY_LIMIT & ~AL_MASK; - if (memStart <= memEnd) addMemory(root, memStart, memEnd); - else unreachable(); // low memory limit already exceeded - } else { - addMemory(root, memStart, memory.size() << 16); - } - ROOT = root; -} - -/** Allocates a block of the specified size. */ -export function allocateBlock(root: Root, size: usize): Block { - let payloadSize = prepareSize(size); - let block = searchBlock(root, payloadSize); - if (!block) { - growMemory(root, payloadSize); - block = changetype(searchBlock(root, payloadSize)); - if (DEBUG) assert(block); // must be found now - } - if (DEBUG) assert((block.mmInfo & ~TAGS_MASK) >= payloadSize); // must fit - removeBlock(root, block); - prepareBlock(root, block, payloadSize); - if (isDefined(ASC_RTRACE)) onalloc(block); - return block; -} - -/** Reallocates a block to the specified size. */ -export function reallocateBlock(root: Root, block: Block, size: usize): Block { - let payloadSize = prepareSize(size); - let blockInfo = block.mmInfo; - let blockSize = blockInfo & ~TAGS_MASK; - - // possibly split and update runtime size if it still fits - if (payloadSize <= blockSize) { - prepareBlock(root, block, payloadSize); - if (isDefined(ASC_RTRACE)) { - if (payloadSize != blockSize) onresize(block, BLOCK_OVERHEAD + blockSize); - } - return block; - } - - // merge with right free block if merger is large enough - let right = GETRIGHT(block); - let rightInfo = right.mmInfo; - if (rightInfo & FREE) { - let mergeSize = blockSize + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); - if (mergeSize >= payloadSize) { - removeBlock(root, right); - block.mmInfo = (blockInfo & TAGS_MASK) | mergeSize; - prepareBlock(root, block, payloadSize); - if (isDefined(ASC_RTRACE)) onresize(block, BLOCK_OVERHEAD + blockSize); - return block; - } - } - - // otherwise move the block - return moveBlock(root, block, size); -} - -/** Moves a block to a new one of the specified size. */ -function moveBlock(root: Root, block: Block, newSize: usize): Block { - let newBlock = allocateBlock(root, newSize); - memory.copy(changetype(newBlock) + BLOCK_OVERHEAD, changetype(block) + BLOCK_OVERHEAD, block.mmInfo & ~TAGS_MASK); - if (changetype(block) >= __heap_base) { - if (isDefined(ASC_RTRACE)) onmove(block, newBlock); - freeBlock(root, block); - } - return newBlock; -} - -/** Frees a block. */ -export function freeBlock(root: Root, block: Block): void { - if (isDefined(ASC_RTRACE)) onfree(block); - block.mmInfo = block.mmInfo | FREE; - insertBlock(root, block); -} - -/** Checks that a used block is valid to be freed or reallocated. */ -function checkUsedBlock(ptr: usize): Block { - let block = changetype(ptr - BLOCK_OVERHEAD); - assert( - ptr != 0 && !(ptr & AL_MASK) && // must exist and be aligned - !(block.mmInfo & FREE) // must be used - ); - return block; -} - -// @ts-ignore: decorator -@global @unsafe -export function __alloc(size: usize): usize { - if (!ROOT) initialize(); - return changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; -} - -// @ts-ignore: decorator -@global @unsafe -export function __realloc(ptr: usize, size: usize): usize { - if (!ROOT) initialize(); - return (ptr < __heap_base - ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) - : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) - ) + BLOCK_OVERHEAD; -} - -// @ts-ignore: decorator -@global @unsafe -export function __free(ptr: usize): void { - if (ptr < __heap_base) return; - if (!ROOT) initialize(); - freeBlock(ROOT, checkUsedBlock(ptr)); -} +import { AL_BITS, AL_SIZE, AL_MASK, DEBUG, BLOCK, BLOCK_OVERHEAD, BLOCK_MAXSIZE } from "./common"; +import { oninit, onalloc, onresize, onmove, onfree } from "./rtrace"; +import { E_ALLOCATION_TOO_LARGE } from "../util/error"; + +// === The TLSF (Two-Level Segregate Fit) memory allocator === +// see: http://www.gii.upv.es/tlsf/ + +// Split into single- and multi-threaded versions, the multi-threaded version just adds basic locks around +// allocation and deallocation. + +// - `ffs(x)` is equivalent to `ctz(x)` with x != 0 +// - `fls(x)` is equivalent to `sizeof(x) * 8 - clz(x) - 1` + +// ╒══════════════ Block size interpretation (32-bit) ═════════════╕ +// 3 2 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits +// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─╫─┴─┴─┴─┤ +// │ | FL │ SB = SL + AL │ ◄─ usize +// └───────────────────────────────────────────────┴───────╨───────┘ +// FL: first level, SL: second level, AL: alignment, SB: small block + +// @ts-ignore: decorator +@inline const SL_BITS: u32 = 4; +// @ts-ignore: decorator +@inline const SL_SIZE: u32 = 1 << SL_BITS; + +// @ts-ignore: decorator +@inline const SB_BITS: u32 = SL_BITS + AL_BITS; +// @ts-ignore: decorator +@inline const SB_SIZE: u32 = 1 << SB_BITS; + +// @ts-ignore: decorator +@inline const FL_BITS: u32 = 31 - SB_BITS; + +// [00]: < 256B (SB) [12]: < 1M +// [01]: < 512B [13]: < 2M +// [02]: < 1K [14]: < 4M +// [03]: < 2K [15]: < 8M +// [04]: < 4K [16]: < 16M +// [05]: < 8K [17]: < 32M +// [06]: < 16K [18]: < 64M +// [07]: < 32K [19]: < 128M +// [08]: < 64K [20]: < 256M +// [09]: < 128K [21]: < 512M +// [10]: < 256K [22]: <= 1G - OVERHEAD +// [11]: < 512K +// VMs limit to 2GB total (currently), making one 1G block max (or three 512M etc.) due to block overhead + +// Tags stored in otherwise unused alignment bits + +// @ts-ignore: decorator +@inline const FREE: usize = 1 << 0; +// @ts-ignore: decorator +@inline const LEFTFREE: usize = 1 << 1; +// @ts-ignore: decorator +@inline const TAGS_MASK: usize = FREE | LEFTFREE; // <= AL_MASK + +// ╒════════════════════ Block layout (32-bit) ════════════════════╕ +// 3 2 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits +// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┼─┤ ┐ +// │ size │L│F│ ◄─┐ info overhead +// ╞>ptr═══════════════════════════════════════════════════════╧═╧═╡ │ ┘ +// │ if free: ◄ prev │ ◄─┤ usize +// ├───────────────────────────────────────────────────────────────┤ │ +// │ if free: next ► │ ◄─┤ +// ├───────────────────────────────────────────────────────────────┤ │ +// │ ... │ │ >= 0 +// ├───────────────────────────────────────────────────────────────┤ │ +// │ if free: back ▲ │ ◄─┘ +// └───────────────────────────────────────────────────────────────┘ >= MIN SIZE +// F: FREE, L: LEFTFREE +@unmanaged export class Block extends BLOCK { + + /** Previous free block, if any. Only valid if free, otherwise part of payload. */ + prev: Block | null; + /** Next free block, if any. Only valid if free, otherwise part of payload. */ + next: Block | null; + + // If the block is free, there is a 'back'reference at its end pointing at its start. +} + +// Block constants. A block must have a minimum size of three pointers so it can hold `prev`, +// `next` and `back` if free. + +// @ts-ignore: decorator +@inline const BLOCK_MINSIZE: usize = ((3 * sizeof() + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; // prev + next + back +// @ts-ignore: decorator +// @inline const BLOCK_MAXSIZE: usize = 1 << (FL_BITS + SB_BITS - 1); // exclusive, lives in common.ts + +/** Gets the left block of a block. Only valid if the left block is free. */ +// @ts-ignore: decorator +@inline function GETFREELEFT(block: Block): Block { + return load(changetype(block) - sizeof()); +} + +/** Gets the right block of a block by advancing to the right by its size. */ +// @ts-ignore: decorator +@inline function GETRIGHT(block: Block): Block { + return changetype(changetype(block) + BLOCK_OVERHEAD + (block.mmInfo & ~TAGS_MASK)); +} + +// ╒═════════════════════ Root layout (32-bit) ════════════════════╕ +// 3 2 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits +// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐ +// │ 0 | flMap S│ ◄────┐ +// ╞═══════════════════════════════════════════════════════════════╡ │ +// │ slMap[0] S │ ◄─┐ │ +// ├───────────────────────────────────────────────────────────────┤ │ │ +// │ slMap[1] │ ◄─┤ │ +// ├───────────────────────────────────────────────────────────────┤ u32 │ +// │ slMap[22] │ ◄─┘ │ +// ╞═══════════════════════════════════════════════════════════════╡ usize +// │ head[0] │ ◄────┤ +// ├───────────────────────────────────────────────────────────────┤ │ +// │ ... │ ◄────┤ +// ├───────────────────────────────────────────────────────────────┤ │ +// │ head[367] │ ◄────┤ +// ╞═══════════════════════════════════════════════════════════════╡ │ +// │ tail │ ◄────┘ +// └───────────────────────────────────────────────────────────────┘ SIZE ┘ +// S: Small blocks map +@unmanaged class Root { + /** First level bitmap. */ + flMap: usize; +} + +// Root constants. Where stuff is stored inside of the root structure. + +// @ts-ignore: decorator +@inline const SL_START: usize = sizeof(); +// @ts-ignore: decorator +@inline const SL_END: usize = SL_START + (FL_BITS << alignof()); +// @ts-ignore: decorator +@inline const HL_START: usize = (SL_END + AL_MASK) & ~AL_MASK; +// @ts-ignore: decorator +@inline const HL_END: usize = HL_START + FL_BITS * SL_SIZE * sizeof(); +// @ts-ignore: decorator +@inline const ROOT_SIZE: usize = HL_END + sizeof(); + +// @ts-ignore: decorator +@lazy export let ROOT: Root = changetype(memory.data(ROOT_SIZE)); // unsafe initializion below + +/** Gets the second level map of the specified first level. */ +// @ts-ignore: decorator +@inline function GETSL(root: Root, fl: usize): u32 { + return load( + changetype(root) + (fl << alignof()), + SL_START + ); +} + +/** Sets the second level map of the specified first level. */ +// @ts-ignore: decorator +@inline function SETSL(root: Root, fl: usize, slMap: u32): void { + store( + changetype(root) + (fl << alignof()), + slMap, + SL_START + ); +} + +/** Gets the head of the free list for the specified combination of first and second level. */ +// @ts-ignore: decorator +@inline function GETHEAD(root: Root, fl: usize, sl: u32): Block | null { + return load( + changetype(root) + (((fl << SL_BITS) + sl) << alignof()), + HL_START + ); +} + +/** Sets the head of the free list for the specified combination of first and second level. */ +// @ts-ignore: decorator +@inline function SETHEAD(root: Root, fl: usize, sl: u32, head: Block | null): void { + store( + changetype(root) + (((fl << SL_BITS) + sl) << alignof()), + head, + HL_START + ); +} + +/** Gets the tail block.. */ +// @ts-ignore: decorator +@inline function GETTAIL(root: Root): Block { + return load( + changetype(root), + HL_END + ); +} + +/** Sets the tail block. */ +// @ts-ignore: decorator +@inline function SETTAIL(root: Root, tail: Block): void { + store( + changetype(root), + tail, + HL_END + ); +} + +/** Inserts a previously used block back into the free list. */ +function insertBlock(root: Root, block: Block): void { + if (DEBUG) assert(block); // cannot be null + let blockInfo = block.mmInfo; + if (DEBUG) assert(blockInfo & FREE); // must be free + + let right = GETRIGHT(block); + let rightInfo = right.mmInfo; + + // merge with right block if also free + if (rightInfo & FREE) { + removeBlock(root, right); + block.mmInfo = blockInfo = blockInfo + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); // keep block tags + right = GETRIGHT(block); + rightInfo = right.mmInfo; + // 'back' is set below + } + + // merge with left block if also free + if (blockInfo & LEFTFREE) { + let left = GETFREELEFT(block); + let leftInfo = left.mmInfo; + if (DEBUG) assert(leftInfo & FREE); // must be free according to right tags + removeBlock(root, left); + block = left; + block.mmInfo = blockInfo = leftInfo + BLOCK_OVERHEAD + (blockInfo & ~TAGS_MASK); // keep left tags + // 'back' is set below + } + + right.mmInfo = rightInfo | LEFTFREE; + // reference to right is no longer used now, hence rightInfo is not synced + + // we now know the size of the block + let size = blockInfo & ~TAGS_MASK; + if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be a valid size + if (DEBUG) assert(changetype(block) + BLOCK_OVERHEAD + size == changetype(right)); // must match + + // set 'back' to itself at the end of block + store(changetype(right) - sizeof(), block); + + // mapping_insert + let fl: usize, sl: u32; + if (size < SB_SIZE) { + fl = 0; + sl = (size >> AL_BITS); + } else { + const inv: usize = sizeof() * 8 - 1; + let boundedSize = min(size, BLOCK_MAXSIZE); + fl = inv - clz(boundedSize); + sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); + fl -= SB_BITS - 1; + } + if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range + + // perform insertion + let head = GETHEAD(root, fl, sl); + block.prev = null; + block.next = head; + if (head) head.prev = block; + SETHEAD(root, fl, sl, block); + + // update first and second level maps + root.flMap |= (1 << fl); + SETSL(root, fl, GETSL(root, fl) | (1 << sl)); +} + +/** Removes a free block from internal lists. */ +function removeBlock(root: Root, block: Block): void { + let blockInfo = block.mmInfo; + if (DEBUG) assert(blockInfo & FREE); // must be free + let size = blockInfo & ~TAGS_MASK; + if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be valid + + // mapping_insert + let fl: usize, sl: u32; + if (size < SB_SIZE) { + fl = 0; + sl = (size >> AL_BITS); + } else { + const inv: usize = sizeof() * 8 - 1; + let boundedSize = min(size, BLOCK_MAXSIZE); + fl = inv - clz(boundedSize); + sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); + fl -= SB_BITS - 1; + } + if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range + + // link previous and next free block + let prev = block.prev; + let next = block.next; + if (prev) prev.next = next; + if (next) next.prev = prev; + + // update head if we are removing it + if (block == GETHEAD(root, fl, sl)) { + SETHEAD(root, fl, sl, next); + + // clear second level map if head is empty now + if (!next) { + let slMap = GETSL(root, fl); + SETSL(root, fl, slMap &= ~(1 << sl)); + + // clear first level map if second level is empty now + if (!slMap) root.flMap &= ~(1 << fl); + } + } + // note: does not alter left/back because it is likely that splitting + // is performed afterwards, invalidating those changes. so, the caller + // must perform those updates. +} + +/** Searches for a free block of at least the specified size. */ +function searchBlock(root: Root, size: usize): Block | null { + // size was already asserted by caller + + // mapping_search + let fl: usize, sl: u32; + if (size < SB_SIZE) { + fl = 0; + sl = (size >> AL_BITS); + } else { + const halfMaxSize = BLOCK_MAXSIZE >> 1; // don't round last fl + const inv: usize = sizeof() * 8 - 1; + const invRound = inv - SL_BITS; + let requestSize = size < halfMaxSize + ? size + (1 << (invRound - clz(size))) - 1 + : size; + fl = inv - clz(requestSize); + sl = ((requestSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); + fl -= SB_BITS - 1; + } + if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range + + // search second level + let slMap = GETSL(root, fl) & (~0 << sl); + let head: Block | null = null; + if (!slMap) { + // search next larger first level + let flMap = root.flMap & (~0 << (fl + 1)); + if (!flMap) { + head = null; + } else { + fl = ctz(flMap); + slMap = GETSL(root, fl); + if (DEBUG) assert(slMap); // can't be zero if fl points here + head = GETHEAD(root, fl, ctz(slMap)); + } + } else { + head = GETHEAD(root, fl, ctz(slMap)); + } + return head; +} + +/** Prepares the specified block before (re-)use, possibly splitting it. */ +function prepareBlock(root: Root, block: Block, size: usize): void { + // size was already asserted by caller + + let blockInfo = block.mmInfo; + if (DEBUG) assert(!((size + BLOCK_OVERHEAD) & AL_MASK)); // size must be aligned so the new block is + + // split if the block can hold another MINSIZE block incl. overhead + let remaining = (blockInfo & ~TAGS_MASK) - size; + if (remaining >= BLOCK_OVERHEAD + BLOCK_MINSIZE) { + block.mmInfo = size | (blockInfo & LEFTFREE); // also discards FREE + + let spare = changetype(changetype(block) + BLOCK_OVERHEAD + size); + spare.mmInfo = (remaining - BLOCK_OVERHEAD) | FREE; // not LEFTFREE + insertBlock(root, spare); // also sets 'back' + + // otherwise tag block as no longer FREE and right as no longer LEFTFREE + } else { + block.mmInfo = blockInfo & ~FREE; + GETRIGHT(block).mmInfo &= ~LEFTFREE; + } +} + +/** Adds more memory to the pool. */ +function addMemory(root: Root, start: usize, end: usize): bool { + if (DEBUG) assert(start <= end); // must be valid + start = ((start + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; + end &= ~AL_MASK; + + let tail = GETTAIL(root); + let tailInfo: usize = 0; + if (tail) { // more memory + if (DEBUG) assert(start >= changetype(tail) + BLOCK_OVERHEAD); + + // merge with current tail if adjacent + const offsetToTail = AL_SIZE; + if (start - offsetToTail == changetype(tail)) { + start -= offsetToTail; + tailInfo = tail.mmInfo; + } else { + // We don't do this, but a user might `memory.grow` manually + // leading to non-adjacent pages managed by TLSF. + } + + } else if (DEBUG) { // first memory + assert(start >= changetype(root) + ROOT_SIZE); // starts after root + } + + // check if size is large enough for a free block and the tail block + let size = end - start; + if (size < BLOCK_OVERHEAD + BLOCK_MINSIZE + BLOCK_OVERHEAD) { + return false; + } + + // left size is total minus its own and the zero-length tail's header + let leftSize = size - 2 * BLOCK_OVERHEAD; + let left = changetype(start); + left.mmInfo = leftSize | FREE | (tailInfo & LEFTFREE); + left.prev = null; + left.next = null; + + // tail is a zero-length used block + tail = changetype(start + BLOCK_OVERHEAD + leftSize); + tail.mmInfo = 0 | LEFTFREE; + SETTAIL(root, tail); + + insertBlock(root, left); // also merges with free left before tail / sets 'back' + + return true; +} + +/** Grows memory to fit at least another block of the specified size. */ +function growMemory(root: Root, size: usize): void { + if (ASC_LOW_MEMORY_LIMIT) { + unreachable(); + return; + } + // Here, both rounding performed in searchBlock ... + const halfMaxSize = BLOCK_MAXSIZE >> 1; + if (size < halfMaxSize) { // don't round last fl + const invRound = (sizeof() * 8 - 1) - SL_BITS; + size += (1 << (invRound - clz(size))) - 1; + } + // and additional BLOCK_OVERHEAD must be taken into account. If we are going + // to merge with the tail block, that's one time, otherwise it's two times. + let pagesBefore = memory.size(); + size += BLOCK_OVERHEAD << usize((pagesBefore << 16) - BLOCK_OVERHEAD != changetype(GETTAIL(root))); + let pagesNeeded = (((size + 0xffff) & ~0xffff) >>> 16); + let pagesWanted = max(pagesBefore, pagesNeeded); // double memory + if (memory.grow(pagesWanted) < 0) { + if (memory.grow(pagesNeeded) < 0) unreachable(); + } + let pagesAfter = memory.size(); + addMemory(root, pagesBefore << 16, pagesAfter << 16); +} + +/** Computes the size (excl. header) of a block. */ +function computeSize(size: usize): usize { + // Size must be large enough and aligned minus preceeding overhead + return size <= BLOCK_MINSIZE + ? BLOCK_MINSIZE + : ((size + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; +} + +/** Prepares and checks an allocation size. */ +function prepareSize(size: usize): usize { + if (size > BLOCK_MAXSIZE) throw new Error(E_ALLOCATION_TOO_LARGE); + return computeSize(size); +} + +/** Initializes the root structure. */ +export function TLSFinitialize(): void { + if (isDefined(ASC_RTRACE)) oninit(__heap_base); + let rootOffset = (__heap_base + AL_MASK) & ~AL_MASK; + let pagesBefore = memory.size(); + let pagesNeeded = ((((rootOffset + ROOT_SIZE) + 0xffff) & ~0xffff) >>> 16); + if (pagesNeeded > pagesBefore && memory.grow(pagesNeeded - pagesBefore) < 0) unreachable(); + let root = ROOT; + root.flMap = 0; + SETTAIL(root, changetype(0)); + for (let fl: usize = 0; fl < FL_BITS; ++fl) { + SETSL(root, fl, 0); + for (let sl: u32 = 0; sl < SL_SIZE; ++sl) { + SETHEAD(root, fl, sl, null); + } + } + let memStart = rootOffset + ROOT_SIZE; + if (ASC_LOW_MEMORY_LIMIT) { + const memEnd = ASC_LOW_MEMORY_LIMIT & ~AL_MASK; + if (memStart <= memEnd) addMemory(root, memStart, memEnd); + else unreachable(); // low memory limit already exceeded + } else { + addMemory(root, memStart, memory.size() << 16); + } +} + +/** Allocates a block of the specified size. */ +export function allocateBlock(root: Root, size: usize): Block { + let payloadSize = prepareSize(size); + let block = searchBlock(root, payloadSize); + if (!block) { + growMemory(root, payloadSize); + block = changetype(searchBlock(root, payloadSize)); + if (DEBUG) assert(block); // must be found now + } + if (DEBUG) assert((block.mmInfo & ~TAGS_MASK) >= payloadSize); // must fit + removeBlock(root, block); + prepareBlock(root, block, payloadSize); + if (isDefined(ASC_RTRACE)) onalloc(block); + return block; +} + +/** Reallocates a block to the specified size. */ +export function reallocateBlock(root: Root, block: Block, size: usize): Block { + let payloadSize = prepareSize(size); + let blockInfo = block.mmInfo; + let blockSize = blockInfo & ~TAGS_MASK; + + // possibly split and update runtime size if it still fits + if (payloadSize <= blockSize) { + prepareBlock(root, block, payloadSize); + if (isDefined(ASC_RTRACE)) { + if (payloadSize != blockSize) onresize(block, BLOCK_OVERHEAD + blockSize); + } + return block; + } + + // merge with right free block if merger is large enough + let right = GETRIGHT(block); + let rightInfo = right.mmInfo; + if (rightInfo & FREE) { + let mergeSize = blockSize + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); + if (mergeSize >= payloadSize) { + removeBlock(root, right); + block.mmInfo = (blockInfo & TAGS_MASK) | mergeSize; + prepareBlock(root, block, payloadSize); + if (isDefined(ASC_RTRACE)) onresize(block, BLOCK_OVERHEAD + blockSize); + return block; + } + } + + // otherwise move the block + return moveBlock(root, block, size); +} + +/** Moves a block to a new one of the specified size. */ +function moveBlock(root: Root, block: Block, newSize: usize): Block { + let newBlock = allocateBlock(root, newSize); + memory.copy(changetype(newBlock) + BLOCK_OVERHEAD, changetype(block) + BLOCK_OVERHEAD, block.mmInfo & ~TAGS_MASK); + if (changetype(block) >= __heap_base) { + if (isDefined(ASC_RTRACE)) onmove(block, newBlock); + freeBlock(root, block); + } + return newBlock; +} + +/** Frees a block. */ +export function freeBlock(root: Root, block: Block): void { + if (isDefined(ASC_RTRACE)) onfree(block); + block.mmInfo = block.mmInfo | FREE; + insertBlock(root, block); +} + +/** Checks that a used block is valid to be freed or reallocated. */ +function checkUsedBlock(ptr: usize): Block { + let block = changetype(ptr - BLOCK_OVERHEAD); + assert( + ptr != 0 && !(ptr & AL_MASK) && // must exist and be aligned + !(block.mmInfo & FREE) // must be used + ); + return block; +} + diff --git a/std/assembly/rt/tlsf-mt.ts b/std/assembly/rt/tlsf-mt.ts new file mode 100644 index 0000000000..e752d333b3 --- /dev/null +++ b/std/assembly/rt/tlsf-mt.ts @@ -0,0 +1,45 @@ +import {BLOCK_OVERHEAD} from "./common"; +import {allocateBlock, freeBlock, reallocateBlock, ROOT, TLSFinitialize, moveBlock, checkUsedBlock} from "./tlsf-base"; +import {TlsfMutex_lock, TlsfMutex_unlock} from './tlsf-mutex' + +const mutex_ptr = memory.data(4, 16); + +// @ts-ignore: decorator +@global @unsafe +export function __alloc(size: usize): usize { + TlsfMutex_lock(mutex_ptr); + + if (!ROOT) TLSFinitialize(); + let r: usize = changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; + + TlsfMutex_unlock(mutex_ptr); + return r; +} + +// @ts-ignore: decorator +@global @unsafe +export function __realloc(ptr: usize, size: usize): usize { + TlsfMutex_lock(mutex_ptr); + + if (!ROOT) TLSFinitialize(); + let r: usize = (ptr < __heap_base + ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) + : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) + ) + BLOCK_OVERHEAD; + + TlsfMutex_unlock(mutex_ptr); + return r; +} + +// @ts-ignore: decorator +@global @unsafe +export function __free(ptr: usize): void { + if (ptr < __heap_base) return; + + TlsfMutex_lock(mutex_ptr); + + if (!ROOT) TLSFinitialize(); + freeBlock(ROOT, checkUsedBlock(ptr)); + + TlsfMutex_unlock(mutex_ptr); +} diff --git a/std/assembly/rt/tlsf-mutex.ts b/std/assembly/rt/tlsf-mutex.ts new file mode 100644 index 0000000000..d323b209b8 --- /dev/null +++ b/std/assembly/rt/tlsf-mutex.ts @@ -0,0 +1,30 @@ +// This just implements a super-simple lock for tlsf-mt.ts + +enum TlsfMutexState { + unlocked, + locked +} + +// Basic spinlock. Spinning is not a performance issue since this only takes as long as an allocation +// @ts-ignore: decorator +@inline +export function TlsfMutex_lock(mutex_ptr: usize): void { + for (; ;) { + // If we succesfully atomically compare and exchange unlocked for locked, we have the mutex + if (atomic.cmpxchg(mutex_ptr, TlsfMutexState.unlocked, TlsfMutexState.locked) === TlsfMutexState.unlocked) + return; + // Wait for unlocked state to try for locked + for (; ;) { + if (atomic.load(mutex_ptr) === TlsfMutexState.unlocked) break; + } + } +} + +// @ts-ignore: decorator +@inline +export function TlsfMutex_unlock(mutex_ptr: usize): void { + if (atomic.cmpxchg(mutex_ptr, TlsfMutexState.locked, TlsfMutexState.unlocked) !== TlsfMutexState.locked) { + // This only happens if someone else unlocked our mutex, or we did it more than once... + throw new Error('Is this the right thing to do here? Mutex in inconsistent state'); + } +} diff --git a/std/assembly/rt/tlsf-st.ts b/std/assembly/rt/tlsf-st.ts new file mode 100644 index 0000000000..970bd6f226 --- /dev/null +++ b/std/assembly/rt/tlsf-st.ts @@ -0,0 +1,28 @@ +import {BLOCK_OVERHEAD} from "./common"; +import {allocateBlock, freeBlock, reallocateBlock, ROOT, TLSFinitialize, moveBlock, checkUsedBlock} from "./tlsf-base"; + +// @ts-ignore: decorator +@global @unsafe +export function __alloc(size: usize): usize { + if (!ROOT) TLSFinitialize(); + + return changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; +} + +// @ts-ignore: decorator +@global @unsafe +export function __realloc(ptr: usize, size: usize): usize { + if (!ROOT) TLSFinitialize(); + return (ptr < __heap_base + ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) + : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) + ) + BLOCK_OVERHEAD; +} + +// @ts-ignore: decorator +@global @unsafe +export function __free(ptr: usize): void { + if (ptr < __heap_base) return; + if (!ROOT) TLSFinitialize(); + freeBlock(ROOT, checkUsedBlock(ptr)); +} From 033783ef87037ca22c09d45aed39544bfc7b9dad Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 9 Dec 2022 21:32:58 -0500 Subject: [PATCH 2/5] add to NOTICE --- NOTICE | 169 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 85 insertions(+), 84 deletions(-) diff --git a/NOTICE b/NOTICE index 93a368305e..cb603b98ae 100644 --- a/NOTICE +++ b/NOTICE @@ -1,84 +1,85 @@ -The following authors have all licensed their contributions to AssemblyScript -under the licensing terms detailed in LICENSE: - -* Daniel Wirtz -* Max Graey -* Igor Sbitnev -* Norton Wang -* Alan Pierce -* Palmer -* Linus Unnebäck -* Joshua Tenner -* Nidin Vinayakan <01@01alchemist.com> -* Aaron Turner -* Willem Wyndham -* Bowen Wang -* Emil Laine -* Stephen Paul Weber -* Jay Phelps -* jhwgh1968 -* Jeffrey Charles -* Vladimir Tikhonov -* Duncan Uszkay -* Surma -* Julien Letellier -* Guido Zuidhof -* ncave <777696+ncave@users.noreply.github.com> -* Andrew Davis -* Maël Nison -* Valeria Viana Gusmao -* Gabor Greif -* Martin Fredriksson -* forcepusher -* Piotr Oleś -* Saúl Cabrera -* Chance Snow -* Peter Salomonsen -* ookangzheng -* yjhmelody -* bnbarak -* Colin Eberhardt -* Ryan Pivovar -* Roman F. <70765447+romdotdog@users.noreply.github.com> -* Joe Pea -* Felipe Gasper -* Congcong Cai -* mooooooi -* Yasushi Ando -* Syed Jafri -* Peter Hayman -* ApsarasX -* Adrien Zinger -* Ruixiang Chen -* Daniel Salvadori -* Jairus Tanaka -* CountBleck -* Abdul Rauf - -Portions of this software are derived from third-party works licensed under -the following terms: - -* TypeScript: https://github.com/Microsoft/TypeScript - - Copyright (c) Microsoft Corporation - Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) - -* Binaryen: https://github.com/WebAssembly/binaryen - - Copyright (c) WebAssembly Community Group participants - Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) - -* musl libc: http://www.musl-libc.org - - Copyright (c) Rich Felker, et al. - The MIT License (https://opensource.org/licenses/MIT) - -* V8: https://developers.google.com/v8/ - - Copyright (c) the V8 project authors - The 3-Clause BSD License (https://opensource.org/licenses/BSD-3-Clause) - -* Arm Optimized Routines: https://github.com/ARM-software/optimized-routines - - Copyright (c) Arm Limited - The MIT License (https://opensource.org/licenses/MIT) +The following authors have all licensed their contributions to AssemblyScript +under the licensing terms detailed in LICENSE: + +* Daniel Wirtz +* Max Graey +* Igor Sbitnev +* Norton Wang +* Alan Pierce +* Palmer +* Linus Unnebäck +* Joshua Tenner +* Nidin Vinayakan <01@01alchemist.com> +* Aaron Turner +* Willem Wyndham +* Bowen Wang +* Emil Laine +* Stephen Paul Weber +* Jay Phelps +* jhwgh1968 +* Jeffrey Charles +* Vladimir Tikhonov +* Duncan Uszkay +* Surma +* Julien Letellier +* Guido Zuidhof +* ncave <777696+ncave@users.noreply.github.com> +* Andrew Davis +* Maël Nison +* Valeria Viana Gusmao +* Gabor Greif +* Martin Fredriksson +* David Schneider +* forcepusher +* Piotr Oleś +* Saúl Cabrera +* Chance Snow +* Peter Salomonsen +* ookangzheng +* yjhmelody +* bnbarak +* Colin Eberhardt +* Ryan Pivovar +* Roman F. <70765447+romdotdog@users.noreply.github.com> +* Joe Pea +* Felipe Gasper +* Congcong Cai +* mooooooi +* Yasushi Ando +* Syed Jafri +* Peter Hayman +* ApsarasX +* Adrien Zinger +* Ruixiang Chen +* Daniel Salvadori +* Jairus Tanaka +* CountBleck +* Abdul Rauf + +Portions of this software are derived from third-party works licensed under +the following terms: + +* TypeScript: https://github.com/Microsoft/TypeScript + + Copyright (c) Microsoft Corporation + Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) + +* Binaryen: https://github.com/WebAssembly/binaryen + + Copyright (c) WebAssembly Community Group participants + Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) + +* musl libc: http://www.musl-libc.org + + Copyright (c) Rich Felker, et al. + The MIT License (https://opensource.org/licenses/MIT) + +* V8: https://developers.google.com/v8/ + + Copyright (c) the V8 project authors + The 3-Clause BSD License (https://opensource.org/licenses/BSD-3-Clause) + +* Arm Optimized Routines: https://github.com/ARM-software/optimized-routines + + Copyright (c) Arm Limited + The MIT License (https://opensource.org/licenses/MIT) From 1dc9695cfc2b39a821c76cf92dfaa9469d0a00e8 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 9 Dec 2022 21:45:27 -0500 Subject: [PATCH 3/5] fix line separatros in tlsf-base.ts --- std/assembly/rt/tlsf-base.ts | 1134 +++++++++++++++++----------------- 1 file changed, 567 insertions(+), 567 deletions(-) diff --git a/std/assembly/rt/tlsf-base.ts b/std/assembly/rt/tlsf-base.ts index c1caf483f6..a188694f4b 100644 --- a/std/assembly/rt/tlsf-base.ts +++ b/std/assembly/rt/tlsf-base.ts @@ -1,567 +1,567 @@ -import { AL_BITS, AL_SIZE, AL_MASK, DEBUG, BLOCK, BLOCK_OVERHEAD, BLOCK_MAXSIZE } from "./common"; -import { oninit, onalloc, onresize, onmove, onfree } from "./rtrace"; -import { E_ALLOCATION_TOO_LARGE } from "../util/error"; - -// === The TLSF (Two-Level Segregate Fit) memory allocator === -// see: http://www.gii.upv.es/tlsf/ - -// Split into single- and multi-threaded versions, the multi-threaded version just adds basic locks around -// allocation and deallocation. - -// - `ffs(x)` is equivalent to `ctz(x)` with x != 0 -// - `fls(x)` is equivalent to `sizeof(x) * 8 - clz(x) - 1` - -// ╒══════════════ Block size interpretation (32-bit) ═════════════╕ -// 3 2 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits -// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─╫─┴─┴─┴─┤ -// │ | FL │ SB = SL + AL │ ◄─ usize -// └───────────────────────────────────────────────┴───────╨───────┘ -// FL: first level, SL: second level, AL: alignment, SB: small block - -// @ts-ignore: decorator -@inline const SL_BITS: u32 = 4; -// @ts-ignore: decorator -@inline const SL_SIZE: u32 = 1 << SL_BITS; - -// @ts-ignore: decorator -@inline const SB_BITS: u32 = SL_BITS + AL_BITS; -// @ts-ignore: decorator -@inline const SB_SIZE: u32 = 1 << SB_BITS; - -// @ts-ignore: decorator -@inline const FL_BITS: u32 = 31 - SB_BITS; - -// [00]: < 256B (SB) [12]: < 1M -// [01]: < 512B [13]: < 2M -// [02]: < 1K [14]: < 4M -// [03]: < 2K [15]: < 8M -// [04]: < 4K [16]: < 16M -// [05]: < 8K [17]: < 32M -// [06]: < 16K [18]: < 64M -// [07]: < 32K [19]: < 128M -// [08]: < 64K [20]: < 256M -// [09]: < 128K [21]: < 512M -// [10]: < 256K [22]: <= 1G - OVERHEAD -// [11]: < 512K -// VMs limit to 2GB total (currently), making one 1G block max (or three 512M etc.) due to block overhead - -// Tags stored in otherwise unused alignment bits - -// @ts-ignore: decorator -@inline const FREE: usize = 1 << 0; -// @ts-ignore: decorator -@inline const LEFTFREE: usize = 1 << 1; -// @ts-ignore: decorator -@inline const TAGS_MASK: usize = FREE | LEFTFREE; // <= AL_MASK - -// ╒════════════════════ Block layout (32-bit) ════════════════════╕ -// 3 2 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits -// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┼─┤ ┐ -// │ size │L│F│ ◄─┐ info overhead -// ╞>ptr═══════════════════════════════════════════════════════╧═╧═╡ │ ┘ -// │ if free: ◄ prev │ ◄─┤ usize -// ├───────────────────────────────────────────────────────────────┤ │ -// │ if free: next ► │ ◄─┤ -// ├───────────────────────────────────────────────────────────────┤ │ -// │ ... │ │ >= 0 -// ├───────────────────────────────────────────────────────────────┤ │ -// │ if free: back ▲ │ ◄─┘ -// └───────────────────────────────────────────────────────────────┘ >= MIN SIZE -// F: FREE, L: LEFTFREE -@unmanaged export class Block extends BLOCK { - - /** Previous free block, if any. Only valid if free, otherwise part of payload. */ - prev: Block | null; - /** Next free block, if any. Only valid if free, otherwise part of payload. */ - next: Block | null; - - // If the block is free, there is a 'back'reference at its end pointing at its start. -} - -// Block constants. A block must have a minimum size of three pointers so it can hold `prev`, -// `next` and `back` if free. - -// @ts-ignore: decorator -@inline const BLOCK_MINSIZE: usize = ((3 * sizeof() + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; // prev + next + back -// @ts-ignore: decorator -// @inline const BLOCK_MAXSIZE: usize = 1 << (FL_BITS + SB_BITS - 1); // exclusive, lives in common.ts - -/** Gets the left block of a block. Only valid if the left block is free. */ -// @ts-ignore: decorator -@inline function GETFREELEFT(block: Block): Block { - return load(changetype(block) - sizeof()); -} - -/** Gets the right block of a block by advancing to the right by its size. */ -// @ts-ignore: decorator -@inline function GETRIGHT(block: Block): Block { - return changetype(changetype(block) + BLOCK_OVERHEAD + (block.mmInfo & ~TAGS_MASK)); -} - -// ╒═════════════════════ Root layout (32-bit) ════════════════════╕ -// 3 2 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits -// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐ -// │ 0 | flMap S│ ◄────┐ -// ╞═══════════════════════════════════════════════════════════════╡ │ -// │ slMap[0] S │ ◄─┐ │ -// ├───────────────────────────────────────────────────────────────┤ │ │ -// │ slMap[1] │ ◄─┤ │ -// ├───────────────────────────────────────────────────────────────┤ u32 │ -// │ slMap[22] │ ◄─┘ │ -// ╞═══════════════════════════════════════════════════════════════╡ usize -// │ head[0] │ ◄────┤ -// ├───────────────────────────────────────────────────────────────┤ │ -// │ ... │ ◄────┤ -// ├───────────────────────────────────────────────────────────────┤ │ -// │ head[367] │ ◄────┤ -// ╞═══════════════════════════════════════════════════════════════╡ │ -// │ tail │ ◄────┘ -// └───────────────────────────────────────────────────────────────┘ SIZE ┘ -// S: Small blocks map -@unmanaged class Root { - /** First level bitmap. */ - flMap: usize; -} - -// Root constants. Where stuff is stored inside of the root structure. - -// @ts-ignore: decorator -@inline const SL_START: usize = sizeof(); -// @ts-ignore: decorator -@inline const SL_END: usize = SL_START + (FL_BITS << alignof()); -// @ts-ignore: decorator -@inline const HL_START: usize = (SL_END + AL_MASK) & ~AL_MASK; -// @ts-ignore: decorator -@inline const HL_END: usize = HL_START + FL_BITS * SL_SIZE * sizeof(); -// @ts-ignore: decorator -@inline const ROOT_SIZE: usize = HL_END + sizeof(); - -// @ts-ignore: decorator -@lazy export let ROOT: Root = changetype(memory.data(ROOT_SIZE)); // unsafe initializion below - -/** Gets the second level map of the specified first level. */ -// @ts-ignore: decorator -@inline function GETSL(root: Root, fl: usize): u32 { - return load( - changetype(root) + (fl << alignof()), - SL_START - ); -} - -/** Sets the second level map of the specified first level. */ -// @ts-ignore: decorator -@inline function SETSL(root: Root, fl: usize, slMap: u32): void { - store( - changetype(root) + (fl << alignof()), - slMap, - SL_START - ); -} - -/** Gets the head of the free list for the specified combination of first and second level. */ -// @ts-ignore: decorator -@inline function GETHEAD(root: Root, fl: usize, sl: u32): Block | null { - return load( - changetype(root) + (((fl << SL_BITS) + sl) << alignof()), - HL_START - ); -} - -/** Sets the head of the free list for the specified combination of first and second level. */ -// @ts-ignore: decorator -@inline function SETHEAD(root: Root, fl: usize, sl: u32, head: Block | null): void { - store( - changetype(root) + (((fl << SL_BITS) + sl) << alignof()), - head, - HL_START - ); -} - -/** Gets the tail block.. */ -// @ts-ignore: decorator -@inline function GETTAIL(root: Root): Block { - return load( - changetype(root), - HL_END - ); -} - -/** Sets the tail block. */ -// @ts-ignore: decorator -@inline function SETTAIL(root: Root, tail: Block): void { - store( - changetype(root), - tail, - HL_END - ); -} - -/** Inserts a previously used block back into the free list. */ -function insertBlock(root: Root, block: Block): void { - if (DEBUG) assert(block); // cannot be null - let blockInfo = block.mmInfo; - if (DEBUG) assert(blockInfo & FREE); // must be free - - let right = GETRIGHT(block); - let rightInfo = right.mmInfo; - - // merge with right block if also free - if (rightInfo & FREE) { - removeBlock(root, right); - block.mmInfo = blockInfo = blockInfo + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); // keep block tags - right = GETRIGHT(block); - rightInfo = right.mmInfo; - // 'back' is set below - } - - // merge with left block if also free - if (blockInfo & LEFTFREE) { - let left = GETFREELEFT(block); - let leftInfo = left.mmInfo; - if (DEBUG) assert(leftInfo & FREE); // must be free according to right tags - removeBlock(root, left); - block = left; - block.mmInfo = blockInfo = leftInfo + BLOCK_OVERHEAD + (blockInfo & ~TAGS_MASK); // keep left tags - // 'back' is set below - } - - right.mmInfo = rightInfo | LEFTFREE; - // reference to right is no longer used now, hence rightInfo is not synced - - // we now know the size of the block - let size = blockInfo & ~TAGS_MASK; - if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be a valid size - if (DEBUG) assert(changetype(block) + BLOCK_OVERHEAD + size == changetype(right)); // must match - - // set 'back' to itself at the end of block - store(changetype(right) - sizeof(), block); - - // mapping_insert - let fl: usize, sl: u32; - if (size < SB_SIZE) { - fl = 0; - sl = (size >> AL_BITS); - } else { - const inv: usize = sizeof() * 8 - 1; - let boundedSize = min(size, BLOCK_MAXSIZE); - fl = inv - clz(boundedSize); - sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); - fl -= SB_BITS - 1; - } - if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range - - // perform insertion - let head = GETHEAD(root, fl, sl); - block.prev = null; - block.next = head; - if (head) head.prev = block; - SETHEAD(root, fl, sl, block); - - // update first and second level maps - root.flMap |= (1 << fl); - SETSL(root, fl, GETSL(root, fl) | (1 << sl)); -} - -/** Removes a free block from internal lists. */ -function removeBlock(root: Root, block: Block): void { - let blockInfo = block.mmInfo; - if (DEBUG) assert(blockInfo & FREE); // must be free - let size = blockInfo & ~TAGS_MASK; - if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be valid - - // mapping_insert - let fl: usize, sl: u32; - if (size < SB_SIZE) { - fl = 0; - sl = (size >> AL_BITS); - } else { - const inv: usize = sizeof() * 8 - 1; - let boundedSize = min(size, BLOCK_MAXSIZE); - fl = inv - clz(boundedSize); - sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); - fl -= SB_BITS - 1; - } - if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range - - // link previous and next free block - let prev = block.prev; - let next = block.next; - if (prev) prev.next = next; - if (next) next.prev = prev; - - // update head if we are removing it - if (block == GETHEAD(root, fl, sl)) { - SETHEAD(root, fl, sl, next); - - // clear second level map if head is empty now - if (!next) { - let slMap = GETSL(root, fl); - SETSL(root, fl, slMap &= ~(1 << sl)); - - // clear first level map if second level is empty now - if (!slMap) root.flMap &= ~(1 << fl); - } - } - // note: does not alter left/back because it is likely that splitting - // is performed afterwards, invalidating those changes. so, the caller - // must perform those updates. -} - -/** Searches for a free block of at least the specified size. */ -function searchBlock(root: Root, size: usize): Block | null { - // size was already asserted by caller - - // mapping_search - let fl: usize, sl: u32; - if (size < SB_SIZE) { - fl = 0; - sl = (size >> AL_BITS); - } else { - const halfMaxSize = BLOCK_MAXSIZE >> 1; // don't round last fl - const inv: usize = sizeof() * 8 - 1; - const invRound = inv - SL_BITS; - let requestSize = size < halfMaxSize - ? size + (1 << (invRound - clz(size))) - 1 - : size; - fl = inv - clz(requestSize); - sl = ((requestSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); - fl -= SB_BITS - 1; - } - if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range - - // search second level - let slMap = GETSL(root, fl) & (~0 << sl); - let head: Block | null = null; - if (!slMap) { - // search next larger first level - let flMap = root.flMap & (~0 << (fl + 1)); - if (!flMap) { - head = null; - } else { - fl = ctz(flMap); - slMap = GETSL(root, fl); - if (DEBUG) assert(slMap); // can't be zero if fl points here - head = GETHEAD(root, fl, ctz(slMap)); - } - } else { - head = GETHEAD(root, fl, ctz(slMap)); - } - return head; -} - -/** Prepares the specified block before (re-)use, possibly splitting it. */ -function prepareBlock(root: Root, block: Block, size: usize): void { - // size was already asserted by caller - - let blockInfo = block.mmInfo; - if (DEBUG) assert(!((size + BLOCK_OVERHEAD) & AL_MASK)); // size must be aligned so the new block is - - // split if the block can hold another MINSIZE block incl. overhead - let remaining = (blockInfo & ~TAGS_MASK) - size; - if (remaining >= BLOCK_OVERHEAD + BLOCK_MINSIZE) { - block.mmInfo = size | (blockInfo & LEFTFREE); // also discards FREE - - let spare = changetype(changetype(block) + BLOCK_OVERHEAD + size); - spare.mmInfo = (remaining - BLOCK_OVERHEAD) | FREE; // not LEFTFREE - insertBlock(root, spare); // also sets 'back' - - // otherwise tag block as no longer FREE and right as no longer LEFTFREE - } else { - block.mmInfo = blockInfo & ~FREE; - GETRIGHT(block).mmInfo &= ~LEFTFREE; - } -} - -/** Adds more memory to the pool. */ -function addMemory(root: Root, start: usize, end: usize): bool { - if (DEBUG) assert(start <= end); // must be valid - start = ((start + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; - end &= ~AL_MASK; - - let tail = GETTAIL(root); - let tailInfo: usize = 0; - if (tail) { // more memory - if (DEBUG) assert(start >= changetype(tail) + BLOCK_OVERHEAD); - - // merge with current tail if adjacent - const offsetToTail = AL_SIZE; - if (start - offsetToTail == changetype(tail)) { - start -= offsetToTail; - tailInfo = tail.mmInfo; - } else { - // We don't do this, but a user might `memory.grow` manually - // leading to non-adjacent pages managed by TLSF. - } - - } else if (DEBUG) { // first memory - assert(start >= changetype(root) + ROOT_SIZE); // starts after root - } - - // check if size is large enough for a free block and the tail block - let size = end - start; - if (size < BLOCK_OVERHEAD + BLOCK_MINSIZE + BLOCK_OVERHEAD) { - return false; - } - - // left size is total minus its own and the zero-length tail's header - let leftSize = size - 2 * BLOCK_OVERHEAD; - let left = changetype(start); - left.mmInfo = leftSize | FREE | (tailInfo & LEFTFREE); - left.prev = null; - left.next = null; - - // tail is a zero-length used block - tail = changetype(start + BLOCK_OVERHEAD + leftSize); - tail.mmInfo = 0 | LEFTFREE; - SETTAIL(root, tail); - - insertBlock(root, left); // also merges with free left before tail / sets 'back' - - return true; -} - -/** Grows memory to fit at least another block of the specified size. */ -function growMemory(root: Root, size: usize): void { - if (ASC_LOW_MEMORY_LIMIT) { - unreachable(); - return; - } - // Here, both rounding performed in searchBlock ... - const halfMaxSize = BLOCK_MAXSIZE >> 1; - if (size < halfMaxSize) { // don't round last fl - const invRound = (sizeof() * 8 - 1) - SL_BITS; - size += (1 << (invRound - clz(size))) - 1; - } - // and additional BLOCK_OVERHEAD must be taken into account. If we are going - // to merge with the tail block, that's one time, otherwise it's two times. - let pagesBefore = memory.size(); - size += BLOCK_OVERHEAD << usize((pagesBefore << 16) - BLOCK_OVERHEAD != changetype(GETTAIL(root))); - let pagesNeeded = (((size + 0xffff) & ~0xffff) >>> 16); - let pagesWanted = max(pagesBefore, pagesNeeded); // double memory - if (memory.grow(pagesWanted) < 0) { - if (memory.grow(pagesNeeded) < 0) unreachable(); - } - let pagesAfter = memory.size(); - addMemory(root, pagesBefore << 16, pagesAfter << 16); -} - -/** Computes the size (excl. header) of a block. */ -function computeSize(size: usize): usize { - // Size must be large enough and aligned minus preceeding overhead - return size <= BLOCK_MINSIZE - ? BLOCK_MINSIZE - : ((size + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; -} - -/** Prepares and checks an allocation size. */ -function prepareSize(size: usize): usize { - if (size > BLOCK_MAXSIZE) throw new Error(E_ALLOCATION_TOO_LARGE); - return computeSize(size); -} - -/** Initializes the root structure. */ -export function TLSFinitialize(): void { - if (isDefined(ASC_RTRACE)) oninit(__heap_base); - let rootOffset = (__heap_base + AL_MASK) & ~AL_MASK; - let pagesBefore = memory.size(); - let pagesNeeded = ((((rootOffset + ROOT_SIZE) + 0xffff) & ~0xffff) >>> 16); - if (pagesNeeded > pagesBefore && memory.grow(pagesNeeded - pagesBefore) < 0) unreachable(); - let root = ROOT; - root.flMap = 0; - SETTAIL(root, changetype(0)); - for (let fl: usize = 0; fl < FL_BITS; ++fl) { - SETSL(root, fl, 0); - for (let sl: u32 = 0; sl < SL_SIZE; ++sl) { - SETHEAD(root, fl, sl, null); - } - } - let memStart = rootOffset + ROOT_SIZE; - if (ASC_LOW_MEMORY_LIMIT) { - const memEnd = ASC_LOW_MEMORY_LIMIT & ~AL_MASK; - if (memStart <= memEnd) addMemory(root, memStart, memEnd); - else unreachable(); // low memory limit already exceeded - } else { - addMemory(root, memStart, memory.size() << 16); - } -} - -/** Allocates a block of the specified size. */ -export function allocateBlock(root: Root, size: usize): Block { - let payloadSize = prepareSize(size); - let block = searchBlock(root, payloadSize); - if (!block) { - growMemory(root, payloadSize); - block = changetype(searchBlock(root, payloadSize)); - if (DEBUG) assert(block); // must be found now - } - if (DEBUG) assert((block.mmInfo & ~TAGS_MASK) >= payloadSize); // must fit - removeBlock(root, block); - prepareBlock(root, block, payloadSize); - if (isDefined(ASC_RTRACE)) onalloc(block); - return block; -} - -/** Reallocates a block to the specified size. */ -export function reallocateBlock(root: Root, block: Block, size: usize): Block { - let payloadSize = prepareSize(size); - let blockInfo = block.mmInfo; - let blockSize = blockInfo & ~TAGS_MASK; - - // possibly split and update runtime size if it still fits - if (payloadSize <= blockSize) { - prepareBlock(root, block, payloadSize); - if (isDefined(ASC_RTRACE)) { - if (payloadSize != blockSize) onresize(block, BLOCK_OVERHEAD + blockSize); - } - return block; - } - - // merge with right free block if merger is large enough - let right = GETRIGHT(block); - let rightInfo = right.mmInfo; - if (rightInfo & FREE) { - let mergeSize = blockSize + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); - if (mergeSize >= payloadSize) { - removeBlock(root, right); - block.mmInfo = (blockInfo & TAGS_MASK) | mergeSize; - prepareBlock(root, block, payloadSize); - if (isDefined(ASC_RTRACE)) onresize(block, BLOCK_OVERHEAD + blockSize); - return block; - } - } - - // otherwise move the block - return moveBlock(root, block, size); -} - -/** Moves a block to a new one of the specified size. */ -function moveBlock(root: Root, block: Block, newSize: usize): Block { - let newBlock = allocateBlock(root, newSize); - memory.copy(changetype(newBlock) + BLOCK_OVERHEAD, changetype(block) + BLOCK_OVERHEAD, block.mmInfo & ~TAGS_MASK); - if (changetype(block) >= __heap_base) { - if (isDefined(ASC_RTRACE)) onmove(block, newBlock); - freeBlock(root, block); - } - return newBlock; -} - -/** Frees a block. */ -export function freeBlock(root: Root, block: Block): void { - if (isDefined(ASC_RTRACE)) onfree(block); - block.mmInfo = block.mmInfo | FREE; - insertBlock(root, block); -} - -/** Checks that a used block is valid to be freed or reallocated. */ -function checkUsedBlock(ptr: usize): Block { - let block = changetype(ptr - BLOCK_OVERHEAD); - assert( - ptr != 0 && !(ptr & AL_MASK) && // must exist and be aligned - !(block.mmInfo & FREE) // must be used - ); - return block; -} - +import { AL_BITS, AL_SIZE, AL_MASK, DEBUG, BLOCK, BLOCK_OVERHEAD, BLOCK_MAXSIZE } from "./common"; +import { oninit, onalloc, onresize, onmove, onfree } from "./rtrace"; +import { E_ALLOCATION_TOO_LARGE } from "../util/error"; + +// === The TLSF (Two-Level Segregate Fit) memory allocator === +// see: http://www.gii.upv.es/tlsf/ + +// Split into single- and multi-threaded versions, the multi-threaded version just adds basic locks around +// allocation and deallocation. + +// - `ffs(x)` is equivalent to `ctz(x)` with x != 0 +// - `fls(x)` is equivalent to `sizeof(x) * 8 - clz(x) - 1` + +// ╒══════════════ Block size interpretation (32-bit) ═════════════╕ +// 3 2 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits +// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┴─┴─┴─╫─┴─┴─┴─┤ +// │ | FL │ SB = SL + AL │ ◄─ usize +// └───────────────────────────────────────────────┴───────╨───────┘ +// FL: first level, SL: second level, AL: alignment, SB: small block + +// @ts-ignore: decorator +@inline const SL_BITS: u32 = 4; +// @ts-ignore: decorator +@inline const SL_SIZE: u32 = 1 << SL_BITS; + +// @ts-ignore: decorator +@inline const SB_BITS: u32 = SL_BITS + AL_BITS; +// @ts-ignore: decorator +@inline const SB_SIZE: u32 = 1 << SB_BITS; + +// @ts-ignore: decorator +@inline const FL_BITS: u32 = 31 - SB_BITS; + +// [00]: < 256B (SB) [12]: < 1M +// [01]: < 512B [13]: < 2M +// [02]: < 1K [14]: < 4M +// [03]: < 2K [15]: < 8M +// [04]: < 4K [16]: < 16M +// [05]: < 8K [17]: < 32M +// [06]: < 16K [18]: < 64M +// [07]: < 32K [19]: < 128M +// [08]: < 64K [20]: < 256M +// [09]: < 128K [21]: < 512M +// [10]: < 256K [22]: <= 1G - OVERHEAD +// [11]: < 512K +// VMs limit to 2GB total (currently), making one 1G block max (or three 512M etc.) due to block overhead + +// Tags stored in otherwise unused alignment bits + +// @ts-ignore: decorator +@inline const FREE: usize = 1 << 0; +// @ts-ignore: decorator +@inline const LEFTFREE: usize = 1 << 1; +// @ts-ignore: decorator +@inline const TAGS_MASK: usize = FREE | LEFTFREE; // <= AL_MASK + +// ╒════════════════════ Block layout (32-bit) ════════════════════╕ +// 3 2 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits +// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┼─┼─┤ ┐ +// │ size │L│F│ ◄─┐ info overhead +// ╞>ptr═══════════════════════════════════════════════════════╧═╧═╡ │ ┘ +// │ if free: ◄ prev │ ◄─┤ usize +// ├───────────────────────────────────────────────────────────────┤ │ +// │ if free: next ► │ ◄─┤ +// ├───────────────────────────────────────────────────────────────┤ │ +// │ ... │ │ >= 0 +// ├───────────────────────────────────────────────────────────────┤ │ +// │ if free: back ▲ │ ◄─┘ +// └───────────────────────────────────────────────────────────────┘ >= MIN SIZE +// F: FREE, L: LEFTFREE +@unmanaged export class Block extends BLOCK { + + /** Previous free block, if any. Only valid if free, otherwise part of payload. */ + prev: Block | null; + /** Next free block, if any. Only valid if free, otherwise part of payload. */ + next: Block | null; + + // If the block is free, there is a 'back'reference at its end pointing at its start. +} + +// Block constants. A block must have a minimum size of three pointers so it can hold `prev`, +// `next` and `back` if free. + +// @ts-ignore: decorator +@inline const BLOCK_MINSIZE: usize = ((3 * sizeof() + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; // prev + next + back +// @ts-ignore: decorator +// @inline const BLOCK_MAXSIZE: usize = 1 << (FL_BITS + SB_BITS - 1); // exclusive, lives in common.ts + +/** Gets the left block of a block. Only valid if the left block is free. */ +// @ts-ignore: decorator +@inline function GETFREELEFT(block: Block): Block { + return load(changetype(block) - sizeof()); +} + +/** Gets the right block of a block by advancing to the right by its size. */ +// @ts-ignore: decorator +@inline function GETRIGHT(block: Block): Block { + return changetype(changetype(block) + BLOCK_OVERHEAD + (block.mmInfo & ~TAGS_MASK)); +} + +// ╒═════════════════════ Root layout (32-bit) ════════════════════╕ +// 3 2 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 bits +// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐ +// │ 0 | flMap S│ ◄────┐ +// ╞═══════════════════════════════════════════════════════════════╡ │ +// │ slMap[0] S │ ◄─┐ │ +// ├───────────────────────────────────────────────────────────────┤ │ │ +// │ slMap[1] │ ◄─┤ │ +// ├───────────────────────────────────────────────────────────────┤ u32 │ +// │ slMap[22] │ ◄─┘ │ +// ╞═══════════════════════════════════════════════════════════════╡ usize +// │ head[0] │ ◄────┤ +// ├───────────────────────────────────────────────────────────────┤ │ +// │ ... │ ◄────┤ +// ├───────────────────────────────────────────────────────────────┤ │ +// │ head[367] │ ◄────┤ +// ╞═══════════════════════════════════════════════════════════════╡ │ +// │ tail │ ◄────┘ +// └───────────────────────────────────────────────────────────────┘ SIZE ┘ +// S: Small blocks map +@unmanaged class Root { + /** First level bitmap. */ + flMap: usize; +} + +// Root constants. Where stuff is stored inside of the root structure. + +// @ts-ignore: decorator +@inline const SL_START: usize = sizeof(); +// @ts-ignore: decorator +@inline const SL_END: usize = SL_START + (FL_BITS << alignof()); +// @ts-ignore: decorator +@inline const HL_START: usize = (SL_END + AL_MASK) & ~AL_MASK; +// @ts-ignore: decorator +@inline const HL_END: usize = HL_START + FL_BITS * SL_SIZE * sizeof(); +// @ts-ignore: decorator +@inline const ROOT_SIZE: usize = HL_END + sizeof(); + +// @ts-ignore: decorator +@lazy export let ROOT: Root = changetype(memory.data(ROOT_SIZE)); // unsafe initializion below + +/** Gets the second level map of the specified first level. */ +// @ts-ignore: decorator +@inline function GETSL(root: Root, fl: usize): u32 { + return load( + changetype(root) + (fl << alignof()), + SL_START + ); +} + +/** Sets the second level map of the specified first level. */ +// @ts-ignore: decorator +@inline function SETSL(root: Root, fl: usize, slMap: u32): void { + store( + changetype(root) + (fl << alignof()), + slMap, + SL_START + ); +} + +/** Gets the head of the free list for the specified combination of first and second level. */ +// @ts-ignore: decorator +@inline function GETHEAD(root: Root, fl: usize, sl: u32): Block | null { + return load( + changetype(root) + (((fl << SL_BITS) + sl) << alignof()), + HL_START + ); +} + +/** Sets the head of the free list for the specified combination of first and second level. */ +// @ts-ignore: decorator +@inline function SETHEAD(root: Root, fl: usize, sl: u32, head: Block | null): void { + store( + changetype(root) + (((fl << SL_BITS) + sl) << alignof()), + head, + HL_START + ); +} + +/** Gets the tail block.. */ +// @ts-ignore: decorator +@inline function GETTAIL(root: Root): Block { + return load( + changetype(root), + HL_END + ); +} + +/** Sets the tail block. */ +// @ts-ignore: decorator +@inline function SETTAIL(root: Root, tail: Block): void { + store( + changetype(root), + tail, + HL_END + ); +} + +/** Inserts a previously used block back into the free list. */ +function insertBlock(root: Root, block: Block): void { + if (DEBUG) assert(block); // cannot be null + let blockInfo = block.mmInfo; + if (DEBUG) assert(blockInfo & FREE); // must be free + + let right = GETRIGHT(block); + let rightInfo = right.mmInfo; + + // merge with right block if also free + if (rightInfo & FREE) { + removeBlock(root, right); + block.mmInfo = blockInfo = blockInfo + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); // keep block tags + right = GETRIGHT(block); + rightInfo = right.mmInfo; + // 'back' is set below + } + + // merge with left block if also free + if (blockInfo & LEFTFREE) { + let left = GETFREELEFT(block); + let leftInfo = left.mmInfo; + if (DEBUG) assert(leftInfo & FREE); // must be free according to right tags + removeBlock(root, left); + block = left; + block.mmInfo = blockInfo = leftInfo + BLOCK_OVERHEAD + (blockInfo & ~TAGS_MASK); // keep left tags + // 'back' is set below + } + + right.mmInfo = rightInfo | LEFTFREE; + // reference to right is no longer used now, hence rightInfo is not synced + + // we now know the size of the block + let size = blockInfo & ~TAGS_MASK; + if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be a valid size + if (DEBUG) assert(changetype(block) + BLOCK_OVERHEAD + size == changetype(right)); // must match + + // set 'back' to itself at the end of block + store(changetype(right) - sizeof(), block); + + // mapping_insert + let fl: usize, sl: u32; + if (size < SB_SIZE) { + fl = 0; + sl = (size >> AL_BITS); + } else { + const inv: usize = sizeof() * 8 - 1; + let boundedSize = min(size, BLOCK_MAXSIZE); + fl = inv - clz(boundedSize); + sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); + fl -= SB_BITS - 1; + } + if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range + + // perform insertion + let head = GETHEAD(root, fl, sl); + block.prev = null; + block.next = head; + if (head) head.prev = block; + SETHEAD(root, fl, sl, block); + + // update first and second level maps + root.flMap |= (1 << fl); + SETSL(root, fl, GETSL(root, fl) | (1 << sl)); +} + +/** Removes a free block from internal lists. */ +function removeBlock(root: Root, block: Block): void { + let blockInfo = block.mmInfo; + if (DEBUG) assert(blockInfo & FREE); // must be free + let size = blockInfo & ~TAGS_MASK; + if (DEBUG) assert(size >= BLOCK_MINSIZE); // must be valid + + // mapping_insert + let fl: usize, sl: u32; + if (size < SB_SIZE) { + fl = 0; + sl = (size >> AL_BITS); + } else { + const inv: usize = sizeof() * 8 - 1; + let boundedSize = min(size, BLOCK_MAXSIZE); + fl = inv - clz(boundedSize); + sl = ((boundedSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); + fl -= SB_BITS - 1; + } + if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range + + // link previous and next free block + let prev = block.prev; + let next = block.next; + if (prev) prev.next = next; + if (next) next.prev = prev; + + // update head if we are removing it + if (block == GETHEAD(root, fl, sl)) { + SETHEAD(root, fl, sl, next); + + // clear second level map if head is empty now + if (!next) { + let slMap = GETSL(root, fl); + SETSL(root, fl, slMap &= ~(1 << sl)); + + // clear first level map if second level is empty now + if (!slMap) root.flMap &= ~(1 << fl); + } + } + // note: does not alter left/back because it is likely that splitting + // is performed afterwards, invalidating those changes. so, the caller + // must perform those updates. +} + +/** Searches for a free block of at least the specified size. */ +function searchBlock(root: Root, size: usize): Block | null { + // size was already asserted by caller + + // mapping_search + let fl: usize, sl: u32; + if (size < SB_SIZE) { + fl = 0; + sl = (size >> AL_BITS); + } else { + const halfMaxSize = BLOCK_MAXSIZE >> 1; // don't round last fl + const inv: usize = sizeof() * 8 - 1; + const invRound = inv - SL_BITS; + let requestSize = size < halfMaxSize + ? size + (1 << (invRound - clz(size))) - 1 + : size; + fl = inv - clz(requestSize); + sl = ((requestSize >> (fl - SL_BITS)) ^ (1 << SL_BITS)); + fl -= SB_BITS - 1; + } + if (DEBUG) assert(fl < FL_BITS && sl < SL_SIZE); // fl/sl out of range + + // search second level + let slMap = GETSL(root, fl) & (~0 << sl); + let head: Block | null = null; + if (!slMap) { + // search next larger first level + let flMap = root.flMap & (~0 << (fl + 1)); + if (!flMap) { + head = null; + } else { + fl = ctz(flMap); + slMap = GETSL(root, fl); + if (DEBUG) assert(slMap); // can't be zero if fl points here + head = GETHEAD(root, fl, ctz(slMap)); + } + } else { + head = GETHEAD(root, fl, ctz(slMap)); + } + return head; +} + +/** Prepares the specified block before (re-)use, possibly splitting it. */ +function prepareBlock(root: Root, block: Block, size: usize): void { + // size was already asserted by caller + + let blockInfo = block.mmInfo; + if (DEBUG) assert(!((size + BLOCK_OVERHEAD) & AL_MASK)); // size must be aligned so the new block is + + // split if the block can hold another MINSIZE block incl. overhead + let remaining = (blockInfo & ~TAGS_MASK) - size; + if (remaining >= BLOCK_OVERHEAD + BLOCK_MINSIZE) { + block.mmInfo = size | (blockInfo & LEFTFREE); // also discards FREE + + let spare = changetype(changetype(block) + BLOCK_OVERHEAD + size); + spare.mmInfo = (remaining - BLOCK_OVERHEAD) | FREE; // not LEFTFREE + insertBlock(root, spare); // also sets 'back' + + // otherwise tag block as no longer FREE and right as no longer LEFTFREE + } else { + block.mmInfo = blockInfo & ~FREE; + GETRIGHT(block).mmInfo &= ~LEFTFREE; + } +} + +/** Adds more memory to the pool. */ +function addMemory(root: Root, start: usize, end: usize): bool { + if (DEBUG) assert(start <= end); // must be valid + start = ((start + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; + end &= ~AL_MASK; + + let tail = GETTAIL(root); + let tailInfo: usize = 0; + if (tail) { // more memory + if (DEBUG) assert(start >= changetype(tail) + BLOCK_OVERHEAD); + + // merge with current tail if adjacent + const offsetToTail = AL_SIZE; + if (start - offsetToTail == changetype(tail)) { + start -= offsetToTail; + tailInfo = tail.mmInfo; + } else { + // We don't do this, but a user might `memory.grow` manually + // leading to non-adjacent pages managed by TLSF. + } + + } else if (DEBUG) { // first memory + assert(start >= changetype(root) + ROOT_SIZE); // starts after root + } + + // check if size is large enough for a free block and the tail block + let size = end - start; + if (size < BLOCK_OVERHEAD + BLOCK_MINSIZE + BLOCK_OVERHEAD) { + return false; + } + + // left size is total minus its own and the zero-length tail's header + let leftSize = size - 2 * BLOCK_OVERHEAD; + let left = changetype(start); + left.mmInfo = leftSize | FREE | (tailInfo & LEFTFREE); + left.prev = null; + left.next = null; + + // tail is a zero-length used block + tail = changetype(start + BLOCK_OVERHEAD + leftSize); + tail.mmInfo = 0 | LEFTFREE; + SETTAIL(root, tail); + + insertBlock(root, left); // also merges with free left before tail / sets 'back' + + return true; +} + +/** Grows memory to fit at least another block of the specified size. */ +function growMemory(root: Root, size: usize): void { + if (ASC_LOW_MEMORY_LIMIT) { + unreachable(); + return; + } + // Here, both rounding performed in searchBlock ... + const halfMaxSize = BLOCK_MAXSIZE >> 1; + if (size < halfMaxSize) { // don't round last fl + const invRound = (sizeof() * 8 - 1) - SL_BITS; + size += (1 << (invRound - clz(size))) - 1; + } + // and additional BLOCK_OVERHEAD must be taken into account. If we are going + // to merge with the tail block, that's one time, otherwise it's two times. + let pagesBefore = memory.size(); + size += BLOCK_OVERHEAD << usize((pagesBefore << 16) - BLOCK_OVERHEAD != changetype(GETTAIL(root))); + let pagesNeeded = (((size + 0xffff) & ~0xffff) >>> 16); + let pagesWanted = max(pagesBefore, pagesNeeded); // double memory + if (memory.grow(pagesWanted) < 0) { + if (memory.grow(pagesNeeded) < 0) unreachable(); + } + let pagesAfter = memory.size(); + addMemory(root, pagesBefore << 16, pagesAfter << 16); +} + +/** Computes the size (excl. header) of a block. */ +function computeSize(size: usize): usize { + // Size must be large enough and aligned minus preceeding overhead + return size <= BLOCK_MINSIZE + ? BLOCK_MINSIZE + : ((size + BLOCK_OVERHEAD + AL_MASK) & ~AL_MASK) - BLOCK_OVERHEAD; +} + +/** Prepares and checks an allocation size. */ +function prepareSize(size: usize): usize { + if (size > BLOCK_MAXSIZE) throw new Error(E_ALLOCATION_TOO_LARGE); + return computeSize(size); +} + +/** Initializes the root structure. */ +export function TLSFinitialize(): void { + if (isDefined(ASC_RTRACE)) oninit(__heap_base); + let rootOffset = (__heap_base + AL_MASK) & ~AL_MASK; + let pagesBefore = memory.size(); + let pagesNeeded = ((((rootOffset + ROOT_SIZE) + 0xffff) & ~0xffff) >>> 16); + if (pagesNeeded > pagesBefore && memory.grow(pagesNeeded - pagesBefore) < 0) unreachable(); + let root = ROOT; + root.flMap = 0; + SETTAIL(root, changetype(0)); + for (let fl: usize = 0; fl < FL_BITS; ++fl) { + SETSL(root, fl, 0); + for (let sl: u32 = 0; sl < SL_SIZE; ++sl) { + SETHEAD(root, fl, sl, null); + } + } + let memStart = rootOffset + ROOT_SIZE; + if (ASC_LOW_MEMORY_LIMIT) { + const memEnd = ASC_LOW_MEMORY_LIMIT & ~AL_MASK; + if (memStart <= memEnd) addMemory(root, memStart, memEnd); + else unreachable(); // low memory limit already exceeded + } else { + addMemory(root, memStart, memory.size() << 16); + } +} + +/** Allocates a block of the specified size. */ +export function allocateBlock(root: Root, size: usize): Block { + let payloadSize = prepareSize(size); + let block = searchBlock(root, payloadSize); + if (!block) { + growMemory(root, payloadSize); + block = changetype(searchBlock(root, payloadSize)); + if (DEBUG) assert(block); // must be found now + } + if (DEBUG) assert((block.mmInfo & ~TAGS_MASK) >= payloadSize); // must fit + removeBlock(root, block); + prepareBlock(root, block, payloadSize); + if (isDefined(ASC_RTRACE)) onalloc(block); + return block; +} + +/** Reallocates a block to the specified size. */ +export function reallocateBlock(root: Root, block: Block, size: usize): Block { + let payloadSize = prepareSize(size); + let blockInfo = block.mmInfo; + let blockSize = blockInfo & ~TAGS_MASK; + + // possibly split and update runtime size if it still fits + if (payloadSize <= blockSize) { + prepareBlock(root, block, payloadSize); + if (isDefined(ASC_RTRACE)) { + if (payloadSize != blockSize) onresize(block, BLOCK_OVERHEAD + blockSize); + } + return block; + } + + // merge with right free block if merger is large enough + let right = GETRIGHT(block); + let rightInfo = right.mmInfo; + if (rightInfo & FREE) { + let mergeSize = blockSize + BLOCK_OVERHEAD + (rightInfo & ~TAGS_MASK); + if (mergeSize >= payloadSize) { + removeBlock(root, right); + block.mmInfo = (blockInfo & TAGS_MASK) | mergeSize; + prepareBlock(root, block, payloadSize); + if (isDefined(ASC_RTRACE)) onresize(block, BLOCK_OVERHEAD + blockSize); + return block; + } + } + + // otherwise move the block + return moveBlock(root, block, size); +} + +/** Moves a block to a new one of the specified size. */ +function moveBlock(root: Root, block: Block, newSize: usize): Block { + let newBlock = allocateBlock(root, newSize); + memory.copy(changetype(newBlock) + BLOCK_OVERHEAD, changetype(block) + BLOCK_OVERHEAD, block.mmInfo & ~TAGS_MASK); + if (changetype(block) >= __heap_base) { + if (isDefined(ASC_RTRACE)) onmove(block, newBlock); + freeBlock(root, block); + } + return newBlock; +} + +/** Frees a block. */ +export function freeBlock(root: Root, block: Block): void { + if (isDefined(ASC_RTRACE)) onfree(block); + block.mmInfo = block.mmInfo | FREE; + insertBlock(root, block); +} + +/** Checks that a used block is valid to be freed or reallocated. */ +function checkUsedBlock(ptr: usize): Block { + let block = changetype(ptr - BLOCK_OVERHEAD); + assert( + ptr != 0 && !(ptr & AL_MASK) && // must exist and be aligned + !(block.mmInfo & FREE) // must be used + ); + return block; +} + From 710add2866d99594711af86f49c1b1937c469cd8 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 9 Dec 2022 21:46:44 -0500 Subject: [PATCH 4/5] fix line seperators in NOTICE --- NOTICE | 170 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/NOTICE b/NOTICE index cb603b98ae..d745fe7ceb 100644 --- a/NOTICE +++ b/NOTICE @@ -1,85 +1,85 @@ -The following authors have all licensed their contributions to AssemblyScript -under the licensing terms detailed in LICENSE: - -* Daniel Wirtz -* Max Graey -* Igor Sbitnev -* Norton Wang -* Alan Pierce -* Palmer -* Linus Unnebäck -* Joshua Tenner -* Nidin Vinayakan <01@01alchemist.com> -* Aaron Turner -* Willem Wyndham -* Bowen Wang -* Emil Laine -* Stephen Paul Weber -* Jay Phelps -* jhwgh1968 -* Jeffrey Charles -* Vladimir Tikhonov -* Duncan Uszkay -* Surma -* Julien Letellier -* Guido Zuidhof -* ncave <777696+ncave@users.noreply.github.com> -* Andrew Davis -* Maël Nison -* Valeria Viana Gusmao -* Gabor Greif -* Martin Fredriksson -* David Schneider -* forcepusher -* Piotr Oleś -* Saúl Cabrera -* Chance Snow -* Peter Salomonsen -* ookangzheng -* yjhmelody -* bnbarak -* Colin Eberhardt -* Ryan Pivovar -* Roman F. <70765447+romdotdog@users.noreply.github.com> -* Joe Pea -* Felipe Gasper -* Congcong Cai -* mooooooi -* Yasushi Ando -* Syed Jafri -* Peter Hayman -* ApsarasX -* Adrien Zinger -* Ruixiang Chen -* Daniel Salvadori -* Jairus Tanaka -* CountBleck -* Abdul Rauf - -Portions of this software are derived from third-party works licensed under -the following terms: - -* TypeScript: https://github.com/Microsoft/TypeScript - - Copyright (c) Microsoft Corporation - Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) - -* Binaryen: https://github.com/WebAssembly/binaryen - - Copyright (c) WebAssembly Community Group participants - Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) - -* musl libc: http://www.musl-libc.org - - Copyright (c) Rich Felker, et al. - The MIT License (https://opensource.org/licenses/MIT) - -* V8: https://developers.google.com/v8/ - - Copyright (c) the V8 project authors - The 3-Clause BSD License (https://opensource.org/licenses/BSD-3-Clause) - -* Arm Optimized Routines: https://github.com/ARM-software/optimized-routines - - Copyright (c) Arm Limited - The MIT License (https://opensource.org/licenses/MIT) +The following authors have all licensed their contributions to AssemblyScript +under the licensing terms detailed in LICENSE: + +* Daniel Wirtz +* Max Graey +* Igor Sbitnev +* Norton Wang +* Alan Pierce +* Palmer +* Linus Unnebäck +* Joshua Tenner +* Nidin Vinayakan <01@01alchemist.com> +* Aaron Turner +* Willem Wyndham +* Bowen Wang +* Emil Laine +* Stephen Paul Weber +* Jay Phelps +* jhwgh1968 +* Jeffrey Charles +* Vladimir Tikhonov +* Duncan Uszkay +* Surma +* Julien Letellier +* Guido Zuidhof +* ncave <777696+ncave@users.noreply.github.com> +* Andrew Davis +* Maël Nison +* Valeria Viana Gusmao +* Gabor Greif +* Martin Fredriksson +* David Schneider +* forcepusher +* Piotr Oleś +* Saúl Cabrera +* Chance Snow +* Peter Salomonsen +* ookangzheng +* yjhmelody +* bnbarak +* Colin Eberhardt +* Ryan Pivovar +* Roman F. <70765447+romdotdog@users.noreply.github.com> +* Joe Pea +* Felipe Gasper +* Congcong Cai +* mooooooi +* Yasushi Ando +* Syed Jafri +* Peter Hayman +* ApsarasX +* Adrien Zinger +* Ruixiang Chen +* Daniel Salvadori +* Jairus Tanaka +* CountBleck +* Abdul Rauf + +Portions of this software are derived from third-party works licensed under +the following terms: + +* TypeScript: https://github.com/Microsoft/TypeScript + + Copyright (c) Microsoft Corporation + Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) + +* Binaryen: https://github.com/WebAssembly/binaryen + + Copyright (c) WebAssembly Community Group participants + Apache License, Version 2.0 (https://opensource.org/licenses/Apache-2.0) + +* musl libc: http://www.musl-libc.org + + Copyright (c) Rich Felker, et al. + The MIT License (https://opensource.org/licenses/MIT) + +* V8: https://developers.google.com/v8/ + + Copyright (c) the V8 project authors + The 3-Clause BSD License (https://opensource.org/licenses/BSD-3-Clause) + +* Arm Optimized Routines: https://github.com/ARM-software/optimized-routines + + Copyright (c) Arm Limited + The MIT License (https://opensource.org/licenses/MIT) From 24f222d6a2bf2f5cd06121e28781a36ba4d3c591 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 9 Dec 2022 22:01:25 -0500 Subject: [PATCH 5/5] fix ROOT some --- std/assembly/rt/tlsf-base.ts | 8 ++++++-- std/assembly/rt/tlsf-mt.ts | 8 ++++---- std/assembly/rt/tlsf-st.ts | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/std/assembly/rt/tlsf-base.ts b/std/assembly/rt/tlsf-base.ts index a188694f4b..f325252ea9 100644 --- a/std/assembly/rt/tlsf-base.ts +++ b/std/assembly/rt/tlsf-base.ts @@ -142,6 +142,9 @@ import { E_ALLOCATION_TOO_LARGE } from "../util/error"; // @ts-ignore: decorator @lazy export let ROOT: Root = changetype(memory.data(ROOT_SIZE)); // unsafe initializion below +// @ts-ignore: decorator +@inline export const ROOT_INIT: usize = memory.data(4); + /** Gets the second level map of the specified first level. */ // @ts-ignore: decorator @inline function GETSL(root: Root, fl: usize): u32 { @@ -486,6 +489,7 @@ export function TLSFinitialize(): void { } else { addMemory(root, memStart, memory.size() << 16); } + store(ROOT_INIT, 1); } /** Allocates a block of the specified size. */ @@ -538,7 +542,7 @@ export function reallocateBlock(root: Root, block: Block, size: usize): Block { } /** Moves a block to a new one of the specified size. */ -function moveBlock(root: Root, block: Block, newSize: usize): Block { +export function moveBlock(root: Root, block: Block, newSize: usize): Block { let newBlock = allocateBlock(root, newSize); memory.copy(changetype(newBlock) + BLOCK_OVERHEAD, changetype(block) + BLOCK_OVERHEAD, block.mmInfo & ~TAGS_MASK); if (changetype(block) >= __heap_base) { @@ -556,7 +560,7 @@ export function freeBlock(root: Root, block: Block): void { } /** Checks that a used block is valid to be freed or reallocated. */ -function checkUsedBlock(ptr: usize): Block { +export function checkUsedBlock(ptr: usize): Block { let block = changetype(ptr - BLOCK_OVERHEAD); assert( ptr != 0 && !(ptr & AL_MASK) && // must exist and be aligned diff --git a/std/assembly/rt/tlsf-mt.ts b/std/assembly/rt/tlsf-mt.ts index e752d333b3..c1d3a1155a 100644 --- a/std/assembly/rt/tlsf-mt.ts +++ b/std/assembly/rt/tlsf-mt.ts @@ -1,5 +1,5 @@ import {BLOCK_OVERHEAD} from "./common"; -import {allocateBlock, freeBlock, reallocateBlock, ROOT, TLSFinitialize, moveBlock, checkUsedBlock} from "./tlsf-base"; +import {allocateBlock, freeBlock, reallocateBlock, ROOT, ROOT_INIT, TLSFinitialize, checkUsedBlock, moveBlock} from "./tlsf-base"; import {TlsfMutex_lock, TlsfMutex_unlock} from './tlsf-mutex' const mutex_ptr = memory.data(4, 16); @@ -9,7 +9,7 @@ const mutex_ptr = memory.data(4, 16); export function __alloc(size: usize): usize { TlsfMutex_lock(mutex_ptr); - if (!ROOT) TLSFinitialize(); + if (!load(ROOT_INIT)) TLSFinitialize(); let r: usize = changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; TlsfMutex_unlock(mutex_ptr); @@ -21,7 +21,7 @@ export function __alloc(size: usize): usize { export function __realloc(ptr: usize, size: usize): usize { TlsfMutex_lock(mutex_ptr); - if (!ROOT) TLSFinitialize(); + if (!load(ROOT_INIT)) TLSFinitialize(); let r: usize = (ptr < __heap_base ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) @@ -38,7 +38,7 @@ export function __free(ptr: usize): void { TlsfMutex_lock(mutex_ptr); - if (!ROOT) TLSFinitialize(); + if (!load(ROOT_INIT)) TLSFinitialize(); freeBlock(ROOT, checkUsedBlock(ptr)); TlsfMutex_unlock(mutex_ptr); diff --git a/std/assembly/rt/tlsf-st.ts b/std/assembly/rt/tlsf-st.ts index 970bd6f226..95dd0e1600 100644 --- a/std/assembly/rt/tlsf-st.ts +++ b/std/assembly/rt/tlsf-st.ts @@ -1,10 +1,10 @@ import {BLOCK_OVERHEAD} from "./common"; -import {allocateBlock, freeBlock, reallocateBlock, ROOT, TLSFinitialize, moveBlock, checkUsedBlock} from "./tlsf-base"; +import {allocateBlock, freeBlock, reallocateBlock, ROOT, ROOT_INIT, TLSFinitialize, checkUsedBlock, moveBlock} from "./tlsf-base"; // @ts-ignore: decorator @global @unsafe export function __alloc(size: usize): usize { - if (!ROOT) TLSFinitialize(); + if (!load(ROOT_INIT)) TLSFinitialize(); return changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; } @@ -12,7 +12,7 @@ export function __alloc(size: usize): usize { // @ts-ignore: decorator @global @unsafe export function __realloc(ptr: usize, size: usize): usize { - if (!ROOT) TLSFinitialize(); + if (!load(ROOT_INIT)) TLSFinitialize(); return (ptr < __heap_base ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) @@ -23,6 +23,6 @@ export function __realloc(ptr: usize, size: usize): usize { @global @unsafe export function __free(ptr: usize): void { if (ptr < __heap_base) return; - if (!ROOT) TLSFinitialize(); + if (!load(ROOT_INIT)) TLSFinitialize(); freeBlock(ROOT, checkUsedBlock(ptr)); }