Skip to content

Commit

Permalink
Implement aligned labels
Browse files Browse the repository at this point in the history
  • Loading branch information
Zoltan Herczeg committed Feb 9, 2025
1 parent abbb179 commit e2f735b
Show file tree
Hide file tree
Showing 12 changed files with 717 additions and 17 deletions.
42 changes: 41 additions & 1 deletion sljit_src/sljitLir.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,19 @@
#define SLJIT_SIMD_TYPE_MASK(m) ((sljit_s32)0xff000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
#define SLJIT_SIMD_TYPE_MASK2(m) ((sljit_s32)0xc0000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))

/* Jump flags. */
/* Label definitions. */

#define SLJIT_LABEL_ALIGNED ((~(sljit_uw)0) - 1)

struct sljit_extended_label {
struct sljit_label label;
sljit_uw index;
sljit_uw data;
};

/* Jump definitions. */

/* Jump flag bits. */
#define JUMP_ADDR 0x1
#define JUMP_MOV_ADDR 0x2
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
Expand Down Expand Up @@ -841,22 +853,34 @@ static SLJIT_INLINE void set_label(struct sljit_label *label, struct sljit_compi
label->next = NULL;
label->u.index = compiler->label_count++;
label->size = compiler->size;

if (compiler->last_label != NULL)
compiler->last_label->next = label;
else
compiler->labels = label;

compiler->last_label = label;
}

static SLJIT_INLINE void set_extended_label(struct sljit_extended_label *ext_label, struct sljit_compiler *compiler, sljit_uw type, sljit_uw data)
{
set_label(&ext_label->label, compiler);
ext_label->index = ext_label->label.u.index;
ext_label->label.u.index = type;
ext_label->data = data;
}

static SLJIT_INLINE void set_jump(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_u32 flags)
{
jump->next = NULL;
jump->flags = flags;
jump->u.label = NULL;

if (compiler->last_jump != NULL)
compiler->last_jump->next = jump;
else
compiler->jumps = jump;

compiler->last_jump = jump;
}

Expand Down Expand Up @@ -2357,6 +2381,22 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_label(struct sljit_compil
CHECK_RETURN_OK;
}

static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_aligned_label(struct sljit_compiler *compiler, sljit_s32 log2_align)
{
SLJIT_UNUSED_ARG(compiler);

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
CHECK_ARGUMENT(log2_align >= SLJIT_LABEL_ALIGN_1 && log2_align <= SLJIT_LABEL_ALIGN_16);
compiler->last_flags = 0;
#endif /* SLJIT_ARGUMENT_CHECKS */

#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
if (SLJIT_UNLIKELY(!!compiler->verbose))
fprintf(compiler->verbose, "label.al%d:\n", 1 << log2_align);
#endif /* SLJIT_VERBOSE */
CHECK_RETURN_OK;
}

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
|| (defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM)
Expand Down
43 changes: 39 additions & 4 deletions sljit_src/sljitLir.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1629,8 +1629,31 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fcopy(struct sljit_compiler *compi

/* Label and jump instructions. */

/* Emits a label which can be the target of jump / mov_addr instructions. */

SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler);

/* Alignment values for sljit_emit_aligned_label. */

#define SLJIT_LABEL_ALIGN_1 0
#define SLJIT_LABEL_ALIGN_2 1
#define SLJIT_LABEL_ALIGN_4 2
#define SLJIT_LABEL_ALIGN_8 3
#define SLJIT_LABEL_ALIGN_16 4

/* Emits a label which address is aligned to a power of 2 value. When some
extra space needs to be added to align the label, that space is filled
with SLJIT_NOP instructions. These labels usually represent the end
of a compilation block, and a new function or some read-only data
(e.g. a jump table) follows it. In these typocal cases the NOPs are
never executed.
log2_align represents the alignment, and its value can
be specified by SLJIT_LABEL_* constants
Note: the constant pool (if present) may be stored before the label. */
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler, sljit_s32 log2_align);

/* The SLJIT_FAST_CALL is a calling method for creating lightweight function
calls. This type of calls preserve the values of all registers and stack
frame. Unlike normal function calls, the enter and return operations must
Expand Down Expand Up @@ -2352,6 +2375,14 @@ static SLJIT_INLINE sljit_uw sljit_get_label_addr(struct sljit_label *label) { r
static SLJIT_INLINE sljit_uw sljit_get_jump_addr(struct sljit_jump *jump) { return jump->addr; }
static SLJIT_INLINE sljit_uw sljit_get_const_addr(struct sljit_const *const_) { return const_->addr; }

/* Returns the absolute address of a label. Same as sljit_get_label_addr,
except on some architectures which stores data on the lower bits of an address. */
#if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
static SLJIT_INLINE sljit_uw sljit_get_label_abs_addr(struct sljit_label *label) { return label->u.addr & ~(sljit_uw)1; }
#else /* !SLJIT_CONFIG_ARM_THUMB2 */
static SLJIT_INLINE sljit_uw sljit_get_label_abs_addr(struct sljit_label *label) { return label->u.addr; }
#endif /* SLJIT_CONFIG_ARM_THUMB2 */

/* Only the address and executable offset are required to perform dynamic
code modifications. See sljit_get_executable_offset function. */
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset);
Expand Down Expand Up @@ -2444,10 +2475,14 @@ static SLJIT_INLINE struct sljit_const *sljit_get_next_const(struct sljit_const

/* A number starting from 0 is assigned to each label, which
represents its creation index. The first label created by the
compiler has index 0, the second has index 1, the third has
index 2, and so on. The returned value is unspecified after
sljit_generate_code() is called. */
static SLJIT_INLINE sljit_uw sljit_get_label_index(struct sljit_label *label) { return label->u.index; }
compiler has index 0, the second one has index 1, the third one
has index 2, and so on. The returned value is unspecified after
sljit_generate_code() is called.
It is recommended to use this function to get the creation index
of a label, since sljit_emit_label() may return with the last label,
if no code is generated since the last sljit_emit_label() call. */
SLJIT_API_FUNC_ATTRIBUTE sljit_uw sljit_get_label_index(struct sljit_label *label);

/* The sljit_jump_has_label() and sljit_jump_has_target() functions
returns non-zero value if a label or target is set for the jump
Expand Down
44 changes: 44 additions & 0 deletions sljit_src/sljitNativeARM_32.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,12 @@ static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_
#endif /* SLJIT_CONFIG_ARM_V6 */
}

static SLJIT_INLINE sljit_ins *process_extended_label(sljit_ins *code_ptr, struct sljit_extended_label *ext_label)
{
SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
return (sljit_ins*)((sljit_uw)code_ptr & ~(ext_label->data));
}

#if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)

static void reduce_code_size(struct sljit_compiler *compiler)
Expand Down Expand Up @@ -945,6 +951,11 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil
SLJIT_ASSERT(!const_ || const_->addr >= word_count);

if (next_min_addr == next_label_size) {
if (label->u.index >= SLJIT_LABEL_ALIGNED) {
code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
*code_ptr = buf_ptr[-1];
}

label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
label->size = (sljit_uw)(code_ptr - code);
label = label->next;
Expand Down Expand Up @@ -1011,6 +1022,9 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil
} while (buf);

if (label && label->size == word_count) {
if (label->u.index >= SLJIT_LABEL_ALIGNED)
code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);

label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
label->size = (sljit_uw)(code_ptr - code);
label = label->next;
Expand Down Expand Up @@ -3156,6 +3170,36 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi
return label;
}

SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler, sljit_s32 log2_align)
{
sljit_uw mask, i;
struct sljit_extended_label *ext_label;

CHECK_ERROR_PTR();
CHECK_PTR(check_sljit_emit_aligned_label(compiler, log2_align));

if (log2_align <= SLJIT_LABEL_ALIGN_4) {
SLJIT_SKIP_CHECKS(compiler);
return sljit_emit_label(compiler);
}

#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
if (SLJIT_UNLIKELY(compiler->cpool_diff != CONST_POOL_EMPTY))
PTR_FAIL_IF(push_cpool(compiler));
#endif /* SLJIT_CONFIG_ARM_V6 */

/* The used space is filled with NOPs. */
mask = ((sljit_uw)1 << log2_align) - sizeof(sljit_ins);

for (i = (mask >> 2); i != 0; i--)
PTR_FAIL_IF(push_inst(compiler, NOP));

ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
PTR_FAIL_IF(!ext_label);
set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
return &ext_label->label;
}

SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
{
struct sljit_jump *jump;
Expand Down
39 changes: 39 additions & 0 deletions sljit_src/sljitNativeARM_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,12 @@ static SLJIT_INLINE void generate_jump_or_mov_addr(struct sljit_jump *jump, slji
buf_ptr[3] = MOVK | ((sljit_ins)((sljit_uw)addr >> 48) << 5) | (3 << 21) | dst;
}

static SLJIT_INLINE sljit_ins *process_extended_label(sljit_ins *code_ptr, struct sljit_extended_label *ext_label)
{
SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
return (sljit_ins*)((sljit_uw)code_ptr & ~(ext_label->data));
}

static void reduce_code_size(struct sljit_compiler *compiler)
{
struct sljit_label *label;
Expand Down Expand Up @@ -529,6 +535,11 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil

/* These structures are ordered by their address. */
if (next_min_addr == next_label_size) {
if (label->u.index >= SLJIT_LABEL_ALIGNED) {
code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
*code_ptr = buf_ptr[-1];
}

label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
label->size = (sljit_uw)(code_ptr - code);
label = label->next;
Expand Down Expand Up @@ -565,6 +576,9 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil
} while (buf);

if (label && label->size == word_count) {
if (label->u.index >= SLJIT_LABEL_ALIGNED)
code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);

label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
label->size = (sljit_uw)(code_ptr - code);
label = label->next;
Expand Down Expand Up @@ -2376,6 +2390,31 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi
return label;
}

SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler, sljit_s32 log2_align)
{
sljit_uw mask, i;
struct sljit_extended_label *ext_label;

CHECK_ERROR_PTR();
CHECK_PTR(check_sljit_emit_aligned_label(compiler, log2_align));

if (log2_align <= SLJIT_LABEL_ALIGN_4) {
SLJIT_SKIP_CHECKS(compiler);
return sljit_emit_label(compiler);
}

/* The used space is filled with NOPs. */
mask = ((sljit_uw)1 << log2_align) - sizeof(sljit_ins);

for (i = (mask >> 2); i != 0; i--)
PTR_FAIL_IF(push_inst(compiler, NOP));

ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
PTR_FAIL_IF(!ext_label);
set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
return &ext_label->label;
}

SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
{
struct sljit_jump *jump;
Expand Down
39 changes: 39 additions & 0 deletions sljit_src/sljitNativeARM_T2_32.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,12 @@ static SLJIT_INLINE void generate_jump_or_mov_addr(struct sljit_jump *jump, slji
jump_inst[1] |= 0xd000;
}

static SLJIT_INLINE sljit_u16 *process_extended_label(sljit_u16 *code_ptr, struct sljit_extended_label *ext_label)
{
SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
return (sljit_u16*)((sljit_uw)code_ptr & ~(ext_label->data));
}

static void reduce_code_size(struct sljit_compiler *compiler)
{
struct sljit_label *label;
Expand Down Expand Up @@ -645,6 +651,11 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil

/* These structures are ordered by their address. */
if (next_min_addr == next_label_size) {
if (label->u.index >= SLJIT_LABEL_ALIGNED) {
code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
*code_ptr = buf_ptr[-1];
}

label->u.addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1;
label->size = (sljit_uw)(code_ptr - code);
label = label->next;
Expand Down Expand Up @@ -682,6 +693,9 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil
} while (buf);

if (label && label->size == half_count) {
if (label->u.index >= SLJIT_LABEL_ALIGNED)
code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);

label->u.addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1;
label->size = (sljit_uw)(code_ptr - code);
label = label->next;
Expand Down Expand Up @@ -2717,6 +2731,31 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi
return label;
}

SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler, sljit_s32 log2_align)
{
sljit_uw mask, i;
struct sljit_extended_label *ext_label;

CHECK_ERROR_PTR();
CHECK_PTR(check_sljit_emit_aligned_label(compiler, log2_align));

if (log2_align <= SLJIT_LABEL_ALIGN_2) {
SLJIT_SKIP_CHECKS(compiler);
return sljit_emit_label(compiler);
}

/* The used space is filled with NOPs. */
mask = ((sljit_uw)1 << log2_align) - sizeof(sljit_u16);

for (i = (mask >> 1); i != 0; i--)
PTR_FAIL_IF(push_inst16(compiler, NOP));

ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
PTR_FAIL_IF(!ext_label);
set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
return &ext_label->label;
}

SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
{
struct sljit_jump *jump;
Expand Down
Loading

0 comments on commit e2f735b

Please sign in to comment.