File 1120-jit-Implement-code-generation-for-ARM.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1120-jit-Implement-code-generation-for-ARM.patch of Package erlang (Revision f55041561f125384b28886aa21e55614)

Currently displaying revision f55041561f125384b28886aa21e55614 , Show latest

From c634176d9f3473605e0c0a60c1917dbf80fbb8cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Fri, 12 Mar 2021 12:46:33 +0100
Subject: [PATCH 20/20] jit: Implement code generation for ARM
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On April 9, 2021, we reached the prompt milestone (with -oldshell
option).

On April 15, 2021, we could run the entire compiler test suite without
crashing.  The test suites for tools and debugger also ran
successfully.

On April 16, 2021, we could run the entire test suites for emulator
and stdlib.

Co-authored-by: John Högberg <john@erlang.org>
Co-authored-by: Sverker Eriksson <sverker@erlang.org>
---
 erts/configure                                |    5 +-
 erts/configure.in                             |    5 +-
 erts/emulator/Makefile.in                     |   16 +-
 erts/emulator/beam/beam_bp.c                  |  109 +-
 erts/emulator/beam/beam_bp.h                  |    3 +-
 erts/emulator/beam/erl_bif_trace.c            |   14 +-
 erts/emulator/beam/erl_nfunc_sched.h          |    6 +-
 erts/emulator/beam/erl_nif.c                  |   54 +-
 erts/emulator/beam/erl_process.h              |    4 +-
 erts/emulator/beam/erl_vm.h                   |    6 +-
 erts/emulator/beam/jit/arm/beam_asm.hpp       | 1555 ++++++++++++++++
 .../emulator/beam/jit/arm/beam_asm_global.cpp |  360 ++++
 .../emulator/beam/jit/arm/beam_asm_module.cpp |  805 ++++++++
 erts/emulator/beam/jit/arm/generators.tab     |  497 +++++
 erts/emulator/beam/jit/arm/instr_arith.cpp    | 1177 ++++++++++++
 erts/emulator/beam/jit/arm/instr_bif.cpp      |  991 ++++++++++
 erts/emulator/beam/jit/arm/instr_bs.cpp       | 1417 ++++++++++++++
 erts/emulator/beam/jit/arm/instr_call.cpp     |  214 +++
 erts/emulator/beam/jit/arm/instr_common.cpp   | 1623 +++++++++++++++++
 erts/emulator/beam/jit/arm/instr_float.cpp    |  162 ++
 erts/emulator/beam/jit/arm/instr_fun.cpp      |  412 +++++
 .../beam/jit/arm/instr_guard_bifs.cpp         |  727 ++++++++
 erts/emulator/beam/jit/arm/instr_map.cpp      |  298 +++
 erts/emulator/beam/jit/arm/instr_msg.cpp      |  385 ++++
 erts/emulator/beam/jit/arm/instr_select.cpp   |  479 +++++
 erts/emulator/beam/jit/arm/instr_trace.cpp    |  217 +++
 erts/emulator/beam/jit/arm/ops.tab            | 1367 ++++++++++++++
 erts/emulator/beam/jit/arm/predicates.tab     |  107 ++
 erts/emulator/beam/jit/arm/process_main.cpp   |  314 ++++
 erts/emulator/beam/jit/beam_asm.h             |  129 +-
 erts/emulator/beam/jit/beam_jit_common.cpp    |    2 +
 erts/emulator/beam/jit/beam_jit_common.hpp    |   10 +
 erts/emulator/beam/jit/beam_jit_main.cpp      |   25 +
 erts/emulator/beam/predicates.tab             |    6 +-
 erts/emulator/internal_doc/BeamAsm.md         |   15 +-
 erts/emulator/utils/beam_makeops              |   31 +-
 36 files changed, 13446 insertions(+), 101 deletions(-)
 create mode 100644 erts/emulator/beam/jit/arm/beam_asm.hpp
 create mode 100644 erts/emulator/beam/jit/arm/beam_asm_global.cpp
 create mode 100644 erts/emulator/beam/jit/arm/beam_asm_module.cpp
 create mode 100644 erts/emulator/beam/jit/arm/generators.tab
 create mode 100644 erts/emulator/beam/jit/arm/instr_arith.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_bif.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_bs.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_call.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_common.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_float.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_fun.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_guard_bifs.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_map.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_msg.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_select.cpp
 create mode 100644 erts/emulator/beam/jit/arm/instr_trace.cpp
 create mode 100644 erts/emulator/beam/jit/arm/ops.tab
 create mode 100644 erts/emulator/beam/jit/arm/predicates.tab
 create mode 100644 erts/emulator/beam/jit/arm/process_main.cpp

diff --git a/erts/configure b/erts/configure
index b96b41af77..8a2d1cb952 100755
--- a/erts/configure
+++ b/erts/configure
@@ -21697,9 +21697,12 @@ if test ${enable_jit} != no; then
         amd64)
            JIT_ARCH=x86
            ;;
+        arm64)
+           JIT_ARCH=arm
+           ;;
         *)
            if test ${enable_jit} = yes; then
-             as_fn_error $? "JIT only works on x86 64-bit" "$LINENO" 5
+             as_fn_error $? "JIT only works on x86 64-bit and ARM 64-bit" "$LINENO" 5
            else
              enable_jit=no
              { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: JIT disabled due to lack to support on $ARCH-$OPSYS" >&5
diff --git a/erts/configure.in b/erts/configure.in
index ee62383db9..2584ffb229 100644
--- a/erts/configure.in
+++ b/erts/configure.in
@@ -2881,9 +2881,12 @@ if test ${enable_jit} != no; then
         amd64)
            JIT_ARCH=x86
            ;;
+        arm64)
+           JIT_ARCH=arm
+           ;;
         *)
            if test ${enable_jit} = yes; then
-             AC_MSG_ERROR([JIT only works on x86 64-bit])
+             AC_MSG_ERROR([JIT only works on x86 64-bit and ARM 64-bit])
            else
              enable_jit=no
              AC_MSG_WARN([JIT disabled due to lack to support on $ARCH-$OPSYS])
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index e4481ce669..ef70d345ec 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -696,7 +696,7 @@ ASMJIT_H = $(wildcard asmjit/*.h) $(wildcard asmjit/core/*.h) $(wildcard asmjit/
 ASMJIT_TTF_H = $(foreach HEADER,$(ASMJIT_H),$(TTF_DIR)/$(HEADER))
 
 ifeq ($(FLAVOR),jit)
-GENERATE+=$(ASMJIT_TTF_H)
+GENERATE+=$(ASMJIT_TTF_H) $(TTF_DIR)/asmjit/asmjit.hpp
 endif
 
 # Preloaded code.
@@ -1013,6 +1013,20 @@ JIT_OBJS = \
     $(OBJDIR)/process_main.o
 
 ifeq ($(JIT_ARCH), x86)
+JIT_OBJS += \
+	$(OBJDIR)/instr_arith.o \
+	$(OBJDIR)/instr_bs.o \
+	$(OBJDIR)/instr_bif.o \
+	$(OBJDIR)/instr_call.o \
+	$(OBJDIR)/instr_common.o \
+	$(OBJDIR)/instr_float.o \
+	$(OBJDIR)/instr_fun.o \
+	$(OBJDIR)/instr_guard_bifs.o \
+	$(OBJDIR)/instr_map.o \
+	$(OBJDIR)/instr_msg.o \
+	$(OBJDIR)/instr_select.o \
+	$(OBJDIR)/instr_trace.o
+else
 JIT_OBJS += \
 	$(OBJDIR)/instr_arith.o \
 	$(OBJDIR)/instr_bs.o \
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 9e8d815ad4..6e926ca11a 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -123,7 +123,8 @@ static void bp_meta_unref(BpMetaTracer *bmt);
 static void bp_count_unref(BpCount *bcp);
 static void bp_time_unref(BpDataTime *bdt);
 static void consolidate_bp_data(Module *modp, ErtsCodeInfo *ci, int local);
-static void uninstall_breakpoint(ErtsCodeInfo *ci);
+static void uninstall_breakpoint(ErtsCodeInfo *ci_rw,
+                                 const ErtsCodeInfo *ci_exec);
 
 /* bp_hash */
 #define BP_TIME_ADD(pi0, pi1)                       \
@@ -188,31 +189,33 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
 	}
 
 	for (fi = 0; fi < num_functions; fi++) {
-            ErtsCodeInfo* ci;
+            const ErtsCodeInfo* ci_exec;
+            ErtsCodeInfo* ci_rw;
             void *w_ptr;
 
-            w_ptr = erts_writable_code_ptr(&module[current]->curr,
-                                           code_hdr->functions[fi]);
-            ci = (ErtsCodeInfo*)w_ptr;
+            ci_exec = code_hdr->functions[fi];
+            w_ptr = erts_writable_code_ptr(&module[current]->curr, ci_exec);
+            ci_rw = (ErtsCodeInfo*)w_ptr;
 
 #ifndef BEAMASM
-	    ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI));
+	    ASSERT(BeamIsOpCode(ci_rw->op, op_i_func_info_IaaI));
 #endif
             switch (specified) {
             case 3:
-                if (ci->mfa.arity != mfa->arity)
+                if (ci_rw->mfa.arity != mfa->arity)
                     continue;
             case 2:
-                if (ci->mfa.function != mfa->function)
+                if (ci_rw->mfa.function != mfa->function)
                     continue;
             case 1:
-                if (ci->mfa.module != mfa->module)
+                if (ci_rw->mfa.module != mfa->module)
                     continue;
             case 0:
                 break;
             }
             /* Store match */
-            f->matching[i].ci = ci;
+            f->matching[i].ci_exec = ci_exec;
+            f->matching[i].ci_rw = ci_rw;
             f->matching[i].mod = module[current];
             i++;
 	}
@@ -260,12 +263,13 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified)
             ASSERT(BeamIsOpCode(ep->trampoline.common.op, op_i_generic_breakpoint));
         }
 
-	f->matching[ne].ci = &ep->info;
-	f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix);
-
-	ne++;
+        f->matching[ne].ci_exec = &ep->info;
+        f->matching[ne].ci_rw = &ep->info;
+        f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix);
 
+        ne++;
     }
+
     f->matched = ne;
 }
 
@@ -288,14 +292,14 @@ erts_consolidate_bp_data(BpFunctions* f, int local)
     ERTS_LC_ASSERT(erts_has_code_write_permission());
 
     for (i = 0; i < n; i++) {
-	consolidate_bp_data(fs[i].mod, fs[i].ci, local);
+        consolidate_bp_data(fs[i].mod, fs[i].ci_rw, local);
     }
 }
 
 static void
-consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
+consolidate_bp_data(Module* modp, ErtsCodeInfo *ci_rw, int local)
 {
-    GenericBp* g = ci->u.gen_bp;
+    GenericBp* g = ci_rw->u.gen_bp;
     GenericBpData* src;
     GenericBpData* dst;
     Uint flags;
@@ -343,14 +347,15 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local)
 	    ASSERT(modp->curr.num_traced_exports >= 0);
 #if !defined(BEAMASM) && defined(DEBUG)
             {
-                BeamInstr instr = *(const BeamInstr*)erts_codeinfo_to_code(ci);
+                BeamInstr instr = *(const BeamInstr*)erts_codeinfo_to_code(ci_rw);
                 ASSERT(!BeamIsOpCode(instr, op_i_generic_breakpoint));
             }
 #endif
-	}
-	ci->u.gen_bp = NULL;
-	Free(g);
-	return;
+        }
+
+        ci_rw->u.gen_bp = NULL;
+        Free(g);
+        return;
     }
 
     /*
@@ -396,11 +401,12 @@ erts_install_breakpoints(BpFunctions* f)
     Uint n = f->matched;
 
     for (i = 0; i < n; i++) {
-	ErtsCodeInfo* ci = f->matching[i].ci;
-	GenericBp* g = ci->u.gen_bp;
-        Module* modp = f->matching[i].mod;
+        const ErtsCodeInfo *ci_exec = f->matching[i].ci_exec;
+        ErtsCodeInfo *ci_rw = f->matching[i].ci_rw;
+        GenericBp *g = ci_rw->u.gen_bp;
+        Module *modp = f->matching[i].mod;
 #ifdef BEAMASM
-        if ((erts_asm_bp_get_flags(ci) & ERTS_ASM_BP_FLAG_BP) == 0 && g) {
+        if ((erts_asm_bp_get_flags(ci_exec) & ERTS_ASM_BP_FLAG_BP) == 0 && g) {
 	    /*
 	     * The breakpoint must be disabled in the active data
 	     * (it will enabled later by switching bp indices),
@@ -409,12 +415,16 @@ erts_install_breakpoints(BpFunctions* f)
 	    ASSERT(g->data[erts_active_bp_ix()].flags == 0);
 	    ASSERT(g->data[erts_staging_bp_ix()].flags != 0);
 
-            erts_asm_bp_set_flag(ci, ERTS_ASM_BP_FLAG_BP);
+            erts_asm_bp_set_flag(ci_rw, ci_exec, ERTS_ASM_BP_FLAG_BP);
             modp->curr.num_breakpoints++;
         }
 #else
-        BeamInstr volatile *pc = (BeamInstr*)erts_codeinfo_to_code(ci);
+        BeamInstr volatile *pc = (BeamInstr*)erts_codeinfo_to_code(ci_rw);
         BeamInstr instr = *pc;
+
+        ASSERT(ci_exec == ci_rw);
+        (void)ci_exec;
+
 	if (!BeamIsOpCode(instr, op_i_generic_breakpoint) && g) {
             BeamInstr br = BeamOpCodeAddr(op_i_generic_breakpoint);
 
@@ -452,30 +462,33 @@ erts_uninstall_breakpoints(BpFunctions* f)
     Uint n = f->matched;
 
     for (i = 0; i < n; i++) {
-        uninstall_breakpoint(f->matching[i].ci);
+        uninstall_breakpoint(f->matching[i].ci_rw, f->matching[i].ci_exec);
     }
 }
 
 #ifdef BEAMASM
 static void
-uninstall_breakpoint(ErtsCodeInfo *ci)
+uninstall_breakpoint(ErtsCodeInfo *ci_rw, const ErtsCodeInfo *ci_exec)
 {
-    if (erts_asm_bp_get_flags(ci) & ERTS_ASM_BP_FLAG_BP) {
-        GenericBp* g = ci->u.gen_bp;
+    if (erts_asm_bp_get_flags(ci_rw) & ERTS_ASM_BP_FLAG_BP) {
+        GenericBp* g = ci_rw->u.gen_bp;
 
         if (g->data[erts_active_bp_ix()].flags == 0) {
-            erts_asm_bp_unset_flag(ci, ERTS_ASM_BP_FLAG_BP);
+            erts_asm_bp_unset_flag(ci_rw, ci_exec, ERTS_ASM_BP_FLAG_BP);
         }
     }
 }
 #else
 static void
-uninstall_breakpoint(ErtsCodeInfo *ci)
+uninstall_breakpoint(ErtsCodeInfo *ci_rw, const ErtsCodeInfo *ci_exec)
 {
-    BeamInstr *pc = (BeamInstr*)erts_codeinfo_to_code(ci);
+    BeamInstr *pc = (BeamInstr*)erts_codeinfo_to_code(ci_rw);
+
+    ASSERT(ci_rw == ci_exec);
+    (void)ci_exec;
 
     if (BeamIsOpCode(*pc, op_i_generic_breakpoint)) {
-        GenericBp* g = ci->u.gen_bp;
+        GenericBp* g = ci_rw->u.gen_bp;
 
         if (g->data[erts_active_bp_ix()].flags == 0) {
             /*
@@ -574,7 +587,7 @@ erts_clear_time_break(BpFunctions* f)
 {
     clear_break(f, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE);
 }
-
+ 
 void
 erts_clear_all_breaks(BpFunctions* f)
 {
@@ -605,16 +618,18 @@ erts_clear_module_break(Module *modp) {
     erts_commit_staged_bp();
 
     for (i = 0; i < n; ++i) {
-        ErtsCodeInfo* ci;
+        const ErtsCodeInfo *ci_exec;
+        ErtsCodeInfo *ci_rw;
         void *w_ptr;
 
-        w_ptr = erts_writable_code_ptr(&modp->curr,
-                                       code_hdr->functions[i]);
-        ci = (ErtsCodeInfo*)w_ptr;
+        ci_exec = code_hdr->functions[i];
+        w_ptr = erts_writable_code_ptr(&modp->curr, ci_exec);
+        ci_rw = (ErtsCodeInfo*)w_ptr;
+
+        uninstall_breakpoint(ci_rw, ci_exec);
+        consolidate_bp_data(modp, ci_rw, 1);
 
-        uninstall_breakpoint(ci);
-        consolidate_bp_data(modp, ci, 1);
-        ASSERT(ci->u.gen_bp == NULL);
+        ASSERT(ci_rw->u.gen_bp == NULL);
     }
 
     return n;
@@ -1367,9 +1382,9 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags,
 
     n = f->matched;
     for (i = 0; i < n; i++) {
-	set_function_break(f->matching[i].ci,
+        set_function_break(f->matching[i].ci_rw,
                            match_spec, break_flags,
-			   count_op, tracer);
+                           count_op, tracer);
     }
 }
 
@@ -1503,7 +1518,7 @@ clear_break(BpFunctions* f, Uint break_flags)
 
     n = f->matched;
     for (i = 0; i < n; i++) {
-	clear_function_break(f->matching[i].ci, break_flags);
+        clear_function_break(f->matching[i].ci_exec, break_flags);
     }
 }
 
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 9941533e2d..4ae068b9b0 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -91,7 +91,8 @@ enum erts_break_op{
 typedef Uint32 ErtsBpIndex;
 
 typedef struct {
-    ErtsCodeInfo *ci;
+    const ErtsCodeInfo *ci_exec;
+    ErtsCodeInfo *ci_rw;
     Module* mod;
 } BpFunction;
 
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 8024687c5e..fc30c7457c 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1442,10 +1442,10 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
     n = finish_bp.e.matched;
 
     for (i = 0; i < n; i++) {
-        ErtsCodeInfo *ci = fp[i].ci;
+        ErtsCodeInfo *ci_rw = fp[i].ci_rw;
         Export* ep;
 
-        ep = ErtsContainerStruct(ci, Export, info);
+        ep = ErtsContainerStruct(ci_rw, Export, info);
 
         if (ep->bif_number != -1) {
             ep->is_bif_traced = !!on;
@@ -1462,7 +1462,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
                 ep->trampoline.trace.address = (BeamInstr) ep->addresses[code_ix];
             }
 
-            erts_set_export_trace(ci, match_prog_set, 0);
+            erts_set_export_trace(ci_rw, match_prog_set, 0);
 
             if (!erts_is_export_trampoline_active(ep, code_ix)) {
                 ep->trampoline.common.op = BeamOpCodeAddr(op_i_generic_breakpoint);
@@ -1474,7 +1474,7 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified,
 	     * Turn off global tracing, either explicitly or implicitly
 	     * before turning on breakpoint tracing.
 	     */
-	    erts_clear_export_trace(ci, 0);
+            erts_clear_export_trace(ci_rw, 0);
             if (BeamIsOpCode(ep->trampoline.common.op, op_i_generic_breakpoint)) {
                 ep->trampoline.common.op = BeamOpCodeAddr(op_trace_jump_W);
 	    }
@@ -1669,7 +1669,7 @@ install_exp_breakpoints(BpFunctions* f)
     Uint i;
 
     for (i = 0; i < ne; i++) {
-        Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
+        Export* ep = ErtsContainerStruct(fp[i].ci_rw, Export, info);
         erts_activate_export_trampoline(ep, code_ix);
     }
 }
@@ -1683,7 +1683,7 @@ uninstall_exp_breakpoints(BpFunctions* f)
     Uint i;
 
     for (i = 0; i < ne; i++) {
-        Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
+        Export* ep = ErtsContainerStruct(fp[i].ci_rw, Export, info);
 
         if (erts_is_export_trampoline_active(ep, code_ix)) {
             ASSERT(BeamIsOpCode(ep->trampoline.common.op, op_trace_jump_W));
@@ -1701,7 +1701,7 @@ clean_export_entries(BpFunctions* f)
     Uint i;
 
     for (i = 0; i < ne; i++) {
-        Export* ep = ErtsContainerStruct(fp[i].ci, Export, info);
+        Export* ep = ErtsContainerStruct(fp[i].ci_rw, Export, info);
 
         if (erts_is_export_trampoline_active(ep, code_ix)) {
             continue;
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index ba4933a5df..bd787f5f77 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -40,8 +40,12 @@ typedef struct {
     struct {
         ErtsCodeInfo info;
 #ifdef BEAMASM
-        // Code used by tracing/nif load
+        /* Code used by tracing/nif load. */
+#  ifdef __aarch64__
+        BeamInstr trace[2];
+#  else
         BeamInstr trace[1];
+#  endif
 #endif
         BeamInstr call_op; /* call_bif || call_nif */
         BeamInstr dfunc;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 0d7127c5ee..176cf99d97 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -4723,13 +4723,16 @@ static void patch_call_nif_early(ErlNifEntry* entry,
     {
         ErlNifFunc* f = &entry->funcs[i];
         const ErtsCodeInfo * const * ci_pp;
-        ErtsCodeInfo* ci;
+        const ErtsCodeInfo *ci_exec;
+        ErtsCodeInfo *ci_rw;
         Eterm f_atom;
 
         erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1);
 
         ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity);
-        ci = erts_writable_code_ptr(this_mi, *ci_pp);
+        ci_exec = *ci_pp;
+
+        ci_rw = erts_writable_code_ptr(this_mi, ci_exec);
 
 #ifndef BEAMASM
         {
@@ -4737,14 +4740,14 @@ static void patch_call_nif_early(ErlNifEntry* entry,
             BeamInstr volatile *code_ptr;
 
             /* `ci` is writable. */
-            code_ptr = (BeamInstr*)erts_codeinfo_to_code(ci);
+            code_ptr = (BeamInstr*)erts_codeinfo_to_code(ci_rw);
 
-            if (ci->u.gen_bp) {
+            if (ci_rw->u.gen_bp) {
                 /*
                  * Function traced, patch the original instruction word
                  * Code write permission protects against racing breakpoint writes.
                  */
-                GenericBp* g = ci->u.gen_bp;
+                GenericBp* g = ci_rw->u.gen_bp;
                 g->orig_instr = BeamSetCodeAddr(g->orig_instr, call_nif_early);
                 if (BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint))
                     continue;
@@ -4756,7 +4759,7 @@ static void patch_call_nif_early(ErlNifEntry* entry,
         }
 #else
         /* See beam_asm.h for details on how the nif load trampoline works */
-        erts_asm_bp_set_flag(ci, ERTS_ASM_BP_FLAG_CALL_NIF_EARLY);
+        erts_asm_bp_set_flag(ci_rw, ci_exec, ERTS_ASM_BP_FLAG_CALL_NIF_EARLY);
 #endif
     }
 }
@@ -4787,15 +4790,25 @@ static void load_nif_1st_finisher(void* vlib)
 
     if (fin) {
         for (i=0; i < lib->entry.num_of_funcs; i++) {
-            ErtsCodeInfo *ci = fin->beam_stubv[i].code_info_rw;
+            const ErtsCodeInfo *ci_exec = fin->beam_stubv[i].code_info_exec;
+            ErtsCodeInfo *ci_rw = fin->beam_stubv[i].code_info_rw;
 
 #ifdef BEAMASM
-            char *code_ptr = (char*)erts_codeinfo_to_code(ci);
-            sys_memcpy(&code_ptr[BEAM_ASM_FUNC_PROLOGUE_SIZE],
+            const char *code_exec = (char*)erts_codeinfo_to_code(ci_exec);
+            char *code_rw = (char*)erts_codeinfo_to_code(ci_rw);
+
+            size_t cpy_sz = sizeof(fin->beam_stubv[0].code.call_nif);
+
+            sys_memcpy(&code_rw[BEAM_ASM_FUNC_PROLOGUE_SIZE],
                        fin->beam_stubv[i].code.call_nif,
-                       sizeof(fin->beam_stubv[0].code.call_nif));
+                       cpy_sz);
+
+            beamasm_flush_icache(&code_exec[BEAM_ASM_FUNC_PROLOGUE_SIZE],
+                                 cpy_sz);
 #else
-            BeamInstr *code_ptr = (BeamInstr*)erts_codeinfo_to_code(ci);
+            BeamInstr *code_ptr = (BeamInstr*)erts_codeinfo_to_code(ci_rw);
+
+            (void)ci_exec;
 
             /* called function */
             code_ptr[1] = fin->beam_stubv[i].code.call_nif[1];
@@ -4844,29 +4857,36 @@ static void load_nif_2nd_finisher(void* vlib)
     fin = lib->finish;
     if (fin) {
         for (i=0; i < lib->entry.num_of_funcs; i++) {
-            ErtsCodeInfo *ci = fin->beam_stubv[i].code_info_rw;
+            const ErtsCodeInfo *ci_exec = fin->beam_stubv[i].code_info_exec;
+            ErtsCodeInfo *ci_rw = fin->beam_stubv[i].code_info_rw;
 
 #ifndef BEAMASM
             BeamInstr volatile *code_ptr;
 
-            code_ptr = (BeamInstr*)erts_codeinfo_to_code(ci);
+            ASSERT(ci_exec == ci_rw);
+            (void)ci_exec;
 
-            if (ci->u.gen_bp) {
+            code_ptr = (BeamInstr*)erts_codeinfo_to_code(ci_rw);
+
+            if (ci_rw->u.gen_bp) {
                 /*
                  * Function traced, patch the original instruction word
                  */
-                GenericBp* g = ci->u.gen_bp;
+                GenericBp* g = ci_rw->u.gen_bp;
                 ASSERT(BeamIsOpCode(g->orig_instr, op_call_nif_early));
                 g->orig_instr = BeamOpCodeAddr(op_call_nif_WWW);
-                if (BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint))
+
+                if (BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)) {
                     continue;
+                }
             }
 
             ASSERT(BeamIsOpCode(code_ptr[0], op_call_nif_early));
             code_ptr[0] = BeamOpCodeAddr(op_call_nif_WWW);
 #else
             /* See beam_asm.h for details on how the nif load trampoline works */
-            erts_asm_bp_unset_flag(ci, ERTS_ASM_BP_FLAG_CALL_NIF_EARLY);
+            erts_asm_bp_unset_flag(ci_rw, ci_exec,
+                                   ERTS_ASM_BP_FLAG_CALL_NIF_EARLY);
 #endif
         }
     }
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 419234686f..588ee0133d 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -661,11 +661,11 @@ typedef struct ErtsSchedulerRegisters_ {
     ErtsCodePtr start_time_i;
     UWord start_time;
 
-#if !defined(NATIVE_ERLANG_STACK) && defined(JIT_HARD_DEBUG)
+#if (!defined(NATIVE_ERLANG_STACK) || defined(__aarch64__)) && defined(JIT_HARD_DEBUG)
     /* Holds the initial thread stack pointer. Used to ensure that everything
      * that is pushed to the stack is also popped. */
     UWord *initial_sp;
-#elif defined(NATIVE_ERLANG_STACK) && defined(DEBUG)
+#elif defined(NATIVE_ERLANG_STACK) && defined(DEBUG) && !defined(__aarch64__)
     /* Raw pointers to the start and end of the stack. Used to test bounds
      * without clobbering any registers. */
     UWord *runtime_stack_start;
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 99946a4189..7c2fe55796 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -70,7 +70,7 @@ typedef enum {
 } ErtsFrameLayout;
 
 ERTS_GLB_INLINE
-const int erts_cp_size(void);
+int erts_cp_size(void);
 
 #if defined(BEAMASM) && defined(ERLANG_FRAME_POINTERS)
 extern ErtsFrameLayout ERTS_WRITE_UNLIKELY(erts_frame_layout);
@@ -100,6 +100,8 @@ extern ErtsFrameLayout ERTS_WRITE_UNLIKELY(erts_frame_layout);
 
 #if defined(BEAMASM) && defined(NATIVE_ERLANG_STACK)
 #define S_REDZONE (CP_SIZE * 3)
+#elif defined(BEAMASM) && defined(__aarch64__)
+#define S_REDZONE (CP_SIZE * 3)
 #elif defined(DEBUG)
 /* Ensure that a redzone won't cause problems in the interpreter. */
 #define S_REDZONE CP_SIZE
@@ -317,7 +319,7 @@ extern void** beam_ops;
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 ERTS_GLB_INLINE
-const int erts_cp_size()
+int erts_cp_size()
 {
     if (erts_frame_layout == ERTS_FRAME_LAYOUT_RA) {
         return 1;
diff --git a/erts/emulator/beam/jit/arm/beam_asm.hpp b/erts/emulator/beam/jit/arm/beam_asm.hpp
new file mode 100644
index 0000000000..c5f9c06358
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/beam_asm.hpp
@@ -0,0 +1,1555 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <queue>
+#include <map>
+#include <functional>
+
+#ifndef ASMJIT_ASMJIT_H_INCLUDED
+#    include <asmjit/asmjit.hpp>
+#endif
+
+#include <asmjit/a64.h>
+
+extern "C"
+{
+#ifdef HAVE_CONFIG_H
+#    include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "beam_catches.h"
+
+#include "beam_asm.h"
+}
+
+#include "beam_jit_common.hpp"
+
+/* Is it safe to STP or LDP `Struct->Field1` and `Struct->Field2`? */
+#define ERTS_CT_ASSERT_FIELD_PAIR(Struct, Field1, Field2)                      \
+    static_assert(std::is_standard_layout<Struct>::value &&                    \
+                  (offsetof(Struct, Field2) - offsetof(Struct, Field1) ==      \
+                   sizeof(((Struct *)nullptr)->Field1)) &&                     \
+                  (sizeof(((Struct *)nullptr)->Field1) ==                      \
+                   sizeof(((Struct *)nullptr)->Field2)))
+
+using namespace asmjit;
+
+class BeamAssembler : public ErrorHandler {
+protected:
+    /* Holds code and relocation information. */
+    CodeHolder code;
+
+    a64::Assembler a;
+
+    FileLogger logger;
+
+    Section *rodata = nullptr;
+
+    /* * * * * * * * * */
+
+    /* Points at x_reg_array inside an ErtsSchedulerRegisters struct, allowing
+     * the aux_regs field to be addressed with an 8-bit displacement. */
+    const arm::Gp scheduler_registers = a64::x19;
+
+    const arm::Gp E = a64::x20;
+    const arm::Gp c_p = a64::x21;
+    const arm::Gp FCALLS = a64::x22;
+    const arm::Gp HTOP = a64::x23;
+
+    /* Local copy of the active code index.
+     *
+     * This is set to ERTS_SAVE_CALLS_CODE_IX when save_calls is active, which
+     * routes us to a common handler routine that calls save_calls before
+     * jumping to the actual code. */
+    const arm::Gp active_code_ix = a64::x24;
+
+    /* X registers */
+#if defined(DEBUG)
+    /*
+     * To ensure that we thoroughly test flushing of caller-save X
+     * registers, define more caller-save X registers in a DEBUG
+     * build.
+     */
+#    define ERTS_HIGHEST_CALLEE_SAVE_XREG 2
+#    define ERTS_HIGHEST_CALLER_SAVE_XREG 5
+    const arm::Gp XREG0 = a64::x25;
+    const arm::Gp XREG1 = a64::x26;
+    const arm::Gp XREG2 = a64::x27;
+
+    /*
+     * Caller-save X registers. Must be flushed before calling C
+     * code.
+     */
+    const arm::Gp XREG3 = a64::x15;
+    const arm::Gp XREG4 = a64::x16;
+    const arm::Gp XREG5 = a64::x17;
+#else
+#    define ERTS_HIGHEST_CALLEE_SAVE_XREG 3
+#    define ERTS_HIGHEST_CALLER_SAVE_XREG 5
+    const arm::Gp XREG0 = a64::x25;
+    const arm::Gp XREG1 = a64::x26;
+    const arm::Gp XREG2 = a64::x27;
+    const arm::Gp XREG3 = a64::x28;
+
+    /*
+     * Caller-save X registers. Must be flushed before calling C
+     * code.
+     */
+    const arm::Gp XREG4 = a64::x15;
+    const arm::Gp XREG5 = a64::x16;
+#endif
+
+#define ERTS_LOWEST_CALLEE_SAVE_XREG (0)
+#define ERTS_LOWEST_CALLER_SAVE_XREG (ERTS_HIGHEST_CALLEE_SAVE_XREG + 1)
+
+    static const int num_register_backed_xregs = 6;
+    const arm::Gp register_backed_xregs[num_register_backed_xregs] =
+            {XREG0, XREG1, XREG2, XREG3, XREG4, XREG5};
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+    const arm::Mem erts_msacc_cache = getSchedulerRegRef(
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.erts_msacc_cache));
+#endif
+
+    /* * * * * * * * * */
+    const arm::Gp ZERO = a64::xzr;
+
+    /*
+     * All of the following registers are caller-save.
+     *
+     * Note that ARG1 is also the register for the return value.
+     */
+    const arm::Gp ARG1 = a64::x0;
+    const arm::Gp ARG2 = a64::x1;
+    const arm::Gp ARG3 = a64::x2;
+    const arm::Gp ARG4 = a64::x3;
+    const arm::Gp ARG5 = a64::x4;
+    const arm::Gp ARG6 = a64::x5;
+    const arm::Gp ARG7 = a64::x6;
+    const arm::Gp ARG8 = a64::x7;
+
+    const arm::Gp TMP1 = a64::x8;
+    const arm::Gp TMP2 = a64::x9;
+    const arm::Gp TMP3 = a64::x10;
+    const arm::Gp TMP4 = a64::x11;
+    const arm::Gp TMP5 = a64::x12;
+    const arm::Gp TMP6 = a64::x13;
+
+    /*
+     * Assume that SUPER_TMP will be destroyed by any helper function.
+     */
+    const arm::Gp SUPER_TMP = a64::x14;
+
+    /* Callee-saved floating-point registers.
+     *
+     * Note that only the bottom 64 bits of these (128-bit) registers are
+     * callee-save, so we cannot pack two floats into each register. */
+    const arm::VecD FREG0 = a64::d8;
+    const arm::VecD FREG1 = a64::d9;
+    const arm::VecD FREG2 = a64::d10;
+    const arm::VecD FREG3 = a64::d11;
+    const arm::VecD FREG4 = a64::d12;
+    const arm::VecD FREG5 = a64::d13;
+    const arm::VecD FREG6 = a64::d14;
+    const arm::VecD FREG7 = a64::d15;
+    static const int num_register_backed_fregs = 8;
+    const arm::VecD register_backed_fregs[num_register_backed_fregs] =
+            {FREG0, FREG1, FREG2, FREG3, FREG4, FREG5, FREG6, FREG7};
+
+    /*
+     * Note that x18 is reserved on Apple platforms and must not be used.
+     */
+
+    const arm::Mem TMP_MEM1q = getSchedulerRegRef(
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.TMP_MEM[0]));
+    const arm::Mem TMP_MEM2q = getSchedulerRegRef(
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.TMP_MEM[1]));
+    const arm::Mem TMP_MEM3q = getSchedulerRegRef(
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.TMP_MEM[2]));
+    const arm::Mem TMP_MEM4q = getSchedulerRegRef(
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.TMP_MEM[3]));
+    const arm::Mem TMP_MEM5q = getSchedulerRegRef(
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.TMP_MEM[4]));
+
+    /* Fill registers with undefined contents to find bugs faster.
+     * A boxed value is most likely to cause noticeable trouble.
+     */
+    static const Uint64 bad_boxed_ptr = 0xcafebad0000002UL;
+
+    /* Number of highest element displacement for stp/ldp. */
+    static const int MAX_LDP_STP_DISPLACEMENT = 0x3F;
+
+public:
+    static bool hasCpuFeature(uint32_t featureId);
+
+    BeamAssembler();
+    BeamAssembler(const std::string &log);
+
+    ~BeamAssembler();
+
+    void *getBaseAddress();
+    size_t getOffset();
+
+protected:
+    void _codegen(JitAllocator *allocator,
+                  const void **executable_ptr,
+                  void **writable_ptr);
+
+    void *getCode(Label label);
+    byte *getCode(char *labelName);
+
+    void handleError(Error err, const char *message, BaseEmitter *origin);
+
+#ifdef JIT_HARD_DEBUG
+    constexpr arm::Mem getInitialSPRef() const {
+        int base = offsetof(ErtsSchedulerRegisters, initial_sp);
+
+        return getSchedulerRegRef(base);
+    }
+#endif
+
+    constexpr arm::Mem getSchedulerRegRef(int offset) const {
+        ASSERT((offset & (sizeof(Eterm) - 1)) == 0);
+        return arm::Mem(scheduler_registers, offset);
+    }
+
+    constexpr arm::Mem getFRef(int index, size_t size = sizeof(UWord)) const {
+        int base = offsetof(ErtsSchedulerRegisters, f_reg_array.d);
+        int offset = index * sizeof(FloatDef);
+
+        ASSERT(0 <= index && index <= 1023);
+        return getSchedulerRegRef(base + offset);
+    }
+
+    constexpr arm::Mem getXRef(int index) const {
+        int base = offsetof(ErtsSchedulerRegisters, x_reg_array.d);
+        int offset = index * sizeof(Eterm);
+
+        ASSERT(0 <= index && index < ERTS_X_REGS_ALLOCATED);
+        return getSchedulerRegRef(base + offset);
+    }
+
+    constexpr arm::Mem getYRef(int index) const {
+        ASSERT(0 <= index && index <= 1023);
+
+        return arm::Mem(E, index * sizeof(Eterm));
+    }
+
+    constexpr arm::Mem getCARRef(arm::Gp Src) const {
+        return arm::Mem(Src, -TAG_PRIMARY_LIST);
+    }
+
+    constexpr arm::Mem getCDRRef(arm::Gp Src,
+                                 size_t size = sizeof(UWord)) const {
+        return arm::Mem(Src, -TAG_PRIMARY_LIST + sizeof(Eterm));
+    }
+
+    /* Loads the X register array into `to`. Remember to sync the registers in
+     * `emit_enter_runtime`. */
+    void load_x_reg_array(arm::Gp to) {
+        int offset = offsetof(ErtsSchedulerRegisters, x_reg_array.d);
+
+        lea(to, getSchedulerRegRef(offset));
+    }
+
+    void load_erl_bits_state(arm::Gp to) {
+        int offset =
+                offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state);
+
+        lea(to, getSchedulerRegRef(offset));
+    }
+
+    void emit_assert_redzone_unused() {
+#ifdef JIT_HARD_DEBUG
+        const int REDZONE_BYTES = S_REDZONE * sizeof(Eterm);
+        Label next = a.newLabel();
+
+        a.sub(SUPER_TMP, E, imm(REDZONE_BYTES));
+        a.cmp(HTOP, SUPER_TMP);
+
+        a.cond_ls().b(next);
+        a.udf(0xbeef);
+
+        a.bind(next);
+#endif
+    }
+
+    /*
+     * Calls an Erlang function.
+     */
+    template<typename Any>
+    void erlang_call(Any Target) {
+        emit_assert_redzone_unused();
+        aligned_call(Target);
+    }
+
+    void branch(arm::Mem target) {
+        a.ldr(SUPER_TMP, target);
+        a.br(SUPER_TMP);
+    }
+
+    template<typename FuncPtr>
+    void aligned_call(FuncPtr(*target)) {
+        mov_imm(SUPER_TMP, target);
+        a.blr(SUPER_TMP);
+    }
+
+    void aligned_call(Label target) {
+        a.bl(target);
+    }
+
+    void aligned_call(arm::Gp target) {
+        a.blr(target);
+    }
+
+    /* Calls the given address. In DEBUG builds, make
+     * sure that the CP is aligned. */
+    template<typename OperandType>
+    void aligned_call(OperandType target) {
+        ERTS_CT_ASSERT(_CPMASK == 3);
+        ASSERT(is_CP(a.offset()));
+        a.ldr(TMP1, target);
+        a.blr(TMP1);
+    }
+
+    void runtime_call(arm::Gp func, unsigned args) {
+        ASSERT(args < 5);
+        a.blr(func);
+    }
+
+    template<typename T>
+    struct function_arity;
+    template<typename T, typename... Args>
+    struct function_arity<T(Args...)>
+            : std::integral_constant<int, sizeof...(Args)> {};
+
+    template<int expected_arity, typename T>
+    void runtime_call(T(*func)) {
+        static_assert(expected_arity == function_arity<T>());
+
+        a.mov(TMP1, func);
+        a.blr(TMP1);
+    }
+
+    /* Explicitly position-independent absolute jump, for use in fragments that
+     * need to be memcpy'd for performance reasons (e.g. NIF stubs) */
+    template<typename T>
+    void pic_jmp(T(*addr)) {
+        a.mov(SUPER_TMP, addr);
+        a.br(SUPER_TMP);
+    }
+
+    constexpr arm::Mem getArgRef(const ArgVal &val) const {
+        switch (val.getType()) {
+        case ArgVal::FReg:
+            return getFRef(val.getValue());
+        case ArgVal::XReg:
+            return getXRef(val.getValue());
+        case ArgVal::YReg:
+            return getYRef(val.getValue());
+        default:
+            ERTS_ASSERT(!"NYI");
+            return arm::Mem();
+        }
+    }
+
+    /* Returns the current code address for the export entry in `Src`
+     *
+     * Export tracing, save_calls, etc is implemented by shared fragments that
+     * assume that the export entry is in ARG1, so we have to copy it over if it
+     * isn't already. */
+    arm::Mem emit_setup_export_call(const arm::Gp &Src) {
+        return emit_setup_export_call(Src, active_code_ix);
+    }
+
+    arm::Mem emit_setup_export_call(const arm::Gp &Src,
+                                    const arm::Gp &CodeIndex) {
+        if (ARG1 != Src) {
+            a.mov(ARG1, Src);
+        }
+        ERTS_CT_ASSERT(offsetof(Export, addresses) == 0);
+        return arm::Mem(ARG1, CodeIndex, arm::lsl(3));
+    }
+
+    enum Update : int {
+        eStack = (1 << 0),
+        eHeap = (1 << 1),
+        eReductions = (1 << 2),
+        eCodeIndex = (1 << 3),
+        eXRegs = (1 << 4)
+    };
+
+    void emit_enter_erlang_frame() {
+        a.str(a64::x30, arm::Mem(E, -8).pre());
+    }
+
+    void emit_leave_erlang_frame() {
+        a.ldr(a64::x30, arm::Mem(E).post(8));
+    }
+
+    void emit_enter_runtime_frame() {
+        a.stp(a64::x29, a64::x30, arm::Mem(a64::sp, -16).pre());
+        a.mov(a64::x29, a64::sp);
+    }
+
+    void emit_leave_runtime_frame() {
+        a.mov(a64::sp, a64::x29);
+        a.ldp(a64::x29, a64::x30, arm::Mem(a64::sp).post(16));
+    }
+
+    /* We keep the first six X registers in machine registers. Some of those
+     * registers are callee-saved and some are caller-saved.
+     *
+     * We ignore the ones above `live` to reduce the save/restore traffic on
+     * these registers. It's enough for this figure to be at least as high as
+     * the number of actually live registers, and we default to all six
+     * registers when we don't know the exact number.
+     *
+     * Furthermore, we only save the callee-save registers when told to sync
+     * sync all registers with the `Update::eXRegs` flag, as this is very
+     * rarely needed. */
+
+    template<int Spec = 0>
+    void emit_enter_runtime(int live = num_register_backed_xregs) {
+        ERTS_CT_ASSERT((Spec & (Update::eReductions | Update::eStack |
+                                Update::eHeap | Update::eXRegs)) == Spec);
+
+        if ((Spec & Update::eStack) && (Spec & Update::eHeap)) {
+            /* Store HTOP and E in one go. */
+            ERTS_CT_ASSERT_FIELD_PAIR(Process, htop, stop);
+            a.stp(HTOP, E, arm::Mem(c_p, offsetof(Process, htop)));
+        } else if (Spec & Update::eStack) {
+            a.str(E, arm::Mem(c_p, offsetof(Process, stop)));
+        } else if (Spec & Update::eHeap) {
+            a.str(HTOP, arm::Mem(c_p, offsetof(Process, htop)));
+        }
+
+        if (Spec & Update::eReductions) {
+            a.str(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls)));
+        }
+
+        /* Save register-backed X registers to the X register array when
+         * needed. The backing registers must NOT be used afterwards.
+         *
+         * In a DEBUG build, the backing X registers will be overwritten with
+         * garbage values. */
+        if (live > 0) {
+            int num_to_save = MIN(live, ERTS_HIGHEST_CALLER_SAVE_XREG + 1);
+            int i;
+
+            if (Spec & Update::eXRegs) {
+                i = ERTS_LOWEST_CALLEE_SAVE_XREG;
+            } else {
+                /* If we don't need to sync the X register array, then we can
+                 * get away with saving only the fragile X registers. */
+                i = ERTS_LOWEST_CALLER_SAVE_XREG;
+            }
+
+#ifdef DEBUG
+            /* Destroy the saved X registers to find bugs sooner.*/
+            if (i < num_to_save) {
+                mov_imm(SUPER_TMP, bad_boxed_ptr + 0x20 + (Spec << 8));
+            }
+#endif
+
+            while (i < num_to_save - 1) {
+                a.stp(register_backed_xregs[i + 0],
+                      register_backed_xregs[i + 1],
+                      getXRef(i));
+
+#ifdef DEBUG
+                a.mov(register_backed_xregs[i + 0], SUPER_TMP);
+                a.mov(register_backed_xregs[i + 1], SUPER_TMP);
+#endif
+
+                i += 2;
+            }
+
+            if (i < num_to_save) {
+                a.str(register_backed_xregs[i], getXRef(i));
+
+#ifdef DEBUG
+                a.mov(register_backed_xregs[i], SUPER_TMP);
+#endif
+            }
+        }
+    }
+
+    template<int Spec = 0>
+    void emit_leave_runtime(int live = num_register_backed_xregs) {
+        ERTS_CT_ASSERT(
+                (Spec & (Update::eReductions | Update::eStack | Update::eHeap |
+                         Update::eXRegs | Update::eCodeIndex)) == Spec);
+
+        if ((Spec & Update::eStack) && (Spec & Update::eHeap)) {
+            /* Load HTOP and E in one go. */
+            ERTS_CT_ASSERT_FIELD_PAIR(Process, htop, stop);
+            a.ldp(HTOP, E, arm::Mem(c_p, offsetof(Process, htop)));
+        } else if (Spec & Update::eHeap) {
+            a.ldr(HTOP, arm::Mem(c_p, offsetof(Process, htop)));
+        } else if (Spec & Update::eStack) {
+            a.ldr(E, arm::Mem(c_p, offsetof(Process, stop)));
+        }
+
+        if (Spec & Update::eReductions) {
+            a.ldr(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls)));
+        }
+
+        if (Spec & Update::eCodeIndex) {
+            /* Updates the local copy of the active code index, retaining
+             * save_calls if active. */
+            mov_imm(SUPER_TMP, &the_active_code_index);
+            a.ldr(SUPER_TMP.w(), arm::Mem(SUPER_TMP));
+            a.cmp(active_code_ix, imm(ERTS_SAVE_CALLS_CODE_IX));
+            a.csel(active_code_ix, active_code_ix, SUPER_TMP, arm::Cond::kEQ);
+        }
+
+        /* Restore register-backed X registers from the X register array when
+         * needed. The register array must NOT be used afterwards.
+         *
+         * In a DEBUG build, the register array will be overwritten with
+         * garbage values. */
+        if (live > 0) {
+            int num_to_restore = MIN(live, ERTS_HIGHEST_CALLER_SAVE_XREG + 1);
+            int i;
+
+            if (Spec & Update::eXRegs) {
+                i = ERTS_LOWEST_CALLEE_SAVE_XREG;
+            } else {
+                /* If we don't need to sync the X register array, then we can
+                 * get away with loading only the fragile X registers. */
+                i = ERTS_LOWEST_CALLER_SAVE_XREG;
+            }
+
+#ifdef DEBUG
+            /* Destroy the restored X registers to find bugs sooner.*/
+            if (i < num_to_restore) {
+                mov_imm(SUPER_TMP, bad_boxed_ptr + 0x80 + (Spec << 8));
+            }
+#endif
+
+            while (i < num_to_restore - 1) {
+                a.ldp(register_backed_xregs[i],
+                      register_backed_xregs[i + 1],
+                      getXRef(i));
+
+#ifdef DEBUG
+                a.stp(SUPER_TMP, SUPER_TMP, getXRef(i));
+#endif
+
+                i += 2;
+            }
+
+            if (i < num_to_restore) {
+                a.ldr(register_backed_xregs[i], getXRef(i));
+
+#ifdef DEBUG
+                a.str(SUPER_TMP, getXRef(i));
+#endif
+            }
+        }
+    }
+
+    void emit_is_boxed(Label Fail, arm::Gp Src) {
+        const int bitNumber = 0;
+        ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED ==
+                       (1 << bitNumber));
+        a.tbnz(Src, imm(bitNumber), Fail);
+    }
+
+    arm::Gp emit_ptr_val(arm::Gp Dst, arm::Gp Src) {
+#if !defined(TAG_LITERAL_PTR)
+        return Src;
+#else
+        /* We intentionally skip TAG_PTR_MASK__ here, as we want to use
+         * plain `emit_boxed_val` when we know the argument can't be a literal,
+         * such as in bit-syntax matching.
+         *
+         * This comes at very little cost as `emit_boxed_val` nearly always has
+         * a displacement. */
+        a.and_(Dst, Src, imm(~TAG_LITERAL_PTR));
+        return Dst;
+#endif
+    }
+
+    void emit_untag_ptr(arm::Gp Dst, arm::Gp Src) {
+        a.and_(Dst, Src, imm(~TAG_PTR_MASK__));
+    }
+
+    constexpr arm::Mem emit_boxed_val(arm::Gp Src, int32_t bytes = 0) const {
+        ASSERT(bytes % sizeof(Eterm) == 0);
+        return arm::Mem(Src, bytes - TAG_PRIMARY_BOXED);
+    }
+
+    void emit_branch_if_not_value(arm::Gp reg, Label lbl) {
+        emit_branch_if_eq(reg, THE_NON_VALUE, lbl);
+    }
+
+    void emit_branch_if_value(arm::Gp reg, Label lbl) {
+        emit_branch_if_ne(reg, THE_NON_VALUE, lbl);
+    }
+
+    void emit_branch_if_eq(arm::Gp reg, Uint value, Label lbl) {
+        if (value == 0) {
+            a.cbz(reg, lbl);
+        } else {
+            a.cmp(reg, imm(value));
+            a.cond_eq().b(lbl);
+        }
+    }
+
+    void emit_branch_if_ne(arm::Gp reg, Uint value, Label lbl) {
+        if (value == 0) {
+            a.cbnz(reg, lbl);
+        } else {
+            a.cmp(reg, imm(value));
+            a.cond_ne().b(lbl);
+        }
+    }
+
+    /* Set the Z flag if Reg1 and Reg2 are both immediates. */
+    void emit_are_both_immediate(arm::Gp Reg1, arm::Gp Reg2) {
+        ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
+        a.and_(SUPER_TMP, Reg1, Reg2);
+        a.and_(SUPER_TMP, SUPER_TMP, imm(_TAG_PRIMARY_MASK));
+        a.cmp(SUPER_TMP, imm(TAG_PRIMARY_IMMED1));
+    }
+
+    /* Set the Z flag if Reg1 and Reg2 are definitely not equal based
+     * on their tags alone. (They may still be equal if both are
+     * immediates and all other bits are equal too.) */
+    void emit_is_unequal_based_on_tags(arm::Gp Reg1, arm::Gp Reg2) {
+        ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
+        ERTS_CT_ASSERT((TAG_PRIMARY_LIST | TAG_PRIMARY_BOXED) ==
+                       TAG_PRIMARY_IMMED1);
+        a.orr(SUPER_TMP, Reg1, Reg2);
+        a.and_(SUPER_TMP, SUPER_TMP, imm(_TAG_PRIMARY_MASK));
+
+        /*
+         * SUPER_TMP will be now be TAG_PRIMARY_IMMED1 if either
+         * one or both registers are immediates, or if one register
+         * is a list and the other a boxed.
+         */
+        a.cmp(SUPER_TMP, imm(TAG_PRIMARY_IMMED1));
+    }
+
+    template<typename T>
+    void mov_imm(arm::Gp to, T value) {
+        static_assert(std::is_integral<T>::value || std::is_pointer<T>::value);
+        if (value) {
+            a.mov(to, imm(value));
+        } else {
+            a.mov(to, ZERO);
+        }
+    }
+
+    void mov_imm(arm::Gp to, std::nullptr_t value) {
+        (void)value;
+        mov_imm(to, 0);
+    }
+
+    void sub(arm::Gp to, arm::Gp src, int64_t val) {
+        if (val < 0) {
+            add(to, src, -val);
+        } else if (val < (1 << 24)) {
+            if (val & 0xFFF) {
+                a.sub(to, src, val & 0xFFF);
+            }
+
+            if (val & 0xFFF000) {
+                a.sub(to, src, val & 0xFFF000);
+            }
+        } else {
+            mov_imm(SUPER_TMP, val);
+            a.sub(to, src, SUPER_TMP);
+        }
+    }
+
+    void add(arm::Gp to, arm::Gp src, int64_t val) {
+        if (val < 0) {
+            sub(to, src, -val);
+        } else if (val < (1 << 24)) {
+            if (val & 0xFFF) {
+                a.add(to, src, val & 0xFFF);
+            }
+
+            if (val & 0xFFF000) {
+                a.add(to, to, val & 0xFFF000);
+            }
+        } else {
+            mov_imm(SUPER_TMP, val);
+            a.add(to, src, SUPER_TMP);
+        }
+    }
+
+    void ldur(arm::Gp reg, arm::Mem mem) {
+        safe_9bit_imm(a64::Inst::kIdLdur, reg, mem);
+    }
+
+    void stur(arm::Gp reg, arm::Mem mem) {
+        safe_9bit_imm(a64::Inst::kIdStur, reg, mem);
+    }
+
+    void safe_9bit_imm(uint32_t instId, arm::Gp reg, arm::Mem mem) {
+        int64_t offset = mem.offset();
+
+        ASSERT(mem.hasBaseReg() && !mem.hasIndex());
+
+        if (Support::isInt9(offset)) {
+            a.emit(instId, reg, mem);
+        } else {
+            lea(SUPER_TMP, mem);
+            a.emit(instId, reg, arm::Mem(SUPER_TMP));
+        }
+    }
+
+    /*
+     * ARM has no LEA instruction. Implement our own to enable us
+     * to use helpers based on getSchedulerRegRef() that return an
+     * arm::Mem class.
+     */
+    void lea(arm::Gp to, arm::Mem mem) {
+        int64_t offset = mem.offset();
+
+        ASSERT(mem.hasBaseReg() && !mem.hasIndex());
+
+        if (offset == 0) {
+            a.mov(to, arm::GpX(mem.baseId()));
+        } else {
+            add(to, arm::GpX(mem.baseId()), offset);
+        }
+    }
+
+public:
+    void embed_rodata(const char *labelName, const char *buff, size_t size);
+    void embed_bss(const char *labelName, size_t size);
+    void embed_zeros(size_t size);
+
+    void setLogger(std::string log);
+    void setLogger(FILE *log);
+
+    template<typename... Ts>
+    void comment(const char *format, Ts... args) {
+        if (logger.file()) {
+            char buff[1024];
+            erts_snprintf(buff, sizeof(buff), format, args...);
+            a.commentf("# %s", buff);
+        }
+    }
+
+    struct AsmRange {
+        ErtsCodePtr start;
+        ErtsCodePtr stop;
+        std::string name;
+
+        /* Not used yet */
+        std::string file;
+        unsigned line;
+    };
+
+    void update_gdb_jit_info(std::string modulename,
+                             std::vector<AsmRange> &functions);
+
+    void embed(void *data, uint32_t size) {
+        a.embed((char *)data, size);
+    }
+};
+
+class BeamGlobalAssembler : public BeamAssembler {
+    typedef void (BeamGlobalAssembler::*emitFptr)(void);
+    typedef void (*fptr)(void);
+
+    /* Please keep this in alphabetical order. */
+#define BEAM_GLOBAL_FUNCS(_)                                                   \
+    _(apply_fun_shared)                                                        \
+    _(arith_compare_shared)                                                    \
+    _(bif_nif_epilogue)                                                        \
+    _(bif_export_trap)                                                         \
+    _(bif_bit_size_body)                                                       \
+    _(bif_bit_size_guard)                                                      \
+    _(bif_byte_size_body)                                                      \
+    _(bif_byte_size_guard)                                                     \
+    _(bif_element_body_shared)                                                 \
+    _(bif_element_guard_shared)                                                \
+    _(bif_is_eq_exact_shared)                                                  \
+    _(bif_is_ne_exact_shared)                                                  \
+    _(bif_tuple_size_body)                                                     \
+    _(bif_tuple_size_guard)                                                    \
+    _(bs_add_guard_shared)                                                     \
+    _(bs_add_body_shared)                                                      \
+    _(bs_get_tail_shared)                                                      \
+    _(bs_size_check_shared)                                                    \
+    _(call_bif_shared)                                                         \
+    _(call_light_bif_shared)                                                   \
+    _(call_nif_yield_helper)                                                   \
+    _(catch_end_shared)                                                        \
+    _(call_nif_early)                                                          \
+    _(call_nif_shared)                                                         \
+    _(check_float_error)                                                       \
+    _(debug_bp)                                                                \
+    _(dispatch_bif)                                                            \
+    _(dispatch_nif)                                                            \
+    _(dispatch_return)                                                         \
+    _(dispatch_save_calls)                                                     \
+    _(export_trampoline)                                                       \
+    _(fconv_shared)                                                            \
+    _(handle_and_error)                                                        \
+    _(handle_call_fun_error)                                                   \
+    _(handle_hd_error)                                                         \
+    _(handle_map_size_error)                                                   \
+    _(handle_not_error)                                                        \
+    _(handle_or_error)                                                         \
+    _(handle_tl_error)                                                         \
+    _(garbage_collect)                                                         \
+    _(generic_bp_global)                                                       \
+    _(generic_bp_local)                                                        \
+    _(i_band_body_shared)                                                      \
+    _(i_bnot_body_shared)                                                      \
+    _(i_bnot_guard_shared)                                                     \
+    _(i_bor_body_shared)                                                       \
+    _(i_bif_body_shared)                                                       \
+    _(i_bif_guard_shared)                                                      \
+    _(i_bsr_body_shared)                                                       \
+    _(i_bsl_body_shared)                                                       \
+    _(i_func_info_shared)                                                      \
+    _(i_length_guard_shared)                                                   \
+    _(i_length_body_shared)                                                    \
+    _(i_loop_rec_shared)                                                       \
+    _(i_new_small_map_lit_shared)                                              \
+    _(i_test_yield_shared)                                                     \
+    _(i_bxor_body_shared)                                                      \
+    _(int_div_rem_body_shared)                                                 \
+    _(int_div_rem_guard_shared)                                                \
+    _(minus_body_shared)                                                       \
+    _(new_map_shared)                                                          \
+    _(update_map_assoc_shared)                                                 \
+    _(unloaded_fun)                                                            \
+    _(plus_body_shared)                                                        \
+    _(process_exit)                                                            \
+    _(process_main)                                                            \
+    _(raise_exception)                                                         \
+    _(raise_exception_shared)                                                  \
+    _(times_body_shared)                                                       \
+    _(times_guard_shared)                                                      \
+    _(unary_minus_body_shared)                                                 \
+    _(update_map_exact_guard_shared)                                           \
+    _(update_map_exact_body_shared)
+
+/* Labels exported from within process_main */
+#define PROCESS_MAIN_LABELS(_)                                                 \
+    _(context_switch)                                                          \
+    _(context_switch_simplified)                                               \
+    _(do_schedule)
+
+#define DECL_ENUM(NAME) NAME,
+
+    enum GlobalLabels : uint32_t {
+        BEAM_GLOBAL_FUNCS(DECL_ENUM) PROCESS_MAIN_LABELS(DECL_ENUM)
+    };
+#undef DECL_ENUM
+
+    static const std::map<GlobalLabels, emitFptr> emitPtrs;
+    static const std::map<GlobalLabels, std::string> labelNames;
+    std::unordered_map<GlobalLabels, Label> labels;
+    std::unordered_map<GlobalLabels, fptr> ptrs;
+
+#define DECL_FUNC(NAME) void emit_##NAME(void);
+
+    BEAM_GLOBAL_FUNCS(DECL_FUNC);
+#undef DECL_FUNC
+
+    template<typename T>
+    void emit_bitwise_fallback_body(T(*func_ptr), const ErtsCodeMFA *mfa);
+
+    void emit_i_length_common(Label fail, int state_size);
+
+    void emit_raise_badarg(const ErtsCodeMFA *mfa);
+
+    void emit_bif_bit_size_helper(Label fail);
+    void emit_bif_byte_size_helper(Label fail);
+    void emit_bif_element_helper(Label fail);
+    void emit_bif_tuple_size_helper(Label fail);
+
+public:
+    BeamGlobalAssembler(JitAllocator *allocator);
+
+    void (*get(GlobalLabels lbl))(void) {
+        ASSERT(ptrs[lbl]);
+        return ptrs[lbl];
+    }
+
+#define GET_CODE(NAME)                                                         \
+    void (*get_##NAME(void))() {                                               \
+        return get(NAME);                                                      \
+    }
+
+    BEAM_GLOBAL_FUNCS(GET_CODE)
+    PROCESS_MAIN_LABELS(GET_CODE)
+#undef GET_CODE
+};
+
+class BeamModuleAssembler : public BeamAssembler {
+    typedef unsigned BeamLabel;
+
+    /* Map of BEAM label number to asmjit Label. These should not be used
+     * directly by most instructions because of displacement limits, use
+     * `resolve_beam_label` instead. */
+    typedef std::unordered_map<BeamLabel, Label> LabelMap;
+    LabelMap rawLabels;
+
+    struct patch {
+        Label where;
+        uint64_t val_offs;
+    };
+
+    struct patch_catch {
+        struct patch patch;
+        Label handler;
+    };
+    std::vector<struct patch_catch> catches;
+
+    /* Map of import entry to patch labels and mfa */
+    struct patch_import {
+        std::vector<struct patch> patches;
+        ErtsCodeMFA mfa;
+    };
+    typedef std::unordered_map<unsigned, struct patch_import> ImportMap;
+    ImportMap imports;
+
+    /* Map of fun entry to patch labels */
+    struct patch_lambda {
+        std::vector<struct patch> patches;
+        Label trampoline;
+    };
+    typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap;
+    LambdaMap lambdas;
+
+    /* Map of literals to patch labels */
+    struct patch_literal {
+        std::vector<struct patch> patches;
+    };
+    typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap;
+    LiteralMap literals;
+
+    /* All string patches */
+    std::vector<struct patch> strings;
+
+    /* All functions that have been seen so far */
+    std::vector<BeamLabel> functions;
+
+    BeamGlobalAssembler *ga;
+
+    /* Used by emit to populate the labelToMFA map */
+    Label currLabel;
+
+    /* Special shared fragments that must reside in each module. */
+    Label codeHeader;
+    Label genericBPTramp;
+
+    /* The module's on_load function, if any. */
+    Label on_load;
+
+    /* The end of the last function. Note that the dispatch table, constants,
+     * and veneers may follow. */
+    Label code_end;
+
+    Eterm mod;
+
+    /* Save the last PC for an error. */
+    size_t last_error_offset = 0;
+
+    static constexpr ptrdiff_t STUB_CHECK_INTERVAL = 4 << 10;
+    size_t last_stub_check_offset = 0;
+
+    enum Displacement : size_t {
+        /* Pessimistic estimate for helper functions, where we don't know the
+         * branch displacement or whether it will be used near label
+         * resolution.
+         *
+         * Note that we subtract the size of one instruction to handle
+         * backward displacements. */
+        dispUnknown = (32 << 10) - sizeof(Uint32) - STUB_CHECK_INTERVAL,
+
+        /* +- 32KB: `tbz`, `tbnz`, `ldr` of 8-byte literal. */
+        disp32K = (32 << 10) - sizeof(Uint32),
+
+        /* +- 1MB: `adr`, `b.cond`, `cb.cond` */
+        disp1MB = (1 << 20) - sizeof(Uint32),
+
+        /* +- 128MB: `b`, `blr` */
+        disp128MB = (128 << 20) - sizeof(Uint32),
+
+        dispMin = dispUnknown,
+        dispMax = disp128MB
+    };
+
+    static_assert(dispMin <= dispUnknown && dispMax >= disp128MB);
+    static_assert(STUB_CHECK_INTERVAL < dispMin / 2);
+
+    struct Veneer {
+        ssize_t latestOffset;
+        Label anchor;
+
+        Label target;
+
+        constexpr bool operator>(const Veneer &other) const {
+            return latestOffset > other.latestOffset;
+        }
+    };
+
+    struct Constant {
+        ssize_t latestOffset;
+        Label anchor;
+
+        ArgVal value;
+
+        constexpr bool operator>(const Constant &other) const {
+            return latestOffset > other.latestOffset;
+        }
+    };
+
+    /* ArgVal -> Constant
+     *
+     * `_pending_constants` points directly into this container, which is
+     * documented to be safe as long as we only insert elements. */
+    std::unordered_multimap<ArgVal, const Constant, ArgVal::Hash> _constants;
+
+    /* Label::id() -> Veneer
+     *
+     * `_pending_veneers` points directly into this container. */
+    std::unordered_multimap<uint32_t, const Veneer> _veneers;
+
+    template<typename T>
+    using PendingStubs =
+            std::priority_queue<std::reference_wrapper<const T>,
+                                std::deque<std::reference_wrapper<const T>>,
+                                std::greater<const T &>>;
+
+    /* All pending stubs, segregated by type and sorted by `latestOffset` in
+     * ascending order.
+     *
+     * We use separate queues to avoid interleaving them, as they have
+     * different sizes and alignment requirements. */
+    PendingStubs<Constant> _pending_constants;
+    PendingStubs<Veneer> _pending_veneers;
+
+    /* Maps code pointers to thunks that jump to them, letting us treat global
+     * fragments as if they were local. */
+    std::unordered_map<void (*)(), Label> _dispatchTable;
+
+public:
+    BeamModuleAssembler(BeamGlobalAssembler *ga,
+                        Eterm mod,
+                        unsigned num_labels);
+    BeamModuleAssembler(BeamGlobalAssembler *ga,
+                        Eterm mod,
+                        unsigned num_labels,
+                        unsigned num_functions);
+
+    bool emit(unsigned op, const Span<ArgVal> &args);
+
+    void codegen(JitAllocator *allocator,
+                 const void **executable_ptr,
+                 void **writable_ptr,
+                 const BeamCodeHeader *in_hdr,
+                 const BeamCodeHeader **out_exec_hdr,
+                 BeamCodeHeader **out_rw_hdr);
+
+    void codegen(JitAllocator *allocator,
+                 const void **executable_ptr,
+                 void **writable_ptr);
+
+    void codegen(char *buff, size_t len);
+
+    ErtsCodePtr getCode(unsigned label);
+    ErtsCodePtr getLambda(unsigned index);
+
+    void *getCode(Label label) {
+        return BeamAssembler::getCode(label);
+    }
+
+    byte *getCode(char *labelName) {
+        return BeamAssembler::getCode(labelName);
+    }
+
+    void embed_vararg_rodata(const Span<ArgVal> &args, a64::Gp reg);
+
+    unsigned getCodeSize() {
+        ASSERT(code.hasBaseAddress());
+        return code.codeSize();
+    }
+
+    void copyCodeHeader(BeamCodeHeader *hdr);
+    BeamCodeHeader *getCodeHeader(void);
+    const ErtsCodeInfo *getOnLoad(void);
+
+    unsigned patchCatches(char *rw_base);
+    void patchLambda(char *rw_base, unsigned index, BeamInstr I);
+    void patchLiteral(char *rw_base, unsigned index, Eterm lit);
+    void patchImport(char *rw_base, unsigned index, BeamInstr I);
+    void patchStrings(char *rw_base, const byte *string);
+
+protected:
+    /* Helpers */
+    void emit_gc_test(const ArgVal &Stack,
+                      const ArgVal &Heap,
+                      const ArgVal &Live);
+    void emit_gc_test_preserve(const ArgVal &Need,
+                               const ArgVal &Live,
+                               arm::Gp term);
+
+    arm::Mem emit_variable_apply(bool includeI);
+    arm::Mem emit_fixed_apply(const ArgVal &arity, bool includeI);
+
+    arm::Gp emit_call_fun();
+
+    arm::Gp emit_is_binary(const ArgVal &Fail,
+                           const ArgVal &Src,
+                           Label next,
+                           Label subbin);
+
+    void emit_get_list(const arm::Gp boxed_ptr,
+                       const ArgVal &Hd,
+                       const ArgVal &Tl);
+
+    void emit_div_rem(const ArgVal &Fail,
+                      const ArgVal &LHS,
+                      const ArgVal &RHS,
+                      const ErtsCodeMFA *error_mfa);
+
+    void emit_i_bif(const ArgVal &Fail, const ArgVal &Bif, const ArgVal &Dst);
+
+    void emit_error(int code);
+
+    int emit_bs_get_field_size(const ArgVal &Size,
+                               int unit,
+                               Label Fail,
+                               const arm::Gp &out);
+
+    void emit_bs_get_utf8(const ArgVal &Ctx, const ArgVal &Fail);
+    void emit_bs_get_utf16(const ArgVal &Ctx,
+                           const ArgVal &Fail,
+                           const ArgVal &Flags);
+
+    void emit_raise_exception();
+    void emit_raise_exception(const ErtsCodeMFA *exp);
+    void emit_raise_exception(Label I, const ErtsCodeMFA *exp);
+
+    void emit_validate(const ArgVal &arity);
+    void emit_bs_skip_bits(const ArgVal &Fail, const ArgVal &Ctx);
+
+    void emit_linear_search(arm::Gp val, Label fail, const Span<ArgVal> &args);
+
+    void emit_float_instr(uint32_t instId,
+                          const ArgVal &LHS,
+                          const ArgVal &RHS,
+                          const ArgVal &Dst);
+
+    void emit_validate_unicode(Label next, Label fail, arm::Gp value);
+
+    void emit_bif_is_eq_ne_exact_immed(const ArgVal &Src,
+                                       const ArgVal &Immed,
+                                       const ArgVal &Dst,
+                                       Eterm fail_value,
+                                       Eterm succ_value);
+
+    void emit_proc_lc_unrequire(void);
+    void emit_proc_lc_require(void);
+
+    void emit_nyi(const char *msg);
+    void emit_nyi(void);
+
+    /* Returns a vector of the untagged and rebased `args`. The adjusted
+     * `comparand` is stored in ARG1. */
+    const std::vector<ArgVal> emit_select_untag(const Span<ArgVal> &args,
+                                                a64::Gp comparand,
+                                                Label fail,
+                                                UWord base,
+                                                int shift);
+
+    void emit_binsearch_nodes(arm::Gp reg,
+                              size_t Left,
+                              size_t Right,
+                              Label fail,
+                              const Span<ArgVal> &args);
+
+    bool emit_optimized_three_way_select(arm::Gp reg,
+                                         Label fail,
+                                         const Span<ArgVal> &args);
+
+#ifdef DEBUG
+    void emit_tuple_assertion(const ArgVal &Src, arm::Gp tuple_reg);
+#endif
+
+#include "beamasm_protos.h"
+
+    /* Resolves a BEAM label.
+     *
+     * When the branch type is not `dispUnknown`, this must be used
+     * _IMMEDIATELY BEFORE_ the instruction that the label is used in. */
+    Label resolve_beam_label(const ArgVal &Label, enum Displacement disp);
+    Label resolve_label(Label target, enum Displacement disp);
+
+    /* Resolves a shared fragment, creating a trampoline that loads the
+     * appropriate address before jumping there.
+     *
+     * When the branch type is not `dispUnknown`, this must be used
+     * _IMMEDIATELY BEFORE_ the instruction that the label is used in. */
+    Label resolve_fragment(void (*fragment)(), enum Displacement disp);
+
+    /* Embeds a constant argument and returns its address. All kinds of
+     * constants are accepted, including labels and export entries.
+     *
+     * When the branch type is not `dispUnknown`, this must be used
+     * _IMMEDIATELY BEFORE_ the instruction that the label is used in. */
+    arm::Mem embed_constant(const ArgVal &value, enum Displacement disp);
+
+    /* Convenience wrapper for embedding raw pointers or immediates. */
+    template<typename T>
+    arm::Mem embed_constant(T data, enum Displacement disp) {
+        static_assert(std::is_integral<T>::value || std::is_pointer<T>::value);
+        return embed_constant(ArgVal(ArgVal::Word, (UWord)data), disp);
+    }
+
+    /* Binds a label and all related veneers that are within reach of it. */
+    void bind_veneer_target(const Label &target);
+
+    void emit_constant(const Constant &constant);
+    void emit_veneer(const Veneer &veneer);
+
+    /* Unconditionally emits all veneers and constants that are due within
+     * `range` bytes. */
+    void flush_pending_stubs(size_t range);
+
+    /* Emits pending veneers when appropriate. Must be called at least once
+     * every `STUB_CHECK_INTERVAL` bytes for veneers and constants to work. */
+    void check_pending_stubs();
+
+    /* Calls the given shared fragment, ensuring that the redzone is unused and
+     * that the return address forms a valid CP. */
+    template<typename Any>
+    void fragment_call(Any Target) {
+        emit_assert_redzone_unused();
+
+#if defined(JIT_HARD_DEBUG)
+        /* Verify that the stack has not grown. */
+        Label next = a.newLabel();
+        a.ldr(SUPER_TMP, getInitialSPRef());
+        a.cmp(a64::sp, SUPER_TMP);
+        a.cond_eq().b(next);
+        a.udf(0xdead);
+        a.bind(next);
+#endif
+
+        a.bl(resolve_fragment((void (*)())Target, disp128MB));
+    }
+
+    template<typename T>
+    struct function_arity;
+    template<typename T, typename... Args>
+    struct function_arity<T(Args...)>
+            : std::integral_constant<int, sizeof...(Args)> {};
+
+    template<int expected_arity, typename T>
+    void runtime_call(T(*func)) {
+        static_assert(expected_arity == function_arity<T>());
+
+        a.bl(resolve_fragment((void (*)())func, disp128MB));
+    }
+
+    bool isRegisterBacked(const ArgVal &arg) {
+        switch (arg.getType()) {
+        case ArgVal::XReg:
+            return arg.getValue() < num_register_backed_xregs;
+        case ArgVal::FReg:
+            return arg.getValue() < num_register_backed_fregs;
+        default:
+            return false;
+        }
+    }
+
+    template<typename RegType = arm::Gp>
+    struct Variable {
+        RegType reg;
+        arm::Mem mem;
+
+        Variable(RegType _r) : Variable(_r, arm::Mem()) {
+        }
+        Variable(RegType _r, arm::Mem _mem) : reg(_r), mem(_mem) {
+        }
+    };
+
+    Variable<arm::Gp> init_destination(const ArgVal &arg, arm::Gp tmp) {
+        /* We don't support storing into GpW since their maximum displacement
+         * is 16K, which means we have to check stubs far more often. */
+        ASSERT(tmp.isGpX());
+
+        if (isRegisterBacked(arg)) {
+            arm::Gp reg = register_backed_xregs[arg.getValue()];
+            return Variable(reg);
+        } else {
+            return Variable(tmp, getArgRef(arg));
+        }
+    }
+
+    Variable<arm::VecD> init_destination(const ArgVal &arg, arm::VecD tmp) {
+        if (isRegisterBacked(arg)) {
+            return Variable(register_backed_fregs[arg.getValue()]);
+        } else {
+            return Variable(tmp, getArgRef(arg));
+        }
+    }
+
+    Variable<arm::Gp> load_source(const ArgVal &arg, arm::Gp tmp) {
+        /* We don't support loading into GpW since their maximum displacement
+         * is 16K, which means we have to check stubs far more often. */
+        ASSERT(tmp.isGpX());
+
+        if (arg.isLiteral()) {
+            a.ldr(tmp, embed_constant(arg, disp32K));
+            return Variable(tmp);
+        } else if (isRegisterBacked(arg)) {
+            arm::Gp xreg = register_backed_xregs[arg.getValue()];
+            return Variable(xreg);
+        } else if (arg.isConstant()) {
+            if ((arg.isImmed() || arg.isWord()) &&
+                Support::isIntOrUInt32(arg.getValue())) {
+                mov_imm(tmp, arg.getValue());
+            } else {
+                a.ldr(tmp, embed_constant(arg, disp32K));
+            }
+
+            return Variable(tmp);
+        } else {
+            /* Register */
+            a.ldr(tmp, getArgRef(arg));
+            return Variable(tmp, getArgRef(arg));
+        }
+    }
+
+    auto load_sources(const ArgVal &Src1,
+                      arm::Gp tmp1,
+                      const ArgVal &Src2,
+                      arm::Gp tmp2) {
+        if (Src1.isRegister() && Src2.isRegister() && !isRegisterBacked(Src1) &&
+            !isRegisterBacked(Src2)) {
+            switch (ArgVal::register_relation(Src1, Src2)) {
+            case ArgVal::Relation::consecutive:
+                safe_ldp(tmp1, tmp2, Src1, Src2);
+                return std::make_pair(Variable(tmp1, getArgRef(Src1)),
+                                      Variable(tmp2, getArgRef(Src2)));
+            case ArgVal::Relation::reverse_consecutive:
+                safe_ldp(tmp2, tmp1, Src2, Src1);
+                return std::make_pair(Variable(tmp1, getArgRef(Src1)),
+                                      Variable(tmp2, getArgRef(Src2)));
+            case ArgVal::Relation::none:
+                break;
+            }
+        }
+
+        return std::make_pair(load_source(Src1, tmp1), load_source(Src2, tmp2));
+    }
+
+    Variable<arm::VecD> load_source(const ArgVal &arg, arm::VecD tmp) {
+        if (isRegisterBacked(arg)) {
+            return Variable<arm::VecD>(register_backed_fregs[arg.getValue()]);
+        } else {
+            /* Register */
+            a.ldr(tmp, getArgRef(arg));
+            return Variable<arm::VecD>(tmp);
+        }
+    }
+
+    void mov_var(const Variable<arm::Gp> &to, const Variable<arm::Gp> &from) {
+        mov_var(to.reg, from);
+    }
+
+    void mov_var(const Variable<arm::Gp> &to, arm::Gp from) {
+        if (to.reg != from) {
+            a.mov(to.reg, from);
+        }
+    }
+
+    void mov_var(arm::Gp to, const Variable<arm::Gp> &from) {
+        if (to != from.reg) {
+            a.mov(to, from.reg);
+        }
+    }
+
+    void flush_var(const Variable<arm::Gp> &to) {
+        if (to.mem.hasBase()) {
+            a.str(to.reg, to.mem);
+        }
+    }
+
+    void flush_var(const Variable<arm::VecD> &to) {
+        if (to.mem.hasBase()) {
+            a.str(to.reg, to.mem);
+        }
+    }
+
+    void flush_vars(const Variable<arm::Gp> &to1,
+                    const Variable<arm::Gp> &to2) {
+        const arm::Mem &mem1 = to1.mem;
+        const arm::Mem &mem2 = to2.mem;
+
+        if (mem1.hasBaseReg() && mem2.hasBaseReg() &&
+            mem1.baseId() == mem2.baseId()) {
+            if (mem1.offset() + 8 == mem2.offset()) {
+                safe_stp(to1.reg, to2.reg, mem1);
+                return;
+            } else if (mem1.offset() == mem2.offset() + 8) {
+                safe_stp(to2.reg, to1.reg, mem2);
+                return;
+            }
+        }
+
+        /* Not possible to optimize with stp. */
+        flush_var(to1);
+        flush_var(to2);
+    }
+
+    void mov_arg(const ArgVal &To, const ArgVal &From) {
+        if (isRegisterBacked(To)) {
+            auto to = init_destination(To, SUPER_TMP);
+            auto from = load_source(From, to.reg);
+            mov_var(to, from);
+            flush_var(to);
+        } else {
+            auto from = load_source(From, SUPER_TMP);
+            auto to = init_destination(To, from.reg);
+            mov_var(to, from);
+            flush_var(to);
+        }
+    }
+
+    void mov_arg(const ArgVal &To, arm::Mem From) {
+        auto to = init_destination(To, SUPER_TMP);
+        a.ldr(to.reg, From);
+        flush_var(to);
+    }
+
+    void mov_arg(arm::Mem To, const ArgVal &From) {
+        auto from = load_source(From, SUPER_TMP);
+        auto to = Variable(from.reg, To);
+        flush_var(to);
+    }
+
+    void mov_arg(arm::Gp to, const ArgVal &from) {
+        auto r = load_source(from, to);
+        if (r.reg != to) {
+            a.mov(to, r.reg);
+        }
+    }
+
+    void mov_arg(const ArgVal &to, arm::Gp from) {
+        auto r = init_destination(to, from);
+        if (r.reg != from) {
+            a.mov(r.reg, from);
+        }
+        flush_var(r);
+    }
+
+    void cmp_arg(arm::Gp gp, const ArgVal &val) {
+        if ((val.isImmed() || val.isWord())) {
+            if (Support::isUInt12((Sint)val.getValue())) {
+                a.cmp(gp, imm(val.getValue()));
+            } else if (Support::isUInt12(-(Sint)val.getValue())) {
+                a.adds(ZERO, gp, imm(-(Sint)val.getValue()));
+            } else {
+                mov_arg(SUPER_TMP, val);
+                a.cmp(gp, SUPER_TMP);
+            }
+        } else {
+            mov_arg(SUPER_TMP, val);
+            a.cmp(gp, SUPER_TMP);
+        }
+    }
+
+    void safe_stp(arm::Gp gp1,
+                  arm::Gp gp2,
+                  const ArgVal &Dst1,
+                  const ArgVal &Dst2) {
+        ASSERT(ArgVal::register_relation(Dst1, Dst2) ==
+               ArgVal::Relation::consecutive);
+        safe_stp(gp1, gp2, getArgRef(Dst1));
+    }
+
+    void safe_stp(arm::Gp gp1, arm::Gp gp2, arm::Mem mem) {
+        int64_t offset = mem.offset();
+
+        ASSERT(gp1.isGpX() && gp2.isGpX());
+
+        if (offset <= sizeof(Eterm) * MAX_LDP_STP_DISPLACEMENT) {
+            a.stp(gp1, gp2, mem);
+        } else {
+            a.str(gp1, mem);
+            a.str(gp2, mem.cloneAdjusted(sizeof(Eterm)));
+        }
+    }
+
+    void safe_ldr(arm::Gp gp, arm::Mem mem) {
+        int64_t offset = mem.offset();
+
+        ASSERT(mem.hasBaseReg() && !mem.hasIndex());
+        ASSERT(gp.isGpX());
+
+        if (offset < sizeof(Eterm) * 4096) {
+            a.ldr(gp, mem);
+        } else {
+            mov_imm(SUPER_TMP, offset);
+            a.add(SUPER_TMP, arm::GpX(mem.baseId()), SUPER_TMP);
+            a.ldr(gp, arm::Mem(SUPER_TMP));
+        }
+    }
+
+    void safe_ldp(arm::Gp gp1,
+                  arm::Gp gp2,
+                  const ArgVal &Src1,
+                  const ArgVal &Src2) {
+        ASSERT(ArgVal::register_relation(Src1, Src2) ==
+               ArgVal::Relation::consecutive);
+
+        safe_ldp(gp1, gp2, getArgRef(Src1));
+    }
+
+    void safe_ldp(arm::Gp gp1, arm::Gp gp2, arm::Mem mem) {
+        int64_t offset = mem.offset();
+
+        ASSERT(gp1.isGpX() && gp2.isGpX());
+        ASSERT(gp1 != gp2);
+
+        if (offset <= sizeof(Eterm) * MAX_LDP_STP_DISPLACEMENT) {
+            a.ldp(gp1, gp2, mem);
+        } else {
+            safe_ldr(gp1, mem);
+            safe_ldr(gp2, mem.cloneAdjusted(sizeof(Eterm)));
+        }
+    }
+};
+
+void beamasm_update_perf_info(std::string modulename,
+                              std::vector<BeamAssembler::AsmRange> &ranges);
diff --git a/erts/emulator/beam/jit/arm/beam_asm_global.cpp b/erts/emulator/beam/jit/arm/beam_asm_global.cpp
new file mode 100644
index 0000000000..723d2199a0
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/beam_asm_global.cpp
@@ -0,0 +1,360 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+using namespace asmjit;
+
+extern "C"
+{
+#include "bif.h"
+#include "beam_common.h"
+}
+
+#define STRINGIFY_(X) #X
+#define STRINGIFY(X) STRINGIFY_(X)
+
+#define DECL_EMIT(NAME) {NAME, &BeamGlobalAssembler::emit_##NAME},
+const std::map<BeamGlobalAssembler::GlobalLabels, BeamGlobalAssembler::emitFptr>
+        BeamGlobalAssembler::emitPtrs = {BEAM_GLOBAL_FUNCS(DECL_EMIT)};
+#undef DECL_EMIT
+
+#define DECL_LABEL_NAME(NAME) {NAME, STRINGIFY(NAME)},
+
+const std::map<BeamGlobalAssembler::GlobalLabels, std::string>
+        BeamGlobalAssembler::labelNames = {BEAM_GLOBAL_FUNCS(
+                DECL_LABEL_NAME) PROCESS_MAIN_LABELS(DECL_LABEL_NAME)};
+#undef DECL_LABEL_NAME
+
+BeamGlobalAssembler::BeamGlobalAssembler(JitAllocator *allocator)
+        : BeamAssembler("beam_asm_global") {
+    labels.reserve(emitPtrs.size());
+
+    /* These labels are defined up-front so global functions can refer to each
+     * other freely without any order dependencies. */
+    for (auto val : labelNames) {
+        std::string name = "global::" + val.second;
+        labels[val.first] = a.newNamedLabel(name.c_str());
+    }
+
+    /* Emit all of the code and bind all of the labels */
+    for (auto val : emitPtrs) {
+        a.align(kAlignCode, 8);
+        a.bind(labels[val.first]);
+        /* This funky syntax calls the function pointer within this instance
+         * of BeamGlobalAssembler */
+        (this->*val.second)();
+    }
+
+    {
+        /* We have no need of the module pointers as we use `getCode(...)` for
+         * everything. */
+        const void *_ignored_exec;
+        void *_ignored_rw;
+        _codegen(allocator, &_ignored_exec, &_ignored_rw);
+    }
+
+#ifndef WIN32
+    std::vector<AsmRange> ranges;
+
+    ranges.reserve(emitPtrs.size());
+
+    for (auto val : emitPtrs) {
+        ErtsCodePtr start = (ErtsCodePtr)getCode(labels[val.first]);
+        ErtsCodePtr stop;
+
+        if (val.first + 1 < emitPtrs.size()) {
+            stop = (ErtsCodePtr)getCode(labels[(GlobalLabels)(val.first + 1)]);
+        } else {
+            stop = (ErtsCodePtr)((char *)getBaseAddress() + code.codeSize());
+        }
+
+        ranges.push_back({.start = start,
+                          .stop = stop,
+                          .name = code.labelEntry(labels[val.first])->name()});
+    }
+
+    update_gdb_jit_info("global", ranges);
+    beamasm_update_perf_info("global", ranges);
+#endif
+
+    /* `this->get_xxx` are populated last to ensure that we crash if we use them
+     * instead of labels in global code. */
+
+    for (auto val : labelNames) {
+        ptrs[val.first] = (fptr)getCode(labels[val.first]);
+    }
+}
+
+/*
+ * ARG3 = (HTOP + bytes needed)
+ * ARG4 = Live registers
+ */
+void BeamGlobalAssembler::emit_garbage_collect() {
+    emit_enter_runtime_frame();
+
+    /* Convert ARG3 to words needed and move it to the correct argument slot. */
+    a.sub(ARG2, ARG3, HTOP);
+    a.lsr(ARG2, ARG2, imm(3));
+
+    /* Save our return address in c_p->i so we can tell where we crashed if we
+     * did so during GC. */
+    a.str(a64::x30, arm::Mem(c_p, offsetof(Process, i)));
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+    a.mov(ARG1, c_p);
+    /* ARG2 is already loaded. */
+    load_x_reg_array(ARG3);
+    /* ARG4 (live registers) is already loaded. */
+    a.mov(ARG5, FCALLS);
+    runtime_call<5>(erts_garbage_collect_nobump);
+    a.sub(FCALLS, FCALLS, ARG1);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+/* Handles trapping to exports from C code, setting registers up in the same
+ * manner a normal call_ext instruction would so that save_calls, tracing, and
+ * so on will work.
+ *
+ * Our return address is on the stack as we always come here from a BIF, so we
+ * must pop it into LR (x30) to convert this into an ordinary call. The
+ * callee will then push LR to the stack in its prologue, cancelling this out.
+ *
+ * Assumes that c_p->current points into the MFA of an export entry. */
+void BeamGlobalAssembler::emit_bif_export_trap() {
+    int export_offset = offsetof(Export, info.mfa);
+
+    a.ldr(ARG1, arm::Mem(c_p, offsetof(Process, current)));
+    a.sub(ARG1, ARG1, export_offset);
+
+    emit_leave_erlang_frame();
+
+    branch(emit_setup_export_call(ARG1));
+}
+
+/* Handles export breakpoints, error handler, jump tracing, and so on.
+ *
+ * We must be careful with LR (x30) and the stack as this runs between the
+ * caller and callee, and the latter pushes LR to the stack as part of its
+ * prologue.
+ *
+ * ARG1 = export entry
+ */
+void BeamGlobalAssembler::emit_export_trampoline() {
+    Label call_bif = a.newLabel(), error_handler = a.newLabel(),
+          jump_trace = a.newLabel();
+
+    /* What are we supposed to do? */
+    a.ldr(TMP1, arm::Mem(ARG1, offsetof(Export, trampoline.common.op)));
+
+    /* We test the generic bp first as it is most likely to be triggered in a
+     * loop. */
+    a.cmp(TMP1, imm(op_i_generic_breakpoint));
+    a.cond_eq().b(labels[generic_bp_global]);
+
+    a.cmp(TMP1, imm(op_call_bif_W));
+    a.cond_eq().b(call_bif);
+
+    a.cmp(TMP1, imm(op_call_error_handler));
+    a.cond_eq().b(error_handler);
+
+    a.cmp(TMP1, imm(op_trace_jump_W));
+    a.cond_eq().b(jump_trace);
+
+    /* Must never happen. */
+    a.udf(0xffff);
+
+    a.bind(call_bif);
+    {
+        /* Emulate a `call_bif` instruction.
+         *
+         * Note that we don't check reductions: yielding here is very tricky
+         * and error-prone, and there's little point in doing so as we can only
+         * land here directly after being scheduled in. */
+        ssize_t func_offset = offsetof(Export, trampoline.bif.address);
+
+        lea(ARG2, arm::Mem(ARG1, offsetof(Export, info.mfa)));
+        a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+        a.ldr(ARG4, arm::Mem(ARG1, func_offset));
+
+        /* `call_bif_shared` assumes that the return address has been pushed to
+         * the stack as part of the prologue, so we have to do that manually
+         * now. */
+        emit_enter_erlang_frame();
+        a.b(labels[call_bif_shared]);
+    }
+
+    a.bind(jump_trace);
+    {
+        a.ldr(TMP1, arm::Mem(ARG1, offsetof(Export, trampoline.trace.address)));
+        a.br(TMP1);
+    }
+
+    a.bind(error_handler);
+    {
+        emit_enter_runtime_frame();
+        emit_enter_runtime<Update::eReductions | Update::eStack |
+                           Update::eHeap | Update::eXRegs>();
+
+        lea(ARG2, arm::Mem(ARG1, offsetof(Export, info.mfa)));
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG3);
+        mov_imm(ARG4, am_undefined_function);
+        runtime_call<4>(call_error_handler);
+
+        /* If there is no error_handler, any number of X registers
+         * can be live. */
+        emit_leave_runtime<Update::eReductions | Update::eStack |
+                           Update::eHeap | Update::eXRegs>();
+        emit_leave_runtime_frame();
+
+        a.cbz(ARG1, labels[process_exit]);
+
+        branch(emit_setup_export_call(ARG1));
+    }
+}
+
+/*
+ * Get the error address implicitly by calling the shared fragment and using
+ * the return address as the error address.
+ */
+void BeamModuleAssembler::emit_raise_exception() {
+    emit_raise_exception(nullptr);
+}
+
+void BeamModuleAssembler::emit_raise_exception(const ErtsCodeMFA *exp) {
+    if (exp) {
+        a.ldr(ARG4, embed_constant(exp, disp32K));
+    } else {
+        a.mov(ARG4, ZERO);
+    }
+
+    fragment_call(ga->get_raise_exception());
+
+    /*
+     * It is important that error address is not equal to a line
+     * instruction that may follow this BEAM instruction. To avoid
+     * that, BeamModuleAssembler::emit() will emit a nop instruction
+     * if necessary.
+     */
+    last_error_offset = getOffset() & -8;
+}
+
+void BeamModuleAssembler::emit_raise_exception(Label I,
+                                               const ErtsCodeMFA *exp) {
+    a.adr(ARG2, I);
+
+    if (exp) {
+        a.ldr(ARG4, embed_constant(exp, disp32K));
+    } else {
+        a.mov(ARG4, ZERO);
+    }
+
+    a.b(resolve_fragment(ga->get_raise_exception_shared(), disp128MB));
+}
+
+void BeamGlobalAssembler::emit_process_exit() {
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    mov_imm(ARG2, 0);
+    mov_imm(ARG4, 0);
+    load_x_reg_array(ARG3);
+    runtime_call<4>(handle_error);
+
+    emit_leave_runtime();
+
+    a.cbz(ARG1, labels[do_schedule]);
+    a.udf(0xdead);
+}
+
+/* You must have already done emit_leave_runtime_frame()! */
+void BeamGlobalAssembler::emit_raise_exception() {
+    a.mov(ARG2, a64::x30);
+    a.b(labels[raise_exception_shared]);
+}
+
+void BeamGlobalAssembler::emit_raise_exception_shared() {
+    Label crash = a.newLabel();
+
+    /* Push a fake CP to ensure that we can handle a topmost frame
+     * with `catch` and an instruction raising and exception.
+     *
+     * The fake CP is discarded by handle_error() before jumping to
+     * a catch handler, and is ignored as a duplicate in stack
+     * traces because it's equal to the error address. */
+    a.str(ARG2, arm::Mem(E, -8).pre());
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+    /* The error address must be a valid CP or NULL. */
+    a.tst(ARG2, imm(_CPMASK));
+    a.cond_ne().b(crash);
+
+    /* ARG2 and ARG4 must be set prior to jumping here! */
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG3);
+    runtime_call<4>(handle_error);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+    a.cbz(ARG1, labels[do_schedule]);
+
+    /* XREG0 = THE_NON_VALUE
+     * XREG1 = class
+     * XREG2 = error reason/thrown value
+     * XREG3 = raw stacktrace. */
+    a.br(ARG1);
+
+    a.bind(crash);
+    a.udf(0xbad);
+}
+
+void BeamModuleAssembler::emit_proc_lc_unrequire(void) {
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    a.mov(ARG1, c_p);
+    mov_imm(ARG2, ERTS_PROC_LOCK_MAIN);
+    runtime_call<2>(erts_proc_lc_unrequire_lock);
+#endif
+}
+
+void BeamModuleAssembler::emit_proc_lc_require(void) {
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    a.mov(ARG1, c_p);
+    mov_imm(ARG2, ERTS_PROC_LOCK_MAIN);
+    runtime_call<4>(erts_proc_lc_require_lock);
+#endif
+}
+
+extern "C"
+{
+    /* GDB puts a breakpoint in this function.
+     *
+     * Has to be on another file than the caller as otherwise gcc may
+     * optimize away the call. */
+    void ERTS_NOINLINE __jit_debug_register_code(void);
+    void ERTS_NOINLINE __jit_debug_register_code(void) {
+    }
+}
diff --git a/erts/emulator/beam/jit/arm/beam_asm_module.cpp b/erts/emulator/beam/jit/arm/beam_asm_module.cpp
new file mode 100644
index 0000000000..6a57e4e7db
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/beam_asm_module.cpp
@@ -0,0 +1,805 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include <algorithm>
+#include <float.h>
+
+#include "beam_asm.hpp"
+using namespace asmjit;
+
+static std::string getAtom(Eterm atom) {
+    Atom *ap = atom_tab(atom_val(atom));
+    return std::string((char *)ap->name, ap->len);
+}
+
+#ifdef BEAMASM_DUMP_SIZES
+#    include <mutex>
+
+typedef std::pair<Uint64, Uint64> op_stats;
+
+static std::unordered_map<char *, op_stats> sizes;
+static std::mutex size_lock;
+
+extern "C" void beamasm_dump_sizes() {
+    std::lock_guard<std::mutex> lock(size_lock);
+
+    std::vector<std::pair<char *, op_stats>> flat(sizes.cbegin(), sizes.cend());
+    double total_size = 0.0;
+
+    for (const auto &op : flat) {
+        total_size += op.second.second;
+    }
+
+    /* Sort instructions by total size, in descending order. */
+    std::sort(
+            flat.begin(),
+            flat.end(),
+            [](std::pair<char *, op_stats> &a, std::pair<char *, op_stats> &b) {
+                return a.second.second > b.second.second;
+            });
+
+    for (const auto &op : flat) {
+        fprintf(stderr,
+                "%34s:\t%zu\t%f\t%zu\t%zu\r\n",
+                op.first,
+                op.second.second,
+                op.second.second / total_size,
+                op.second.first,
+                op.second.first ? (op.second.second / op.second.first) : 0);
+    }
+}
+#endif
+
+ErtsCodePtr BeamModuleAssembler::getCode(BeamLabel label) {
+    ASSERT(label < rawLabels.size() + 1);
+    return (ErtsCodePtr)getCode(rawLabels[label]);
+}
+
+ErtsCodePtr BeamModuleAssembler::getLambda(unsigned index) {
+    const auto &lambda = lambdas[index];
+    return (ErtsCodePtr)getCode(lambda.trampoline);
+}
+
+BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga,
+                                         Eterm mod,
+                                         unsigned num_labels)
+        : BeamAssembler(getAtom(mod)) {
+    this->ga = ga;
+    this->mod = mod;
+
+    _veneers.reserve(num_labels + 1);
+    rawLabels.reserve(num_labels + 1);
+
+    for (unsigned i = 1; i < num_labels; i++) {
+        Label lbl;
+
+#ifdef DEBUG
+        std::string lblName = "label_" + std::to_string(i);
+        lbl = a.newNamedLabel(lblName.data());
+#else
+        lbl = a.newLabel();
+#endif
+
+        rawLabels[i] = lbl;
+    }
+}
+
+BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga,
+                                         Eterm mod,
+                                         unsigned num_labels,
+                                         unsigned num_functions)
+        : BeamModuleAssembler(ga, mod, num_labels) {
+    codeHeader = a.newLabel();
+    a.align(kAlignCode, 8);
+    a.bind(codeHeader);
+
+    embed_zeros(sizeof(BeamCodeHeader) +
+                sizeof(ErtsCodeInfo *) * num_functions);
+
+    /* Setup the early_nif/breakpoint trampoline. */
+    genericBPTramp = a.newLabel();
+    a.bind(genericBPTramp);
+    {
+        Label bp_and_nif = a.newLabel(), bp_only = a.newLabel(),
+              nif_only = a.newLabel();
+
+        a.cmp(ARG1, imm(ERTS_ASM_BP_FLAG_BP_NIF_CALL_NIF_EARLY));
+        a.cond_eq().b(bp_and_nif);
+        ERTS_CT_ASSERT((1 << 0) == ERTS_ASM_BP_FLAG_CALL_NIF_EARLY);
+        a.tbnz(ARG1, imm(0), nif_only);
+        ERTS_CT_ASSERT((1 << 1) == ERTS_ASM_BP_FLAG_BP);
+        a.tbnz(ARG1, imm(1), bp_only);
+        a.ret(a64::x30);
+
+        a.bind(bp_and_nif);
+        {
+            emit_enter_runtime_frame();
+            a.bl(resolve_fragment(ga->get_generic_bp_local(), disp128MB));
+            emit_leave_runtime_frame();
+
+            a.b(nif_only);
+        }
+
+        a.bind(bp_only);
+        {
+            emit_enter_runtime_frame();
+            a.bl(resolve_fragment(ga->get_generic_bp_local(), disp128MB));
+            emit_leave_runtime_frame();
+
+            a.ret(a64::x30);
+        }
+
+        a.bind(nif_only);
+        a.b(resolve_fragment(ga->get_call_nif_early(), disp128MB));
+    }
+
+#ifdef DEBUG
+    last_stub_check_offset = a.offset();
+#endif
+}
+
+void BeamModuleAssembler::embed_vararg_rodata(const Span<ArgVal> &args,
+                                              a64::Gp reg) {
+    /* Short sequences are inlined in the .text section for slightly better
+     * speed. */
+    bool inlineData = args.size() <= 6;
+
+    Label data = a.newLabel(), next = a.newLabel();
+
+    if (inlineData) {
+        a.adr(reg, data);
+        a.b(next);
+    } else {
+        Label pointer = a.newLabel();
+
+        a.ldr(reg, arm::Mem(pointer));
+        a.b(next);
+
+        a.align(kAlignCode, 8);
+        a.bind(pointer);
+        a.embedLabel(data, 8);
+
+        a.section(rodata);
+    }
+
+    a.align(kAlignData, 8);
+    a.bind(data);
+
+    for (const ArgVal &arg : args) {
+        union {
+            BeamInstr as_beam;
+            char as_char[1];
+        } data;
+
+        a.align(kAlignData, 8);
+        switch (arg.getType()) {
+        case ArgVal::Literal: {
+            auto &patches = literals[arg.getValue()].patches;
+            Label patch = a.newLabel();
+
+            a.bind(patch);
+            a.embedUInt64(LLONG_MAX);
+            patches.push_back({patch, 0});
+            break;
+        }
+        case ArgVal::XReg:
+            data.as_beam = make_loader_x_reg(arg.getValue());
+            a.embed(&data.as_char, sizeof(data.as_beam));
+            break;
+        case ArgVal::YReg:
+            data.as_beam = make_loader_y_reg(arg.getValue());
+            a.embed(&data.as_char, sizeof(data.as_beam));
+            break;
+        case ArgVal::Label:
+            a.embedLabel(rawLabels[arg.getValue()]);
+            break;
+        case ArgVal::Immediate:
+        case ArgVal::Word:
+            data.as_beam = arg.getValue();
+            a.embed(&data.as_char, sizeof(data.as_beam));
+            break;
+        default:
+            ERTS_ASSERT(!"error");
+        }
+    }
+
+    if (!inlineData) {
+        a.section(code.textSection());
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_i_nif_padding() {
+    const size_t minimum_size = sizeof(UWord[BEAM_NATIVE_MIN_FUNC_SZ]);
+    size_t prev_func_start, diff;
+
+    prev_func_start = code.labelOffsetFromBase(rawLabels[functions.back() + 1]);
+    diff = a.offset() - prev_func_start;
+
+    if (diff < minimum_size) {
+        embed_zeros(minimum_size - diff);
+    }
+}
+
+void BeamModuleAssembler::emit_i_breakpoint_trampoline() {
+    /* This little prologue is used by nif loading and tracing to insert
+     * alternative instructions. */
+    Label next = a.newLabel();
+
+    emit_enter_erlang_frame();
+
+    /* This branch is modified to jump to the MOVZ instruction when the
+     * breakpoint is enabled. */
+    a.b(next);
+
+    /* This instruction is updated with the current flag. */
+    a.movz(ARG1, imm(ERTS_ASM_BP_FLAG_NONE));
+
+    if (genericBPTramp.isValid()) {
+        a.bl(resolve_label(genericBPTramp, disp128MB));
+    } else {
+        /* NIF or BIF stub; we're not going to use this trampoline as-is, but
+         * we need to reserve space for it. */
+        a.udf(0xB1F);
+    }
+
+    a.bind(next);
+
+    ASSERT((a.offset() - code.labelOffsetFromBase(currLabel)) ==
+           BEAM_ASM_FUNC_PROLOGUE_SIZE);
+}
+
+static void i_emit_nyi(char *msg) {
+    erts_exit(ERTS_ERROR_EXIT, "NYI: %s\n", msg);
+}
+
+void BeamModuleAssembler::emit_nyi(const char *msg) {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, imm(msg));
+    runtime_call<1>(i_emit_nyi);
+
+    /* Never returns */
+}
+
+void BeamModuleAssembler::emit_nyi() {
+    emit_nyi("<unspecified>");
+}
+
+bool BeamModuleAssembler::emit(unsigned specific_op, const Span<ArgVal> &args) {
+    check_pending_stubs();
+
+#ifdef BEAMASM_DUMP_SIZES
+    size_t before = a.offset();
+#endif
+
+    comment(opc[specific_op].name);
+
+#define InstrCnt()
+    switch (specific_op) {
+#include "beamasm_emit.h"
+    default:
+        ERTS_ASSERT(0 && "Invalid instruction");
+        break;
+    }
+
+    if (getOffset() == last_error_offset) {
+        /*
+         * The previous PC where an exception may occur is equal to the
+         * current offset, which is also the offset of the next
+         * instruction. If the next instruction happens to be a
+         * line instruction, the location for the exception will
+         * be that line instruction, which is probably wrong.
+         * To avoid that, bump the instruction offset.
+         */
+        a.nop();
+    }
+
+#ifdef BEAMASM_DUMP_SIZES
+    {
+        std::lock_guard<std::mutex> lock(size_lock);
+
+        sizes[opc[specific_op].name].first++;
+        sizes[opc[specific_op].name].second += a.offset() - before;
+    }
+#endif
+
+    return true;
+}
+
+/*
+ * Here follows meta instructions.
+ */
+
+void BeamGlobalAssembler::emit_i_func_info_shared() {
+    /* a64::x30 now points 4 bytes into the ErtsCodeInfo struct for the
+     * function. Put the address of the MFA into ARG1. */
+    a.add(ARG1, a64::x30, offsetof(ErtsCodeInfo, mfa) - 4);
+    mov_imm(TMP1, EXC_FUNCTION_CLAUSE);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+    a.str(ARG1, arm::Mem(c_p, offsetof(Process, current)));
+
+    mov_imm(ARG2, 0);
+    mov_imm(ARG4, 0);
+
+    a.b(labels[raise_exception_shared]);
+}
+
+void BeamModuleAssembler::emit_i_func_info(const ArgVal &Label,
+                                           const ArgVal &Module,
+                                           const ArgVal &Function,
+                                           const ArgVal &Arity) {
+    ErtsCodeInfo info;
+
+    functions.push_back(Label.getValue());
+
+    info.mfa.module = Module.getValue();
+    info.mfa.function = Function.getValue();
+    info.mfa.arity = Arity.getValue();
+    info.u.gen_bp = NULL;
+
+    comment("%T:%T/%d", info.mfa.module, info.mfa.function, info.mfa.arity);
+
+    /* This is an ErtsCodeInfo structure that has a valid ARM opcode as its `op`
+     * field, which *calls* the `function_clause` fragment so we can trace it
+     * back to this particular function.
+     *
+     * We avoid using the `fragment_call` helper to ensure a constant layout,
+     * as it adds code in certain debug configurations. */
+    if (codeHeader.isValid()) {
+        a.bl(resolve_fragment(ga->get_i_func_info_shared(), disp128MB));
+    } else {
+        a.nop();
+    }
+
+    a.align(kAlignCode, sizeof(UWord));
+    a.embed(&info.u.gen_bp, sizeof(info.u.gen_bp));
+    a.embed(&info.mfa, sizeof(info.mfa));
+}
+
+void BeamModuleAssembler::emit_label(const ArgVal &Label) {
+    currLabel = rawLabels[Label.getValue()];
+    bind_veneer_target(currLabel);
+}
+
+void BeamModuleAssembler::emit_aligned_label(const ArgVal &Label,
+                                             const ArgVal &Alignment) {
+    ASSERT(Alignment.isWord());
+    a.align(kAlignCode, Alignment.getValue());
+    emit_label(Label);
+}
+
+void BeamModuleAssembler::emit_on_load() {
+    on_load = currLabel;
+}
+
+void BeamModuleAssembler::bind_veneer_target(const Label &target) {
+    auto veneer_range = _veneers.equal_range(target.id());
+    for (auto it = veneer_range.first; it != veneer_range.second; it++) {
+        const Veneer &veneer = it->second;
+
+        ASSERT(veneer.target == target);
+
+        if (!code.isLabelBound(veneer.anchor)) {
+            ASSERT(a.offset() <= veneer.latestOffset);
+            a.bind(veneer.anchor);
+
+            /* TODO: remove from pending stubs? */
+        }
+    }
+
+    a.bind(target);
+}
+
+void BeamModuleAssembler::emit_int_code_end() {
+    /* This label is used to figure out the end of the last function */
+    code_end = a.newLabel();
+    a.bind(code_end);
+
+    emit_nyi("int_code_end");
+
+    /* We emit the dispatch table before all remaining stubs to bind veneers
+     * directly in the table itself, avoiding a painful extra jump.
+     *
+     * Since the table is potentially very large, we'll emit all stubs that are
+     * due within it so we won't have to check on every iteration. */
+    flush_pending_stubs(_dispatchTable.size() * sizeof(Uint32[8]) +
+                        dispUnknown);
+
+    for (auto pair : _dispatchTable) {
+        bind_veneer_target(pair.second);
+
+        a.mov(SUPER_TMP, imm(pair.first));
+        a.br(SUPER_TMP);
+    }
+
+    /* Emit all remaining stubs. */
+    flush_pending_stubs(dispMax);
+}
+
+void BeamModuleAssembler::emit_line(const ArgVal &) {
+    /*
+     * There is no need to align the line instruction. In the loaded
+     * code, the type of the pointer will be void* and that pointer
+     * will only be used in comparisons.
+     */
+}
+
+void BeamModuleAssembler::emit_func_line(const ArgVal &Loc) {
+    emit_line(Loc);
+}
+
+void BeamModuleAssembler::emit_empty_func_line() {
+}
+
+/*
+ * Here follows stubs for instructions that should never be called.
+ */
+
+void BeamModuleAssembler::emit_i_debug_breakpoint() {
+    emit_nyi("i_debug_breakpoint should never be called");
+}
+
+void BeamModuleAssembler::emit_i_generic_breakpoint() {
+    emit_nyi("i_generic_breakpoint should never be called");
+}
+
+void BeamModuleAssembler::emit_trace_jump(const ArgVal &) {
+    emit_nyi("trace_jump should never be called");
+}
+
+void BeamModuleAssembler::emit_call_error_handler() {
+    emit_nyi("call_error_handler should never be called");
+}
+
+unsigned BeamModuleAssembler::patchCatches(char *rw_base) {
+    unsigned catch_no = BEAM_CATCHES_NIL;
+
+    for (const auto &c : catches) {
+        const auto &patch = c.patch;
+        ErtsCodePtr handler;
+
+        handler = (ErtsCodePtr)getCode(c.handler);
+        catch_no = beam_catches_cons(handler, catch_no, nullptr);
+
+        /* Patch the `mov` instruction with the catch tag */
+        auto offset = code.labelOffsetFromBase(patch.where);
+        auto where = (Eterm *)&rw_base[offset];
+
+        ASSERT(LLONG_MAX == *where);
+        Eterm catch_term = make_catch(catch_no);
+
+        /* With the current tag scheme, more than 33 million
+         * catches can exist at once. */
+        ERTS_ASSERT(catch_term >> 31 == 0);
+
+        *where = catch_term;
+    }
+
+    return catch_no;
+}
+
+void BeamModuleAssembler::patchImport(char *rw_base,
+                                      unsigned index,
+                                      BeamInstr I) {
+    for (const auto &patch : imports[index].patches) {
+        auto offset = code.labelOffsetFromBase(patch.where);
+        auto where = (Eterm *)&rw_base[offset];
+
+        ASSERT(LLONG_MAX == *where);
+        *where = I + patch.val_offs;
+    }
+}
+
+void BeamModuleAssembler::patchLambda(char *rw_base,
+                                      unsigned index,
+                                      BeamInstr I) {
+    for (const auto &patch : lambdas[index].patches) {
+        auto offset = code.labelOffsetFromBase(patch.where);
+        auto where = (Eterm *)&rw_base[offset];
+
+        ASSERT(LLONG_MAX == *where);
+        *where = I + patch.val_offs;
+    }
+}
+
+void BeamModuleAssembler::patchLiteral(char *rw_base,
+                                       unsigned index,
+                                       Eterm lit) {
+    for (const auto &patch : literals[index].patches) {
+        auto offset = code.labelOffsetFromBase(patch.where);
+        auto where = (Eterm *)&rw_base[offset];
+
+        ASSERT(LLONG_MAX == *where);
+        *where = lit + patch.val_offs;
+    }
+}
+
+void BeamModuleAssembler::patchStrings(char *rw_base,
+                                       const byte *string_table) {
+    for (const auto &patch : strings) {
+        auto offset = code.labelOffsetFromBase(patch.where);
+        auto where = (const byte **)&rw_base[offset];
+
+        ASSERT(LLONG_MAX == (Eterm)*where);
+        *where = string_table + patch.val_offs;
+    }
+}
+
+Label BeamModuleAssembler::resolve_beam_label(const ArgVal &Lbl,
+                                              enum Displacement disp) {
+    ASSERT(Lbl.isLabel());
+    return resolve_label(rawLabels[Lbl.getValue()], disp);
+}
+
+Label BeamModuleAssembler::resolve_label(Label target, enum Displacement disp) {
+    ssize_t currOffset = a.offset();
+
+    ssize_t minOffset = currOffset - disp;
+    ssize_t maxOffset = currOffset + disp;
+
+    ASSERT(disp >= dispMin && disp <= dispMax);
+    ASSERT(target.isValid());
+
+    if (code.isLabelBound(target)) {
+        ssize_t targetOffset = code.labelOffsetFromBase(target);
+
+        /* Backward reference: skip veneers if it's already in range. */
+        if (targetOffset >= minOffset) {
+            return target;
+        }
+    }
+
+    /* If a previously created veneer is reachable from this point, we can use
+     * it instead of creating a new one. */
+    auto range = _veneers.equal_range(target.id());
+    for (auto it = range.first; it != range.second; it++) {
+        const Veneer &veneer = it->second;
+
+        if (code.isLabelBound(veneer.anchor)) {
+            ssize_t veneerOffset = code.labelOffsetFromBase(veneer.anchor);
+
+            if (veneerOffset >= minOffset && veneerOffset <= maxOffset) {
+                return veneer.anchor;
+            }
+        } else if (veneer.latestOffset <= maxOffset) {
+            return veneer.anchor;
+        }
+    }
+
+    auto it = _veneers.emplace(target.id(),
+                               Veneer{.latestOffset = maxOffset,
+                                      .anchor = a.newLabel(),
+                                      .target = target});
+
+    const Veneer &veneer = it->second;
+    _pending_veneers.emplace(veneer);
+
+    return veneer.anchor;
+}
+
+Label BeamModuleAssembler::resolve_fragment(void (*fragment)(),
+                                            enum Displacement disp) {
+    Label target;
+
+    auto it = _dispatchTable.find(fragment);
+
+    if (it != _dispatchTable.end()) {
+        target = it->second;
+    } else {
+        target = a.newLabel();
+        _dispatchTable.emplace(fragment, target);
+    }
+
+    return resolve_label(target, disp);
+}
+
+arm::Mem BeamModuleAssembler::embed_constant(const ArgVal &value,
+                                             enum Displacement disp) {
+    ssize_t currOffset = a.offset();
+
+    ssize_t minOffset = currOffset - disp;
+    ssize_t maxOffset = currOffset + disp;
+
+    ASSERT(disp >= dispMin && disp <= dispMax);
+    ASSERT(value.isConstant());
+
+    /* If a previously embedded constant is reachable from this point, we
+     * can use it instead of creating a new one. */
+    auto range = _constants.equal_range(value);
+    for (auto it = range.first; it != range.second; it++) {
+        const Constant &constant = it->second;
+
+        if (code.isLabelBound(constant.anchor)) {
+            ssize_t constOffset = code.labelOffsetFromBase(constant.anchor);
+
+            if (constOffset >= minOffset && constOffset <= maxOffset) {
+                return arm::Mem(constant.anchor);
+            }
+        } else if (constant.latestOffset <= maxOffset) {
+            return arm::Mem(constant.anchor);
+        }
+    }
+
+    auto it = _constants.emplace(value,
+                                 Constant{.latestOffset = maxOffset,
+                                          .anchor = a.newLabel(),
+                                          .value = value});
+
+    const Constant &constant = it->second;
+    _pending_constants.emplace(constant);
+
+    return arm::Mem(constant.anchor);
+}
+
+void BeamModuleAssembler::emit_i_flush_stubs() {
+    /* Flush all stubs that are due within the next two check intervals
+     * to prevent them from being emitted inside function prologues or
+     * NIF padding. */
+    flush_pending_stubs(STUB_CHECK_INTERVAL * 2);
+    last_stub_check_offset = a.offset();
+}
+
+void BeamModuleAssembler::check_pending_stubs() {
+    size_t currOffset = a.offset();
+
+    /* We shouldn't let too much space pass between checks. */
+    ASSERT((last_stub_check_offset + dispMin) >= currOffset);
+
+    if ((last_stub_check_offset + STUB_CHECK_INTERVAL) < currOffset) {
+        last_stub_check_offset = currOffset;
+
+        flush_pending_stubs(STUB_CHECK_INTERVAL * 2);
+    }
+}
+
+void BeamModuleAssembler::flush_pending_stubs(size_t range) {
+    size_t effective_offset;
+    Label next;
+
+    effective_offset = a.offset() + range;
+
+    while (!_pending_veneers.empty()) {
+        const Veneer &veneer = _pending_veneers.top();
+
+        if (veneer.latestOffset > effective_offset) {
+            break;
+        }
+
+        if (!code.isLabelBound(veneer.anchor)) {
+            if (!next.isValid()) {
+                next = a.newLabel();
+
+                comment("Begin stub section");
+                a.b(next);
+            }
+
+            emit_veneer(veneer);
+
+            effective_offset = a.offset() + range;
+        }
+
+        _pending_veneers.pop();
+    }
+
+    while (!_pending_constants.empty()) {
+        const Constant &constant = _pending_constants.top();
+
+        if (constant.latestOffset > effective_offset) {
+            break;
+        }
+
+        /* Unlike veneers, we never bind constants ahead of time. */
+        ASSERT(!code.isLabelBound(constant.anchor));
+
+        if (!next.isValid()) {
+            next = a.newLabel();
+
+            comment("Begin stub section");
+            a.b(next);
+        }
+
+        emit_constant(constant);
+
+        effective_offset = a.offset() + range;
+
+        _pending_constants.pop();
+    }
+
+    if (next.isValid()) {
+        comment("End stub section");
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_veneer(const Veneer &veneer) {
+    const Label &anchor = veneer.anchor;
+    const Label &target = veneer.target;
+    bool directBranch;
+
+    ASSERT(!code.isLabelBound(anchor));
+    a.bind(anchor);
+
+    /* Prefer direct branches when possible. */
+    if (code.isLabelBound(target)) {
+        auto targetOffset = code.labelOffsetFromBase(target);
+        directBranch = (a.offset() - targetOffset) <= disp128MB;
+    } else {
+        directBranch = false;
+    }
+
+#ifdef DEBUG
+    directBranch &= (a.offset() % 512) >= 256;
+#endif
+
+    if (ERTS_LIKELY(directBranch)) {
+        a.b(target);
+    } else {
+        Label pointer = a.newLabel();
+
+        a.ldr(SUPER_TMP, arm::Mem(pointer));
+        a.br(SUPER_TMP);
+
+        a.align(kAlignCode, 8);
+        a.bind(pointer);
+        a.embedLabel(veneer.target);
+    }
+}
+
+void BeamModuleAssembler::emit_constant(const Constant &constant) {
+    const Label &anchor = constant.anchor;
+    const ArgVal &value = constant.value;
+    auto rawValue = value.getValue();
+
+    ASSERT(!code.isLabelBound(anchor));
+    a.align(kAlignData, 8);
+    a.bind(anchor);
+
+    if (value.isImmed() || value.isWord()) {
+        a.embedUInt64(rawValue);
+    } else if (value.isLabel()) {
+        a.embedLabel(rawLabels[rawValue]);
+    } else {
+        a.embedUInt64(LLONG_MAX);
+
+        switch (value.getType()) {
+        case ArgVal::BytePtr:
+            strings.push_back({anchor, rawValue});
+            break;
+        case ArgVal::Catch: {
+            auto handler = rawLabels[rawValue];
+            catches.push_back({{anchor, 0}, handler});
+            break;
+        }
+        case ArgVal::Export:
+            imports[rawValue].patches.push_back({anchor, 0});
+            break;
+        case ArgVal::FunEntry:
+            lambdas[rawValue].patches.push_back({anchor, 0});
+            break;
+        case ArgVal::Literal:
+            literals[rawValue].patches.push_back({anchor, 0});
+            break;
+        default:
+            ASSERT(!"error");
+        }
+    }
+}
diff --git a/erts/emulator/beam/jit/arm/generators.tab b/erts/emulator/beam/jit/arm/generators.tab
new file mode 100644
index 0000000000..abe030f51f
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/generators.tab
@@ -0,0 +1,497 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2020. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+// Generate the fastest instruction to fetch an integer from a binary.
+gen.get_integer2(Fail, Ms, Live, Size, Unit, Flags, Dst) {
+    BeamOp* op;
+    UWord bits;
+
+    $NewBeamOp(S, op);
+    $NativeEndian(Flags);
+
+    if (Size.type == TAG_i) {
+        if (!beam_load_safe_mul(Size.val, Unit.val, &bits)) {
+            $BeamOpNameArity(op, jump, 1);
+            op->a[0] = Fail;
+
+            return op;
+        } else if (bits <= 1023) {
+            $BeamOpNameArity(op, i_bs_get_fixed_integer, 6);
+            op->a[0] = Ms;
+            op->a[1] = Fail;
+            op->a[2] = Live;
+            op->a[3].type = TAG_u;
+            op->a[3].val = Flags.val;
+            op->a[4].type = TAG_u;
+            op->a[4].val = bits;
+            op->a[5] = Dst;
+
+            return op;
+        }
+    }
+
+    $BeamOpNameArity(op, i_bs_get_integer, 6);
+    op->a[0] = Ms;
+    op->a[1] = Fail;
+    op->a[2] = Live;
+    op->a[3].type = TAG_u;
+    op->a[3].val = (Unit.val << 3) | Flags.val;
+    op->a[4] = Size;
+    op->a[5] = Dst;
+
+    return op;
+}
+
+gen.select_tuple_arity(Src, Fail, Size, Rest) {
+    BeamOp* op;
+    BeamOpArg *tmp;
+    int size = Size.val / 2;
+    int arity = Size.val + 3;
+    int i, j;
+
+    /* Verify the validity of the list. */
+    if (Size.val % 2 != 0) {
+        return NULL;
+    }
+
+    for (i = 0; i < Size.val; i += 2) {
+        if (Rest[i].type != TAG_u || Rest[i+1].type != TAG_f) {
+            return NULL;
+        }
+    }
+
+    /*
+     * Generate the generic instruction.
+     * Assumption:
+     *   Few different tuple arities to select on (fewer than 20).
+     *   Use linear scan approach.
+     */
+    $NewBeamOp(S, op);
+    $BeamOpNameArity(op, i_select_tuple_arity, 3);
+    $BeamOpArity(op, arity);
+    op->next = NULL;
+    op->a[0] = Src;
+    op->a[1] = Fail;
+
+    /* Variable argument count */
+    op->a[2].type = TAG_u;
+    op->a[2].val = Size.val;
+
+    tmp = (BeamOpArg*)erts_alloc(ERTS_ALC_T_LOADER_TMP,
+                                sizeof(BeamOpArg) * arity);
+
+    for (i = 3; i < arity; i += 2) {
+        tmp[i - 2]      = Rest[i - 2];
+        tmp[i - 3].type = TAG_v;
+        tmp[i - 3].val  = make_arityval(Rest[i - 3].val);
+    }
+
+    /* Sort the values to make them useful for a binary or sentinel search. */
+    beam_load_sort_select_vals(tmp, size);
+
+    j = 3;
+    for (i = 3; i < arity; i += 2) {
+        op->a[j + size] = tmp[i - 2];
+        op->a[j]        = tmp[i - 3];
+        j++;
+    }
+
+    erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp);
+
+    return op;
+}
+
+gen.split_values(Src, TypeFail, Fail, Size, Rest) {
+    BeamOp* op1;
+    BeamOp* op2;
+    BeamOp* is_integer;
+    int i;
+
+    ASSERT(Size.val >= 2 && Size.val % 2 == 0);
+
+    $NewBeamOp(S, is_integer);
+    $BeamOpNameArity(is_integer, is_integer, 2);
+    is_integer->a[0] = TypeFail;
+    is_integer->a[1] = Src;
+
+    $NewBeamOp(S, op1);
+    $BeamOpNameArity(op1, select_val, 3);
+    $BeamOpArity(op1, 3 + Size.val);
+    op1->a[0] = Src;
+    /* We use [] instead of a failure label to indicate a fallthrough
+     * to next instruction if none of the values match. That way, we
+     * don't have to create a new label dynamically, which would be
+     * tricky in BeamAsm. */
+    op1->a[1].type = TAG_n;
+    op1->a[1].val = 0;
+    op1->a[2].type = TAG_u;
+    op1->a[2].val = 0;
+
+    $NewBeamOp(S, op2);
+    $BeamOpNameArity(op2, select_val, 3);
+    $BeamOpArity(op2, 3 + Size.val);
+    op2->a[0] = Src;
+    op2->a[1] = Fail;
+    op2->a[2].type = TAG_u;
+    op2->a[2].val = 0;
+
+    /*
+     * Split the list.
+     */
+
+    ASSERT(Size.type == TAG_u);
+    for (i = 0; i < Size.val; i += 2) {
+        BeamOp* op = (Rest[i].type == TAG_q) ? op2 : op1;
+        int dst = 3 + op->a[2].val;
+
+        ASSERT(Rest[i+1].type == TAG_f);
+        op->a[dst] = Rest[i];
+        op->a[dst+1] = Rest[i+1];
+        op->arity += 2;
+        op->a[2].val += 2;
+    }
+    ASSERT(op1->a[2].val > 0);
+    ASSERT(op2->a[2].val > 0);
+
+    /*
+     * Order the instruction sequence appropriately.
+     */
+
+    if (TypeFail.val == Fail.val) {
+        /*
+         * select_val Src [] S ... (small numbers)
+         * is_integer Fail S
+         * select_val Src Fail S ... (bignums)
+         */
+        op1->next = is_integer;
+        is_integer->next = op2;
+    } else {
+        /*
+         * is_integer TypeFail S
+         * select_val Src [] S ... (small numbers)
+         * select_val Src Fail S ... (bignums)
+         */
+        is_integer->next = op1;
+        op1->next = op2;
+	op1 = is_integer;
+    }
+    op2->next = NULL;
+
+    return op1;
+}
+
+//
+// Generate a select_val instruction.  We know that a jump table
+// is not suitable, and that all values are of the same type
+// (integer or atoms).
+//
+gen.select_val(Src, Fail, Size, Rest) {
+    BeamOp* op;
+    BeamOpArg *tmp;
+    int arity = Size.val + 3;
+    int size = Size.val / 2;
+    int i, j;
+
+    $NewBeamOp(S, op);
+    op->next = NULL;
+
+    /* Use linear search for small search spaces */
+    if (size <= 10) {
+        $BeamOpNameArity(op, i_select_val_lins, 3);
+    } else {
+        $BeamOpNameArity(op, i_select_val_bins, 3);
+    }
+
+    $BeamOpArity(op, arity);
+    op->a[0] = Src;
+    op->a[1] = Fail;
+
+    /* Variable argument count */
+    op->a[2].type = TAG_u;
+    op->a[2].val = Size.val;
+
+    tmp = (BeamOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(BeamOpArg)*(arity));
+
+    for (i = 3; i < arity; i++) {
+        tmp[i-3] = Rest[i-3];
+    }
+
+    /* Sort the values to make them useful for a binary or sentinel search. */
+    beam_load_sort_select_vals(tmp, size);
+
+    j = 3;
+    for (i = 3; i < arity; i += 2) {
+        op->a[j]      = tmp[i-3];
+        op->a[j+size] = tmp[i-2];
+        j++;
+    }
+
+    erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp);
+
+#ifdef DEBUG
+    for (i = 0; i < size - 1; i++) {
+        ASSERT(op->a[i+3].val <= op->a[i+4].val);
+    }
+#endif
+
+    return op;
+}
+
+//
+// Generate a select_val instruction for big numbers.
+//
+gen.select_literals(Src, Fail, Size, Rest) {
+    BeamOp* op;
+    BeamOp* jump;
+    BeamOp** prev_next = &op;
+
+    int i;
+
+    for (i = 0; i < Size.val; i += 2) {
+        BeamOp* op;
+        ASSERT(Rest[i].type == TAG_q);
+
+        $NewBeamOp(S, op);
+        $BeamOpNameArity(op, is_ne_exact, 3);
+        op->a[0] = Rest[i+1];
+        op->a[1] = Src;
+        op->a[2] = Rest[i];
+        *prev_next = op;
+        prev_next = &op->next;
+    }
+
+    $NewBeamOp(S, jump);
+    $BeamOpNameArity(jump, jump, 1);
+    jump->next = NULL;
+    jump->a[0] = Fail;
+    *prev_next = jump;
+    return op;
+}
+
+gen.new_small_map_lit(Dst, Live, Size, Rest) {
+    unsigned size = Size.val;
+    Uint lit;
+    unsigned i;
+    BeamOp* op;
+    BeamOpArg* dst;
+    Eterm* tmp;
+    Eterm* thp;
+    Eterm keys;
+
+    $NewBeamOp(S, op);
+    $BeamOpNameArity(op, i_new_small_map_lit, 4);
+    $BeamOpArity(op, 4 + size/2);
+    op->next = NULL;
+
+    tmp = thp = erts_alloc(ERTS_ALC_T_LOADER_TMP, (1 + size/2) * sizeof(*tmp));
+    keys = make_tuple(thp);
+    *thp++ = make_arityval(size/2);
+
+    dst = op->a+4;
+
+    for (i = 0; i < size; i += 2) {
+        switch (Rest[i].type) {
+        case TAG_a:
+            *thp++ = Rest[i].val;
+            ASSERT(is_atom(Rest[i].val));
+            break;
+        case TAG_i:
+            *thp++ = make_small(Rest[i].val);
+            break;
+        case TAG_n:
+            *thp++ = NIL;
+            break;
+        case TAG_q:
+            *thp++ = beamfile_get_literal(&S->beam, Rest[i].val);
+            break;
+        }
+        *dst++ = Rest[i + 1];
+    }
+
+    lit = beamfile_add_literal(&S->beam, keys);
+    erts_free(ERTS_ALC_T_LOADER_TMP, tmp);
+
+    op->a[0] = Dst;
+    op->a[1] = Live;
+    op->a[2].type = TAG_q;
+    op->a[2].val = lit;
+    op->a[3].type = TAG_u;
+    op->a[3].val = size / 2;
+
+    return op;
+}
+
+// Generate the fastest instruction to fetch a binary from a binary.
+gen.get_binary2(Fail, Ms, Live, Size, Unit, Flags, Dst) {
+    BeamOp* op;
+
+    $NewBeamOp(S, op);
+    $NativeEndian(Flags);
+
+    if (Size.type == TAG_a && Size.val == am_all) {
+        $BeamOpNameArity(op, i_bs_get_binary_all2, 5);
+        op->a[0] = Ms;
+        op->a[1] = Fail;
+        op->a[2] = Live;
+        op->a[3] = Unit;
+        op->a[4] = Dst;
+    } else {
+        $BeamOpNameArity(op, i_bs_get_binary2, 6);
+        op->a[0] = Ms;
+        op->a[1] = Fail;
+        op->a[2] = Live;
+        op->a[3] = Size;
+        op->a[4].type = TAG_u;
+        op->a[4].val = (Unit.val << 3) | Flags.val;
+        op->a[5] = Dst;
+    }
+
+    op->next = NULL;
+    return op;
+}
+
+gen.allocate_heap_zero(Ns, Nh, Live) {
+    BeamOp* alloc;
+    BeamOp* init;
+    int i;
+
+    $NewBeamOp(S, alloc);
+    $NewBeamOp(S, init);
+
+    $BeamOpNameArity(alloc, allocate_heap, 3);
+    alloc->a[0] = Ns;
+    alloc->a[1] = Nh;
+    alloc->a[2] = Live;
+    alloc->next = init;
+
+    $BeamOpNameArity(init, init_yregs, 1);
+    $BeamOpArity(init, Ns.val+1);
+    init->a[0] = Ns;
+    for (i = 0; i < Ns.val; i++) {
+	init->a[i+1].type = TAG_y;
+	init->a[i+1].val = i;
+    }
+
+    return alloc;
+}
+
+gen.jump_tab(Src, Fail, Size, Rest) {
+    Sint min, max;
+    Sint i;
+    Sint size;
+    Sint arity;
+    int fixed_args;
+    BeamOp* op;
+
+    /*
+     * The rule in ops.tab must ensure that there are more than
+     * two values.
+     */
+    ASSERT(Size.val > 2);
+    ASSERT(Size.val % 2 == 0);
+
+    /* Calculate the minimum and maximum values and size of jump table. */
+    ASSERT(Rest[0].type == TAG_i);
+    min = max = Rest[0].val;
+    for (i = 2; i < Size.val; i += 2) {
+        ASSERT(Rest[i].type == TAG_i && Rest[i+1].type == TAG_f);
+        if (Rest[i].val < min) {
+            min = Rest[i].val;
+        } else if (max < Rest[i].val) {
+            max = Rest[i].val;
+        }
+    }
+    size = max - min + 1;
+
+    /* Allocate structure and fill in the fixed fields. */
+    $NewBeamOp(S, op);
+    op->next = NULL;
+    $BeamOpNameArity(op, i_jump_on_val, 4);
+    fixed_args = op->arity;
+    arity = fixed_args + size;
+    $BeamOpArity(op, arity);
+    op->a[0] = Src;
+    op->a[1] = Fail;
+    op->a[2].type = TAG_u;
+    op->a[2].val = min;
+    op->a[3].type = TAG_u;
+    op->a[3].val = size;
+
+    /* Fill in the jump table. */
+    for (i = fixed_args; i < arity; i++) {
+        op->a[i] = Fail;
+    }
+
+    for (i = 0; i < Size.val; i += 2) {
+        Sint index = fixed_args + Rest[i].val - min;
+        ASSERT(fixed_args <= index && index < arity);
+        op->a[index] = Rest[i+1];
+    }
+
+    return op;
+}
+
+gen.func_end(Func_Label, Entry_Label) {
+    BeamOp *op = NULL;
+
+    if (S->labels[Entry_Label.val].lambda_index != -1) {
+        int index = S->labels[Entry_Label.val].lambda_index;
+        BeamFile_LambdaEntry *lambda_entry;
+        BeamOp *lambda;
+
+        lambda_entry = &S->beam.lambdas.entries[index];
+
+        $NewBeamOp(S, lambda);
+        $BeamOpNameArity(lambda, i_lambda_trampoline, 4);
+
+        lambda->a[0].type = TAG_u;
+        lambda->a[0].val = index;
+        lambda->a[1] = Entry_Label;
+        lambda->a[2].type = TAG_u;
+        lambda->a[2].val = lambda_entry->arity;
+        lambda->a[3].type = TAG_u;
+        lambda->a[3].val = lambda_entry->num_free;
+
+        lambda->next = op;
+        op = lambda;
+    }
+
+    if (S->may_load_nif) {
+        BeamOp *padding;
+
+        $NewBeamOp(S, padding);
+        $BeamOpNameArity(padding, i_nif_padding, 0);
+
+        padding->next = op;
+        op = padding;
+    }
+
+    if (op == NULL) {
+        /* Workaround for the fact that we have to return an instruction from
+         * generators. The loader will remove this dummy instruction. */
+        $NewBeamOp(S, op);
+        $BeamOpNameArity(op, delete_me, 0);
+
+        op->next = NULL;
+    }
+
+    return op;
+}
diff --git a/erts/emulator/beam/jit/arm/instr_arith.cpp b/erts/emulator/beam/jit/arm/instr_arith.cpp
new file mode 100644
index 0000000000..4d68ed7ae9
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_arith.cpp
@@ -0,0 +1,1177 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "erl_bif_table.h"
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_plus_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_Plus, 2};
+
+    Label error = a.newLabel();
+
+    /* Save original arguments for the error path. */
+    a.stp(ARG2, ARG3, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_mixed_plus);
+
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        /* emit_enter_runtime() was done in the module code. */
+        emit_leave_runtime(0);
+
+        /* Place the original arguments in X registers. */
+        a.ldp(XREG0, XREG1, TMP_MEM1q);
+        mov_imm(ARG4, &bif_mfa);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_i_plus(const ArgVal &Fail,
+                                      const ArgVal &Live,
+                                      const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    Label next = a.newLabel();
+
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    bool rhs_is_small = RHS.isImmed() && is_small(RHS.getValue());
+
+    if (rhs_is_small && Support::isUInt12(RHS.getValue())) {
+        comment("add small constant with overflow check");
+        Uint cleared_tag = RHS.getValue() & ~_TAG_IMMED1_MASK;
+        a.adds(ARG1, lhs.reg, imm(cleared_tag));
+    } else {
+        comment("addition with overflow check");
+        a.and_(TMP1, rhs.reg, imm(~_TAG_IMMED1_MASK));
+        a.adds(ARG1, lhs.reg, TMP1);
+    }
+
+    if (rhs_is_small) {
+        a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+    } else if (LHS.isImmed() && is_small(LHS.getValue())) {
+        a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK));
+    } else {
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        a.and_(TMP1, lhs.reg, rhs.reg);
+        a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    }
+
+    /* Test for not overflow AND small operands. */
+    a.ccmp(TMP1, imm(_TAG_IMMED1_SMALL), 0, arm::Cond::kVC);
+    a.cond_eq().b(next);
+
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    if (Fail.getValue() != 0) {
+        emit_enter_runtime(Live.getValue());
+        a.mov(ARG1, c_p);
+        runtime_call<3>(erts_mixed_plus);
+        emit_leave_runtime(Live.getValue());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_plus_body_shared());
+        emit_leave_runtime(Live.getValue());
+    }
+
+    a.bind(next);
+    mov_arg(Dst, ARG1);
+}
+
+/*
+ * ARG2 = Src
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_unary_minus_body_shared() {
+    Label error = a.newLabel();
+
+    /* Save original argument for the error path. */
+    a.str(ARG2, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_unary_minus);
+
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        static const ErtsCodeMFA bif_mfa = {am_erlang, am_Minus, 1};
+
+        /* emit_enter_runtime() was done in the module code. */
+        emit_leave_runtime(0);
+
+        /* Place the original argument in an X registers. */
+        a.ldr(XREG0, TMP_MEM1q);
+        mov_imm(ARG4, &bif_mfa);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_i_unary_minus(const ArgVal &Fail,
+                                             const ArgVal &Live,
+                                             const ArgVal &Src,
+                                             const ArgVal &Dst) {
+    Label next = a.newLabel();
+    auto src = load_source(Src, ARG2);
+
+    a.mov(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.and_(TMP2, src.reg, imm(~_TAG_IMMED1_MASK));
+    a.subs(ARG1, TMP1, TMP2);
+
+    /* Test for not overflow AND small operands. */
+    a.ccmp(TMP2, imm(_TAG_IMMED1_SMALL), 0, arm::Cond::kVC);
+    a.cond_eq().b(next);
+
+    mov_var(ARG2, src);
+    if (Fail.getValue() != 0) {
+        emit_enter_runtime(Live.getValue());
+        a.mov(ARG1, c_p);
+        runtime_call<2>(erts_unary_minus);
+        emit_leave_runtime(Live.getValue());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_unary_minus_body_shared());
+        emit_leave_runtime(Live.getValue());
+    }
+
+    a.bind(next);
+    mov_arg(Dst, ARG1);
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_minus_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_Minus, 2};
+
+    Label error = a.newLabel();
+
+    /* Save original arguments for the error path. */
+    a.stp(ARG2, ARG3, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_mixed_minus);
+
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        /* emit_enter_runtime() was done in the module code. */
+        emit_leave_runtime(0);
+
+        /* Place the original arguments in X registers. */
+        a.ldp(XREG0, XREG1, TMP_MEM1q);
+        mov_imm(ARG4, &bif_mfa);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_i_minus(const ArgVal &Fail,
+                                       const ArgVal &Live,
+                                       const ArgVal &LHS,
+                                       const ArgVal &RHS,
+                                       const ArgVal &Dst) {
+    Label next = a.newLabel();
+
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    bool rhs_is_small = RHS.isImmed() && is_small(RHS.getValue());
+
+    if (rhs_is_small && Support::isUInt12(RHS.getValue())) {
+        comment("subtract small constant with overflow check");
+        Uint cleared_tag = RHS.getValue() & ~_TAG_IMMED1_MASK;
+        a.subs(ARG1, lhs.reg, imm(cleared_tag));
+    } else {
+        comment("subtraction with overflow check");
+        a.and_(TMP1, rhs.reg, imm(~_TAG_IMMED1_MASK));
+        a.subs(ARG1, lhs.reg, TMP1);
+    }
+
+    if (rhs_is_small) {
+        a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+    } else if (LHS.isImmed() && is_small(LHS.getValue())) {
+        a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK));
+    } else {
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        a.and_(TMP1, lhs.reg, rhs.reg);
+        a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    }
+
+    /* Test for not overflow AND small operands. */
+    a.ccmp(TMP1, imm(_TAG_IMMED1_SMALL), 0, arm::Cond::kVC);
+    a.cond_eq().b(next);
+
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    if (Fail.getValue() != 0) {
+        emit_enter_runtime(Live.getValue());
+        a.mov(ARG1, c_p);
+        runtime_call<3>(erts_mixed_minus);
+        emit_leave_runtime(Live.getValue());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_minus_body_shared());
+        emit_leave_runtime(Live.getValue());
+    }
+
+    a.bind(next);
+    mov_arg(Dst, ARG1);
+}
+
+/* ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The result is returned in ARG1 (set to THE_NON_VALUE if
+ * the call failed).
+ */
+void BeamGlobalAssembler::emit_times_guard_shared() {
+    Label generic = a.newLabel();
+
+    /* Speculatively untag and multiply. */
+    a.and_(TMP1, ARG2, imm(~_TAG_IMMED1_MASK));
+    a.asr(TMP2, ARG3, imm(_TAG_IMMED1_SIZE));
+    a.mul(TMP3, TMP1, TMP2);
+    a.smulh(TMP4, TMP1, TMP2);
+
+    /* Check that both operands are small integers. */
+    ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+    a.and_(TMP1, ARG2, ARG3);
+    a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(generic);
+
+    /* The high 65 bits of result will all be the same if no overflow
+     * occurred. Another way to say that is that the sign bit of the
+     * low 64 bits repeated 64 times must be equal to the high 64 bits
+     * of the product. */
+    a.cmp(TMP4, TMP3, arm::asr(63));
+    a.cond_ne().b(generic);
+
+    a.orr(ARG1, TMP3, imm(_TAG_IMMED1_SMALL));
+    a.ret(a64::x30);
+
+    a.bind(generic);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_mixed_times);
+
+    emit_leave_runtime();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+/* ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_times_body_shared() {
+    Label generic = a.newLabel(), error = a.newLabel();
+
+    /* Speculatively untag and multiply. */
+    a.and_(TMP1, ARG2, imm(~_TAG_IMMED1_MASK));
+    a.asr(TMP2, ARG3, imm(_TAG_IMMED1_SIZE));
+    a.mul(TMP3, TMP1, TMP2);
+    a.smulh(TMP4, TMP1, TMP2);
+
+    /* Check that both operands are integers. */
+    ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+    a.and_(TMP1, ARG2, ARG3);
+    a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(generic);
+
+    /* The high 65 bits of result will all be the same if no overflow
+     * occurred. Another way to say that is that the sign bit of the
+     * low 64 bits repeated 64 times must be equal to the high 64 bits
+     * of the product. */
+    a.cmp(TMP4, TMP3, arm::asr(63));
+    a.cond_ne().b(generic);
+
+    a.orr(ARG1, TMP3, imm(_TAG_IMMED1_SMALL));
+    a.ret(a64::x30);
+
+    a.bind(generic);
+
+    /* Save original arguments for the error path. */
+    a.stp(ARG2, ARG3, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_mixed_times);
+
+    emit_leave_runtime();
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        static const ErtsCodeMFA bif_mfa = {am_erlang, am_Times, 2};
+
+        /* Place the original arguments in x-registers. */
+        a.ldp(XREG0, XREG1, TMP_MEM1q);
+        mov_imm(ARG4, &bif_mfa);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_i_times(const ArgVal &Fail,
+                                       const ArgVal &Live,
+                                       const ArgVal &LHS,
+                                       const ArgVal &RHS,
+                                       const ArgVal &Dst) {
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    if (Fail.getValue() != 0) {
+        fragment_call(ga->get_times_guard_shared());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        fragment_call(ga->get_times_body_shared());
+    }
+
+    mov_arg(Dst, ARG1);
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * Quotient is returned in ARG1, remainder in ARG2.
+ * Error is indicated by the Z flag.
+ */
+void BeamGlobalAssembler::emit_int_div_rem_guard_shared() {
+    Label exit = a.newLabel(), generic = a.newLabel();
+
+    /* Speculatively go ahead with the division. */
+    a.asr(TMP1, ARG2, imm(_TAG_IMMED1_SIZE));
+    a.asr(TMP2, ARG3, imm(_TAG_IMMED1_SIZE));
+    a.sdiv(TMP3, TMP1, TMP2);
+    a.msub(TMP4, TMP3, TMP2, TMP1);
+
+    a.cmp(ARG3, imm(make_small(0)));
+    a.cond_eq().b(exit);
+
+    /* Check whether both operands are small integers. */
+    ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+    a.and_(TMP1, ARG2, ARG3);
+    a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(generic);
+
+    /* MIN_SMALL divided by -1 will overflow, and we'll need to fall
+     * back to the generic handler in that case. */
+    a.asr(TMP1, TMP3, imm(SMALL_BITS - 1));
+    a.cmp(TMP1, imm(1));
+    a.cond_ge().b(generic);
+
+    /* The Z flag is now clear (meaning no error). */
+
+    mov_imm(TMP1, _TAG_IMMED1_SMALL);
+    arm::Shift tagShift = arm::lsl(_TAG_IMMED1_SIZE);
+    a.orr(ARG1, TMP1, TMP3, tagShift);
+    a.orr(ARG2, TMP1, TMP4, tagShift);
+
+    a.bind(exit);
+    { a.ret(a64::x30); }
+
+    a.bind(generic);
+    {
+        emit_enter_runtime_frame();
+        emit_enter_runtime();
+
+        a.mov(ARG1, c_p);
+        lea(ARG4, TMP_MEM4q);
+        lea(ARG5, TMP_MEM5q);
+        runtime_call<5>(erts_int_div_rem);
+
+        emit_leave_runtime();
+        emit_leave_runtime_frame();
+
+        a.tst(ARG1, ARG1);
+        a.ldp(ARG1, ARG2, TMP_MEM4q);
+
+        a.ret(a64::x30);
+    }
+}
+
+/* ARG2 = LHS
+ * ARG3 = RHS
+ * ARG4 = error MFA
+ *
+ * Quotient is returned in ARG1, remainder in ARG2.
+ */
+void BeamGlobalAssembler::emit_int_div_rem_body_shared() {
+    Label div_zero = a.newLabel(), generic_div = a.newLabel(),
+          generic_error = a.newLabel();
+
+    /* Speculatively go ahead with the division. */
+    a.asr(TMP1, ARG2, imm(_TAG_IMMED1_SIZE));
+    a.asr(TMP2, ARG3, imm(_TAG_IMMED1_SIZE));
+    a.sdiv(TMP3, TMP1, TMP2);
+    a.msub(TMP4, TMP3, TMP2, TMP1);
+
+    a.cmp(ARG3, imm(make_small(0)));
+    a.cond_eq().b(div_zero);
+
+    /* Check whether both operands are small integers. */
+    ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+    a.and_(TMP1, ARG2, ARG3);
+    a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(generic_div);
+
+    /* MIN_SMALL divided by -1 will overflow, and we'll need to fall
+     * back to the generic handler in that case. */
+    a.asr(TMP1, TMP3, imm(SMALL_BITS - 1));
+    a.cmp(TMP1, imm(1));
+    a.cond_ge().b(generic_div);
+
+    mov_imm(TMP1, _TAG_IMMED1_SMALL);
+    arm::Shift tagShift = arm::lsl(_TAG_IMMED1_SIZE);
+    a.orr(ARG1, TMP1, TMP3, tagShift);
+    a.orr(ARG2, TMP1, TMP4, tagShift);
+
+    a.ret(a64::x30);
+
+    a.bind(generic_div);
+    {
+        emit_enter_runtime_frame();
+        emit_enter_runtime();
+
+        /* Save MFA and original arguments for the error path. */
+        a.stp(ARG2, ARG3, TMP_MEM1q);
+        a.str(ARG4, TMP_MEM3q);
+
+        a.mov(ARG1, c_p);
+        lea(ARG4, TMP_MEM4q);
+        lea(ARG5, TMP_MEM5q);
+        runtime_call<5>(erts_int_div_rem);
+
+        emit_leave_runtime();
+        emit_leave_runtime_frame();
+
+        a.tst(ARG1, ARG1);
+        a.ldp(ARG1, ARG2, TMP_MEM4q);
+        a.cond_eq().b(generic_error);
+
+        a.ret(a64::x30);
+    }
+
+    a.bind(div_zero);
+    {
+        mov_imm(TMP1, EXC_BADARITH);
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+        a.mov(XREG0, ARG2);
+        a.mov(XREG1, ARG3);
+        a.b(labels[raise_exception]);
+    }
+
+    a.bind(generic_error);
+    {
+        a.ldp(XREG0, XREG1, TMP_MEM1q);
+        a.ldr(ARG4, TMP_MEM3q); // MFA
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_div_rem(const ArgVal &Fail,
+                                       const ArgVal &LHS,
+                                       const ArgVal &RHS,
+                                       const ErtsCodeMFA *error_mfa) {
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    if (Fail.getValue() != 0) {
+        fragment_call(ga->get_int_div_rem_guard_shared());
+        a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.mov(ARG4, imm(error_mfa));
+        fragment_call(ga->get_int_div_rem_body_shared());
+    }
+}
+
+void BeamModuleAssembler::emit_i_rem_div(const ArgVal &Fail,
+                                         const ArgVal &Live,
+                                         const ArgVal &LHS,
+                                         const ArgVal &RHS,
+                                         const ArgVal &Remainder,
+                                         const ArgVal &Quotient) {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_rem, 2};
+
+    emit_div_rem(Fail, LHS, RHS, &bif_mfa);
+
+    mov_arg(Remainder, ARG2);
+    mov_arg(Quotient, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_div_rem(const ArgVal &Fail,
+                                         const ArgVal &Live,
+                                         const ArgVal &LHS,
+                                         const ArgVal &RHS,
+                                         const ArgVal &Quotient,
+                                         const ArgVal &Remainder) {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_div, 2};
+
+    emit_div_rem(Fail, LHS, RHS, &bif_mfa);
+
+    mov_arg(Quotient, ARG1);
+    mov_arg(Remainder, ARG2);
+}
+
+void BeamModuleAssembler::emit_i_int_div(const ArgVal &Fail,
+                                         const ArgVal &Live,
+                                         const ArgVal &LHS,
+                                         const ArgVal &RHS,
+                                         const ArgVal &Quotient) {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_div, 2};
+
+    emit_div_rem(Fail, LHS, RHS, &bif_mfa);
+
+    mov_arg(Quotient, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_rem(const ArgVal &Fail,
+                                     const ArgVal &Live,
+                                     const ArgVal &LHS,
+                                     const ArgVal &RHS,
+                                     const ArgVal &Remainder) {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_rem, 2};
+
+    emit_div_rem(Fail, LHS, RHS, &bif_mfa);
+
+    mov_arg(Remainder, ARG2);
+}
+
+void BeamModuleAssembler::emit_i_m_div(const ArgVal &Fail,
+                                       const ArgVal &Live,
+                                       const ArgVal &LHS,
+                                       const ArgVal &RHS,
+                                       const ArgVal &Dst) {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_Div, 2};
+
+    Label next = a.newLabel();
+
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    emit_enter_runtime(Live.getValue());
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_mixed_div);
+
+    emit_leave_runtime(Live.getValue());
+
+    a.cmp(ARG1, imm(THE_NON_VALUE));
+
+    if (Fail.getValue() != 0) {
+        a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.cond_ne().b(next);
+
+        mov_arg(XREG0, LHS);
+        mov_arg(XREG1, RHS);
+
+        emit_raise_exception(&bif_mfa);
+    }
+
+    a.bind(next);
+    mov_arg(Dst, ARG1);
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+template<typename T>
+void BeamGlobalAssembler::emit_bitwise_fallback_body(T(*func_ptr),
+                                                     const ErtsCodeMFA *mfa) {
+    Label error = a.newLabel();
+
+    emit_enter_runtime_frame();
+
+    /* Save original arguments for the error path. */
+    a.stp(ARG2, ARG3, TMP_MEM1q);
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(func_ptr);
+
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        /* emit_enter_runtime() was done in the module code. */
+        emit_leave_runtime(0);
+
+        /* Place the original arguments in X registers. */
+        a.ldp(XREG0, XREG1, TMP_MEM1q);
+        a.mov(ARG4, imm(mfa));
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamGlobalAssembler::emit_i_band_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_band, 2};
+    emit_bitwise_fallback_body(erts_band, &bif_mfa);
+}
+
+void BeamModuleAssembler::emit_i_band(const ArgVal &Fail,
+                                      const ArgVal &Live,
+                                      const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    Label next = a.newLabel();
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    auto dst = init_destination(Dst, ARG1);
+
+    /* TAG & TAG = TAG, so we don't need to tag it again. */
+    a.and_(ARG1, lhs.reg, rhs.reg);
+
+    /* All other term types has at least one zero in the low 4
+     * bits. Therefore, the result will be a small iff both operands
+     * are small. */
+    ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+    a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_eq().b(next);
+
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    if (Fail.getValue() != 0) {
+        emit_enter_runtime(Live.getValue());
+        a.mov(ARG1, c_p);
+        runtime_call<3>(erts_band);
+        emit_leave_runtime(Live.getValue());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_i_band_body_shared());
+        emit_leave_runtime(Live.getValue());
+    }
+
+    a.bind(next);
+    {
+        mov_var(dst, ARG1);
+        flush_var(dst);
+    }
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * Result is returned in RET.
+ */
+void BeamGlobalAssembler::emit_i_bor_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_bor, 2};
+    emit_bitwise_fallback_body(erts_bor, &bif_mfa);
+}
+
+void BeamModuleAssembler::emit_i_bor(const ArgVal &Fail,
+                                     const ArgVal &Live,
+                                     const ArgVal &LHS,
+                                     const ArgVal &RHS,
+                                     const ArgVal &Dst) {
+    Label generic = a.newLabel(), next = a.newLabel();
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    auto dst = init_destination(Dst, ARG1);
+
+    /* TAG | TAG = TAG, so we don't need to tag it again. */
+    a.orr(ARG1, lhs.reg, rhs.reg);
+
+    if (RHS.isImmed() && is_small(RHS.getValue())) {
+        a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+    } else if (LHS.isImmed() && is_small(LHS.getValue())) {
+        a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK));
+    } else {
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        a.and_(TMP1, lhs.reg, rhs.reg);
+        a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    }
+
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_eq().b(next);
+
+    a.bind(generic);
+    {
+        mov_var(ARG2, lhs);
+        mov_var(ARG3, rhs);
+
+        if (Fail.getValue() != 0) {
+            emit_enter_runtime(Live.getValue());
+            a.mov(ARG1, c_p);
+            runtime_call<3>(erts_bor);
+            emit_leave_runtime(Live.getValue());
+            emit_branch_if_not_value(ARG1,
+                                     resolve_beam_label(Fail, dispUnknown));
+        } else {
+            emit_enter_runtime(Live.getValue());
+            fragment_call(ga->get_i_bor_body_shared());
+            emit_leave_runtime(Live.getValue());
+        }
+    }
+
+    a.bind(next);
+    {
+        mov_var(dst, ARG1);
+        flush_var(dst);
+    }
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_i_bxor_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_bxor, 2};
+    emit_bitwise_fallback_body(erts_bxor, &bif_mfa);
+}
+
+void BeamModuleAssembler::emit_i_bxor(const ArgVal &Fail,
+                                      const ArgVal &Live,
+                                      const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    Label next = a.newLabel();
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    auto dst = init_destination(Dst, ARG1);
+
+    /* TAG ^ TAG = 0, so we'll need to tag it again. */
+    a.eor(ARG1, lhs.reg, rhs.reg);
+    a.orr(ARG1, ARG1, imm(_TAG_IMMED1_SMALL));
+
+    if (RHS.isImmed() && is_small(RHS.getValue())) {
+        a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+    } else if (LHS.isImmed() && is_small(LHS.getValue())) {
+        a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK));
+    } else {
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        a.and_(TMP1, lhs.reg, rhs.reg);
+        a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    }
+
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_eq().b(next);
+
+    mov_var(ARG2, lhs);
+    mov_var(ARG3, rhs);
+
+    if (Fail.getValue() != 0) {
+        emit_enter_runtime(Live.getValue());
+        runtime_call<3>(erts_bxor);
+        emit_leave_runtime(Live.getValue());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_i_bxor_body_shared());
+        emit_leave_runtime(Live.getValue());
+    }
+
+    a.bind(next);
+    {
+        mov_var(dst, ARG1);
+        flush_var(dst);
+    }
+}
+
+/*
+ * ARG1 = Src
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1. Error is indicated by
+ * THE_NON_VALUE.
+ */
+void BeamGlobalAssembler::emit_i_bnot_guard_shared() {
+    emit_enter_runtime_frame();
+
+    /* Undo the speculative inversion in module code. */
+    a.eor(ARG2, ARG1, imm(~_TAG_IMMED1_MASK));
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_bnot);
+
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+/*
+ * ARG1 = Src
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_i_bnot_body_shared() {
+    Label error = a.newLabel();
+
+    emit_enter_runtime_frame();
+
+    /* Undo the speculative inversion in module code. */
+    a.eor(ARG2, ARG1, imm(~_TAG_IMMED1_MASK));
+
+    /* Save original arguments for the error path. */
+    a.str(ARG2, TMP_MEM1q);
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_bnot);
+
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        static const ErtsCodeMFA bif_mfa = {am_erlang, am_bnot, 1};
+
+        /* emit_enter_runtime() was done in the module code. */
+        emit_leave_runtime(0);
+
+        /* Place the original arguments in X registers. */
+        a.ldr(XREG0, TMP_MEM1q);
+        mov_imm(ARG4, &bif_mfa);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bnot(const ArgVal &Fail,
+                                      const ArgVal &Live,
+                                      const ArgVal &Src,
+                                      const ArgVal &Dst) {
+    Label next = a.newLabel();
+    auto src = load_source(Src, TMP2);
+    auto dst = init_destination(Dst, ARG1);
+
+    /* Invert everything except the tag so we don't have to tag it again. */
+    a.eor(ARG1, src.reg, imm(~_TAG_IMMED1_MASK));
+
+    a.and_(TMP1, src.reg, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_eq().b(next);
+
+    if (Fail.getValue() != 0) {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_i_bnot_guard_shared());
+        emit_leave_runtime(Live.getValue());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        emit_enter_runtime(Live.getValue());
+        fragment_call(ga->get_i_bnot_body_shared());
+        emit_leave_runtime(Live.getValue());
+    }
+
+    a.bind(next);
+    mov_var(dst, ARG1);
+    flush_var(dst);
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_i_bsr_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_bsr, 2};
+    emit_bitwise_fallback_body(erts_bsr, &bif_mfa);
+}
+
+void BeamModuleAssembler::emit_i_bsr(const ArgVal &Fail,
+                                     const ArgVal &Live,
+                                     const ArgVal &LHS,
+                                     const ArgVal &RHS,
+                                     const ArgVal &Dst) {
+    Label generic = a.newLabel(), next = a.newLabel();
+    auto lhs = load_source(LHS, ARG2);
+    auto dst = init_destination(Dst, ARG1);
+
+    if (RHS.isImmed() && is_small(RHS.getValue())) {
+        Sint shift = signed_val(RHS.getValue());
+
+        if (shift >= 0 && shift < SMALL_BITS - 1) {
+            a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+            a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+            a.cond_ne().b(generic);
+
+            /* We don't need to clear the mask after shifting because
+             * _TAG_IMMED1_SMALL will set all the bits anyway. */
+            ERTS_CT_ASSERT(_TAG_IMMED1_MASK == _TAG_IMMED1_SMALL);
+            a.asr(TMP1, lhs.reg, imm(shift));
+            a.orr(dst.reg, TMP1, imm(_TAG_IMMED1_SMALL));
+
+            a.b(next);
+        } else {
+            /* Constant shift is negative or too big to fit the `asr`
+             * instruction; fall back to the generic path. */
+        }
+    }
+
+    a.bind(generic);
+    {
+        mov_var(ARG2, lhs);
+        mov_arg(ARG3, RHS);
+
+        if (Fail.getValue() != 0) {
+            emit_enter_runtime(Live.getValue());
+            a.mov(ARG1, c_p);
+            runtime_call<3>(erts_bsr);
+            emit_leave_runtime(Live.getValue());
+            emit_branch_if_not_value(ARG1,
+                                     resolve_beam_label(Fail, dispUnknown));
+        } else {
+            emit_enter_runtime(Live.getValue());
+            fragment_call(ga->get_i_bsr_body_shared());
+            emit_leave_runtime(Live.getValue());
+        }
+
+        mov_var(dst, ARG1);
+    }
+
+    a.bind(next);
+    flush_var(dst);
+}
+
+/*
+ * ARG2 = LHS
+ * ARG3 = RHS
+ *
+ * The module code must have executed emit_enter_runtime()
+ * before calling this function.
+ *
+ * The result is returned in ARG1.
+ */
+void BeamGlobalAssembler::emit_i_bsl_body_shared() {
+    static const ErtsCodeMFA bif_mfa = {am_erlang, am_bsl, 2};
+    emit_bitwise_fallback_body(erts_bsl, &bif_mfa);
+}
+
+static int count_leading_zeroes(UWord value) {
+    const int word_bits = sizeof(value) * CHAR_BIT;
+
+    if (value == 0) {
+        return word_bits;
+    }
+
+    return Support::clz(value);
+}
+
+void BeamModuleAssembler::emit_i_bsl(const ArgVal &Fail,
+                                     const ArgVal &Live,
+                                     const ArgVal &LHS,
+                                     const ArgVal &RHS,
+                                     const ArgVal &Dst) {
+    Label generic = a.newLabel(), next = a.newLabel();
+    auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);
+    auto dst = init_destination(Dst, ARG1);
+
+    bool inline_shift = true;
+    if (LHS.isImmed() && RHS.isImmed()) {
+        /* The compiler should've optimized this away, so we'll fall
+         * back to the generic path to simplify the inline
+         * implementation. */
+        inline_shift = false;
+    } else if (LHS.isLiteral() || RHS.isLiteral()) {
+        /* At least one argument is not a small. */
+        inline_shift = false;
+    } else if (LHS.isImmed() && !is_small(LHS.getValue())) {
+        /* Invalid constant. */
+        inline_shift = false;
+    } else if (RHS.isImmed() &&
+               (!is_small(RHS.getValue()) || signed_val(RHS.getValue()) < 0 ||
+                signed_val(RHS.getValue()) >= SMALL_BITS - 1)) {
+        /* Constant shift is invalid or always produces a bignum. */
+        inline_shift = false;
+    }
+
+    if (inline_shift) {
+        /* shiftLimit will be calculated as the number of leading sign
+         * bits not counting the sign bit. Note that this value is one
+         * lower than the number of leading zeros as used by the
+         * x86_64 JIT. */
+        Operand shiftLimit, shiftCount;
+
+        ASSERT(!(LHS.isImmed() && RHS.isImmed()));
+        if (LHS.isRegister()) {
+            /* Count the number of leading sign bits so we can test
+             * whether the shift will overflow. (The count does not
+             * include the sign bit.) To ensure that the tag bits are
+             * not counted, we must make sure that the topmost tag bit
+             * is equal to the inverted value of the sign bit. */
+            ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+            a.eor(TMP1, lhs.reg, lhs.reg, arm::lsr(64 - _TAG_IMMED1_SIZE));
+            a.cls(ARG4, TMP1);
+            shiftLimit = ARG4;
+
+            a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+            a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+            a.cond_ne().b(generic);
+        } else {
+            UWord value = LHS.getValue();
+
+            if (signed_val(value) < 0) {
+                value ^= ~(UWord)_TAG_IMMED1_MASK;
+            }
+
+            shiftLimit = imm(count_leading_zeroes(value) - 1);
+        }
+
+        if (RHS.isRegister()) {
+            /* Negate the tag bits and then rotate them out, forcing the
+             * comparison below to fail for non-smalls. */
+            ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+            a.eor(ARG5, rhs.reg, imm(_TAG_IMMED1_SMALL));
+            a.ror(ARG5, ARG5, imm(_TAG_IMMED1_SIZE));
+            shiftCount = ARG5;
+
+            /* Fall back to generic path when the shift magnitude is negative or
+             * greater than the leading zero count.
+             *
+             * The raw emit form is used since `shiftLimit` may be a register
+             * or immediate, and the `cmp` helper doesn't accept untyped
+             * `Operand`s. */
+            a.emit(a64::Inst::kIdCmp, ARG5, shiftLimit);
+            a.cond_hi().b(generic);
+        } else {
+            ASSERT(!shiftLimit.isImm());
+
+            shiftCount = imm(signed_val(RHS.getValue()));
+
+            a.emit(a64::Inst::kIdCmp, shiftLimit, shiftCount);
+            a.cond_lo().b(generic);
+        }
+
+        a.and_(TMP1, lhs.reg, imm(~_TAG_IMMED1_MASK));
+        a.emit(a64::Inst::kIdLsl, TMP1, TMP1, shiftCount);
+        a.orr(dst.reg, TMP1, imm(_TAG_IMMED1_SMALL));
+
+        flush_var(dst);
+        a.b(next);
+    }
+
+    a.bind(generic);
+    {
+        mov_var(ARG2, lhs);
+        mov_var(ARG3, rhs);
+
+        if (Fail.getValue() != 0) {
+            emit_enter_runtime(Live.getValue());
+            a.mov(ARG1, c_p);
+            runtime_call<3>(erts_bsl);
+            emit_leave_runtime(Live.getValue());
+            emit_branch_if_not_value(ARG1,
+                                     resolve_beam_label(Fail, dispUnknown));
+        } else {
+            emit_enter_runtime(Live.getValue());
+            fragment_call(ga->get_i_bsl_body_shared());
+            emit_leave_runtime(Live.getValue());
+        }
+
+        mov_var(dst, ARG1);
+        flush_var(dst);
+    }
+
+    a.bind(next);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_bif.cpp b/erts/emulator/beam/jit/arm/instr_bif.cpp
new file mode 100644
index 0000000000..199a517e0d
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_bif.cpp
@@ -0,0 +1,991 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "beam_common.h"
+#include "code_ix.h"
+#include "erl_bif_table.h"
+#include "erl_nfunc_sched.h"
+#include "bif.h"
+#include "erl_msacc.h"
+}
+
+/* ARG2 = argument vector, ARG4 (!) = bif function pointer
+ *
+ * Result is returned in ARG1 (will be THE_NON_VALUE if the BIF call failed). */
+void BeamGlobalAssembler::emit_i_bif_guard_shared() {
+    /* We use the X register array for the arguments for the BIF. The
+     * actual contents of the first three X registers are kept safe in
+     * callee-saved machine registers (XREG0 through XREG2).
+     */
+    ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 2);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    lea(ARG2, getXRef(0));
+    mov_imm(ARG3, 0);
+    runtime_call(ARG4, 3); /* ARG3 is never used by guard BIFs. */
+
+    emit_leave_runtime<Update::eReductions>();
+    emit_leave_runtime_frame();
+    a.ret(a64::x30);
+}
+
+/* ARG2 = argument vector, ARG4 (!) = bif function pointer
+ *
+ * Result is returned in RET. */
+void BeamGlobalAssembler::emit_i_bif_body_shared() {
+    Label error = a.newLabel();
+
+    /* See comment in emit_i_bif_guard_shared. */
+    ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 2);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eReductions>();
+
+    /* Save current BIF for the error path. */
+    a.mov(ARG1, c_p);
+    lea(ARG2, getXRef(0));
+    a.str(ARG4, TMP_MEM1q);
+    mov_imm(ARG3, 0); /* ARG3 is never used by guard BIFs. */
+
+    runtime_call(ARG4, 3);
+    emit_branch_if_not_value(ARG1, error);
+
+    emit_leave_runtime<Update::eReductions>();
+
+    emit_leave_runtime_frame();
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        /* Find the correct MFA from the BIF's function address. */
+        a.ldr(ARG1, TMP_MEM1q);
+        runtime_call<1>(ubif2mfa);
+
+        /* The argument registers must be reloaded on error, as the machine
+         * registers may contain garbage, which will later be swapped into the
+         * register array in the `raise_exception` fragment. */
+        emit_leave_runtime<Update::eReductions | Update::eXRegs>(3);
+        emit_leave_runtime_frame();
+
+        a.mov(ARG4, ARG1);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bif1(const ArgVal &Src1,
+                                      const ArgVal &Fail,
+                                      const ArgVal &Bif,
+                                      const ArgVal &Dst) {
+    auto src1 = load_source(Src1, TMP1);
+
+    a.str(src1.reg, getXRef(0));
+
+    emit_i_bif(Fail, Bif, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bif2(const ArgVal &Src1,
+                                      const ArgVal &Src2,
+                                      const ArgVal &Fail,
+                                      const ArgVal &Bif,
+                                      const ArgVal &Dst) {
+    auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2);
+
+    a.stp(src1.reg, src2.reg, getXRef(0));
+
+    emit_i_bif(Fail, Bif, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bif3(const ArgVal &Src1,
+                                      const ArgVal &Src2,
+                                      const ArgVal &Src3,
+                                      const ArgVal &Fail,
+                                      const ArgVal &Bif,
+                                      const ArgVal &Dst) {
+    auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2);
+    auto src3 = load_source(Src3, TMP3);
+
+    a.stp(src1.reg, src2.reg, getXRef(0));
+    a.str(src3.reg, getXRef(2));
+
+    emit_i_bif(Fail, Bif, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bif(const ArgVal &Fail,
+                                     const ArgVal &Bif,
+                                     const ArgVal &Dst) {
+    mov_arg(ARG4, Bif);
+
+    if (Fail.getValue() != 0) {
+        fragment_call(ga->get_i_bif_guard_shared());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        fragment_call(ga->get_i_bif_body_shared());
+    }
+
+    mov_arg(Dst, ARG1);
+}
+
+/*
+ * Emit code for guard BIFs that can't fail (e.g. is_list/1).  We
+ * don't need to test for failure.
+ */
+
+void BeamModuleAssembler::emit_nofail_bif1(const ArgVal &Src1,
+                                           const ArgVal &Bif,
+                                           const ArgVal &Dst) {
+    auto src1 = load_source(Src1, TMP1);
+
+    a.str(src1.reg, getXRef(0));
+
+    mov_arg(ARG4, Bif);
+    fragment_call(ga->get_i_bif_guard_shared());
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_nofail_bif2(const ArgVal &Src1,
+                                           const ArgVal &Src2,
+                                           const ArgVal &Bif,
+                                           const ArgVal &Dst) {
+    auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2);
+
+    a.stp(src1.reg, src2.reg, getXRef(0));
+
+    mov_arg(ARG4, Bif);
+    fragment_call(ga->get_i_bif_guard_shared());
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_length_setup(const ArgVal &Fail,
+                                              const ArgVal &Live,
+                                              const ArgVal &Src) {
+    mov_arg(TMP1, Src);
+    mov_imm(TMP2, make_small(0));
+
+    /* Store trap state after the currently live registers. There are
+     * 3 extra registers beyond the ordinary ones that we're free to
+     * use for whatever purpose. */
+    ERTS_CT_ASSERT(ERTS_X_REGS_ALLOCATED - MAX_REG >= 3);
+    mov_arg(ArgVal(ArgVal::XReg, Live.getValue() + 0), TMP1);
+    mov_arg(ArgVal(ArgVal::XReg, Live.getValue() + 1), TMP2);
+
+    /* Store original argument. This is only needed for exceptions and can be
+     * safely skipped in guards. */
+    if (Fail.getValue() == 0) {
+        mov_arg(ArgVal(ArgVal::XReg, Live.getValue() + 2), TMP1);
+    }
+}
+
+/* ARG2 = live registers, ARG3 = entry address
+ *
+ * Result is returned in RET. */
+void BeamGlobalAssembler::emit_i_length_common(Label fail, int state_size) {
+    Label trap_or_error = a.newLabel();
+
+    ASSERT(state_size >= 2 && state_size <= ERTS_X_REGS_ALLOCATED - MAX_REG);
+
+    /* Save arguments for error/trapping path. */
+    a.stp(ARG2, ARG3, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eReductions | Update::eXRegs>();
+
+    a.mov(ARG1, c_p);
+    lea(TMP1, getXRef(0));
+    a.add(ARG2, TMP1, ARG2, arm::lsl(3));
+    runtime_call<2>(erts_trapping_length_1);
+
+    emit_branch_if_not_value(ARG1, trap_or_error);
+
+    emit_leave_runtime<Update::eReductions | Update::eXRegs>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+
+    a.bind(trap_or_error);
+    {
+        a.ldp(ARG2, ARG3, TMP_MEM1q);
+        a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+        a.cmp(TMP1, imm(TRAP));
+        a.cond_ne().b(fail);
+
+        emit_leave_runtime<Update::eReductions | Update::eXRegs>();
+        emit_leave_runtime_frame();
+
+        /* The trap state is stored in the registers above the current live
+         * ones, so we add the state size (in words) to keep it alive. */
+        a.add(ARG2, ARG2, imm(state_size));
+
+        a.str(ZERO, arm::Mem(c_p, offsetof(Process, current)));
+        a.str(ARG2, arm::Mem(c_p, offsetof(Process, arity)));
+
+        /* We'll find our way back through the entry address (ARG3). */
+        a.b(labels[context_switch_simplified]);
+    }
+}
+
+/* ARG2 = live registers, ARG3 = entry address
+ *
+ * Result is returned in RET. */
+void BeamGlobalAssembler::emit_i_length_body_shared() {
+    Label error = a.newLabel();
+    /* `state_size = 3` to include the original argument. */
+    emit_i_length_common(error, 3);
+
+    a.bind(error);
+    {
+        static const ErtsCodeMFA bif_mfa = {am_erlang, am_length, 1};
+
+        /* Move the original argument to x0. It's stored in the third word of
+         * the trap state. */
+        lea(TMP1, getXRef(0));
+        a.add(ARG2, TMP1, ARG2, arm::lsl(3));
+        a.ldr(TMP1, arm::Mem(ARG2, sizeof(Eterm[2])));
+
+        emit_leave_runtime<Update::eReductions | Update::eXRegs>();
+        emit_leave_runtime_frame();
+
+        a.mov(XREG0, TMP1);
+
+        mov_imm(ARG4, &bif_mfa);
+        emit_raise_exception();
+    }
+}
+
+/* ARG2 = live registers, ARG3 = entry address
+ *
+ * Result is returned in ARG. Error is indicated by THE_NON_VALUE. */
+void BeamGlobalAssembler::emit_i_length_guard_shared() {
+    Label error = a.newLabel();
+
+    emit_i_length_common(error, 2);
+
+    a.bind(error);
+    {
+        emit_leave_runtime<Update::eReductions | Update::eXRegs>();
+        emit_leave_runtime_frame();
+
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_i_length(const ArgVal &Fail,
+                                        const ArgVal &Live,
+                                        const ArgVal &Dst) {
+    Label entry = a.newLabel();
+
+    a.bind(entry);
+
+    mov_arg(ARG2, Live);
+    a.adr(ARG3, entry);
+    if (Fail.getValue() != 0) {
+        fragment_call(ga->get_i_length_guard_shared());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        fragment_call(ga->get_i_length_body_shared());
+    }
+
+    mov_arg(Dst, ARG1);
+}
+
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+
+static Eterm debug_call_light_bif(Process *c_p,
+                                  Eterm *reg,
+                                  ErtsCodePtr I,
+                                  ErtsBifFunc vbf) {
+    Eterm result;
+
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    {
+        ERTS_CHK_MBUF_SZ(c_p);
+        ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+        result = vbf(c_p, reg, I);
+        ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+        ERTS_CHK_MBUF_SZ(c_p);
+
+        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+        ERTS_HOLE_CHECK(c_p);
+    }
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+
+    return result;
+}
+#endif
+
+/* It is important that the below code is as optimized as possible.
+ * When doing any changes, make sure to look at the estone bif_dispatch
+ * benchmark to make sure you don't introduce any regressions.
+ *
+ * ARG3 = entry
+ * ARG4 = export entry
+ * ARG8 = BIF pointer
+ */
+void BeamGlobalAssembler::emit_call_light_bif_shared() {
+    /* We use the HTOP, FCALLS, and XREG1 registers as they are not
+     * used on the runtime-stack and are caller save. */
+
+    arm::Gp I = HTOP, exp = FCALLS;
+
+    Label error = a.newLabel(), trace = a.newLabel(), trap = a.newLabel(),
+          yield = a.newLabel(), call_save_calls = a.newLabel(),
+          call_bif = a.newLabel(), gc_after_bif_call = a.newLabel(),
+          check_bif_return = a.newLabel();
+
+    /* Check if we should trace this bif call */
+    a.ldr(TMP1.w(), arm::Mem(ARG4, offsetof(Export, is_bif_traced)));
+    a.cbnz(TMP1, trace);
+
+    a.subs(FCALLS, FCALLS, imm(1));
+    a.cond_le().b(yield);
+    {
+        emit_enter_runtime_frame();
+        emit_enter_runtime<Update::eReductions | Update::eStack |
+                           Update::eHeap | Update::eXRegs>();
+
+        /* Spill the arguments we may need on the error path. */
+        a.mov(I, ARG3);
+        a.mov(exp, ARG4);
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+        {
+            Label skip_msacc = a.newLabel();
+
+            a.ldr(TMP1, erts_msacc_cache);
+            a.cbz(TMP1, skip_msacc);
+
+            /* The values of the X registers are in the X register
+             * array, so we can use XREG0 to save the contents of ARG1
+             * during the call. */
+            a.mov(XREG0, ARG1);
+            a.ldr(ARG1, erts_msacc_cache);
+            a.ldr(ARG2, arm::Mem(ARG4, offsetof(Export, info.mfa.module)));
+            a.mov(ARG3, XREG0);
+            runtime_call<3>(erts_msacc_set_bif_state);
+
+            a.mov(ARG3, I);
+            a.mov(ARG1, XREG0);
+
+            a.bind(skip_msacc);
+        }
+#endif
+
+        /* Check if we need to call save_calls */
+        emit_branch_if_eq(active_code_ix,
+                          ERTS_SAVE_CALLS_CODE_IX,
+                          call_save_calls);
+        a.bind(call_bif);
+
+        a.ldr(ARG1, arm::Mem(c_p, offsetof(Process, mbuf)));
+        a.str(ARG1, TMP_MEM1q);
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+        a.mov(ARG4, ARG8);
+        runtime_call<4>(debug_call_light_bif);
+#else
+        runtime_call(ARG8, 3);
+#endif
+
+        /* ERTS_IS_GC_DESIRED_INTERNAL */
+        {
+            a.ldr(ARG2, arm::Mem(c_p, offsetof(Process, stop)));
+            a.mov(ARG3, ARG1);
+            a.ldr(ARG5, arm::Mem(c_p, offsetof(Process, htop)));
+
+            /* Test whether binary heap size should trigger gc */
+            a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, bin_vheap_sz)));
+            a.ldr(TMP2, arm::Mem(c_p, offsetof(Process, off_heap.overhead)));
+            a.cmp(TMP2, TMP1);
+            a.cond_hi().b(gc_after_bif_call);
+
+            /* Test whether GC is forced. */
+            a.ldr(TMP1.w(), arm::Mem(c_p, offsetof(Process, flags)));
+            a.tbnz(TMP1, Support::ctz(F_FORCE_GC), gc_after_bif_call);
+
+            /* Test if heap fragment size is larger than remaining heap size. */
+            a.sub(TMP1, ARG2, ARG5);
+            a.asr(TMP1, TMP1, imm(3));
+            a.ldr(TMP2, arm::Mem(c_p, offsetof(Process, mbuf_sz)));
+            a.cmp(TMP1, TMP2);
+            a.cond_lt().b(gc_after_bif_call);
+        }
+        /*
+           ARG2 is set to E
+           ARG3 is set to bif return
+           ARG5 is set to HTOP
+
+           HTOP is exp
+           E_saved|E is I
+        */
+        a.bind(check_bif_return);
+        emit_branch_if_not_value(ARG3, trap);
+
+        a.mov(HTOP, ARG5);
+        a.mov(E, ARG2);
+
+        /* We must update the active code index in case another process has
+         * loaded new code, as the result of this BIF may be observable on both
+         * ends.
+         *
+         * It doesn't matter whether the BIF modifies anything; if process A
+         * loads new code and calls erlang:monotonic_time/0 soon after, we'd
+         * break the illusion of atomic upgrades if process B still ran old code
+         * after seeing a later timestamp from its own call to
+         * erlang:monotonic_time/0. */
+
+        emit_leave_runtime<Update::eReductions | Update::eCodeIndex |
+                           Update::eXRegs>();
+        emit_leave_runtime_frame();
+
+        a.mov(XREG0, ARG3);
+        a.ret(a64::x30);
+
+        a.bind(call_save_calls);
+        {
+            /* Stash the bif function pointer */
+            a.str(ARG8, TMP_MEM1q);
+
+            a.mov(ARG1, c_p);
+            a.mov(ARG2, exp);
+            runtime_call<2>(save_calls);
+
+            /* Restore bif pointer and ARG3 to the values expected by the bif
+             * call */
+            a.ldr(ARG8, TMP_MEM1q);
+            a.mov(ARG3, I);
+
+            a.b(call_bif);
+        }
+
+        a.bind(trap);
+        {
+            a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+            emit_branch_if_ne(TMP1, TRAP, error);
+
+            emit_leave_runtime<Update::eHeap | Update::eStack | Update::eXRegs |
+                               Update::eReductions | Update::eCodeIndex>();
+            emit_leave_runtime_frame();
+
+            /* Push our return address to the Erlang stack and trap out.
+             *
+             * The BIF_TRAP macros all set up c_p->arity and c_p->current, so
+             * we can use a simplified context switch. */
+            emit_enter_erlang_frame();
+            a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+            a.b(labels[context_switch_simplified]);
+        }
+
+        a.bind(error);
+        {
+            a.mov(ARG4, exp);
+            a.mov(ARG2, I);
+
+            emit_leave_runtime<Update::eHeap | Update::eStack | Update::eXRegs |
+                               Update::eReductions | Update::eCodeIndex>();
+            emit_leave_runtime_frame();
+
+            /* raise_exception_shared expects current PC in ARG2 and MFA in
+             * ARG4. */
+            add(ARG4, ARG4, offsetof(Export, info.mfa));
+            a.b(labels[raise_exception_shared]);
+        }
+
+        a.bind(gc_after_bif_call);
+        {
+            a.mov(ARG1, c_p);
+            a.ldr(ARG2, TMP_MEM1q);
+            /* ARG3 already contains result */
+            load_x_reg_array(ARG4);
+            a.ldr(ARG5, arm::Mem(exp, offsetof(Export, info.mfa.arity)));
+            runtime_call<5>(erts_gc_after_bif_call_lhf);
+            a.ldr(ARG2, arm::Mem(c_p, offsetof(Process, stop)));
+            a.mov(ARG3, ARG1);
+            a.ldr(ARG5, arm::Mem(c_p, offsetof(Process, htop)));
+            a.b(check_bif_return);
+        }
+    }
+
+    a.bind(trace);
+    {
+        /* Call the export entry instead of the BIF. */
+        branch(emit_setup_export_call(ARG4));
+    }
+
+    a.bind(yield);
+    {
+        a.ldr(ARG2, arm::Mem(ARG4, offsetof(Export, info.mfa.arity)));
+        lea(ARG4, arm::Mem(ARG4, offsetof(Export, info.mfa)));
+        a.str(ARG2, arm::Mem(c_p, offsetof(Process, arity)));
+        a.str(ARG4, arm::Mem(c_p, offsetof(Process, current)));
+
+        /* We'll find our way back through ARG3 (entry address). */
+        a.b(labels[context_switch_simplified]);
+    }
+}
+
+void BeamModuleAssembler::emit_call_light_bif(const ArgVal &Bif,
+                                              const ArgVal &Exp) {
+    Label entry = a.newLabel();
+
+    a.bind(entry);
+
+    mov_arg(ARG4, Exp);
+    mov_arg(ARG8, Bif);
+    a.adr(ARG3, entry);
+
+    fragment_call(ga->get_call_light_bif_shared());
+}
+
+void BeamModuleAssembler::emit_send() {
+    Label entry = a.newLabel();
+
+    /* This is essentially a mirror of call_light_bif, there's no point to
+     * specializing send/2 anymore. We do it here because it's far more work to
+     * do it in the loader. */
+    a.bind(entry);
+
+    a.ldr(ARG4, embed_constant(BIF_TRAP_EXPORT(BIF_send_2), disp32K));
+    a.ldr(ARG8, embed_constant(send_2, disp32K));
+    a.adr(ARG3, entry);
+
+    fragment_call(ga->get_call_light_bif_shared());
+}
+
+void BeamGlobalAssembler::emit_bif_nif_epilogue(void) {
+    Label check_trap = a.newLabel(), trap = a.newLabel(), error = a.newLabel();
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+    {
+        Label skip_msacc = a.newLabel();
+
+        a.ldr(TMP1, erts_msacc_cache);
+        a.cbz(TMP1, skip_msacc);
+
+        /* The values of the X registers are in the X register array,
+         * so we can use XREG0 to save the contents of ARG1 during the
+         * call. */
+        a.mov(XREG0, ARG1);
+        a.ldr(ARG1, erts_msacc_cache);
+        mov_imm(ARG2, ERTS_MSACC_STATE_EMULATOR);
+        mov_imm(ARG3, 1);
+        runtime_call<3>(erts_msacc_set_state_m__);
+        a.mov(ARG1, XREG0);
+
+        a.bind(skip_msacc);
+    }
+#endif
+
+    /* Another process may have loaded new code and somehow notified us through
+     * this call, so we must update the active code index. */
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions | Update::eCodeIndex>();
+
+    emit_branch_if_not_value(ARG1, check_trap);
+
+    comment("Do return and dispatch to it");
+    a.mov(XREG0, ARG1);
+
+    emit_leave_erlang_frame();
+    a.ret(a64::x30);
+
+    a.bind(check_trap);
+    a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+    a.cmp(TMP1, imm(TRAP));
+    a.cond_ne().b(error);
+    {
+        comment("yield");
+
+        comment("test trap to hibernate");
+        a.ldr(TMP1.w(), arm::Mem(c_p, offsetof(Process, flags)));
+        a.tbz(TMP1, imm(Support::ctz(F_HIBERNATE_SCHED)), trap);
+
+        comment("do hibernate trap");
+        a.and_(TMP1, TMP1, imm(~F_HIBERNATE_SCHED));
+        a.str(TMP1.w(), arm::Mem(c_p, offsetof(Process, flags)));
+        a.b(labels[do_schedule]);
+    }
+
+    a.bind(trap);
+    {
+        comment("do normal trap");
+
+        /* The BIF_TRAP macros all set up c_p->arity and c_p->current, so we
+         * can use a simplified context switch. */
+        a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+        a.b(labels[context_switch_simplified]);
+    }
+
+    a.bind(error);
+    {
+        a.mov(ARG2, E);
+
+        emit_enter_runtime();
+
+        a.mov(ARG1, c_p);
+        runtime_call<2>(erts_printable_return_address);
+
+        emit_leave_runtime();
+
+        a.mov(ARG2, ARG1);
+        a.ldr(ARG4, arm::Mem(c_p, offsetof(Process, current)));
+        a.b(labels[raise_exception_shared]);
+    }
+}
+
+/* Used by call_bif, dispatch_bif, and export_trampoline.
+ *
+ * Note that we don't check reductions here as we may have jumped here through
+ * interpreted code (e.g. an ErtsNativeFunc or export entry) and it's very
+ * tricky to yield back. Reductions are checked in module code instead.
+ *
+ * ARG2 = BIF MFA
+ * ARG3 = I (rip), doesn't need to point past an MFA
+ * ARG4 = function to be called */
+void BeamGlobalAssembler::emit_call_bif_shared(void) {
+    /* "Heavy" BIFs need up-to-date values for `c_p->i`, `c_p->current`, and
+     * `c_p->arity`. */
+
+    emit_enter_runtime_frame();
+    a.str(ARG2, arm::Mem(c_p, offsetof(Process, current)));
+    /* `call_bif` wants arity in ARG5. */
+    a.ldr(ARG5, arm::Mem(ARG2, offsetof(ErtsCodeMFA, arity)));
+    a.str(ARG5, arm::Mem(c_p, offsetof(Process, arity)));
+    a.str(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+
+    /* The corresponding leave can be found in the epilogue. */
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+    {
+        Label skip_msacc = a.newLabel();
+
+        a.ldr(TMP1, erts_msacc_cache);
+        a.cbz(TMP1, skip_msacc);
+
+        /* The values of the X registers are in the X register array,
+         * so we can use XREG0 through XREG2 to save the contents of
+         * the ARG* registers during the call. */
+        a.mov(XREG0, ARG3);
+        a.mov(XREG1, ARG4);
+        a.mov(XREG2, ARG5);
+
+        a.ldr(ARG1, erts_msacc_cache);
+        a.ldr(ARG2, arm::Mem(ARG2, offsetof(ErtsCodeMFA, module)));
+        a.mov(ARG3, ARG4);
+        runtime_call<3>(erts_msacc_set_bif_state);
+
+        a.mov(ARG3, XREG0);
+        a.mov(ARG4, XREG1);
+        a.mov(ARG5, XREG2);
+
+        a.bind(skip_msacc);
+    }
+#endif
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    /* ARG3 (I), ARG4 (func), and ARG5 (arity) have already been provided. */
+    runtime_call<5>(beam_jit_call_bif);
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+    /* The values of the X registers are in the X register array, so we can use
+     * XREG0 to save the contents of ARG1 during the call. */
+    a.mov(XREG0, ARG1);
+    lea(ARG1, erts_msacc_cache);
+    runtime_call<1>(erts_msacc_update_cache);
+    a.mov(ARG1, XREG0);
+#endif
+
+    emit_leave_runtime_frame();
+    emit_bif_nif_epilogue();
+}
+
+void BeamGlobalAssembler::emit_dispatch_bif(void) {
+    /* c_p->i points into the trampoline of a ErtsNativeFunc, right after the
+     * `info` structure. */
+    a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+
+    ERTS_CT_ASSERT(offsetof(ErtsNativeFunc, trampoline.trace) ==
+                   sizeof(ErtsCodeInfo));
+
+    ssize_t mfa_offset = offsetof(ErtsNativeFunc, trampoline.trace) -
+                         offsetof(ErtsNativeFunc, trampoline.info.mfa);
+
+    a.sub(ARG2, ARG3, imm(mfa_offset));
+
+    ssize_t dfunc_offset = offsetof(ErtsNativeFunc, trampoline.dfunc) -
+                           offsetof(ErtsNativeFunc, trampoline.trace);
+    a.ldr(ARG4, arm::Mem(ARG3, dfunc_offset));
+
+    a.b(labels[call_bif_shared]);
+}
+
+/* This is only used for opcode compatibility with the interpreter, it's never
+ * actually called. */
+void BeamModuleAssembler::emit_call_bif(const ArgVal &Func) {
+    (void)Func;
+
+    emit_nyi("emit_call_bif");
+}
+
+void BeamModuleAssembler::emit_call_bif_mfa(const ArgVal &M,
+                                            const ArgVal &F,
+                                            const ArgVal &A) {
+    BeamInstr func;
+    Export *e;
+
+    e = erts_active_export_entry(M.getValue(), F.getValue(), A.getValue());
+    ASSERT(e != NULL && e->bif_number != -1);
+
+    func = (BeamInstr)bif_table[e->bif_number].f;
+
+    a.adr(ARG3, currLabel);
+    a.sub(ARG2, ARG3, imm(sizeof(ErtsCodeMFA)));
+    a.mov(ARG4, imm(func));
+
+    a.b(resolve_fragment(ga->get_call_bif_shared(), disp128MB));
+}
+
+void BeamGlobalAssembler::emit_call_nif_early() {
+    a.mov(ARG2, a64::x30);
+    a.sub(ARG2, ARG2, imm(BEAM_ASM_BP_RETURN_OFFSET + sizeof(ErtsCodeInfo)));
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_call_nif_early);
+
+    emit_leave_runtime();
+
+    /* Emulate `emit_call_nif`, loading the current (phony) instruction
+     * pointer into ARG3.
+     *
+     * Note that we "inherit" the frame that was pushed to the stack prior to
+     * running the breakpoint instruction, discarding the current content of
+     * LR (x30). */
+    a.mov(ARG3, ARG1);
+    a.b(labels[call_nif_shared]);
+}
+
+/* Used by call_nif, call_nif_early, and dispatch_nif.
+ *
+ * Note that we don't check reductions here as we may have jumped here through
+ * interpreted code (e.g. an ErtsNativeFunc or export entry) and it's very
+ * tricky to yield back. Reductions are checked in module code instead.
+ *
+ * ARG3 = current I, just past the end of an ErtsCodeInfo. */
+void BeamGlobalAssembler::emit_call_nif_shared(void) {
+    /* The corresponding leave can be found in the epilogue. */
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+    { /* ToDO: x86 -> arm */
+        Label skip_msacc = a.newLabel();
+
+        a.ldr(TMP1, erts_msacc_cache);
+        a.cbz(TMP1, skip_msacc);
+
+        /* The values of the X registers are in the X register array,
+         * so we can use XREG0 to save the contents of ARG3 during the
+         * call. */
+        a.mov(XREG0, ARG3);
+        a.ldr(ARG1, erts_msacc_cache);
+        mov_imm(ARG2, ERTS_MSACC_STATE_NIF);
+        mov_imm(ARG3, 1);
+        runtime_call<3>(erts_msacc_set_state_m__);
+        a.mov(ARG3, XREG0);
+
+        a.bind(skip_msacc);
+    }
+#endif
+
+    a.mov(ARG1, c_p);
+    a.mov(ARG2, ARG3);
+    load_x_reg_array(ARG3);
+    a.ldr(ARG4, arm::Mem(ARG2, 8 + BEAM_ASM_FUNC_PROLOGUE_SIZE));
+    a.ldr(ARG5, arm::Mem(ARG2, 16 + BEAM_ASM_FUNC_PROLOGUE_SIZE));
+    a.ldr(ARG6, arm::Mem(ARG2, 24 + BEAM_ASM_FUNC_PROLOGUE_SIZE));
+    runtime_call<5>(beam_jit_call_nif);
+
+    emit_bif_nif_epilogue();
+}
+
+void BeamGlobalAssembler::emit_dispatch_nif(void) {
+    /* c_p->i points into the trampoline of a ErtsNativeFunc, right after the
+     * `info` structure.
+     *
+     * ErtsNativeFunc already follows the NIF call layout, so we don't need to
+     * do anything beyond loading the address. */
+    a.ldr(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+    a.b(labels[call_nif_shared]);
+}
+
+void BeamGlobalAssembler::emit_call_nif_yield_helper() {
+    Label yield = a.newLabel();
+
+    a.subs(FCALLS, FCALLS, imm(1));
+    a.cond_le().b(yield);
+    a.b(labels[call_nif_shared]);
+
+    a.bind(yield);
+    {
+        int mfa_offset = sizeof(ErtsCodeMFA);
+        int arity_offset = offsetof(ErtsCodeMFA, arity) - mfa_offset;
+
+        a.ldur(TMP1, arm::Mem(ARG3, arity_offset));
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, arity)));
+
+        a.sub(TMP1, ARG3, imm(mfa_offset));
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, current)));
+
+        /* Yield to `dispatch` rather than `entry` to avoid pushing too many
+         * frames to the stack. See `emit_call_nif` for details. */
+        a.add(ARG3, ARG3, imm(BEAM_ASM_FUNC_PROLOGUE_SIZE + sizeof(UWord[4])));
+        a.b(labels[context_switch_simplified]);
+    }
+}
+
+/* WARNING: This stub is memcpy'd, so all code herein must be explicitly
+ * position-independent. */
+void BeamModuleAssembler::emit_call_nif(const ArgVal &Func,
+                                        const ArgVal &NifMod,
+                                        const ArgVal &DirtyFunc) {
+    Label entry = a.newLabel(), dispatch = a.newLabel();
+
+#ifdef DEBUG
+    size_t entry_offset = a.offset();
+#endif
+
+    ASSERT(BEAM_ASM_FUNC_PROLOGUE_SIZE ==
+           (a.offset() - code.labelOffsetFromBase(currLabel)));
+
+    /* The start of this function must mimic the layout of ErtsNativeFunc.
+     *
+     * We jump here on the very first entry. */
+    a.bind(entry);
+    {
+        a.b(dispatch); /* call_op */
+
+        a.align(kAlignCode, 8);
+
+        /* ErtsNativeFunc.func */
+        a.embedUInt64(Func.getValue());
+
+        /* ErtsNativeFunc.m */
+        a.embedUInt64(NifMod.getValue());
+
+        /* ErtsNativeFunc.dfunc */
+        a.embedUInt64(DirtyFunc.getValue());
+    }
+
+    /* `emit_call_nif_yield_helper` relies on this to compute the address of
+     * `dispatch` */
+    ASSERT(a.offset() == entry_offset + sizeof(UWord[4]));
+
+    a.bind(dispatch);
+    {
+        a.adr(ARG3, currLabel);
+        pic_jmp(ga->get_call_nif_yield_helper());
+    }
+}
+
+static ErtsCodePtr get_on_load_address(Process *c_p, Eterm module) {
+    const Module *modp = erts_get_module(module, erts_active_code_ix());
+
+    if (modp && modp->on_load) {
+        const BeamCodeHeader *hdr = (modp->on_load)->code_hdr;
+
+        if (hdr) {
+            return erts_codeinfo_to_code(hdr->on_load);
+        }
+    }
+
+    c_p->freason = BADARG;
+
+    return NULL;
+}
+
+/* Implements the internal and undocumented erlang:call_on_load_function/1,
+ * which is very tricky to implement as a BIF. */
+void BeamModuleAssembler::emit_i_call_on_load_function() {
+    static ErtsCodeMFA mfa = {am_erlang, am_call_on_load_function, 1};
+    Label next = a.newLabel();
+
+    a.mov(ARG2, XREG0);
+
+    /* The first X register must be preserved for the error path. */
+    emit_enter_runtime(1);
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(get_on_load_address);
+
+    emit_leave_runtime(1);
+
+    a.cbnz(ARG1, next);
+    emit_raise_exception(&mfa);
+
+    a.bind(next);
+    erlang_call(ARG1);
+}
+
+void BeamModuleAssembler::emit_i_load_nif() {
+    static ErtsCodeMFA mfa = {am_erlang, am_load_nif, 2};
+
+    Label entry = a.newLabel(), next = a.newLabel(), schedule = a.newLabel();
+
+    a.bind(entry);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>(2);
+
+    a.mov(ARG1, c_p);
+    a.adr(ARG2, currLabel);
+    load_x_reg_array(ARG3);
+    runtime_call<3>(beam_jit_load_nif);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>(2);
+
+    a.cmp(ARG1, imm(RET_NIF_yield));
+    a.cond_eq().b(schedule);
+
+    a.cmp(ARG1, imm(RET_NIF_success));
+    a.cond_eq().b(next);
+
+    emit_raise_exception(currLabel, &mfa);
+
+    a.bind(schedule);
+    {
+        a.adr(ARG3, entry);
+        a.b(resolve_fragment(ga->get_context_switch_simplified(), disp128MB));
+    }
+
+    a.bind(next);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_bs.cpp b/erts/emulator/beam/jit/arm/instr_bs.cpp
new file mode 100644
index 0000000000..c8cdc34989
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_bs.cpp
@@ -0,0 +1,1417 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "erl_binary.h"
+#include "erl_bits.h"
+#include "beam_common.h"
+}
+
+/* Clobbers TMP1+TMP2
+ *
+ * If max_size > 0, we jump to the fail label when Size > max_size
+ *
+ * Returns -1 when the field check always fails, 1 if it may fail, and 0 if it
+ * never fails. */
+int BeamModuleAssembler::emit_bs_get_field_size(const ArgVal &Size,
+                                                int unit,
+                                                Label fail,
+                                                const arm::Gp &out) {
+    if (Size.isImmed()) {
+        if (is_small(Size.getValue())) {
+            Sint sval = signed_val(Size.getValue());
+
+            if (sval < 0) {
+                /* badarg */
+            } else if (sval > (MAX_SMALL / unit)) {
+                /* system_limit */
+            } else {
+                mov_imm(out, sval * unit);
+                return 0;
+            }
+        }
+
+        a.b(fail);
+        return -1;
+    } else {
+        auto size_reg = load_source(Size, TMP2);
+
+        /* Negating the tag bits lets us guard against non-smalls, negative
+         * numbers, and overflow with a single `tst` instruction. */
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        ASSERT(unit <= 1024);
+
+        a.eor(out, size_reg.reg, imm(_TAG_IMMED1_SMALL));
+        a.tst(out, imm(0xFFF0000000000000UL | _TAG_IMMED1_MASK));
+
+        if (unit == 0) {
+            /* Silly but legal.*/
+            mov_imm(out, 0);
+        } else if (Support::isPowerOf2(unit)) {
+            int trailing_bits = Support::ctz<Eterm>(unit);
+
+            /* The tag bits were cleared out by the argument check, so all we
+             * need to do is shift the result into place. */
+            if (trailing_bits < _TAG_IMMED1_SIZE) {
+                a.lsr(out, out, imm(_TAG_IMMED1_SIZE - trailing_bits));
+            } else if (trailing_bits > _TAG_IMMED1_SIZE) {
+                a.lsl(out, out, imm(trailing_bits - _TAG_IMMED1_SIZE));
+            }
+        } else {
+            if (unit >= (1 << _TAG_IMMED1_SIZE)) {
+                mov_imm(TMP1, unit >> _TAG_IMMED1_SIZE);
+            } else {
+                a.lsr(out, out, imm(_TAG_IMMED1_SIZE));
+                mov_imm(TMP1, unit);
+            }
+
+            a.mul(out, out, TMP1);
+        }
+
+        a.cond_ne().b(fail);
+
+        return 1;
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_init_heap(const ArgVal &Size,
+                                              const ArgVal &Heap,
+                                              const ArgVal &Live,
+                                              const ArgVal &Dst) {
+    mov_arg(ARG4, Size);
+    mov_arg(ARG5, Heap);
+    mov_arg(ARG6, Live);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue());
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    load_erl_bits_state(ARG3);
+    runtime_call<6>(beam_jit_bs_init);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue());
+
+    mov_arg(Dst, ARG1);
+}
+
+/* Set the error reason when a size check has failed. */
+void BeamGlobalAssembler::emit_bs_size_check_shared() {
+    emit_enter_runtime_frame();
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(beam_jit_bs_field_size_argument_error);
+
+    emit_leave_runtime(0);
+    emit_leave_runtime_frame();
+
+    mov_imm(ARG4, 0);
+    a.b(labels[raise_exception]);
+}
+
+void BeamModuleAssembler::emit_i_bs_init_fail_heap(const ArgVal &Size,
+                                                   const ArgVal &Heap,
+                                                   const ArgVal &Fail,
+                                                   const ArgVal &Live,
+                                                   const ArgVal &Dst) {
+    Label fail;
+
+    if (Fail.getValue() != 0) {
+        fail = resolve_beam_label(Fail, dispUnknown);
+    } else {
+        fail = a.newLabel();
+    }
+
+    if (emit_bs_get_field_size(Size, 1, fail, ARG4) >= 0) {
+        mov_arg(ARG5, Heap);
+        mov_arg(ARG6, Live);
+
+        emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                           Update::eReductions>(Live.getValue());
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        load_erl_bits_state(ARG3);
+        runtime_call<6>(beam_jit_bs_init);
+
+        emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                           Update::eReductions>(Live.getValue());
+
+        mov_arg(Dst, ARG1);
+    }
+
+    if (Fail.getValue() == 0) {
+        Label next = a.newLabel();
+
+        a.b(next);
+
+        a.bind(fail);
+        {
+            mov_arg(ARG2, Size);
+            fragment_call(ga->get_bs_size_check_shared());
+        }
+
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_init(const ArgVal &Size,
+                                         const ArgVal &Live,
+                                         const ArgVal &Dst) {
+    const ArgVal Heap(ArgVal::Word, 0);
+    emit_i_bs_init_heap(Size, Heap, Live, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bs_init_fail(const ArgVal &Size,
+                                              const ArgVal &Fail,
+                                              const ArgVal &Live,
+                                              const ArgVal &Dst) {
+    const ArgVal Heap(ArgVal::Word, 0);
+    emit_i_bs_init_fail_heap(Size, Heap, Fail, Live, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bs_init_bits(const ArgVal &NumBits,
+                                              const ArgVal &Live,
+                                              const ArgVal &Dst) {
+    const ArgVal heap(ArgVal::Word, 0);
+    emit_i_bs_init_bits_heap(NumBits, heap, Live, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bs_init_bits_heap(const ArgVal &NumBits,
+                                                   const ArgVal &Alloc,
+                                                   const ArgVal &Live,
+                                                   const ArgVal &Dst) {
+    mov_arg(ARG4, NumBits);
+    mov_arg(ARG5, Alloc);
+    mov_arg(ARG6, Live);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue());
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    load_erl_bits_state(ARG3);
+    runtime_call<6>(beam_jit_bs_init_bits);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue());
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_init_bits_fail(const ArgVal &NumBits,
+                                                   const ArgVal &Fail,
+                                                   const ArgVal &Live,
+                                                   const ArgVal &Dst) {
+    const ArgVal Heap(ArgVal::Word, 0);
+
+    emit_i_bs_init_bits_fail_heap(NumBits, Heap, Fail, Live, Dst);
+}
+
+void BeamModuleAssembler::emit_i_bs_init_bits_fail_heap(const ArgVal &NumBits,
+                                                        const ArgVal &Alloc,
+                                                        const ArgVal &Fail,
+                                                        const ArgVal &Live,
+                                                        const ArgVal &Dst) {
+    Label fail;
+
+    if (Fail.getValue() != 0) {
+        fail = resolve_beam_label(Fail, dispUnknown);
+    } else {
+        fail = a.newLabel();
+    }
+
+    if (emit_bs_get_field_size(NumBits, 1, fail, ARG4) >= 0) {
+        mov_arg(ARG5, Alloc);
+        mov_arg(ARG6, Live);
+
+        emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                           Update::eReductions>(Live.getValue());
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        load_erl_bits_state(ARG3);
+        runtime_call<6>(beam_jit_bs_init_bits);
+
+        emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                           Update::eReductions>(Live.getValue());
+
+        mov_arg(Dst, ARG1);
+    }
+
+    if (Fail.getValue() == 0) {
+        Label next = a.newLabel();
+
+        a.b(next);
+        a.bind(fail);
+        {
+            mov_arg(ARG2, NumBits);
+            fragment_call(ga->get_bs_size_check_shared());
+        }
+
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_bs_put_string(const ArgVal &Size,
+                                             const ArgVal &Ptr) {
+    mov_arg(ARG2, Ptr);
+    mov_arg(ARG3, Size);
+
+    emit_enter_runtime();
+
+    load_erl_bits_state(ARG1);
+    runtime_call<3>(erts_new_bs_put_string);
+
+    emit_leave_runtime();
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_integer_imm(const ArgVal &Src,
+                                                        const ArgVal &Fail,
+                                                        const ArgVal &Sz,
+                                                        const ArgVal &Flags) {
+    mov_arg(ARG2, Src);
+    mov_arg(ARG3, Sz);
+    mov_arg(ARG4, Flags);
+
+    emit_enter_runtime();
+
+    load_erl_bits_state(ARG1);
+    runtime_call<4>(erts_new_bs_put_integer);
+
+    emit_leave_runtime();
+
+    if (Fail.getValue() != 0) {
+        a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+    } else {
+        Label next = a.newLabel();
+
+        a.cbnz(ARG1, next);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_integer(const ArgVal &Fail,
+                                                    const ArgVal &Sz,
+                                                    const ArgVal &Flags,
+                                                    const ArgVal &Src) {
+    int unit = Flags.getValue() >> 3;
+    Label next, fail;
+
+    if (Fail.getValue() != 0) {
+        fail = resolve_beam_label(Fail, dispUnknown);
+    } else {
+        fail = a.newLabel();
+        next = a.newLabel();
+    }
+
+    /* Clobbers RET + ARG3 */
+    if (emit_bs_get_field_size(Sz, unit, fail, ARG3) >= 0) {
+        mov_arg(ARG2, Src);
+        mov_arg(ARG4, Flags);
+
+        emit_enter_runtime();
+
+        load_erl_bits_state(ARG1);
+        runtime_call<4>(erts_new_bs_put_integer);
+
+        emit_leave_runtime();
+
+        if (Fail.getValue() != 0) {
+            a.cbz(ARG1, fail);
+        } else {
+            a.cbnz(ARG1, next);
+        }
+    }
+
+    if (Fail.getValue() == 0) {
+        a.bind(fail);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_binary(const ArgVal &Fail,
+                                                   const ArgVal &Sz,
+                                                   const ArgVal &Flags,
+                                                   const ArgVal &Src) {
+    int unit = Flags.getValue() >> 3;
+    Label next, fail;
+
+    if (Fail.getValue() != 0) {
+        fail = resolve_beam_label(Fail, dispUnknown);
+    } else {
+        fail = a.newLabel();
+        next = a.newLabel();
+    }
+
+    if (emit_bs_get_field_size(Sz, unit, fail, ARG3) >= 0) {
+        mov_arg(ARG2, Src);
+
+        emit_enter_runtime<Update::eReductions>();
+
+        a.mov(ARG1, c_p);
+        runtime_call<3>(erts_new_bs_put_binary);
+
+        emit_leave_runtime<Update::eReductions>();
+
+        if (Fail.getValue() != 0) {
+            a.cbz(ARG1, fail);
+        } else {
+            a.cbnz(ARG1, next);
+        }
+    }
+
+    if (Fail.getValue() == 0) {
+        a.bind(fail);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_binary_all(const ArgVal &Src,
+                                                       const ArgVal &Fail,
+                                                       const ArgVal &Unit) {
+    mov_arg(ARG2, Src);
+    mov_arg(ARG3, Unit);
+
+    emit_enter_runtime<Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_new_bs_put_binary_all);
+
+    emit_leave_runtime<Update::eReductions>();
+
+    if (Fail.getValue() == 0) {
+        Label next = a.newLabel();
+
+        a.cbnz(ARG1, next);
+        emit_error(BADARG);
+        a.bind(next);
+    } else {
+        a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+    }
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_binary_imm(const ArgVal &Fail,
+                                                       const ArgVal &Sz,
+                                                       const ArgVal &Src) {
+    mov_arg(ARG2, Src);
+    mov_arg(ARG3, Sz);
+
+    emit_enter_runtime<Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_new_bs_put_binary);
+
+    emit_leave_runtime<Update::eReductions>();
+
+    if (Fail.getValue() == 0) {
+        Label next = a.newLabel();
+
+        a.cbnz(ARG1, next);
+        emit_error(BADARG);
+        a.bind(next);
+    } else {
+        a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+    }
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_float(const ArgVal &Fail,
+                                                  const ArgVal &Sz,
+                                                  const ArgVal &Flags,
+                                                  const ArgVal &Src) {
+    int unit = Flags.getValue() >> 3;
+    Label next, fail;
+
+    if (Fail.getValue() != 0) {
+        fail = resolve_beam_label(Fail, dispUnknown);
+    } else {
+        fail = a.newLabel();
+        next = a.newLabel();
+    }
+
+    if (emit_bs_get_field_size(Sz, unit, fail, ARG3) >= 0) {
+        mov_arg(ARG2, Src);
+        mov_arg(ARG4, Flags);
+
+        emit_enter_runtime();
+
+        a.mov(ARG1, c_p);
+        runtime_call<4>(erts_new_bs_put_float);
+
+        emit_leave_runtime();
+
+        if (Fail.getValue() != 0) {
+            a.cbz(ARG1, fail);
+        } else {
+            a.cbnz(ARG1, next);
+        }
+    }
+
+    if (Fail.getValue() == 0) {
+        a.bind(fail);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_new_bs_put_float_imm(const ArgVal &Fail,
+                                                      const ArgVal &Sz,
+                                                      const ArgVal &Flags,
+                                                      const ArgVal &Src) {
+    mov_arg(ARG2, Src);
+    mov_arg(ARG3, Sz);
+    mov_arg(ARG4, Flags);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<4>(erts_new_bs_put_float);
+
+    emit_leave_runtime();
+
+    if (Fail.getValue() != 0) {
+        a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+    } else {
+        Label next = a.newLabel();
+
+        a.cbnz(ARG1, next);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_start_match3(const ArgVal &Src,
+                                                 const ArgVal &Live,
+                                                 const ArgVal &Fail,
+                                                 const ArgVal &Dst) {
+    Label is_binary = a.newLabel(), next = a.newLabel();
+
+    mov_arg(ARG2, Src);
+
+    if (Fail.getValue() != 0) {
+        emit_is_boxed(resolve_beam_label(Fail, dispUnknown), ARG2);
+    } else {
+        /* bs_start_match3 may not throw, and the compiler will only emit {f,0}
+         * when it knows that the source is a match state or binary, so we're
+         * free to skip the binary tests. */
+    }
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, ARG2);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr, 0));
+
+    a.and_(TMP1, TMP1, imm(_HEADER_SUBTAG_MASK));
+    a.cmp(TMP1, imm(BIN_MATCHSTATE_SUBTAG));
+    a.cond_eq().b(next);
+
+    if (Fail.getValue() != 0) {
+        comment("is_binary_header");
+        a.cmp(TMP1, _TAG_HEADER_SUB_BIN);
+        a.cond_eq().b(is_binary);
+        ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN);
+        a.and_(TMP1, TMP1, imm(~4));
+        a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN));
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    }
+
+    a.bind(is_binary);
+    {
+        /* Src is not guaranteed to be inside the live range, so we need to
+         * stash it during GC. */
+        emit_gc_test_preserve(ArgVal(ArgVal::Word, ERL_BIN_MATCHSTATE_SIZE(0)),
+                              Live,
+                              ARG2);
+
+        emit_enter_runtime<Update::eStack | Update::eHeap>(Live.getValue());
+
+        a.mov(ARG1, c_p);
+        /* ARG2 was set above */
+        runtime_call<2>(erts_bs_start_match_3);
+
+        emit_leave_runtime<Update::eStack | Update::eHeap>(Live.getValue());
+
+        a.add(ARG2, ARG1, imm(TAG_PRIMARY_BOXED));
+    }
+
+    a.bind(next);
+    mov_arg(Dst, ARG2);
+}
+
+void BeamModuleAssembler::emit_i_bs_match_string(const ArgVal &Ctx,
+                                                 const ArgVal &Fail,
+                                                 const ArgVal &Bits,
+                                                 const ArgVal &Ptr) {
+    const int position_offset = offsetof(ErlBinMatchState, mb.offset);
+    const int size_offset = offsetof(ErlBinMatchState, mb.size);
+    const int base_offset = offsetof(ErlBinMatchState, mb.base);
+
+    const UWord size = Bits.getValue();
+
+    {
+        auto ctx_reg = load_source(Ctx, TMP1);
+
+        a.ldur(TMP2, emit_boxed_val(ctx_reg.reg, position_offset));
+        a.ldur(TMP3, emit_boxed_val(ctx_reg.reg, size_offset));
+        a.add(TMP4, TMP2, imm(size));
+        a.cmp(TMP4, TMP3);
+        a.cond_hi().b(resolve_beam_label(Fail, disp1MB));
+
+        /* ARG4 = mb->offset & 7 */
+        a.and_(ARG4, TMP2, imm(7));
+
+        /* ARG3 = mb->base + (mb->offset >> 3) */
+        a.lsr(TMP2, TMP2, imm(3));
+        a.ldur(TMP1, emit_boxed_val(ctx_reg.reg, base_offset));
+        a.add(ARG3, TMP1, TMP2);
+    }
+
+    emit_enter_runtime();
+
+    mov_arg(ARG1, Ptr);
+    mov_imm(ARG2, 0);
+    mov_imm(ARG5, size);
+    runtime_call<5>(erts_cmp_bits);
+
+    emit_leave_runtime();
+    a.cbnz(ARG1, resolve_beam_label(Fail, disp1MB));
+
+    {
+        auto ctx_reg = load_source(Ctx, TMP1);
+
+        a.ldur(TMP2, emit_boxed_val(ctx_reg.reg, position_offset));
+        a.add(TMP2, TMP2, imm(size));
+        a.stur(TMP2, emit_boxed_val(ctx_reg.reg, position_offset));
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_get_position(const ArgVal &Ctx,
+                                                 const ArgVal &Dst) {
+    const int position_offset = offsetof(ErlBinMatchState, mb.offset);
+    auto ctx_reg = load_source(Ctx, TMP1);
+    auto dst_reg = init_destination(Dst, TMP2);
+
+    /* Match contexts can never be literals, so we can skip clearing literal
+     * tags. */
+    a.ldur(dst_reg.reg, emit_boxed_val(ctx_reg.reg, position_offset));
+    a.lsl(dst_reg.reg, dst_reg.reg, imm(_TAG_IMMED1_SIZE));
+    a.orr(dst_reg.reg, dst_reg.reg, imm(_TAG_IMMED1_SMALL));
+
+    flush_var(dst_reg);
+}
+
+void BeamModuleAssembler::emit_i_bs_get_fixed_integer(const ArgVal &Ctx,
+                                                      const ArgVal &Fail,
+                                                      const ArgVal &Live,
+                                                      const ArgVal &Flags,
+                                                      const ArgVal &Bits,
+                                                      const ArgVal &Dst) {
+    auto ctx = load_source(Ctx, TMP1);
+    int flags, bits;
+
+    flags = Flags.getValue();
+    bits = Bits.getValue();
+
+    if (bits >= SMALL_BITS) {
+        emit_gc_test_preserve(ArgVal(ArgVal::Word, BIG_NEED_FOR_BITS(bits)),
+                              Live,
+                              ctx.reg);
+    }
+
+    lea(ARG4, emit_boxed_val(ctx.reg, offsetof(ErlBinMatchState, mb)));
+
+    if (bits >= SMALL_BITS) {
+        emit_enter_runtime<Update::eHeap>(Live.getValue());
+    } else {
+        emit_enter_runtime(Live.getValue());
+    }
+
+    a.mov(ARG1, c_p);
+    a.mov(ARG2, bits);
+    a.mov(ARG3, flags);
+    /* ARG4 set above. */
+    runtime_call<4>(erts_bs_get_integer_2);
+
+    if (bits >= SMALL_BITS) {
+        emit_leave_runtime<Update::eHeap>(Live.getValue());
+    } else {
+        emit_leave_runtime(Live.getValue());
+    }
+
+    emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_get_integer(const ArgVal &Ctx,
+                                                const ArgVal &Fail,
+                                                const ArgVal &Live,
+                                                const ArgVal &FlagsAndUnit,
+                                                const ArgVal &Sz,
+                                                const ArgVal &Dst) {
+    Label fail;
+    int unit;
+
+    fail = resolve_beam_label(Fail, dispUnknown);
+    unit = FlagsAndUnit.getValue() >> 3;
+
+    if (emit_bs_get_field_size(Sz, unit, fail, ARG5) >= 0) {
+        mov_arg(ARG3, Ctx);
+        mov_arg(ARG4, FlagsAndUnit);
+        mov_arg(ARG6, Live);
+
+        emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                           Update::eReductions>(Live.getValue());
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        runtime_call<6>(beam_jit_bs_get_integer);
+
+        emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                           Update::eReductions>(Live.getValue());
+
+        emit_branch_if_not_value(ARG1, fail);
+        mov_arg(Dst, ARG1);
+    }
+}
+
+void BeamModuleAssembler::emit_bs_test_tail2(const ArgVal &Fail,
+                                             const ArgVal &Ctx,
+                                             const ArgVal &Offset) {
+    const int position_offset = offsetof(ErlBinMatchState, mb.offset);
+    const int size_offset = offsetof(ErlBinMatchState, mb.size);
+
+    auto ctx_reg = load_source(Ctx, TMP1);
+
+    ASSERT(Offset.isWord());
+
+    a.ldur(TMP2, emit_boxed_val(ctx_reg.reg, size_offset));
+    a.ldur(TMP3, emit_boxed_val(ctx_reg.reg, position_offset));
+    a.sub(TMP2, TMP2, TMP3);
+
+    if (Offset.getValue() != 0) {
+        a.cmp(TMP2, imm(Offset.getValue()));
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.cbnz(TMP2, resolve_beam_label(Fail, disp1MB));
+    }
+}
+
+void BeamModuleAssembler::emit_bs_set_position(const ArgVal &Ctx,
+                                               const ArgVal &Pos) {
+    const int position_offset = offsetof(ErlBinMatchState, mb.offset);
+    auto [ctx, pos] = load_sources(Ctx, TMP1, Pos, TMP2);
+
+    a.lsr(TMP2, pos.reg, imm(_TAG_IMMED1_SIZE));
+    a.stur(TMP2, emit_boxed_val(ctx.reg, position_offset));
+}
+
+void BeamModuleAssembler::emit_i_bs_get_binary_all2(const ArgVal &Ctx,
+                                                    const ArgVal &Fail,
+                                                    const ArgVal &Live,
+                                                    const ArgVal &Unit,
+                                                    const ArgVal &Dst) {
+    unsigned unit = Unit.getValue();
+
+    mov_arg(ARG1, Ctx);
+
+    /* Ctx is not guaranteed to be inside the live range, so we need to stash
+     * it during GC. */
+    emit_gc_test_preserve(ArgVal(ArgVal::Word, EXTRACT_SUB_BIN_HEAP_NEED),
+                          Live,
+                          ARG1);
+
+    /* Make field fetching slightly more compact by pre-loading the match
+     * buffer into the right argument slot for `erts_bs_get_binary_all_2`. */
+    lea(ARG2, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb)));
+
+    ERTS_CT_ASSERT_FIELD_PAIR(ErlBinMatchBuffer, offset, size);
+    a.ldp(TMP2, TMP3, arm::Mem(ARG2, offsetof(ErlBinMatchBuffer, offset)));
+
+    /* Remainder = Size - Offset */
+    a.sub(TMP1, TMP3, TMP2);
+
+    /* Unit may be 1 if compiling with +no_bsm3, which lacks the
+     * bs_get_tail instruction. */
+    if (unit > 1) {
+        if ((unit & (unit - 1))) {
+            mov_imm(TMP2, unit);
+
+            a.udiv(TMP3, TMP1, TMP2);
+            a.msub(TMP1, TMP3, TMP2, TMP1);
+
+            a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB));
+        } else {
+            a.tst(TMP1, imm(unit - 1));
+            a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+        }
+    }
+
+    emit_enter_runtime<Update::eHeap>(Live.getValue());
+
+    a.mov(ARG1, c_p);
+    /* ARG2 was set above. */
+    runtime_call<2>(erts_bs_get_binary_all_2);
+
+    emit_leave_runtime<Update::eHeap>(Live.getValue());
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamGlobalAssembler::emit_bs_get_tail_shared() {
+    lea(TMP1, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb)));
+
+    ERTS_CT_ASSERT_FIELD_PAIR(ErlBinMatchBuffer, orig, base);
+    a.ldp(ARG2, ARG3, arm::Mem(TMP1, offsetof(ErlBinMatchBuffer, orig)));
+
+    ERTS_CT_ASSERT_FIELD_PAIR(ErlBinMatchBuffer, offset, size);
+    a.ldp(ARG4, TMP1, arm::Mem(TMP1, offsetof(ErlBinMatchBuffer, offset)));
+
+    lea(ARG1, arm::Mem(c_p, offsetof(Process, htop)));
+
+    /* Extracted size = mb->size - mb->offset */
+    a.sub(ARG5, TMP1, ARG4);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eHeap>();
+
+    runtime_call<5>(erts_extract_sub_binary);
+
+    emit_leave_runtime<Update::eHeap>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+void BeamModuleAssembler::emit_bs_get_tail(const ArgVal &Ctx,
+                                           const ArgVal &Dst,
+                                           const ArgVal &Live) {
+    mov_arg(ARG1, Ctx);
+
+    /* Ctx is not guaranteed to be inside the live range, so we need to stash
+     * it during GC. */
+    emit_gc_test_preserve(ArgVal(ArgVal::Word, EXTRACT_SUB_BIN_HEAP_NEED),
+                          Live,
+                          ARG1);
+
+    fragment_call(ga->get_bs_get_tail_shared());
+
+    mov_arg(Dst, ARG1);
+}
+
+/* Bits to skip are passed in ARG1 */
+void BeamModuleAssembler::emit_bs_skip_bits(const ArgVal &Fail,
+                                            const ArgVal &Ctx) {
+    const int position_offset = offsetof(ErlBinMatchState, mb.offset);
+    const int size_offset = offsetof(ErlBinMatchState, mb.size);
+
+    auto ctx_reg = load_source(Ctx, TMP1);
+
+    a.ldur(TMP2, emit_boxed_val(ctx_reg.reg, position_offset));
+    a.ldur(TMP3, emit_boxed_val(ctx_reg.reg, size_offset));
+
+    a.add(TMP2, TMP2, ARG1);
+    a.cmp(TMP2, TMP3);
+    a.cond_hi().b(resolve_beam_label(Fail, disp1MB));
+
+    a.stur(TMP2, emit_boxed_val(ctx_reg.reg, position_offset));
+}
+
+void BeamModuleAssembler::emit_i_bs_skip_bits2(const ArgVal &Ctx,
+                                               const ArgVal &Bits,
+                                               const ArgVal &Fail,
+                                               const ArgVal &Unit) {
+    Label fail = resolve_beam_label(Fail, dispUnknown);
+
+    if (emit_bs_get_field_size(Bits, Unit.getValue(), fail, ARG1) >= 0) {
+        emit_bs_skip_bits(Fail, Ctx);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_skip_bits_imm2(const ArgVal &Fail,
+                                                   const ArgVal &Ctx,
+                                                   const ArgVal &Bits) {
+    mov_arg(ARG1, Bits);
+    emit_bs_skip_bits(Fail, Ctx);
+}
+
+void BeamModuleAssembler::emit_i_bs_get_binary2(const ArgVal &Ctx,
+                                                const ArgVal &Fail,
+                                                const ArgVal &Live,
+                                                const ArgVal &Size,
+                                                const ArgVal &Flags,
+                                                const ArgVal &Dst) {
+    Label fail;
+    int unit;
+
+    fail = resolve_beam_label(Fail, dispUnknown);
+    unit = Flags.getValue() >> 3;
+
+    if (emit_bs_get_field_size(Size, unit, fail, ARG2) >= 0) {
+        a.str(ARG2, TMP_MEM1q);
+
+        mov_arg(ARG4, Ctx);
+
+        /* Ctx is not guaranteed to be inside the live range, so we need to
+         * stash it during GC. */
+        emit_gc_test_preserve(ArgVal(ArgVal::Word, EXTRACT_SUB_BIN_HEAP_NEED),
+                              Live,
+                              ARG4);
+
+        lea(ARG4, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb)));
+
+        emit_enter_runtime<Update::eHeap>(Live.getValue());
+
+        a.mov(ARG1, c_p);
+        a.ldr(ARG2, TMP_MEM1q);
+        mov_imm(ARG3, Flags.getValue());
+        runtime_call<4>(erts_bs_get_binary_2);
+
+        emit_leave_runtime<Update::eHeap>(Live.getValue());
+
+        emit_branch_if_not_value(ARG1, fail);
+
+        mov_arg(Dst, ARG1);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_get_float2(const ArgVal &Ctx,
+                                               const ArgVal &Fail,
+                                               const ArgVal &Live,
+                                               const ArgVal &Sz,
+                                               const ArgVal &Flags,
+                                               const ArgVal &Dst) {
+    Label fail;
+    Sint unit;
+
+    fail = resolve_beam_label(Fail, dispUnknown);
+    unit = Flags.getValue() >> 3;
+
+    mov_arg(ARG4, Ctx);
+
+    /* Ctx is not guaranteed to be inside the live range, so we need to stash
+     * it during GC. */
+    emit_gc_test_preserve(ArgVal(ArgVal::Word, FLOAT_SIZE_OBJECT), Live, ARG4);
+
+    if (emit_bs_get_field_size(Sz, unit, fail, ARG2) >= 0) {
+        lea(ARG4, emit_boxed_val(ARG4, offsetof(ErlBinMatchState, mb)));
+
+        emit_enter_runtime<Update::eHeap>(Live.getValue());
+
+        a.mov(ARG1, c_p);
+        mov_imm(ARG3, Flags.getValue());
+        runtime_call<4>(erts_bs_get_float_2);
+
+        emit_leave_runtime<Update::eHeap>(Live.getValue());
+
+        emit_branch_if_not_value(ARG1, fail);
+
+        mov_arg(Dst, ARG1);
+    }
+}
+
+void BeamModuleAssembler::emit_i_bs_utf8_size(const ArgVal &Src,
+                                              const ArgVal &Dst) {
+    auto src_reg = load_source(Src, TMP1);
+    auto dst_reg = init_destination(Dst, TMP2);
+
+    Label next = a.newLabel();
+
+    /* Note that the source and destination registers could be the
+     * same register. Therefore, we must copy the source register
+     * before writing to the destination register. */
+    a.lsr(TMP1, src_reg.reg, imm(_TAG_IMMED1_SIZE));
+    mov_imm(dst_reg.reg, make_small(1));
+    a.cmp(TMP1, imm(0x7F));
+    a.cond_ls().b(next);
+
+    mov_imm(dst_reg.reg, make_small(2));
+    a.cmp(TMP1, imm(0x7FFUL));
+    a.cond_ls().b(next);
+
+    a.cmp(TMP1, imm(0x10000UL));
+    mov_imm(TMP2, make_small(3));
+    mov_imm(TMP3, make_small(4));
+    a.csel(dst_reg.reg, TMP2, TMP3, arm::Cond::kLO);
+
+    a.bind(next);
+    flush_var(dst_reg);
+}
+
+void BeamModuleAssembler::emit_i_bs_put_utf8(const ArgVal &Fail,
+                                             const ArgVal &Src) {
+    mov_arg(ARG2, Src);
+
+    emit_enter_runtime();
+
+    load_erl_bits_state(ARG1);
+    runtime_call<2>(erts_bs_put_utf8);
+
+    emit_leave_runtime();
+
+    if (Fail.getValue() != 0) {
+        a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+    } else {
+        Label next = a.newLabel();
+
+        a.cbnz(ARG1, next);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_bs_get_utf8(const ArgVal &Ctx,
+                                           const ArgVal &Fail) {
+    mov_arg(ARG1, Ctx);
+    lea(ARG1, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb)));
+
+    emit_enter_runtime();
+
+    runtime_call<1>(erts_bs_get_utf8);
+
+    emit_leave_runtime();
+
+    emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+}
+
+void BeamModuleAssembler::emit_i_bs_get_utf8(const ArgVal &Ctx,
+                                             const ArgVal &Fail,
+                                             const ArgVal &Dst) {
+    emit_bs_get_utf8(Ctx, Fail);
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_skip_utf8(const ArgVal &Ctx,
+                                              const ArgVal &Fail) {
+    emit_bs_get_utf8(Ctx, Fail);
+}
+
+void BeamModuleAssembler::emit_i_bs_utf16_size(const ArgVal &Src,
+                                               const ArgVal &Dst) {
+    auto src_reg = load_source(Src, TMP1);
+    auto dst_reg = init_destination(Dst, TMP2);
+
+    /* erts_bs_put_utf16 errors out whenever something's fishy, so we can
+     * return garbage (2 or 4) if our input is not a small. */
+    a.asr(TMP1, src_reg.reg, imm(_TAG_IMMED1_SIZE));
+    a.cmp(TMP1, imm(0x10000UL));
+    mov_imm(TMP1, make_small(2));
+    mov_imm(TMP2, make_small(4));
+    a.csel(dst_reg.reg, TMP1, TMP2, arm::Cond::kLO);
+
+    flush_var(dst_reg);
+}
+
+void BeamModuleAssembler::emit_bs_put_utf16(const ArgVal &Fail,
+                                            const ArgVal &Flags,
+                                            const ArgVal &Src) {
+    mov_arg(ARG2, Src);
+    mov_arg(ARG3, Flags);
+
+    emit_enter_runtime();
+
+    load_erl_bits_state(ARG1);
+    runtime_call<3>(erts_bs_put_utf16);
+
+    emit_leave_runtime();
+
+    if (Fail.getValue() != 0) {
+        a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+    } else {
+        Label next = a.newLabel();
+
+        a.cbnz(ARG1, next);
+        emit_error(BADARG);
+        a.bind(next);
+    }
+}
+
+void BeamModuleAssembler::emit_bs_get_utf16(const ArgVal &Ctx,
+                                            const ArgVal &Fail,
+                                            const ArgVal &Flags) {
+    auto ctx_reg = load_source(Ctx, TMP1);
+
+    lea(ARG1, emit_boxed_val(ctx_reg.reg, offsetof(ErlBinMatchState, mb)));
+
+    emit_enter_runtime();
+
+    mov_imm(ARG2, Flags.getValue());
+    runtime_call<2>(erts_bs_get_utf16);
+
+    emit_leave_runtime();
+
+    emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+}
+
+void BeamModuleAssembler::emit_i_bs_get_utf16(const ArgVal &Ctx,
+                                              const ArgVal &Fail,
+                                              const ArgVal &Flags,
+                                              const ArgVal &Dst) {
+    emit_bs_get_utf16(Ctx, Fail, Flags);
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_skip_utf16(const ArgVal &Ctx,
+                                               const ArgVal &Fail,
+                                               const ArgVal &Flags) {
+    emit_bs_get_utf16(Ctx, Fail, Flags);
+}
+
+void BeamModuleAssembler::emit_validate_unicode(Label next,
+                                                Label fail,
+                                                arm::Gp value) {
+    ASSERT(value != TMP2);
+
+    a.and_(TMP2, value, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP2, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(fail);
+
+    mov_imm(TMP2, make_small(0xD800UL));
+    a.cmp(value, TMP2);
+    a.cond_lo().b(next);
+
+    mov_imm(TMP2, make_small(0xDFFFUL));
+    a.cmp(value, TMP2);
+    a.cond_ls().b(fail);
+
+    mov_imm(TMP2, make_small(0x10FFFFUL));
+    a.cmp(value, TMP2);
+    a.cond_hi().b(fail);
+
+    a.b(next);
+}
+
+void BeamModuleAssembler::emit_i_bs_validate_unicode(const ArgVal &Fail,
+                                                     const ArgVal &Src) {
+    auto src_reg = load_source(Src, TMP1);
+    Label fail, next = a.newLabel();
+
+    if (Fail.getValue() != 0) {
+        fail = resolve_beam_label(Fail, dispUnknown);
+    } else {
+        fail = a.newLabel();
+    }
+
+    emit_validate_unicode(next, fail, src_reg.reg);
+
+    if (Fail.getValue() == 0) {
+        a.bind(fail);
+        emit_error(BADARG);
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_i_bs_validate_unicode_retract(const ArgVal &Fail,
+                                                             const ArgVal &Src,
+                                                             const ArgVal &Ms) {
+    Label fail = a.newLabel(), next = a.newLabel();
+    auto src_reg = load_source(Src, TMP1);
+
+    emit_validate_unicode(next, fail, src_reg.reg);
+
+    a.bind(fail);
+    {
+        const int position_offset = offsetof(ErlBinMatchState, mb.offset);
+        auto ctx_reg = load_source(Ms, TMP2);
+
+        a.ldur(TMP1, emit_boxed_val(ctx_reg.reg, position_offset));
+        a.sub(TMP1, TMP1, imm(32));
+        a.stur(TMP1, emit_boxed_val(ctx_reg.reg, position_offset));
+
+        if (Fail.getValue() != 0) {
+            a.b(resolve_beam_label(Fail, disp128MB));
+        } else {
+            emit_error(BADARG);
+        }
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_bs_test_unit(const ArgVal &Fail,
+                                            const ArgVal &Ctx,
+                                            const ArgVal &Unit) {
+    auto ctx_reg = load_source(Ctx, TMP1);
+    unsigned int unit = Unit.getValue();
+
+    a.ldur(TMP2,
+           emit_boxed_val(ctx_reg.reg, offsetof(ErlBinMatchState, mb.size)));
+    a.ldur(TMP3,
+           emit_boxed_val(ctx_reg.reg, offsetof(ErlBinMatchState, mb.offset)));
+
+    a.sub(TMP1, TMP2, TMP3);
+
+    if ((unit & (unit - 1))) {
+        mov_imm(TMP2, unit);
+
+        a.udiv(TMP3, TMP1, TMP2);
+        a.msub(TMP1, TMP3, TMP2, TMP1);
+
+        a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.tst(TMP1, imm(unit - 1));
+        a.cond_ne().b(resolve_beam_label(Fail, dispUnknown));
+    }
+}
+
+/* ARG2 = current `Size`,
+ * ARG3 = elements to `Add`,
+ * ARG4 = element `Unit`
+ *
+ * Error is indicated through cond_ne() */
+void BeamGlobalAssembler::emit_bs_add_guard_shared() {
+    Label error = a.newLabel();
+
+    /* Since `Unit` is guaranteed to be less than 1024, we can check overflow
+     * and negative numbers by testing whether any of the highest 12 value bits
+     * are set on either argument. */
+    a.orr(TMP1, ARG2, ARG3);
+    a.tst(TMP1, imm(0xFFF0000000000000UL));
+
+    a.and_(TMP1, ARG2, ARG3);
+    a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    a.ccmp(TMP1, imm(_TAG_IMMED1_SMALL), imm(0), imm(arm::Cond::kEQ));
+    a.cond_ne().b(error);
+
+    /* Return `Size` + `Add` * `Unit`
+     *
+     * Note that the result takes the tag bits from `Size` */
+    a.and_(TMP2, ARG3, imm(~_TAG_IMMED1_SMALL));
+    a.mul(TMP2, TMP2, ARG4);
+    a.add(ARG1, TMP2, ARG2);
+
+    a.bind(error);
+    a.ret(a64::x30);
+}
+
+/* ARG2 = current `Size`,
+ * ARG3 = elements to `Add`,
+ * ARG4 = element `Unit` */
+void BeamGlobalAssembler::emit_bs_add_body_shared() {
+    Label error = a.newLabel();
+
+    /* Since `Unit` is guaranteed to be less than 1024, we can check overflow
+     * and negative numbers by testing whether any of the highest 12 value bits
+     * are set on either argument. */
+    a.orr(TMP1, ARG2, ARG3);
+    a.tst(TMP1, imm(0xFFF0000000000000UL));
+
+    a.and_(TMP1, ARG2, ARG3);
+    a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    a.ccmp(TMP1, imm(_TAG_IMMED1_SMALL), imm(0), imm(arm::Cond::kEQ));
+    a.cond_ne().b(error);
+
+    /* Return `Size` + `Add` * `Unit`
+     *
+     * Note that the result takes the tag bits from `Size` */
+    a.and_(TMP2, ARG3, imm(~_TAG_IMMED1_SMALL));
+    a.mul(TMP2, TMP2, ARG4);
+    a.add(ARG1, TMP2, ARG2);
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        emit_enter_runtime_frame();
+        emit_enter_runtime(0);
+
+        a.mov(ARG1, c_p);
+        runtime_call<3>(beam_jit_bs_add_argument_error);
+
+        emit_leave_runtime(0);
+        emit_leave_runtime_frame();
+
+        mov_imm(ARG4, 0);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_bs_add(const ArgVal &Fail,
+                                      const ArgVal &Size,
+                                      const ArgVal &Add,
+                                      const ArgVal &Unit,
+                                      const ArgVal &Dst) {
+    ASSERT(Unit.isWord() && Unit.getValue() < 1024);
+
+    mov_arg(ARG2, Size);
+    mov_arg(ARG3, Add);
+    mov_arg(ARG4, Unit);
+
+    if (Fail.getValue() == 0) {
+        fragment_call(ga->get_bs_add_body_shared());
+    } else {
+        fragment_call(ga->get_bs_add_guard_shared());
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    }
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_append(const ArgVal &Fail,
+                                           const ArgVal &ExtraHeap,
+                                           const ArgVal &Live,
+                                           const ArgVal &Unit,
+                                           const ArgVal &Size,
+                                           const ArgVal &Bin,
+                                           const ArgVal &Dst) {
+    mov_arg(ARG3, Live);
+    mov_arg(ARG4, Size);
+    mov_arg(ARG5, ExtraHeap);
+    mov_arg(ARG6, Unit);
+
+    mov_arg(ArgVal(ArgVal::XReg, Live.getValue()), Bin);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue() + 1);
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<6>(erts_bs_append);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue() + 1);
+
+    if (Fail.getValue() != 0) {
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        Label next = a.newLabel();
+
+        emit_branch_if_value(ARG1, next);
+        /* The error has been prepared in `erts_bs_append` */
+        emit_raise_exception();
+
+        a.bind(next);
+    }
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_private_append(const ArgVal &Fail,
+                                                   const ArgVal &Unit,
+                                                   const ArgVal &Size,
+                                                   const ArgVal &Src,
+                                                   const ArgVal &Dst) {
+    mov_arg(ARG2, Src);
+    mov_arg(ARG3, Size);
+    mov_arg(ARG4, Unit);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<4>(erts_bs_private_append);
+
+    emit_leave_runtime();
+
+    if (Fail.getValue() != 0) {
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        Label next = a.newLabel();
+
+        emit_branch_if_value(ARG1, next);
+        /* The error has been prepared in `erts_bs_private_append` */
+        emit_raise_exception();
+
+        a.bind(next);
+    }
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_bs_init_writable() {
+    ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 0);
+
+    a.mov(ARG1, c_p);
+    a.mov(ARG2, XREG0);
+
+    /* We have an implicit liveness of 0, so we don't need to stash X
+     * registers. */
+    emit_enter_runtime<Update::eReductions | Update::eStack | Update::eHeap>(0);
+
+    runtime_call<2>(erts_bs_init_writable);
+
+    emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(0);
+
+    a.mov(XREG0, ARG1);
+}
+
+/* Old compatibility instructions for <= OTP-21. Kept in order to be able to
+ * load old code. While technically we could remove these in OTP-24, we've
+ * decided to keep them until at least OTP-25 to make things easier for
+ * users. */
+void BeamModuleAssembler::emit_i_bs_start_match2(const ArgVal &Src,
+                                                 const ArgVal &Fail,
+                                                 const ArgVal &Live,
+                                                 const ArgVal &Slots,
+                                                 const ArgVal &Dst) {
+    mov_arg(ARG1, Src);
+    mov_arg(ARG2, Live);
+    mov_arg(ARG3, Slots);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue());
+
+    a.mov(ARG4, c_p);
+    load_x_reg_array(ARG5);
+    runtime_call<5>(beam_jit_bs_start_match2);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(Live.getValue());
+
+    emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_bs_save2(const ArgVal &Ctx,
+                                          const ArgVal &Slot) {
+    const int slot_offset = offsetof(ErlBinMatchState, save_offset) +
+                            (sizeof(Eterm) * Slot.getValue());
+    const int mb_offset = offsetof(ErlBinMatchState, mb.offset);
+    auto ctx_reg = load_source(Ctx, TMP1);
+
+    a.ldur(TMP2, emit_boxed_val(ctx_reg.reg, mb_offset));
+    a.stur(TMP2, emit_boxed_val(ctx_reg.reg, slot_offset));
+}
+
+void BeamModuleAssembler::emit_i_bs_restore2(const ArgVal &Ctx,
+                                             const ArgVal &Slot) {
+    const int slot_offset = offsetof(ErlBinMatchState, save_offset) +
+                            (sizeof(Eterm) * Slot.getValue());
+    const int mb_offset = offsetof(ErlBinMatchState, mb.offset);
+
+    auto ctx_reg = load_source(Ctx, TMP1);
+
+    a.ldur(TMP2, emit_boxed_val(ctx_reg.reg, slot_offset));
+    a.stur(TMP2, emit_boxed_val(ctx_reg.reg, mb_offset));
+}
+
+void BeamModuleAssembler::emit_bs_context_to_binary(const ArgVal &Src) {
+    mov_arg(ARG1, Src);
+
+    emit_enter_runtime();
+
+    runtime_call<1>(beam_jit_bs_context_to_binary);
+
+    emit_leave_runtime();
+}
diff --git a/erts/emulator/beam/jit/arm/instr_call.cpp b/erts/emulator/beam/jit/arm/instr_call.cpp
new file mode 100644
index 0000000000..a141ef5fd1
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_call.cpp
@@ -0,0 +1,214 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "beam_common.h"
+}
+
+void BeamGlobalAssembler::emit_dispatch_return() {
+    a.mov(ARG3, a64::x30);
+    a.str(ZERO, arm::Mem(c_p, offsetof(Process, current)));
+    mov_imm(TMP1, 1);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, arity)));
+    a.b(labels[context_switch_simplified]);
+}
+
+void BeamModuleAssembler::emit_return() {
+    emit_leave_erlang_frame();
+
+#ifdef JIT_HARD_DEBUG
+    /* Validate return address and {x,0} */
+    emit_validate(ArgVal(ArgVal::Word, 1));
+#endif
+
+    /* The reduction test is kept in module code because moving it to a shared
+     * fragment caused major performance regressions in dialyzer. */
+    a.subs(FCALLS, FCALLS, imm(1));
+    a.cond_mi().b(resolve_fragment(ga->get_dispatch_return(), disp1MB));
+
+    a.ret(a64::x30);
+}
+
+void BeamModuleAssembler::emit_i_call(const ArgVal &CallTarget) {
+    erlang_call(resolve_beam_label(CallTarget, disp128MB));
+}
+
+void BeamModuleAssembler::emit_i_call_last(const ArgVal &CallTarget,
+                                           const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+    emit_i_call_only(CallTarget);
+}
+
+void BeamModuleAssembler::emit_i_call_only(const ArgVal &CallTarget) {
+    emit_leave_erlang_frame();
+    a.b(resolve_beam_label(CallTarget, disp128MB));
+}
+
+/* Handles save_calls. When the active code index is ERTS_SAVE_CALLS_CODE_IX,
+ * all remote calls will land here.
+ *
+ * Export entry is in ARG1, return address is in LR (x30). Both of these must
+ * be preserved since this runs between caller and callee. */
+void BeamGlobalAssembler::emit_dispatch_save_calls() {
+    a.str(ARG1, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime();
+
+    a.mov(ARG2, ARG1);
+    a.mov(ARG1, c_p);
+    runtime_call<2>(save_calls);
+
+    emit_leave_runtime();
+    emit_leave_runtime_frame();
+
+    a.ldr(ARG1, TMP_MEM1q);
+
+    /* Keep going with the actual code index. */
+    a.mov(TMP1, imm(&the_active_code_index));
+    a.ldr(TMP1.w(), arm::Mem(TMP1));
+
+    branch(emit_setup_export_call(ARG1, TMP1));
+}
+
+void BeamModuleAssembler::emit_i_call_ext(const ArgVal &Exp) {
+    mov_arg(ARG1, Exp);
+
+    arm::Mem target = emit_setup_export_call(ARG1);
+    erlang_call(target);
+}
+
+void BeamModuleAssembler::emit_i_call_ext_only(const ArgVal &Exp) {
+    mov_arg(ARG1, Exp);
+
+    arm::Mem target = emit_setup_export_call(ARG1);
+    emit_leave_erlang_frame();
+    branch(target);
+}
+
+void BeamModuleAssembler::emit_i_call_ext_last(const ArgVal &Exp,
+                                               const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+    emit_i_call_ext_only(Exp);
+}
+
+static ErtsCodeMFA apply3_mfa = {am_erlang, am_apply, 3};
+
+arm::Mem BeamModuleAssembler::emit_variable_apply(bool includeI) {
+    Label dispatch = a.newLabel(), entry = a.newLabel();
+
+    a.bind(entry);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>(3);
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+
+    if (includeI) {
+        a.adr(ARG3, entry);
+    } else {
+        mov_imm(ARG3, 0);
+    }
+
+    mov_imm(ARG4, 0);
+
+    comment("apply()");
+    runtime_call<4>(apply);
+
+    /* Any number of X registers can be live at this point. */
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+    a.cbnz(ARG1, dispatch);
+    emit_raise_exception(entry, &apply3_mfa);
+
+    a.bind(dispatch);
+    return emit_setup_export_call(ARG1);
+}
+
+void BeamModuleAssembler::emit_i_apply() {
+    arm::Mem target = emit_variable_apply(false);
+    erlang_call(target);
+}
+
+void BeamModuleAssembler::emit_i_apply_last(const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+    emit_i_apply_only();
+}
+
+void BeamModuleAssembler::emit_i_apply_only() {
+    arm::Mem target = emit_variable_apply(true);
+
+    emit_leave_erlang_frame();
+    branch(target);
+}
+
+arm::Mem BeamModuleAssembler::emit_fixed_apply(const ArgVal &Arity,
+                                               bool includeI) {
+    Label dispatch = a.newLabel(), entry = a.newLabel();
+
+    a.bind(entry);
+
+    mov_arg(ARG3, Arity);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>(
+            Arity.getValue() + 2);
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+
+    if (includeI) {
+        a.adr(ARG4, entry);
+    } else {
+        mov_imm(ARG4, 0);
+    }
+
+    mov_imm(ARG5, 0);
+
+    runtime_call<5>(fixed_apply);
+
+    /* We will need to reload all X registers in case there has been
+     * an error. */
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+    a.cbnz(ARG1, dispatch);
+    emit_raise_exception(entry, &apply3_mfa);
+
+    a.bind(dispatch);
+
+    return emit_setup_export_call(ARG1);
+}
+
+void BeamModuleAssembler::emit_apply(const ArgVal &Arity) {
+    arm::Mem target = emit_fixed_apply(Arity, false);
+    erlang_call(target);
+}
+
+void BeamModuleAssembler::emit_apply_last(const ArgVal &Arity,
+                                          const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+
+    arm::Mem target = emit_fixed_apply(Arity, true);
+
+    emit_leave_erlang_frame();
+    branch(target);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_common.cpp b/erts/emulator/beam/jit/arm/instr_common.cpp
new file mode 100644
index 0000000000..44c4e1fc21
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_common.cpp
@@ -0,0 +1,1623 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Notes.
+ *
+ * The immediate operand for the and, orr, eor, and tst instructions
+ * follow special rules.
+ *
+ * For our purposes, only bit patterns consisting of 1 through 63 ones
+ * at any position in a word are possible to encode as an
+ * immediate. Other patterns must be loaded into a tempoary register.
+ *
+ * Here are some examples of possible immediate values:
+ *
+ *    0b00000011
+ *    0b00001111
+ *    0b00111100
+ *
+ *    0xFFFFFFFFFFFFFFF0
+ *    0x100000000000000F
+ *
+ * The last one is possible because it is the pattern 0x1F
+ * (0b00011111) rotated right one position.
+ *
+ * Here is an example of mask that is not a possible to encode as an
+ * immediate:
+ *
+ *    0b111011
+ *
+ * For more about the encoding rules, see:
+ *
+ * https://stackoverflow.com/questions/30904718/range-of-immediate-values-in-armv8-a64-assembly
+ *
+ */
+
+#include <algorithm>
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "erl_bif_table.h"
+#include "big.h"
+#include "beam_catches.h"
+#include "beam_common.h"
+#include "code_ix.h"
+}
+
+using namespace asmjit;
+
+/* Helpers */
+
+void BeamModuleAssembler::emit_error(int reason) {
+    mov_imm(TMP1, reason);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+    emit_raise_exception();
+}
+
+void BeamModuleAssembler::emit_gc_test_preserve(const ArgVal &Need,
+                                                const ArgVal &Live,
+                                                arm::Gp term) {
+    const int32_t bytes_needed = (Need.getValue() + S_RESERVED) * sizeof(Eterm);
+    Label after_gc_check = a.newLabel();
+
+    add(ARG3, HTOP, bytes_needed);
+    a.cmp(ARG3, E);
+    a.cond_ls().b(after_gc_check);
+
+    ASSERT(Live.getValue() < ERTS_X_REGS_ALLOCATED);
+    mov_arg(ArgVal(ArgVal::XReg, Live.getValue()), term);
+
+    mov_imm(ARG4, Live.getValue() + 1);
+    fragment_call(ga->get_garbage_collect());
+
+    mov_arg(term, ArgVal(ArgVal::XReg, Live.getValue()));
+
+    a.bind(after_gc_check);
+}
+
+void BeamModuleAssembler::emit_gc_test(const ArgVal &Ns,
+                                       const ArgVal &Nh,
+                                       const ArgVal &Live) {
+    int32_t bytes_needed =
+            (Ns.getValue() + Nh.getValue() + S_RESERVED) * sizeof(Eterm);
+    Label after_gc_check = a.newLabel();
+
+    add(ARG3, HTOP, bytes_needed);
+    a.cmp(ARG3, E);
+    a.cond_ls().b(after_gc_check);
+
+    mov_imm(ARG4, Live.getValue());
+    fragment_call(ga->get_garbage_collect());
+
+    a.bind(after_gc_check);
+}
+
+void BeamModuleAssembler::emit_validate(const ArgVal &arity) {
+#ifdef DEBUG
+    Label next = a.newLabel(), crash = a.newLabel();
+
+    /* Crash if the Erlang heap is not word-aligned */
+    a.tst(HTOP, imm(sizeof(Eterm) - 1));
+    a.cond_ne().b(crash);
+
+    /* Crash if the Erlang stack is not word-aligned */
+    a.tst(E, imm(sizeof(Eterm) - 1));
+    a.cond_ne().b(crash);
+
+    /* Crash if we've overrun the stack */
+    lea(TMP1, arm::Mem(E, -(int32_t)(S_REDZONE * sizeof(Eterm))));
+    a.cmp(HTOP, TMP1);
+    a.cond_hi().b(crash);
+
+    a.b(next);
+
+    a.bind(crash);
+    a.udf(0xbad);
+    a.bind(next);
+
+#    ifdef JIT_HARD_DEBUG
+    emit_enter_runtime_frame();
+
+    for (unsigned i = 0; i < arity.getValue(); i++) {
+        mov_arg(ARG1, ArgVal(ArgVal::XReg, i));
+
+        emit_enter_runtime();
+        runtime_call<1>(beam_jit_validate_term);
+        emit_leave_runtime();
+    }
+
+    emit_leave_runtime_frame();
+#    endif
+
+#endif
+}
+
+/* Instrs */
+
+void BeamModuleAssembler::emit_i_validate(const ArgVal &Arity) {
+    emit_validate(Arity);
+}
+
+void BeamModuleAssembler::emit_allocate_heap(const ArgVal &NeedStack,
+                                             const ArgVal &NeedHeap,
+                                             const ArgVal &Live) {
+    ASSERT(NeedStack.isWord() && NeedStack.getValue() <= MAX_REG);
+    ArgVal needed = NeedStack;
+
+    emit_gc_test(needed, NeedHeap, Live);
+
+    if (needed.getValue() > 0) {
+        sub(E, E, needed.getValue() * sizeof(Eterm));
+    }
+}
+
+void BeamModuleAssembler::emit_allocate(const ArgVal &NeedStack,
+                                        const ArgVal &Live) {
+    emit_allocate_heap(NeedStack, ArgVal(ArgVal::Word, 0), Live);
+}
+
+void BeamModuleAssembler::emit_deallocate(const ArgVal &Deallocate) {
+    ASSERT(Deallocate.isWord() && Deallocate.getValue() <= 1023);
+
+    if (Deallocate.getValue() > 0) {
+        add(E, E, Deallocate.getValue() * sizeof(Eterm));
+    }
+}
+
+void BeamModuleAssembler::emit_test_heap(const ArgVal &Nh, const ArgVal &Live) {
+    emit_gc_test(ArgVal(ArgVal::Word, 0), Nh, Live);
+}
+
+void BeamModuleAssembler::emit_normal_exit() {
+    /* This is implictly global; it does not normally appear in modules and
+     * doesn't require size optimization. */
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+    emit_proc_lc_unrequire();
+
+    mov_imm(TMP1, EXC_NORMAL);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+    a.str(ZERO, arm::Mem(c_p, offsetof(Process, arity)));
+    a.mov(ARG1, c_p);
+    mov_imm(ARG2, am_normal);
+    runtime_call<2>(erts_do_exit_process);
+
+    emit_proc_lc_require();
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.b(resolve_fragment(ga->get_do_schedule(), disp128MB));
+}
+
+void BeamModuleAssembler::emit_continue_exit() {
+    /* This is implictly global; it does not normally appear in modules and
+     * doesn't require size optimization. */
+
+    emit_enter_runtime<Update::eReductions | Update::eStack | Update::eHeap>(0);
+    emit_proc_lc_unrequire();
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(erts_continue_exit_process);
+
+    emit_proc_lc_require();
+    emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(0);
+
+    a.b(resolve_fragment(ga->get_do_schedule(), disp128MB));
+}
+
+void BeamModuleAssembler::emit_get_list(const ArgVal &Src,
+                                        const ArgVal &Hd,
+                                        const ArgVal &Tl) {
+    auto src = load_source(Src, TMP1);
+    auto hd = init_destination(Hd, TMP2);
+    auto tl = init_destination(Tl, TMP3);
+    arm::Gp cons_ptr = emit_ptr_val(TMP1, src.reg);
+
+    /* The `ldp` instruction does not accept a negative offset, so we
+     * will need subtract the LIST tag beforehand. (This also nicely
+     * take care of the potential overwriting issue when Src == Hd.) */
+    a.sub(TMP1, cons_ptr, imm(TAG_PRIMARY_LIST));
+    if (hd.reg == tl.reg) {
+        /* ldp with two identical registers is an illegal
+         * instruction. Produce the same result at the interpreter. */
+        a.ldr(tl.reg, arm::Mem(TMP1, sizeof(Eterm)));
+        flush_var(tl);
+    } else {
+        a.ldp(hd.reg, tl.reg, arm::Mem(TMP1));
+        flush_vars(hd, tl);
+    }
+}
+
+void BeamModuleAssembler::emit_get_hd(const ArgVal &Src, const ArgVal &Hd) {
+    auto src = load_source(Src, TMP1);
+    auto hd = init_destination(Hd, TMP2);
+    arm::Gp cons_ptr = emit_ptr_val(TMP1, src.reg);
+
+    a.ldur(hd.reg, getCARRef(cons_ptr));
+    flush_var(hd);
+}
+
+void BeamModuleAssembler::emit_get_tl(const ArgVal &Src, const ArgVal &Tl) {
+    auto src = load_source(Src, TMP1);
+    auto tl = init_destination(Tl, TMP2);
+    arm::Gp cons_ptr = emit_ptr_val(TMP1, src.reg);
+
+    a.ldur(tl.reg, getCDRRef(cons_ptr));
+    flush_var(tl);
+}
+
+void BeamModuleAssembler::emit_i_get(const ArgVal &Src, const ArgVal &Dst) {
+    mov_arg(ARG2, Src);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_pd_hash_get);
+
+    emit_leave_runtime();
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_get_hash(const ArgVal &Src,
+                                          const ArgVal &Hash,
+                                          const ArgVal &Dst) {
+    mov_arg(ARG2, Hash);
+    mov_arg(ARG3, Src);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_pd_hash_get_with_hx);
+
+    emit_leave_runtime();
+
+    mov_arg(Dst, ARG1);
+}
+
+/* Store the untagged pointer to a tuple in ARG1. */
+void BeamModuleAssembler::emit_load_tuple_ptr(const ArgVal &Src) {
+    auto src = load_source(Src, ARG1);
+    emit_untag_ptr(ARG1, src.reg);
+}
+
+#ifdef DEBUG
+/* Emit an assertion to ensure that tuple_reg points into the same
+ * tuple as Src. */
+void BeamModuleAssembler::emit_tuple_assertion(const ArgVal &Src,
+                                               arm::Gp tuple_reg) {
+    Label ok = a.newLabel(), fatal = a.newLabel();
+    ASSERT(tuple_reg != TMP1);
+    mov_arg(TMP1, Src);
+    emit_is_boxed(fatal, TMP1);
+    emit_untag_ptr(TMP1, TMP1);
+    a.cmp(TMP1, tuple_reg);
+    a.cond_eq().b(ok);
+
+    a.bind(fatal);
+    a.udf(0xaaaa);
+    a.bind(ok);
+}
+#endif
+
+/* Fetch an element from the tuple pointed to by the untagged pointer
+ * in ARG1. */
+void BeamModuleAssembler::emit_i_get_tuple_element(const ArgVal &Src,
+                                                   const ArgVal &Element,
+                                                   const ArgVal &Dst) {
+#ifdef DEBUG
+    emit_tuple_assertion(Src, ARG1);
+#endif
+
+    auto dst = init_destination(Dst, TMP1);
+    safe_ldr(dst.reg, arm::Mem(ARG1, Element.getValue()));
+    flush_var(dst);
+}
+
+/* Fetch two consecutive tuple elements from the tuple pointed to by
+ * the boxed pointer in ARG1. */
+void BeamModuleAssembler::emit_get_two_tuple_elements(const ArgVal &Src,
+                                                      const ArgVal &Element,
+                                                      const ArgVal &Dst1,
+                                                      const ArgVal &Dst2) {
+#ifdef DEBUG
+    emit_tuple_assertion(Src, ARG1);
+#endif
+
+    auto dst1 = init_destination(Dst1, TMP1);
+    auto dst2 = init_destination(Dst2, TMP2);
+
+    arm::Mem element_ptr = arm::Mem(ARG1, Element.getValue());
+    safe_ldp(dst1.reg, dst2.reg, element_ptr);
+    flush_vars(dst1, dst2);
+}
+
+void BeamModuleAssembler::emit_init(const ArgVal &Y) {
+    mov_imm(TMP1, NIL);
+    a.str(TMP1, getArgRef(Y));
+}
+
+void BeamModuleAssembler::emit_init_yregs(const ArgVal &Size,
+                                          const Span<ArgVal> &args) {
+    unsigned count = Size.getValue();
+    ASSERT(count == args.size());
+
+    unsigned i = 0;
+
+    mov_imm(TMP1, NIL);
+
+    while (i < count) {
+        unsigned slots = 1;
+        unsigned first_y = args[i].getValue();
+
+        while (i + slots < count) {
+            const ArgVal &current_y = args[i + slots];
+            if (first_y + slots != current_y.getValue()) {
+                break;
+            }
+            slots++;
+        }
+
+        i += slots;
+
+        /* Now first_y is the number of the first y register to be initialized
+         * and slots is the number of y registers to be initialized. */
+        while (slots >= 2) {
+            /* `stp` can only address the first 64 Y registers. */
+            if (first_y <= MAX_LDP_STP_DISPLACEMENT) {
+                a.stp(TMP1, TMP1, getYRef(first_y));
+                first_y += 2;
+                slots -= 2;
+            } else {
+                a.str(TMP1, getYRef(first_y));
+                first_y += 1;
+                slots -= 1;
+            }
+        }
+
+        if (slots == 1) {
+            a.str(TMP1, getYRef(first_y));
+        }
+    }
+}
+
+void BeamModuleAssembler::emit_trim(const ArgVal &Words,
+                                    const ArgVal &Remaining) {
+    ASSERT(Words.isWord() && Words.getValue() <= 1023);
+
+    if (Words.getValue() > 0) {
+        add(E, E, Words.getValue() * sizeof(Eterm));
+    }
+}
+
+void BeamModuleAssembler::emit_i_move(const ArgVal &Src, const ArgVal &Dst) {
+    mov_arg(Dst, Src);
+}
+
+void BeamModuleAssembler::emit_store_two_xregs(const ArgVal &Src1,
+                                               const ArgVal &Dst1,
+                                               const ArgVal &Src2,
+                                               const ArgVal &Dst2) {
+    auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2);
+    safe_stp(src1.reg, src2.reg, Dst1, Dst2);
+}
+
+void BeamModuleAssembler::emit_load_two_xregs(const ArgVal &Src1,
+                                              const ArgVal &Dst1,
+                                              const ArgVal &Src2,
+                                              const ArgVal &Dst2) {
+    ASSERT(ArgVal::register_relation(Src1, Src2) ==
+           ArgVal::Relation::consecutive);
+    auto dst1 = init_destination(Dst1, TMP1);
+    auto dst2 = init_destination(Dst2, TMP2);
+
+    safe_ldp(dst1.reg, dst2.reg, Src1, Src2);
+    flush_vars(dst1, dst2);
+}
+
+void BeamModuleAssembler::emit_move_two_yregs(const ArgVal &Src1,
+                                              const ArgVal &Dst1,
+                                              const ArgVal &Src2,
+                                              const ArgVal &Dst2) {
+    /* Optimize fetching of source Y registers. */
+    switch (ArgVal::register_relation(Src1, Src2)) {
+    case ArgVal::Relation::consecutive:
+        safe_ldp(TMP1, TMP2, Src1, Src2);
+        break;
+    case ArgVal::Relation::reverse_consecutive:
+        safe_ldp(TMP2, TMP1, Src2, Src1);
+        break;
+    case ArgVal::Relation::none:
+        a.ldr(TMP1, getArgRef(Src1));
+        a.ldr(TMP2, getArgRef(Src2));
+        break;
+    }
+
+    /* Destination registers are always in consecutive order. */
+    safe_stp(TMP1, TMP2, Dst1, Dst2);
+}
+
+void BeamModuleAssembler::emit_swap(const ArgVal &R1, const ArgVal &R2) {
+    if (isRegisterBacked(R1)) {
+        auto r1 = load_source(R1, ZERO);
+        mov_arg(TMP1, R2);
+        mov_arg(R2, R1);
+        a.mov(r1.reg, TMP1);
+    } else if (isRegisterBacked(R2)) {
+        return emit_swap(R2, R1);
+    } else {
+        switch (ArgVal::register_relation(R1, R2)) {
+        case ArgVal::Relation::consecutive:
+            safe_ldp(TMP1, TMP2, R1, R2);
+            safe_stp(TMP2, TMP1, R1, R2);
+            break;
+        case ArgVal::Relation::reverse_consecutive:
+            safe_ldp(TMP1, TMP2, R2, R1);
+            safe_stp(TMP2, TMP1, R2, R1);
+            break;
+        case ArgVal::Relation::none:
+            a.ldr(TMP1, getArgRef(R1));
+            a.ldr(TMP2, getArgRef(R2));
+            a.str(TMP1, getArgRef(R2));
+            a.str(TMP2, getArgRef(R1));
+            break;
+        }
+    }
+}
+
+void BeamModuleAssembler::emit_swap2(const ArgVal &R1,
+                                     const ArgVal &R2,
+                                     const ArgVal &R3) {
+    auto [arg1, arg2] = load_sources(R1, TMP1, R2, TMP2);
+    auto arg3 = load_source(R3, TMP3);
+
+    mov_var(TMP4, arg1);
+    mov_var(arg1, arg2);
+    mov_var(arg2, arg3);
+    mov_var(arg3, TMP4);
+
+    flush_vars(arg1, arg2);
+    flush_var(arg3);
+}
+
+void BeamModuleAssembler::emit_swap3(const ArgVal &R1,
+                                     const ArgVal &R2,
+                                     const ArgVal &R3,
+                                     const ArgVal &R4) {
+    auto [arg1, arg2] = load_sources(R1, TMP1, R2, TMP2);
+    auto [arg3, arg4] = load_sources(R3, TMP3, R4, TMP4);
+
+    mov_var(TMP5, arg1);
+    mov_var(arg1, arg2);
+    mov_var(arg2, arg3);
+    mov_var(arg3, arg4);
+    mov_var(arg4, TMP5);
+
+    flush_vars(arg1, arg2);
+    flush_vars(arg3, arg4);
+}
+
+void BeamModuleAssembler::emit_swap4(const ArgVal &R1,
+                                     const ArgVal &R2,
+                                     const ArgVal &R3,
+                                     const ArgVal &R4,
+                                     const ArgVal &R5) {
+    auto [arg1, arg2] = load_sources(R1, TMP1, R2, TMP2);
+    auto [arg3, arg4] = load_sources(R3, TMP3, R4, TMP4);
+    auto arg5 = load_source(R5, TMP5);
+
+    mov_var(TMP6, arg1);
+    mov_var(arg1, arg2);
+    mov_var(arg2, arg3);
+    mov_var(arg3, arg4);
+    mov_var(arg4, arg5);
+    mov_var(arg5, TMP6);
+
+    flush_vars(arg1, arg2);
+    flush_vars(arg3, arg4);
+    flush_var(arg5);
+}
+
+void BeamModuleAssembler::emit_node(const ArgVal &Dst) {
+    a.ldr(TMP1, embed_constant(&erts_this_node, disp32K));
+    a.ldr(TMP1, arm::Mem(TMP1));
+    mov_arg(Dst, arm::Mem(TMP1, offsetof(ErlNode, sysname)));
+}
+
+void BeamModuleAssembler::emit_put_list(const ArgVal &Hd,
+                                        const ArgVal &Tl,
+                                        const ArgVal &Dst) {
+    auto [hd, tl] = load_sources(Hd, TMP1, Tl, TMP2);
+    auto dst = init_destination(Dst, TMP3);
+
+    a.stp(hd.reg, tl.reg, arm::Mem(HTOP).post(sizeof(Eterm[2])));
+    a.sub(dst.reg, HTOP, imm(sizeof(Eterm[2]) - TAG_PRIMARY_LIST));
+
+    flush_var(dst);
+}
+
+void BeamModuleAssembler::emit_put_list2(const ArgVal &Hd1,
+                                         const ArgVal &Hd2,
+                                         const ArgVal &Tl,
+                                         const ArgVal &Dst) {
+    const arm::Mem put_cons = arm::Mem(HTOP).post(sizeof(Eterm[2]));
+
+    auto [hd1, hd2] = load_sources(Hd1, TMP1, Hd2, TMP2);
+    auto tl = load_source(Tl, TMP3);
+    auto dst = init_destination(Dst, TMP4);
+
+    a.stp(hd1.reg, tl.reg, put_cons);
+    a.sub(dst.reg, HTOP, imm(sizeof(Eterm[2]) - TAG_PRIMARY_LIST));
+
+    a.stp(hd2.reg, dst.reg, put_cons);
+    a.sub(dst.reg, HTOP, imm(sizeof(Eterm[2]) - TAG_PRIMARY_LIST));
+
+    flush_var(dst);
+}
+
+void BeamModuleAssembler::emit_put_tuple2(const ArgVal &Dst,
+                                          const ArgVal &Arity,
+                                          const Span<ArgVal> &args) {
+    ASSERT(arityval(Arity.getValue()) == args.size());
+
+    std::vector<ArgVal> data;
+    data.reserve(args.size() + 1);
+    data.push_back(Arity);
+
+    bool dst_is_src = false;
+    for (auto arg : args) {
+        data.push_back(arg);
+        dst_is_src |= (arg == Dst);
+    }
+
+    if (dst_is_src) {
+        a.add(TMP1, HTOP, TAG_PRIMARY_BOXED);
+    } else {
+        auto ptr = init_destination(Dst, TMP1);
+        a.add(ptr.reg, HTOP, TAG_PRIMARY_BOXED);
+        flush_var(ptr);
+    }
+
+    size_t size = data.size();
+    unsigned i;
+    for (i = 0; i < size - 1; i += 2) {
+        if ((i % 128) == 0) {
+            check_pending_stubs();
+        }
+
+        auto [first, second] = load_sources(data[i], TMP2, data[i + 1], TMP3);
+        a.stp(first.reg, second.reg, arm::Mem(HTOP).post(sizeof(Eterm[2])));
+    }
+
+    if (i < size) {
+        mov_arg(arm::Mem(HTOP).post(sizeof(Eterm)), data[i]);
+    }
+
+    if (dst_is_src) {
+        auto ptr = init_destination(Dst, TMP1);
+        mov_var(ptr, TMP1);
+        flush_var(ptr);
+    }
+}
+
+void BeamModuleAssembler::emit_self(const ArgVal &Dst) {
+    mov_arg(Dst, arm::Mem(c_p, offsetof(Process, common.id)));
+}
+
+void BeamModuleAssembler::emit_set_tuple_element(const ArgVal &Element,
+                                                 const ArgVal &Tuple,
+                                                 const ArgVal &Offset) {
+    auto tuple = load_source(Tuple, TMP1);
+    auto element = load_source(Element, TMP2);
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, tuple.reg);
+    arm::Mem boxed_val = emit_boxed_val(boxed_ptr, Offset.getValue());
+
+    stur(element.reg, boxed_val);
+}
+
+void BeamModuleAssembler::emit_is_nonempty_list(const ArgVal &Fail,
+                                                const ArgVal &Src) {
+    auto list_ptr = load_source(Src, TMP1);
+    const int bitNumber = 1;
+
+    ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST == (1 << bitNumber));
+    a.tbnz(list_ptr.reg, bitNumber, resolve_beam_label(Fail, disp32K));
+}
+
+void BeamModuleAssembler::emit_jump(const ArgVal &Fail) {
+    a.b(resolve_beam_label(Fail, disp128MB));
+}
+
+void BeamModuleAssembler::emit_is_atom(const ArgVal &Fail, const ArgVal &Src) {
+    auto src = load_source(Src, TMP1);
+
+    a.and_(TMP1, src.reg, imm(_TAG_IMMED2_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED2_ATOM));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+void BeamModuleAssembler::emit_is_boolean(const ArgVal &Fail,
+                                          const ArgVal &Src) {
+    /* Since am_true and am_false differ by a single bit, we can simplify the
+     * check by clearing said bit and comparing against the lesser one. */
+    ERTS_CT_ASSERT(am_false == make_atom(0));
+    ERTS_CT_ASSERT(am_true == make_atom(1));
+
+    auto src = load_source(Src, TMP1);
+    a.and_(TMP1, src.reg, imm(~(am_true & ~_TAG_IMMED1_MASK)));
+    a.cmp(TMP1, imm(am_false));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+arm::Gp BeamModuleAssembler::emit_is_binary(const ArgVal &Fail,
+                                            const ArgVal &Src,
+                                            Label next,
+                                            Label subbin) {
+    auto src = load_source(Src, ARG1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+
+    arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP1, imm(_TAG_HEADER_SUB_BIN));
+    a.cond_eq().b(subbin);
+
+    ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN);
+    a.and_(TMP1, TMP1, imm(~4));
+    a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN));
+    a.cond_eq().b(next);
+    a.b(resolve_beam_label(Fail, disp128MB));
+
+    return boxed_ptr;
+}
+
+void BeamModuleAssembler::emit_is_binary(const ArgVal &Fail,
+                                         const ArgVal &Src) {
+    Label next = a.newLabel(), subbin = a.newLabel();
+    arm::Gp boxed_ptr;
+
+    boxed_ptr = emit_is_binary(Fail, Src, next, subbin);
+
+    a.bind(subbin);
+    {
+        /* emit_is_binary() has already removed the literal tag (if
+         * applicable) from the copy of Src. */
+        a.ldrb(TMP1.w(),
+               emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize)));
+        a.cbnz(TMP1, resolve_beam_label(Fail, disp1MB));
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_bitstring(const ArgVal &Fail,
+                                            const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    (void)emit_is_binary(Fail, Src, next, next);
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_float(const ArgVal &Fail, const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+
+    a.cmp(TMP1, imm(HEADER_FLONUM));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_function(const ArgVal &Fail,
+                                           const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.cmp(TMP1, imm(HEADER_FUN));
+    a.cond_eq().b(next);
+    a.cmp(TMP1, imm(HEADER_EXPORT));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_function2(const ArgVal &Fail,
+                                            const ArgVal &Src,
+                                            const ArgVal &Arity) {
+    if (Arity.getType() != ArgVal::Immediate) {
+        /*
+         * Non-literal arity - extremely uncommon. Generate simple code.
+         */
+        mov_arg(ARG2, Src);
+        mov_arg(ARG3, Arity);
+
+        emit_enter_runtime();
+
+        a.mov(ARG1, c_p);
+        runtime_call<3>(erl_is_function);
+
+        emit_leave_runtime();
+
+        a.cmp(ARG1, imm(am_true));
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+        return;
+    }
+
+    unsigned arity = unsigned_val(Arity.getValue());
+    if (arity > MAX_ARG) {
+        /* Arity is negative or too large. */
+        a.b(resolve_beam_label(Fail, disp128MB));
+
+        return;
+    }
+
+    Label next = a.newLabel();
+    Label fun = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP2, emit_boxed_val(boxed_ptr));
+    a.cmp(TMP2, imm(HEADER_FUN));
+    a.cond_eq().b(fun);
+    a.cmp(TMP2, imm(HEADER_EXPORT));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    comment("Check arity of export fun");
+    a.ldur(TMP2, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    a.ldr(TMP2, arm::Mem(TMP2, offsetof(Export, info.mfa.arity)));
+    emit_branch_if_ne(TMP2, arity, resolve_beam_label(Fail, dispUnknown));
+    a.b(next);
+
+    comment("Check arity of fun");
+    a.bind(fun);
+    {
+        a.ldur(TMP2, emit_boxed_val(boxed_ptr, offsetof(ErlFunThing, arity)));
+        emit_branch_if_ne(TMP2, arity, resolve_beam_label(Fail, dispUnknown));
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_integer(const ArgVal &Fail,
+                                          const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP2, imm(_TAG_IMMED1_SMALL));
+    a.cond_eq().b(next);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+
+    /* The following value (0b111011) is not possible to use as
+     * an immediate operand for 'and'. See the note at the beginning
+     * of the file.
+     */
+    mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT);
+    a.and_(TMP1, TMP1, TMP2);
+    a.cmp(TMP1, imm(_TAG_HEADER_POS_BIG));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_list(const ArgVal &Fail, const ArgVal &Src) {
+    auto src = load_source(Src, TMP1);
+
+    a.tst(src.reg, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST));
+    a.mov(TMP2, NIL);
+    a.ccmp(src.reg, TMP2, 4, arm::Cond::kNE);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+void BeamModuleAssembler::emit_is_map(const ArgVal &Fail, const ArgVal &Src) {
+    auto src = load_source(Src, TMP1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP1, imm(_TAG_HEADER_MAP));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+void BeamModuleAssembler::emit_is_nil(const ArgVal &Fail, const ArgVal &Src) {
+    auto src = load_source(Src, TMP1);
+    a.cmp(src.reg, imm(NIL));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+void BeamModuleAssembler::emit_is_number(const ArgVal &Fail,
+                                         const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP2, imm(_TAG_IMMED1_SMALL));
+    a.cond_eq().b(next);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+
+    /* The following value (0b111011) is not possible to use as
+     * an immediate operand for 'and'. See the note at the beginning
+     * of the file.
+     */
+    mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT);
+    a.and_(TMP2, TMP1, TMP2);
+    a.cmp(TMP2, imm(_TAG_HEADER_POS_BIG));
+
+    a.mov(TMP3, imm(HEADER_FLONUM));
+    a.ccmp(TMP1, TMP3, 4, arm::Cond::kNE);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_pid(const ArgVal &Fail, const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP2, imm(_TAG_IMMED1_PID));
+    a.cond_eq().b(next);
+
+    /* Reuse TMP2 as the important bits are still available. */
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP2, emit_boxed_val(boxed_ptr));
+    a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP2, imm(_TAG_HEADER_EXTERNAL_PID));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_port(const ArgVal &Fail, const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP2, imm(_TAG_IMMED1_PORT));
+    a.cond_eq().b(next);
+
+    /* Reuse TMP2 as the important bits are still available. */
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP2, emit_boxed_val(boxed_ptr));
+    a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP2, imm(_TAG_HEADER_EXTERNAL_PORT));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_reference(const ArgVal &Fail,
+                                            const ArgVal &Src) {
+    Label next = a.newLabel();
+
+    auto src = load_source(Src, TMP1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP1, imm(_TAG_HEADER_EXTERNAL_REF));
+    a.ccmp(TMP1, imm(_TAG_HEADER_REF), 4, arm::Cond::kNE);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+/* Note: This instruction leaves the untagged pointer to the tuple in
+ * ARG1. */
+void BeamModuleAssembler::emit_i_is_tagged_tuple(const ArgVal &Fail,
+                                                 const ArgVal &Src,
+                                                 const ArgVal &Arity,
+                                                 const ArgVal &Tag) {
+    auto src = load_source(Src, ARG1);
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+    emit_untag_ptr(ARG1, src.reg);
+
+    /* It is tempting to fetch the both the header word and the first
+     * element of the tuple with ldp, but that is potentially unsafe
+     * if Src is the empty tuple. To make ldp safe, we would have to
+     * ensure that an empty tuple in a heap fragment, literal area, or
+     * persistent term is always followed by one word of allocated
+     * memory. */
+    a.ldr(TMP1, arm::Mem(ARG1));
+    cmp_arg(TMP1, Arity);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.ldr(TMP2, arm::Mem(ARG1, sizeof(Eterm)));
+    cmp_arg(TMP2, Tag);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+/* Note: This instruction leaves the untagged pointer to the tuple in
+ * ARG2. */
+void BeamModuleAssembler::emit_i_is_tagged_tuple_ff(const ArgVal &NotTuple,
+                                                    const ArgVal &NotRecord,
+                                                    const ArgVal &Src,
+                                                    const ArgVal &Arity,
+                                                    const ArgVal &Tag) {
+    Label correct_arity = a.newLabel();
+    auto src = load_source(Src, ARG1);
+
+    emit_is_boxed(resolve_beam_label(NotTuple, dispUnknown), src.reg);
+    emit_untag_ptr(ARG1, src.reg);
+
+    /* It is tempting to fetch the both the header word and the first
+     * element of the tuple with ldp, but that is potentially unsafe
+     * if Src is the empty tuple. */
+    a.ldr(TMP1, arm::Mem(ARG1));
+
+    cmp_arg(TMP1, Arity);
+    a.cond_eq().b(correct_arity);
+
+    /* Not a tuple or the wrong arity. Decide which. */
+    ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0);
+    a.tst(TMP1, imm(_TAG_HEADER_MASK));
+    a.cond_eq().b(resolve_beam_label(NotRecord, disp1MB));
+    a.b(resolve_beam_label(NotTuple, disp128MB));
+
+    a.bind(correct_arity);
+    {
+        a.ldr(TMP2, arm::Mem(ARG1, sizeof(Eterm)));
+        cmp_arg(TMP2, Tag);
+        a.cond_ne().b(resolve_beam_label(NotRecord, disp1MB));
+    }
+}
+
+/* Note: This instruction leaves the untagged pointer to the tuple in
+ * ARG1. */
+void BeamModuleAssembler::emit_i_is_tuple(const ArgVal &Fail,
+                                          const ArgVal &Src) {
+    auto src = load_source(Src, ARG1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+    emit_untag_ptr(ARG1, src.reg);
+
+    a.ldr(TMP1, arm::Mem(ARG1));
+    ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0);
+    a.tst(TMP1, imm(_TAG_HEADER_MASK));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+/* Note: This instruction leaves the untagged pointer to the tuple in
+ * ARG1. */
+void BeamModuleAssembler::emit_i_is_tuple_of_arity(const ArgVal &Fail,
+                                                   const ArgVal &Src,
+                                                   const ArgVal &Arity) {
+    auto src = load_source(Src, ARG1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+    emit_untag_ptr(ARG1, src.reg);
+
+    a.ldr(TMP1, arm::Mem(ARG1));
+    cmp_arg(TMP1, Arity);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+/* Note: This instruction leaves the untagged pointer to the tuple in
+ * ARG1. */
+void BeamModuleAssembler::emit_i_test_arity(const ArgVal &Fail,
+                                            const ArgVal &Src,
+                                            const ArgVal &Arity) {
+    auto src = load_source(Src, ARG1);
+    emit_untag_ptr(ARG1, src.reg);
+
+    a.ldr(TMP1, arm::Mem(ARG1));
+    cmp_arg(TMP1, Arity);
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+}
+
+void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail,
+                                           const ArgVal &X,
+                                           const ArgVal &Y) {
+    auto x = load_source(X, ARG1);
+
+    if (Y.isImmed()) {
+        /* If the second operand is a known to be an immediate, we can
+         * fail immediately if the operands are not equal. */
+        cmp_arg(x.reg, Y);
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+        return;
+    }
+
+    /* Both operands are registers. */
+    Label next = a.newLabel();
+    auto y = load_source(Y, ARG2);
+
+    a.cmp(x.reg, y.reg);
+    a.cond_eq().b(next);
+
+    /* The terms could still be equal if both operands are pointers
+     * having the same tag. */
+    emit_is_unequal_based_on_tags(x.reg, y.reg);
+    a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+    /* Both operands are pointers having the same tag. Must do a
+     * deeper comparison. */
+    mov_var(ARG1, x);
+    mov_var(ARG2, y);
+
+    emit_enter_runtime();
+
+    runtime_call<2>(eq);
+
+    emit_leave_runtime();
+
+    a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail,
+                                           const ArgVal &X,
+                                           const ArgVal &Y) {
+    auto x = load_source(X, ARG1);
+
+    if (Y.isImmed()) {
+        /* If the second operand is a known to be an immediate, we can
+         * fail immediately if the operands are equal. */
+        cmp_arg(x.reg, Y);
+        a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+        return;
+    }
+
+    /* Both operands are registers. */
+    Label next = a.newLabel();
+    auto y = load_source(Y, ARG2);
+
+    a.cmp(x.reg, y.reg);
+    a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+    /* Test whether the terms are definitely unequal based on the tags
+     * alone. */
+    emit_is_unequal_based_on_tags(x.reg, y.reg);
+    a.cond_eq().b(next);
+
+    /* Both operands are pointers having the same tag. Must do a
+     * deeper comparison. */
+    mov_var(ARG1, x);
+    mov_var(ARG2, y);
+
+    emit_enter_runtime();
+
+    runtime_call<2>(eq);
+
+    emit_leave_runtime();
+
+    a.cbnz(ARG1, resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_eq(const ArgVal &Fail,
+                                     const ArgVal &X,
+                                     const ArgVal &Y) {
+    Label next = a.newLabel();
+
+    auto x = load_source(X, ARG1);
+    auto y = load_source(Y, ARG2);
+
+    a.cmp(x.reg, y.reg);
+    a.cond_eq().b(next);
+
+    /* We can skip deep comparisons when both args are immediates. */
+    emit_are_both_immediate(x.reg, y.reg);
+    a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+    mov_var(ARG1, x);
+    mov_var(ARG2, y);
+    fragment_call(ga->get_arith_compare_shared());
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_ne(const ArgVal &Fail,
+                                     const ArgVal &X,
+                                     const ArgVal &Y) {
+    Label next = a.newLabel();
+
+    auto x = load_source(X, ARG1);
+    auto y = load_source(Y, ARG2);
+
+    a.cmp(x.reg, y.reg);
+    a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+    /* We can skip deep comparisons when both args are immediates. */
+    emit_are_both_immediate(x.reg, y.reg);
+    a.cond_eq().b(next);
+
+    mov_var(ARG1, x);
+    mov_var(ARG2, y);
+    fragment_call(ga->get_arith_compare_shared());
+    a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+    a.bind(next);
+}
+
+/*
+ * ARG1 = LHS
+ * ARG2 = RHS
+ *
+ * Result is returned in the flags.
+ */
+void BeamGlobalAssembler::emit_arith_compare_shared() {
+    Label atom_compare = a.newLabel(), generic_compare = a.newLabel();
+
+    /* Are both floats?
+     *
+     * This is done first as relative comparisons on atoms doesn't make much
+     * sense. */
+    a.orr(TMP1, ARG1, ARG2);
+    emit_is_boxed(atom_compare, TMP1);
+
+    arm::Gp boxed_ptr1 = emit_ptr_val(TMP1, ARG1);
+    a.ldur(TMP3, emit_boxed_val(boxed_ptr1));
+    arm::Gp boxed_ptr2 = emit_ptr_val(TMP2, ARG2);
+    a.ldur(TMP4, emit_boxed_val(boxed_ptr2));
+
+    mov_imm(TMP5, HEADER_FLONUM);
+    a.cmp(TMP3, TMP5);
+    a.ccmp(TMP4, TMP5, 0, arm::Cond::kEQ);
+    a.cond_ne().b(generic_compare);
+
+    a.ldur(a64::d0, emit_boxed_val(boxed_ptr1, sizeof(Eterm)));
+    a.ldur(a64::d1, emit_boxed_val(boxed_ptr2, sizeof(Eterm)));
+    a.fcmpe(a64::d0, a64::d1);
+    a.ret(a64::x30);
+
+    a.bind(atom_compare);
+    {
+        a.and_(TMP1, ARG1, imm(_TAG_IMMED2_MASK));
+        a.and_(TMP2, ARG2, imm(_TAG_IMMED2_MASK));
+        a.sub(TMP1, TMP1, imm(_TAG_IMMED2_ATOM));
+        a.sub(TMP2, TMP2, imm(_TAG_IMMED2_ATOM));
+        a.orr(TMP1, TMP1, TMP2);
+        a.cbnz(TMP1, generic_compare);
+
+        emit_enter_runtime_frame();
+        emit_enter_runtime();
+
+        runtime_call<2>(erts_cmp_atoms);
+
+        emit_leave_runtime();
+        emit_leave_runtime_frame();
+
+        /* Note: erts_cmp_atoms() returns int, not Sint. */
+        a.tst(ARG1.w(), ARG1.w());
+        a.ret(a64::x30);
+    }
+
+    a.bind(generic_compare);
+    {
+        emit_enter_runtime_frame();
+        emit_enter_runtime();
+
+        comment("erts_cmp_compound(X, Y, 0, 0);");
+        mov_imm(ARG3, 0);
+        mov_imm(ARG4, 0);
+        runtime_call<4>(erts_cmp_compound);
+
+        emit_leave_runtime();
+        emit_leave_runtime_frame();
+
+        a.tst(ARG1, ARG1);
+
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_is_lt(const ArgVal &Fail,
+                                     const ArgVal &LHS,
+                                     const ArgVal &RHS) {
+    Label generic = a.newLabel(), next = a.newLabel();
+
+    auto lhs = load_source(LHS, ARG1);
+    auto rhs = load_source(RHS, ARG2);
+
+    a.cmp(lhs.reg, rhs.reg);
+    a.cond_eq().b(resolve_beam_label(Fail, disp1MB));
+
+    /* Relative comparisons are overwhelmingly likely to be used on smalls, so
+     * we'll specialize those and keep the rest in a shared fragment. */
+
+    if (RHS.isImmed() && is_small(RHS.getValue())) {
+        a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+    } else if (LHS.isImmed() && is_small(LHS.getValue())) {
+        a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK));
+    } else {
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        a.and_(TMP1, lhs.reg, rhs.reg);
+        a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    }
+
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(generic);
+
+    a.cmp(lhs.reg, rhs.reg);
+    a.cond_lt().b(next);
+    a.b(resolve_beam_label(Fail, disp128MB));
+
+    a.bind(generic);
+    {
+        mov_var(ARG1, lhs);
+        mov_var(ARG2, rhs);
+        fragment_call(ga->get_arith_compare_shared());
+        a.cond_ge().b(resolve_beam_label(Fail, disp1MB));
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_is_ge(const ArgVal &Fail,
+                                     const ArgVal &LHS,
+                                     const ArgVal &RHS) {
+    Label generic = a.newLabel(), next = a.newLabel();
+
+    auto lhs = load_source(LHS, ARG1);
+    auto rhs = load_source(RHS, ARG2);
+
+    a.cmp(lhs.reg, rhs.reg);
+    a.cond_eq().b(next);
+
+    /* Relative comparisons are overwhelmingly likely to be used on smalls, so
+     * we'll specialize those and keep the rest in a shared fragment. */
+
+    if (RHS.isImmed() && is_small(RHS.getValue())) {
+        a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK));
+    } else if (LHS.isImmed() && is_small(LHS.getValue())) {
+        a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK));
+    } else {
+        ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+        a.and_(TMP1, lhs.reg, rhs.reg);
+        a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK));
+    }
+
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(generic);
+
+    a.cmp(lhs.reg, rhs.reg);
+    a.cond_ge().b(next);
+    a.b(resolve_beam_label(Fail, disp128MB));
+
+    a.bind(generic);
+    {
+        mov_var(ARG1, lhs);
+        mov_var(ARG2, rhs);
+        fragment_call(ga->get_arith_compare_shared());
+        a.cond_lt().b(resolve_beam_label(Fail, disp1MB));
+    }
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_badmatch(const ArgVal &Src) {
+    mov_arg(arm::Mem(c_p, offsetof(Process, fvalue)), Src);
+    emit_error(BADMATCH);
+}
+
+void BeamModuleAssembler::emit_case_end(const ArgVal &Src) {
+    mov_arg(arm::Mem(c_p, offsetof(Process, fvalue)), Src);
+    emit_error(EXC_CASE_CLAUSE);
+}
+
+void BeamModuleAssembler::emit_system_limit_body() {
+    emit_error(SYSTEM_LIMIT);
+}
+
+void BeamModuleAssembler::emit_if_end() {
+    emit_error(EXC_IF_CLAUSE);
+}
+
+void BeamModuleAssembler::emit_catch(const ArgVal &Y, const ArgVal &Handler) {
+    a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, catches)));
+    a.add(TMP1, TMP1, imm(1));
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, catches)));
+
+    mov_arg(Y, Handler);
+}
+
+void BeamGlobalAssembler::emit_catch_end_shared() {
+    Label not_throw = a.newLabel(), not_error = a.newLabel(),
+          after_gc = a.newLabel();
+
+    /* XREG0 = THE_NON_VALUE
+     * XREG1 = class
+     * XREG2 = error reason/thrown value
+     * XREG3 = raw stacktrace. */
+
+    mov_imm(TMP1, NIL);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, fvalue)));
+
+    a.mov(XREG0, XREG2);
+    a.cmp(XREG1, imm(am_throw));
+
+    a.cond_ne().b(not_throw);
+
+    /* Return thrown value. */
+    a.ret(a64::x30);
+
+    a.bind(not_throw);
+    {
+        emit_enter_runtime_frame();
+
+        a.mov(ARG1, XREG0);
+
+        a.cmp(XREG1, imm(am_error));
+        a.cond_ne().b(not_error);
+
+        a.mov(ARG2, XREG0);
+        a.mov(ARG3, XREG3);
+
+        /* This is an error, attach a stacktrace to the reason. */
+        ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 2);
+        emit_enter_runtime<Update::eStack | Update::eHeap>(2);
+
+        a.mov(ARG1, c_p);
+        runtime_call<3>(add_stacktrace);
+
+        emit_leave_runtime<Update::eStack | Update::eHeap>(2);
+
+        /* Fall through! */
+    }
+
+    /* Error term from exit/1 or stack backtrace from error/{1,2,3} is
+     * now in ARG1. */
+    a.bind(not_error);
+    {
+        const int32_t bytes_needed = (3 + S_RESERVED) * sizeof(Eterm);
+        add(ARG3, HTOP, bytes_needed);
+        a.cmp(ARG3, E);
+
+        a.cond_ls().b(after_gc);
+        {
+            /* Preserve stacktrace / reason. */
+            a.mov(XREG0, ARG1);
+
+            mov_imm(ARG4, 1);
+            a.bl(labels[garbage_collect]);
+
+            a.mov(ARG1, XREG0);
+        }
+        a.bind(after_gc);
+
+        a.add(XREG0, HTOP, imm(TAG_PRIMARY_BOXED));
+        mov_imm(TMP1, make_arityval(2));
+        mov_imm(TMP2, am_EXIT);
+        a.stp(TMP1, TMP2, arm::Mem(HTOP).post(2 * sizeof(Eterm)));
+        a.str(ARG1, arm::Mem(HTOP).post(sizeof(Eterm)));
+    }
+
+    emit_leave_runtime_frame();
+    a.ret(a64::x30);
+}
+
+void BeamModuleAssembler::emit_catch_end(const ArgVal &Y) {
+    Label next = a.newLabel();
+
+    /* XREG0 = THE_NON_VALUE
+     * XREG1 = class
+     * XREG2 = error reason/thrown value
+     * XREG3 = raw stacktrace. */
+
+    emit_try_end(Y);
+    emit_branch_if_value(XREG0, next);
+    fragment_call(ga->get_catch_end_shared());
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_try_end(const ArgVal &Y) {
+    a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, catches)));
+    a.sub(TMP1, TMP1, imm(1));
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, catches)));
+    emit_init(Y);
+}
+
+void BeamModuleAssembler::emit_try_case(const ArgVal &Y) {
+    /* XREG0 = THE_NON_VALUE
+     * XREG1 = class
+     * XREG2 = error reason/thrown value
+     * XREG3 = raw stacktrace.
+     *
+     * These need to be shifted down one step. */
+    a.mov(XREG0, XREG1);
+    a.mov(XREG1, XREG2);
+    a.mov(XREG2, XREG3);
+
+    a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, catches)));
+    a.sub(TMP1, TMP1, imm(1));
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, catches)));
+    mov_imm(TMP1, NIL);
+    mov_arg(Y, TMP1);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, fvalue)));
+}
+
+void BeamModuleAssembler::emit_try_case_end(const ArgVal &Src) {
+    mov_arg(arm::Mem(c_p, offsetof(Process, fvalue)), Src);
+    emit_error(EXC_TRY_CLAUSE);
+}
+
+void BeamModuleAssembler::emit_raise(const ArgVal &Trace, const ArgVal &Value) {
+    mov_arg(TMP1, Value);
+    mov_arg(ARG2, Trace);
+
+    /* This is an error, attach a stacktrace to the reason. */
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, fvalue)));
+    a.str(ARG2, arm::Mem(c_p, offsetof(Process, ftrace)));
+
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_sanitize_freason);
+
+    emit_leave_runtime(0);
+
+    emit_raise_exception();
+}
+
+void BeamModuleAssembler::emit_build_stacktrace() {
+    a.mov(ARG2, XREG0);
+
+    emit_enter_runtime<Update::eStack | Update::eHeap>(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(build_stacktrace);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap>(0);
+
+    a.mov(XREG0, ARG1);
+}
+
+/* This instruction has the same semantics as the erlang:raise/3 BIF,
+ * except that it can rethrow a raw stack backtrace. */
+void BeamModuleAssembler::emit_raw_raise() {
+    Label next = a.newLabel();
+
+    a.mov(ARG1, XREG2);
+    a.mov(ARG2, XREG0);
+    a.mov(ARG3, XREG1);
+    a.mov(ARG4, c_p);
+
+    emit_enter_runtime(0);
+    runtime_call<4>(raw_raise);
+    emit_leave_runtime(0);
+
+    a.cbnz(ARG1, next);
+
+    emit_raise_exception();
+
+    a.bind(next);
+    mov_imm(XREG0, am_badarg);
+}
+
+static size_t TEST_YIELD_RETURN_OFFSET = sizeof(Uint32[3]);
+
+void BeamGlobalAssembler::emit_i_test_yield_shared() {
+    int mfa_offset = sizeof(ErtsCodeMFA) + BEAM_ASM_FUNC_PROLOGUE_SIZE;
+
+    /* Yield return address is in LR (x30). */
+    a.sub(ARG2, a64::x30, imm(TEST_YIELD_RETURN_OFFSET + mfa_offset));
+    a.str(ARG2, arm::Mem(c_p, offsetof(Process, current)));
+
+    a.ldr(ARG2, arm::Mem(ARG2, offsetof(ErtsCodeMFA, arity)));
+    a.str(ARG2, arm::Mem(c_p, offsetof(Process, arity)));
+
+    a.mov(ARG3, a64::x30);
+    a.b(labels[context_switch_simplified]);
+}
+
+void BeamModuleAssembler::emit_i_test_yield() {
+    Label next = a.newLabel();
+
+    /* When present, this is guaranteed to be the first instruction after the
+     * breakpoint trampoline. */
+
+#ifdef DEBUG
+    size_t startPos = a.offset();
+    ASSERT(startPos % 8 == 0);
+#endif
+
+    a.subs(FCALLS, FCALLS, imm(1));
+    a.cond_gt().b(next);
+
+    /* We avoid using the `fragment_call` helper to ensure a constant layout,
+     * as it adds code in certain debug configurations. */
+    a.bl(resolve_fragment(ga->get_i_test_yield_shared(), disp128MB));
+    a.bind(next);
+
+    ASSERT(a.offset() - startPos == TEST_YIELD_RETURN_OFFSET);
+}
+
+void BeamModuleAssembler::emit_i_yield() {
+    mov_imm(XREG0, am_true);
+    fragment_call(ga->get_dispatch_return());
+}
+
+void BeamModuleAssembler::emit_i_perf_counter() {
+    Label next = a.newLabel(), small = a.newLabel();
+
+    emit_enter_runtime_frame();
+    runtime_call<0>(erts_sys_time_data__.r.o.perf_counter);
+    emit_leave_runtime_frame();
+
+    a.asr(TMP1, ARG1, imm(SMALL_BITS - 1));
+    a.add(TMP1, TMP1, imm(1));
+    a.cmp(TMP1, imm(1));
+    a.cond_ls().b(small);
+
+    {
+        a.mov(XREG0, ARG1);
+
+        emit_gc_test(ArgVal(ArgVal::Word, 0),
+                     ArgVal(ArgVal::Word, ERTS_MAX_UINT64_HEAP_SIZE),
+                     ArgVal(ArgVal::Word, 0));
+
+        mov_imm(TMP1, make_pos_bignum_header(1));
+        a.stp(TMP1, XREG0, arm::Mem(HTOP).post(sizeof(Eterm[2])));
+        a.sub(XREG0, HTOP, imm(sizeof(Eterm[2]) - TAG_PRIMARY_BOXED));
+        a.b(next);
+    }
+
+    a.bind(small);
+    {
+        a.lsl(ARG1, ARG1, imm(_TAG_IMMED1_SIZE));
+        a.orr(XREG0, ARG1, imm(_TAG_IMMED1_SMALL));
+    }
+
+    a.bind(next);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_float.cpp b/erts/emulator/beam/jit/arm/instr_float.cpp
new file mode 100644
index 0000000000..b1c4220100
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_float.cpp
@@ -0,0 +1,162 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "big.h"
+}
+
+/* Checks whether d0 contains a finite value.
+ *
+ * Clobbers d30 and d31. */
+void BeamGlobalAssembler::emit_check_float_error() {
+    Label double_max = a.newLabel(), error = a.newLabel();
+
+    a.fabs(a64::d30, a64::d0);
+    a.ldr(a64::d31, arm::Mem(double_max));
+    a.fcmp(a64::d30, a64::d31);
+    a.cond_hi().b(error);
+    a.ret(a64::x30);
+
+    a.align(kAlignCode, 8);
+    a.bind(double_max);
+    a.embedUInt64(0x7FEFFFFFFFFFFFFFul);
+
+    a.bind(error);
+    {
+        mov_imm(ARG4, 0);
+        mov_imm(TMP1, EXC_BADARITH);
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_float_instr(uint32_t instId,
+                                           const ArgVal &LHS,
+                                           const ArgVal &RHS,
+                                           const ArgVal &Dst) {
+    auto lhs = load_source(LHS, a64::d0);
+    auto rhs = load_source(RHS, a64::d1);
+    auto dst = init_destination(Dst, a64::d2);
+
+    a.emit(instId, a64::d0, lhs.reg, rhs.reg);
+    fragment_call(ga->get_check_float_error());
+    a.mov(dst.reg, a64::d0);
+    flush_var(dst);
+}
+
+/* * * * */
+
+void BeamModuleAssembler::emit_fload(const ArgVal &Src, const ArgVal &Dst) {
+    auto src = load_source(Src, TMP1);
+    auto dst = init_destination(Dst, a64::d0);
+    arm::Gp float_ptr = emit_ptr_val(TMP1, src.reg);
+
+    a.ldur(dst.reg, emit_boxed_val(float_ptr, sizeof(Eterm)));
+    flush_var(dst);
+}
+
+void BeamModuleAssembler::emit_fstore(const ArgVal &Src, const ArgVal &Dst) {
+    auto src = load_source(Src, a64::d0);
+    auto dst = init_destination(Dst, TMP1);
+
+    a.add(dst.reg, HTOP, imm(TAG_PRIMARY_BOXED));
+
+    mov_imm(TMP2, HEADER_FLONUM);
+    a.str(TMP2, arm::Mem(HTOP).post(sizeof(Eterm)));
+    a.str(src.reg, arm::Mem(HTOP).post(sizeof(Eterm)));
+
+    flush_var(dst);
+}
+
+static double handle_fconv(Eterm src) {
+    if (is_small(src)) {
+        return (double)signed_val(src);
+    } else if (is_float(src)) {
+        double res;
+        GET_DOUBLE(src, *(FloatDef *)&res);
+        return res;
+    } else if (is_big(src)) {
+        double res;
+        if (big_to_double(src, &res) < 0) {
+            return NAN;
+        }
+        return res;
+    } else {
+        return NAN;
+    }
+}
+
+void BeamGlobalAssembler::emit_fconv_shared() {
+    emit_enter_runtime_frame();
+    emit_enter_runtime();
+
+    runtime_call<1>(handle_fconv);
+
+    emit_leave_runtime();
+    emit_leave_runtime_frame();
+
+    a.b(labels[check_float_error]);
+}
+
+void BeamModuleAssembler::emit_fconv(const ArgVal &Src, const ArgVal &Dst) {
+    auto dst = init_destination(Dst, a64::d0);
+
+    mov_arg(ARG1, Src);
+    fragment_call(ga->get_fconv_shared());
+    a.mov(dst.reg, a64::d0);
+    flush_var(dst);
+}
+
+void BeamModuleAssembler::emit_i_fadd(const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    emit_float_instr(a64::Inst::kIdFadd_v, LHS, RHS, Dst);
+}
+
+void BeamModuleAssembler::emit_i_fsub(const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    emit_float_instr(a64::Inst::kIdFsub_v, LHS, RHS, Dst);
+}
+
+void BeamModuleAssembler::emit_i_fmul(const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    emit_float_instr(a64::Inst::kIdFmul_v, LHS, RHS, Dst);
+}
+
+void BeamModuleAssembler::emit_i_fdiv(const ArgVal &LHS,
+                                      const ArgVal &RHS,
+                                      const ArgVal &Dst) {
+    emit_float_instr(a64::Inst::kIdFdiv_v, LHS, RHS, Dst);
+}
+
+void BeamModuleAssembler::emit_i_fnegate(const ArgVal &Src, const ArgVal &Dst) {
+    auto src = load_source(Src, a64::d0);
+    auto dst = init_destination(Dst, a64::d1);
+
+    a.fneg(a64::d0, src.reg);
+    fragment_call(ga->get_check_float_error());
+    a.mov(dst.reg, a64::d0);
+    flush_var(dst);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_fun.cpp b/erts/emulator/beam/jit/arm/instr_fun.cpp
new file mode 100644
index 0000000000..bd04162ba2
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_fun.cpp
@@ -0,0 +1,412 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2021-2021. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+/* Calls to functions that are being purged (but haven't finished) land here.
+ *
+ * Keep in mind that this runs in the limbo between caller and callee. It must
+ * not clobber LR (x30).
+ *
+ * ARG3 = arity
+ * ARG4 = fun thing
+ * ARG5 = address of the call_fun instruction that got us here. Note that we
+ *        can't use LR (x30) for this because tail calls point elsehwere. */
+void BeamGlobalAssembler::emit_unloaded_fun() {
+    Label error = a.newLabel();
+
+    a.str(ARG5, TMP_MEM1q);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eHeap | Update::eStack | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    /* ARG3 and ARG4 have already been set. */
+    runtime_call<4>(beam_jit_handle_unloaded_fun);
+
+    emit_leave_runtime<Update::eHeap | Update::eStack | Update::eXRegs |
+                       Update::eReductions | Update::eCodeIndex>();
+    emit_leave_runtime_frame();
+
+    a.cbz(ARG1, error);
+
+    a.ldr(TMP1, emit_setup_export_call(ARG1));
+    a.br(TMP1);
+
+    a.bind(error);
+    {
+        a.ldr(ARG2, TMP_MEM1q);
+        mov_imm(ARG4, nullptr);
+        a.b(labels[raise_exception_shared]);
+    }
+}
+
+/* Handles errors for `call_fun` and `i_lambda_trampoline`. Assumes that we're
+ * running on the Erlang stack with a valid stack frame.
+ *
+ * ARG3 = arity
+ * ARG4 = fun thing
+ * ARG5 = address of the call_fun instruction that got us here. Note that we
+ *        can't use LR (x30) for this because tail calls point elsehwere. */
+void BeamGlobalAssembler::emit_handle_call_fun_error() {
+    Label bad_arity = a.newLabel(), bad_fun = a.newLabel();
+
+    emit_is_boxed(bad_fun, ARG4);
+
+    arm::Gp fun_thing = emit_ptr_val(TMP1, ARG4);
+    a.ldur(TMP1, emit_boxed_val(fun_thing));
+    a.cmp(TMP1, imm(HEADER_FUN));
+    a.cond_eq().b(bad_arity);
+    a.cmp(TMP1, imm(HEADER_EXPORT));
+    a.cond_eq().b(bad_arity);
+
+    a.bind(bad_fun);
+    {
+        /* Not a fun. This is only reachable through `call_fun` */
+        mov_imm(TMP1, EXC_BADFUN);
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+        a.str(ARG4, arm::Mem(c_p, offsetof(Process, fvalue)));
+
+        a.mov(ARG2, ARG5);
+        mov_imm(ARG4, nullptr);
+        a.b(labels[raise_exception_shared]);
+    }
+
+    a.bind(bad_arity);
+    {
+        /* Bad arity. This is reachable through `call_fun` when we have an
+         * export fun, and `i_lambda_trampoline` when we have a local one. */
+
+        a.stp(ARG4, ARG5, TMP_MEM1q);
+
+        emit_enter_runtime<Update::eHeap | Update::eStack | Update::eXRegs>();
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        /* ARG3 is already set */
+        runtime_call<3>(beam_jit_build_argument_list);
+
+        emit_leave_runtime<Update::eHeap | Update::eStack | Update::eXRegs>();
+
+        a.ldr(XREG0, TMP_MEM1q);
+        a.mov(XREG1, ARG1);
+
+        /* Create the {Fun, Args} tuple. */
+        {
+            const int32_t bytes_needed = (3 + S_RESERVED) * sizeof(Eterm);
+            Label after_gc = a.newLabel();
+
+            add(ARG3, HTOP, bytes_needed);
+            a.cmp(ARG3, E);
+            a.cond_ls().b(after_gc);
+            {
+                mov_imm(ARG4, 2);
+                a.bl(labels[garbage_collect]);
+            }
+            a.bind(after_gc);
+
+            a.add(ARG1, HTOP, imm(TAG_PRIMARY_BOXED));
+
+            mov_imm(TMP1, make_arityval(2));
+            a.str(TMP1, arm::Mem(HTOP).post(sizeof(Eterm)));
+            a.stp(XREG0, XREG1, arm::Mem(HTOP).post(sizeof(Eterm[2])));
+        }
+
+        a.mov(TMP1, imm(EXC_BADARITY));
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+        a.str(ARG1, arm::Mem(c_p, offsetof(Process, fvalue)));
+
+        a.ldr(ARG2, TMP_MEM2q);
+        mov_imm(ARG4, nullptr);
+        a.b(labels[raise_exception_shared]);
+    }
+}
+
+/* `call_fun` instructions land here to check arity and set up their
+ * environment before jumping to the actual implementation.
+ *
+ * Keep in mind that this runs in the limbo between caller and callee. It must
+ * not clobber LR (x30).
+ *
+ * ARG3 = arity
+ * ARG4 = fun thing
+ * ARG5 = current PC */
+void BeamModuleAssembler::emit_i_lambda_trampoline(const ArgVal &Index,
+                                                   const ArgVal &Lbl,
+                                                   const ArgVal &Arity,
+                                                   const ArgVal &NumFree) {
+    const ssize_t env_offset = offsetof(ErlFunThing, env) - TAG_PRIMARY_BOXED;
+    const ssize_t fun_arity = Arity.getValue() - NumFree.getValue();
+    const ssize_t total_arity = Arity.getValue();
+    Label error = a.newLabel();
+
+    auto &lambda = lambdas[Index.getValue()];
+    lambda.trampoline = a.newLabel();
+    a.bind(lambda.trampoline);
+
+    a.cmp(ARG3, imm(fun_arity));
+    a.cond_ne().b(error);
+
+    if (NumFree.getValue() == 1) {
+        auto first = init_destination(ArgVal(ArgVal::XReg, fun_arity), TMP1);
+
+        /* Don't bother untagging when there's only a single element, it's
+         * guaranteed to be within range of LDUR. */
+        emit_ptr_val(ARG4, ARG4);
+        a.ldur(first.reg, arm::Mem(ARG4, env_offset));
+        flush_var(first);
+    } else if (NumFree.getValue() >= 2) {
+        ssize_t i;
+
+        emit_ptr_val(ARG4, ARG4);
+        a.add(ARG4, ARG4, imm(env_offset));
+
+        for (i = fun_arity; i < total_arity - 1; i += 2) {
+            auto first = init_destination(ArgVal(ArgVal::XReg, i), TMP1);
+            auto second = init_destination(ArgVal(ArgVal::XReg, i + 1), TMP2);
+
+            a.ldp(first.reg, second.reg, arm::Mem(ARG4).post(sizeof(Eterm[2])));
+            flush_vars(first, second);
+        }
+
+        if (i < total_arity) {
+            auto last = init_destination(ArgVal(ArgVal::XReg, i), TMP1);
+            a.ldr(last.reg, arm::Mem(ARG4));
+            flush_var(last);
+        }
+    }
+
+    a.b(resolve_beam_label(Lbl, disp128MB));
+
+    a.bind(error);
+    {
+        emit_enter_erlang_frame();
+        a.b(resolve_fragment(ga->get_handle_call_fun_error(), disp128MB));
+    }
+}
+
+void BeamModuleAssembler::emit_i_make_fun3(const ArgVal &Fun,
+                                           const ArgVal &Dst,
+                                           const ArgVal &Arity,
+                                           const ArgVal &NumFree,
+                                           const Span<ArgVal> &env) {
+    const ssize_t num_free = NumFree.getValue();
+    ssize_t i;
+
+    ASSERT(num_free == env.size());
+
+    mov_arg(ARG2, Fun);
+    mov_arg(ARG3, Arity);
+    mov_arg(ARG4, NumFree);
+
+    emit_enter_runtime<Update::eHeap>();
+
+    a.mov(ARG1, c_p);
+    runtime_call<4>(new_fun_thing);
+
+    emit_leave_runtime<Update::eHeap>();
+
+    if (num_free) {
+        comment("Move fun environment");
+    }
+
+    for (i = 0; i < num_free - 1; i += 2) {
+        ssize_t offset = offsetof(ErlFunThing, env) + i * sizeof(Eterm);
+
+        if ((i % 128) == 0) {
+            check_pending_stubs();
+        }
+
+        auto [first, second] = load_sources(env[i], TMP1, env[i + 1], TMP2);
+        safe_stp(first.reg, second.reg, arm::Mem(ARG1, offset));
+    }
+
+    if (i < num_free) {
+        ssize_t offset = offsetof(ErlFunThing, env) + i * sizeof(Eterm);
+        mov_arg(arm::Mem(ARG1, offset), env[i]);
+    }
+
+    comment("Create boxed ptr");
+    auto dst = init_destination(Dst, TMP1);
+    a.orr(dst.reg, ARG1, imm(TAG_PRIMARY_BOXED));
+    flush_var(dst);
+}
+
+void BeamGlobalAssembler::emit_apply_fun_shared() {
+    Label finished = a.newLabel();
+
+    /* Put the arity and fun into the right registers for `call_fun`, and stash
+     * the argument list in ARG5 for the error path. We'll bump the arity as
+     * we go through the argument list. */
+    mov_imm(ARG3, 0);
+    a.mov(ARG4, XREG0);
+    a.mov(ARG5, XREG1);
+
+    {
+        Label unpack_next = a.newLabel(), malformed_list = a.newLabel(),
+              raise_error = a.newLabel();
+
+        /* apply/2 is rarely used on a hot code path, so we'll simplify things
+         * by switching to the runtime environment where we can operate
+         * entirely on the X register array.
+         *
+         * Note that we don't have any live registers at this point. */
+        emit_enter_runtime<Update::eXRegs>(0);
+
+        a.mov(TMP1, ARG5);
+        lea(TMP2, getXRef(0));
+
+        a.bind(unpack_next);
+        {
+            a.cmp(TMP1, imm(NIL));
+            a.cond_eq().b(finished);
+
+            ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST == (1 << 1));
+            a.tbnz(TMP1, imm(1), malformed_list);
+
+            emit_ptr_val(TMP1, TMP1);
+            a.sub(TMP1, TMP1, imm(TAG_PRIMARY_LIST));
+            a.ldp(TMP3, TMP1, arm::Mem(TMP1));
+            a.str(TMP3, arm::Mem(TMP2).post(sizeof(Eterm)));
+
+            /* We bail at MAX_REG-1 rather than MAX_REG as the highest register
+             * is reserved for the loader. */
+            a.add(ARG3, ARG3, imm(1));
+            a.cmp(ARG3, imm(MAX_REG - 1));
+            a.cond_lo().b(unpack_next);
+        }
+
+        a.mov(TMP1, imm(SYSTEM_LIMIT));
+        a.b(raise_error);
+
+        a.bind(malformed_list);
+        a.mov(TMP1, imm(BADARG));
+
+        a.bind(raise_error);
+        {
+            static const ErtsCodeMFA apply_mfa = {am_erlang, am_apply, 2};
+
+            emit_leave_runtime<Update::eXRegs>(0);
+
+            a.mov(XREG0, ARG4);
+            a.mov(XREG1, ARG5);
+
+            a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+            mov_imm(ARG4, &apply_mfa);
+            a.b(labels[raise_exception]);
+        }
+    }
+
+    a.bind(finished);
+    {
+        emit_leave_runtime<Update::eXRegs>();
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_i_apply_fun() {
+    fragment_call(ga->get_apply_fun_shared());
+    erlang_call(emit_call_fun());
+}
+
+void BeamModuleAssembler::emit_i_apply_fun_last(const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+    emit_i_apply_fun_only();
+}
+
+void BeamModuleAssembler::emit_i_apply_fun_only() {
+    fragment_call(ga->get_apply_fun_shared());
+    emit_leave_erlang_frame();
+    a.br(emit_call_fun());
+}
+
+/* Asssumes that:
+ *   ARG3 = arity
+ *   ARG4 = fun thing */
+arm::Gp BeamModuleAssembler::emit_call_fun() {
+    Label exported = a.newLabel(), next = a.newLabel();
+
+    /* Load the error fragment into TMP3 so we can CSEL ourselves there on
+     * error. */
+    a.adr(TMP3, resolve_fragment(ga->get_handle_call_fun_error(), disp1MB));
+
+    /* The `handle_call_fun_error` and `i_lambda_trampoline` fragments expect
+     * current PC in ARG5. */
+    a.adr(ARG5, next);
+
+    /* As emit_is_boxed(), but explicitly sets ZF so we can rely on that for
+     * error checking in `next`. */
+    a.tst(ARG4, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED));
+    a.cond_ne().b(next);
+
+    arm::Gp fun_thing = emit_ptr_val(TMP2, ARG4);
+    a.ldur(TMP1, emit_boxed_val(fun_thing));
+    a.cmp(TMP1, imm(HEADER_EXPORT));
+    a.cond_eq().b(exported);
+    a.cmp(TMP1, imm(HEADER_FUN));
+    a.cond_ne().b(next);
+
+    a.ldur(TMP1, emit_boxed_val(fun_thing, offsetof(ErlFunThing, fe)));
+    a.ldr(TMP1, arm::Mem(TMP1, offsetof(ErlFunEntry, address)));
+    a.b(next);
+
+    a.bind(exported);
+    {
+        a.ldur(ARG1, emit_boxed_val(fun_thing, sizeof(Eterm)));
+
+        a.ldr(TMP1, arm::Mem(ARG1, offsetof(Export, info.mfa.arity)));
+        a.cmp(TMP1, ARG3);
+
+        a.ldr(TMP1, emit_setup_export_call(ARG1));
+    }
+
+    /* Assumes that ZF is set on success and clear on error, overwriting our
+     * destination with the error fragment's address. */
+    a.bind(next);
+    a.csel(TMP1, TMP1, TMP3, imm(arm::Cond::kEQ));
+
+    return TMP1;
+}
+
+void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Arity) {
+    mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue()));
+    mov_imm(ARG3, Arity.getValue());
+
+    erlang_call(emit_call_fun());
+}
+
+void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Arity,
+                                               const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+
+    mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue()));
+    mov_imm(ARG3, Arity.getValue());
+
+    arm::Gp target = emit_call_fun();
+    emit_leave_erlang_frame();
+    a.br(target);
+}
+
+/* Psuedo-instruction for signalling lambda load errors. Never actually runs. */
+void BeamModuleAssembler::emit_i_lambda_error(const ArgVal &Dummy) {
+    emit_nyi("emit_i_lambda_error");
+}
diff --git a/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp b/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp
new file mode 100644
index 0000000000..7b0dbbdd3f
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp
@@ -0,0 +1,727 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Guard BIF calls using the generic bif1, bif2, and bif3 instructions
+ * are expensive. Not only are there two indirect calls (one to the
+ * fragment, one to the BIF itself), but the caller-saved X registers
+ * must also be saved and restored, and the BIF operands that are
+ * usually in CPU registers must be written out to memory.
+ *
+ * Therefore, guard BIFs that are used fairly frequently and can
+ * be implemented entirely in assembly language without any calls to
+ * C function are implemented in this source file.
+ */
+
+#include <algorithm>
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "erl_bif_table.h"
+#include "big.h"
+#include "beam_catches.h"
+#include "beam_common.h"
+#include "code_ix.h"
+#include "erl_map.h"
+}
+
+using namespace asmjit;
+
+/* Raise a badarg exception for the given MFA. */
+void BeamGlobalAssembler::emit_raise_badarg(const ErtsCodeMFA *mfa) {
+    mov_imm(TMP1, BADARG);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+    mov_imm(ARG4, mfa);
+    a.b(labels[raise_exception]);
+}
+
+/* ================================================================
+ *  '=:='/2
+ *  '=/='/2
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_bif_is_eq_exact_shared() {
+    Label succ = a.newLabel(), fail = a.newLabel();
+
+    a.cmp(ARG1, ARG2);
+    a.cond_eq().b(succ);
+
+    /* The terms could still be equal if both operands are pointers
+     * having the same tag. */
+    emit_is_unequal_based_on_tags(ARG1, ARG2);
+    a.cond_eq().b(fail);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime();
+
+    runtime_call<2>(eq);
+
+    emit_leave_runtime();
+    emit_leave_runtime_frame();
+
+    a.cbz(ARG1, fail);
+
+    a.bind(succ);
+    {
+        mov_imm(ARG1, am_true);
+        a.ret(a64::x30);
+    }
+
+    a.bind(fail);
+    {
+        mov_imm(ARG1, am_false);
+        a.ret(a64::x30);
+    }
+}
+
+void BeamGlobalAssembler::emit_bif_is_ne_exact_shared() {
+    Label succ = a.newLabel(), fail = a.newLabel();
+
+    a.cmp(ARG1, ARG2);
+    a.cond_eq().b(fail);
+
+    emit_is_unequal_based_on_tags(ARG1, ARG2);
+    a.cond_eq().b(succ);
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime();
+
+    runtime_call<2>(eq);
+
+    emit_leave_runtime();
+    emit_leave_runtime_frame();
+
+    a.cbnz(ARG1, fail);
+
+    a.bind(succ);
+    {
+        mov_imm(ARG1, am_true);
+        a.ret(a64::x30);
+    }
+
+    a.bind(fail);
+    {
+        mov_imm(ARG1, am_false);
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgVal &Src,
+                                                        const ArgVal &Immed,
+                                                        const ArgVal &Dst,
+                                                        Eterm fail_value,
+                                                        Eterm succ_value) {
+    auto src = load_source(Src, TMP1);
+    auto dst = init_destination(Dst, TMP2);
+
+    cmp_arg(src.reg, Immed);
+    mov_imm(TMP3, succ_value);
+    mov_imm(TMP4, fail_value);
+    a.csel(dst.reg, TMP3, TMP4, arm::Cond::kEQ);
+    flush_var(dst);
+}
+
+void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgVal &LHS,
+                                               const ArgVal &RHS,
+                                               const ArgVal &Dst) {
+    if (RHS.isImmed()) {
+        emit_bif_is_eq_ne_exact_immed(LHS, RHS, Dst, am_false, am_true);
+    } else {
+        auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2);
+        auto dst = init_destination(Dst, ARG1);
+
+        mov_var(ARG1, lhs);
+        mov_var(ARG2, rhs);
+        fragment_call(ga->get_bif_is_eq_exact_shared());
+        mov_var(dst, ARG1);
+        flush_var(dst);
+    }
+}
+
+void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgVal &LHS,
+                                               const ArgVal &RHS,
+                                               const ArgVal &Dst) {
+    if (RHS.isImmed()) {
+        emit_bif_is_eq_ne_exact_immed(LHS, RHS, Dst, am_true, am_false);
+    } else {
+        auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2);
+        auto dst = init_destination(Dst, ARG1);
+
+        mov_var(ARG1, lhs);
+        mov_var(ARG2, rhs);
+        fragment_call(ga->get_bif_is_ne_exact_shared());
+        mov_var(dst, ARG1);
+        flush_var(dst);
+    }
+}
+
+/* ================================================================
+ *  and/2
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_handle_and_error() {
+    static ErtsCodeMFA mfa = {am_erlang, am_and, 2};
+    emit_raise_badarg(&mfa);
+}
+
+void BeamModuleAssembler::emit_bif_and(const ArgVal &Fail,
+                                       const ArgVal &Src1,
+                                       const ArgVal &Src2,
+                                       const ArgVal &Dst) {
+    static const Uint diff_bit = am_true - am_false;
+    Label next = a.newLabel();
+
+    auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2);
+    auto dst = init_destination(Dst, TMP3);
+
+    ERTS_CT_ASSERT(am_false == make_atom(0));
+    ERTS_CT_ASSERT(am_true == make_atom(1));
+
+    a.and_(TMP3, src1.reg, imm(_TAG_IMMED2_MASK | ~diff_bit));
+    a.and_(TMP4, src2.reg, imm(_TAG_IMMED2_MASK | ~diff_bit));
+    a.cmp(TMP3, imm(_TAG_IMMED2_ATOM));
+    a.ccmp(TMP3, TMP4, 0, arm::Cond::kEQ);
+
+    if (Fail.getValue()) {
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.cond_eq().b(next);
+        mov_var(XREG0, src1);
+        mov_var(XREG1, src2);
+        fragment_call(ga->get_handle_or_error());
+    }
+
+    a.bind(next);
+    {
+        a.and_(dst.reg, src1.reg, src2.reg);
+        flush_var(dst);
+    }
+}
+
+/* ================================================================
+ *  bit_size/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_bif_bit_size_helper(Label fail) {
+    Label not_sub_bin = a.newLabel();
+    arm::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1);
+
+    emit_is_boxed(fail, boxed_ptr);
+
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP1, imm(_TAG_HEADER_SUB_BIN));
+    a.cond_ne().b(not_sub_bin);
+
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    a.ldurb(TMP2.w(), emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize)));
+
+    mov_imm(ARG1, _TAG_IMMED1_SMALL);
+    a.add(TMP1, TMP2, TMP1, arm::lsl(3));
+    a.bfi(ARG1, TMP1, imm(_TAG_IMMED1_SIZE), imm(SMALL_BITS));
+    a.ret(a64::x30);
+
+    a.bind(not_sub_bin);
+    ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN);
+    a.and_(TMP1, TMP1, imm(~4));
+    a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN));
+    a.cond_ne().b(fail);
+
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    mov_imm(ARG1, _TAG_IMMED1_SMALL);
+    a.bfi(ARG1, TMP1, imm(_TAG_IMMED1_SIZE + 3), imm(SMALL_BITS - 3));
+
+    a.ret(a64::x30);
+}
+
+void BeamGlobalAssembler::emit_bif_bit_size_body() {
+    Label error = a.newLabel();
+
+    emit_bif_bit_size_helper(error);
+
+    a.bind(error);
+    {
+        static ErtsCodeMFA mfa = {am_erlang, am_bit_size, 1};
+        a.mov(XREG0, ARG1);
+        emit_raise_badarg(&mfa);
+    }
+}
+
+void BeamGlobalAssembler::emit_bif_bit_size_guard() {
+    Label error = a.newLabel();
+
+    emit_bif_bit_size_helper(error);
+
+    a.bind(error);
+    {
+        mov_imm(ARG1, THE_NON_VALUE);
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_bif_bit_size(const ArgVal &Fail,
+                                            const ArgVal &Src,
+                                            const ArgVal &Dst) {
+    auto src = load_source(Src, ARG1);
+    auto dst = init_destination(Dst, ARG1);
+
+    mov_var(ARG1, src);
+
+    if (Fail.getValue() == 0) {
+        fragment_call(ga->get_bif_bit_size_body());
+    } else {
+        fragment_call(ga->get_bif_bit_size_guard());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    }
+
+    mov_var(dst, ARG1);
+    flush_var(dst);
+}
+
+/* ================================================================
+ *  byte_size/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_bif_byte_size_helper(Label fail) {
+    Label not_sub_bin = a.newLabel();
+    arm::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1);
+
+    emit_is_boxed(fail, boxed_ptr);
+
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP1, imm(_TAG_HEADER_SUB_BIN));
+    a.cond_ne().b(not_sub_bin);
+
+    a.ldurb(TMP2.w(), emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize)));
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    a.cmp(TMP2, imm(0));
+    a.cinc(TMP1, TMP1, arm::Cond::kNE);
+
+    mov_imm(ARG1, _TAG_IMMED1_SMALL);
+    a.bfi(ARG1, TMP1, imm(_TAG_IMMED1_SIZE), imm(SMALL_BITS));
+    a.ret(a64::x30);
+
+    a.bind(not_sub_bin);
+    ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN);
+    a.and_(TMP1, TMP1, imm(~4));
+    a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN));
+    a.cond_ne().b(fail);
+
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    mov_imm(ARG1, _TAG_IMMED1_SMALL);
+    a.bfi(ARG1, TMP1, imm(_TAG_IMMED1_SIZE), imm(SMALL_BITS));
+
+    a.ret(a64::x30);
+}
+
+void BeamGlobalAssembler::emit_bif_byte_size_body() {
+    Label error = a.newLabel();
+
+    emit_bif_byte_size_helper(error);
+
+    a.bind(error);
+    {
+        static ErtsCodeMFA mfa = {am_erlang, am_byte_size, 1};
+        a.mov(XREG0, ARG1);
+        emit_raise_badarg(&mfa);
+    }
+}
+
+void BeamGlobalAssembler::emit_bif_byte_size_guard() {
+    Label error = a.newLabel();
+
+    emit_bif_byte_size_helper(error);
+
+    a.bind(error);
+    {
+        mov_imm(ARG1, THE_NON_VALUE);
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_bif_byte_size(const ArgVal &Fail,
+                                             const ArgVal &Src,
+                                             const ArgVal &Dst) {
+    auto src = load_source(Src, ARG1);
+    auto dst = init_destination(Dst, ARG1);
+
+    mov_var(ARG1, src);
+
+    if (Fail.getValue() == 0) {
+        fragment_call(ga->get_bif_byte_size_body());
+    } else {
+        fragment_call(ga->get_bif_byte_size_guard());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    }
+
+    mov_var(dst, ARG1);
+    flush_var(dst);
+}
+
+/* ================================================================
+ *  element/2
+ * ================================================================
+ */
+
+/* ARG1 = Position (1-based)
+ * ARG2 = Tuple
+ *
+ * Will return the result in ARG1, or jump to the label `fail` if
+ * the operation fails.
+ */
+void BeamGlobalAssembler::emit_bif_element_helper(Label fail) {
+    a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    a.cond_ne().b(fail);
+
+    /* Ensure that ARG2 contains a tuple. */
+    emit_is_boxed(fail, ARG2);
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, ARG2);
+    lea(TMP1, emit_boxed_val(boxed_ptr));
+    a.ldr(TMP2, arm::Mem(TMP1));
+    ERTS_CT_ASSERT(make_arityval(0) == 0);
+    a.tst(TMP2, imm(_TAG_HEADER_MASK));
+    a.cond_ne().b(fail);
+
+    /* Ensure that the position points within the tuple. */
+    a.lsr(TMP2, TMP2, imm(_HEADER_ARITY_OFFS));
+    a.asr(TMP3, ARG1, imm(_TAG_IMMED1_SIZE));
+    a.cmp(TMP3, imm(1));
+    a.cond_mi().b(fail);
+    a.cmp(TMP2, TMP3);
+    a.cond_lo().b(fail);
+
+    a.ldr(ARG1, arm::Mem(TMP1, TMP3, arm::lsl(3)));
+    a.ret(a64::x30);
+}
+
+void BeamGlobalAssembler::emit_bif_element_body_shared() {
+    Label error = a.newLabel();
+
+    emit_bif_element_helper(error);
+
+    a.bind(error);
+    {
+        static ErtsCodeMFA mfa = {am_erlang, am_element, 2};
+        a.mov(XREG0, ARG1);
+        a.mov(XREG1, ARG2);
+        emit_raise_badarg(&mfa);
+    }
+}
+
+void BeamGlobalAssembler::emit_bif_element_guard_shared() {
+    Label error = a.newLabel();
+
+    emit_bif_element_helper(error);
+
+    a.bind(error);
+    {
+        mov_imm(ARG1, THE_NON_VALUE);
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_bif_element(const ArgVal &Fail,
+                                           const ArgVal &Pos,
+                                           const ArgVal &Tuple,
+                                           const ArgVal &Dst) {
+    mov_arg(ARG1, Pos);
+    mov_arg(ARG2, Tuple);
+
+    if (Fail.getValue() != 0) {
+        fragment_call(ga->get_bif_element_guard_shared());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        fragment_call(ga->get_bif_element_body_shared());
+    }
+
+    auto dst = init_destination(Dst, ARG1);
+    mov_var(dst, ARG1);
+    flush_var(dst);
+}
+
+/* ================================================================
+ *  hd/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_handle_hd_error() {
+    static ErtsCodeMFA mfa = {am_erlang, am_hd, 1};
+    emit_raise_badarg(&mfa);
+}
+
+void BeamModuleAssembler::emit_bif_hd(const ArgVal &Src, const ArgVal &Hd) {
+    Label good_cons = a.newLabel();
+    auto src = load_source(Src, TMP1);
+    auto hd = init_destination(Hd, TMP2);
+    const int bitNumber = 1;
+
+    ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST == (1 << bitNumber));
+
+    a.tbz(src.reg, bitNumber, good_cons);
+    mov_var(XREG0, src);
+    fragment_call(ga->get_handle_hd_error());
+
+    a.bind(good_cons);
+    {
+        arm::Gp cons_ptr = emit_ptr_val(TMP1, src.reg);
+        a.ldur(hd.reg, getCARRef(cons_ptr));
+        flush_var(hd);
+    }
+}
+
+/* ================================================================
+ *  map_size/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_handle_map_size_error() {
+    static ErtsCodeMFA mfa = {am_erlang, am_map_size, 1};
+    mov_imm(TMP1, BADMAP);
+    a.str(TMP1, arm::Mem(c_p, offsetof(Process, freason)));
+    a.str(XREG0, arm::Mem(c_p, offsetof(Process, fvalue)));
+    mov_imm(ARG4, &mfa);
+    a.b(labels[raise_exception]);
+}
+
+void BeamModuleAssembler::emit_bif_map_size(const ArgVal &Fail,
+                                            const ArgVal &Src,
+                                            const ArgVal &Dst) {
+    Label error = a.newLabel(), good_map = a.newLabel();
+    auto src = load_source(Src, TMP1);
+    auto dst = init_destination(Dst, TMP2);
+
+    if (Fail.getValue() == 0) {
+        emit_is_boxed(error, src.reg);
+    } else {
+        emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+    }
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP3, src.reg);
+    a.ldur(TMP4, emit_boxed_val(boxed_ptr));
+    a.and_(TMP4, TMP4, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP4, imm(_TAG_HEADER_MAP));
+
+    if (Fail.getValue() == 0) {
+        a.cond_eq().b(good_map);
+        a.bind(error);
+        {
+            mov_var(XREG0, src);
+            fragment_call(ga->get_handle_map_size_error());
+        }
+    } else {
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+        a.bind(error); /* Never referenced. */
+    }
+
+    a.bind(good_map);
+    {
+        ERTS_CT_ASSERT(offsetof(flatmap_t, size) == sizeof(Eterm));
+        a.ldur(TMP1, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+        mov_imm(dst.reg, _TAG_IMMED1_SMALL);
+        a.bfi(dst.reg, TMP1, imm(_TAG_IMMED1_SIZE), imm(SMALL_BITS));
+        flush_var(dst);
+    }
+}
+
+/* ================================================================
+ *  not/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_handle_not_error() {
+    static ErtsCodeMFA mfa = {am_erlang, am_not, 1};
+    emit_raise_badarg(&mfa);
+}
+
+void BeamModuleAssembler::emit_bif_not(const ArgVal &Fail,
+                                       const ArgVal &Src,
+                                       const ArgVal &Dst) {
+    Label next = a.newLabel();
+    auto src = load_source(Src, TMP1);
+    auto dst = init_destination(Dst, TMP2);
+    ERTS_CT_ASSERT(am_false == make_atom(0));
+    ERTS_CT_ASSERT(am_true == make_atom(1));
+    Uint diff_bit = am_true - am_false;
+
+    a.and_(TMP3, src.reg, imm(_TAG_IMMED2_MASK | ~diff_bit));
+    a.cmp(TMP3, imm(_TAG_IMMED2_ATOM));
+
+    if (Fail.getValue() == 0) {
+        a.cond_eq().b(next);
+        mov_var(XREG0, src);
+        fragment_call(ga->get_handle_not_error());
+    } else {
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    }
+
+    a.bind(next);
+    {
+        a.eor(dst.reg, src.reg, imm(diff_bit));
+        flush_var(dst);
+    }
+}
+
+/* ================================================================
+ *  or/2
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_handle_or_error() {
+    static ErtsCodeMFA mfa = {am_erlang, am_or, 2};
+    emit_raise_badarg(&mfa);
+}
+
+void BeamModuleAssembler::emit_bif_or(const ArgVal &Fail,
+                                      const ArgVal &Src1,
+                                      const ArgVal &Src2,
+                                      const ArgVal &Dst) {
+    static const Uint diff_bit = am_true - am_false;
+    Label next = a.newLabel();
+
+    auto [src1, src2] = load_sources(Src1, TMP1, Src2, TMP2);
+    auto dst = init_destination(Dst, TMP3);
+
+    ERTS_CT_ASSERT(am_false == make_atom(0));
+    ERTS_CT_ASSERT(am_true == make_atom(1));
+
+    a.and_(TMP3, src1.reg, imm(_TAG_IMMED2_MASK | ~diff_bit));
+    a.and_(TMP4, src2.reg, imm(_TAG_IMMED2_MASK | ~diff_bit));
+    a.cmp(TMP3, imm(_TAG_IMMED2_ATOM));
+    a.ccmp(TMP3, TMP4, 0, arm::Cond::kEQ);
+
+    if (Fail.getValue()) {
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.cond_eq().b(next);
+        mov_var(XREG0, src1);
+        mov_var(XREG1, src2);
+        fragment_call(ga->get_handle_or_error());
+    }
+
+    a.bind(next);
+    {
+        a.orr(dst.reg, src1.reg, src2.reg);
+        flush_var(dst);
+    }
+}
+
+/* ================================================================
+ *  tl/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_handle_tl_error() {
+    static ErtsCodeMFA mfa = {am_erlang, am_tl, 1};
+    emit_raise_badarg(&mfa);
+}
+
+void BeamModuleAssembler::emit_bif_tl(const ArgVal &Src, const ArgVal &Tl) {
+    Label good_cons = a.newLabel();
+    auto src = load_source(Src, TMP1);
+    auto tl = init_destination(Tl, TMP2);
+    const int bitNumber = 1;
+
+    ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST == (1 << bitNumber));
+
+    a.tbz(src.reg, bitNumber, good_cons);
+    mov_var(XREG0, src);
+    fragment_call(ga->get_handle_tl_error());
+
+    a.bind(good_cons);
+    {
+        arm::Gp cons_ptr = emit_ptr_val(TMP1, src.reg);
+        a.ldur(tl.reg, getCDRRef(cons_ptr));
+        flush_var(tl);
+    }
+}
+
+/* ================================================================
+ *  tuple_size/1
+ * ================================================================
+ */
+
+void BeamGlobalAssembler::emit_bif_tuple_size_helper(Label fail) {
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, ARG1);
+
+    emit_is_boxed(fail, boxed_ptr);
+
+    ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr));
+    a.tst(TMP1, imm(_TAG_HEADER_MASK));
+    a.cond_ne().b(fail);
+
+    ERTS_CT_ASSERT(_HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE > 0);
+    ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
+    a.lsr(TMP1, TMP1, _HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE);
+    a.orr(ARG1, TMP1, imm(_TAG_IMMED1_SMALL));
+
+    a.ret(a64::x30);
+}
+
+void BeamGlobalAssembler::emit_bif_tuple_size_body() {
+    Label error = a.newLabel();
+
+    emit_bif_tuple_size_helper(error);
+
+    a.bind(error);
+    {
+        static ErtsCodeMFA mfa = {am_erlang, am_tuple_size, 1};
+        a.mov(XREG0, ARG1);
+        emit_raise_badarg(&mfa);
+    }
+}
+
+void BeamGlobalAssembler::emit_bif_tuple_size_guard() {
+    Label error = a.newLabel();
+
+    emit_bif_tuple_size_helper(error);
+
+    a.bind(error);
+    {
+        mov_imm(ARG1, THE_NON_VALUE);
+        a.ret(a64::x30);
+    }
+}
+
+void BeamModuleAssembler::emit_bif_tuple_size(const ArgVal &Fail,
+                                              const ArgVal &Src,
+                                              const ArgVal &Dst) {
+    auto src = load_source(Src, ARG1);
+    auto dst = init_destination(Dst, ARG1);
+
+    mov_var(ARG1, src);
+
+    if (Fail.getValue() == 0) {
+        fragment_call(ga->get_bif_tuple_size_body());
+    } else {
+        fragment_call(ga->get_bif_tuple_size_guard());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    }
+
+    mov_var(dst, ARG1);
+    flush_var(dst);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_map.cpp b/erts/emulator/beam/jit/arm/instr_map.cpp
new file mode 100644
index 0000000000..1018fef413
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_map.cpp
@@ -0,0 +1,298 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+#include <algorithm>
+#include "beam_asm.hpp"
+
+using namespace asmjit;
+
+extern "C"
+{
+#include "erl_map.h"
+#include "beam_common.h"
+}
+
+void BeamModuleAssembler::emit_ensure_map(const ArgVal &map) {
+    Label next = a.newLabel(), badmap = a.newLabel();
+
+    auto map_reg = load_source(map, TMP1);
+
+    emit_is_boxed(badmap, map_reg.reg);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP2, map_reg.reg);
+    a.ldur(TMP2, emit_boxed_val(boxed_ptr));
+    a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP2, imm(_TAG_HEADER_MAP));
+    a.cond_eq().b(next);
+
+    a.bind(badmap);
+    {
+        a.stur(map_reg.reg, arm::Mem(c_p, offsetof(Process, fvalue)));
+        emit_error(BADMAP);
+    }
+
+    a.bind(next);
+}
+
+void BeamGlobalAssembler::emit_new_map_shared() {
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<5>(erts_gc_new_map);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+void BeamModuleAssembler::emit_new_map(const ArgVal &Dst,
+                                       const ArgVal &Live,
+                                       const ArgVal &Size,
+                                       const Span<ArgVal> &args) {
+    embed_vararg_rodata(args, ARG5);
+
+    mov_imm(ARG3, Live.getValue());
+    mov_imm(ARG4, args.size());
+    fragment_call(ga->get_new_map_shared());
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamGlobalAssembler::emit_i_new_small_map_lit_shared() {
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<5>(erts_gc_new_small_map_lit);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+void BeamModuleAssembler::emit_i_new_small_map_lit(const ArgVal &Dst,
+                                                   const ArgVal &Live,
+                                                   const ArgVal &Keys,
+                                                   const ArgVal &Size,
+                                                   const Span<ArgVal> &args) {
+    ASSERT(Size.getValue() == args.size());
+
+    embed_vararg_rodata(args, ARG5);
+
+    ASSERT(Keys.isLiteral());
+    mov_arg(ARG3, Keys);
+    mov_imm(ARG4, Live.getValue());
+
+    fragment_call(ga->get_i_new_small_map_lit_shared());
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_i_get_map_element(const ArgVal &Fail,
+                                                 const ArgVal &Src,
+                                                 const ArgVal &Key,
+                                                 const ArgVal &Dst) {
+    mov_arg(ARG1, Src);
+    mov_arg(ARG2, Key);
+
+    emit_enter_runtime();
+
+    runtime_call<2>(get_map_element);
+
+    emit_leave_runtime();
+
+    emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+
+    /*
+     * Don't store the result if the destination is the scratch X register.
+     * (This instruction was originally a has_map_fields instruction.)
+     */
+    if (!(Dst.getType() == ArgVal::XReg && Dst.getValue() == SCRATCH_X_REG)) {
+        mov_arg(Dst, ARG1);
+    }
+}
+
+void BeamModuleAssembler::emit_i_get_map_elements(const ArgVal &Fail,
+                                                  const ArgVal &Src,
+                                                  const ArgVal &Size,
+                                                  const Span<ArgVal> &args) {
+    ASSERT(Size.getValue() == args.size());
+
+    embed_vararg_rodata(args, ARG5);
+
+    mov_arg(ARG1, Src);
+    a.mov(ARG3, E);
+
+    emit_enter_runtime<Update::eXRegs>();
+
+    mov_imm(ARG4, args.size() / 3);
+    load_x_reg_array(ARG2);
+    runtime_call<5>(beam_jit_get_map_elements);
+
+    emit_leave_runtime<Update::eXRegs>();
+
+    a.cbz(ARG1, resolve_beam_label(Fail, disp1MB));
+}
+
+void BeamModuleAssembler::emit_i_get_map_element_hash(const ArgVal &Fail,
+                                                      const ArgVal &Src,
+                                                      const ArgVal &Key,
+                                                      const ArgVal &Hx,
+                                                      const ArgVal &Dst) {
+    mov_arg(ARG1, Src);
+    mov_arg(ARG2, Key);
+    mov_arg(ARG3, Hx);
+
+    emit_enter_runtime();
+
+    runtime_call<3>(get_map_element_hash);
+
+    emit_leave_runtime();
+
+    emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+
+    /*
+     * Don't store the result if the destination is the scratch X register.
+     * (This instruction was originally a has_map_fields instruction.)
+     */
+    if (!(Dst.getType() == ArgVal::XReg && Dst.getValue() == SCRATCH_X_REG)) {
+        mov_arg(Dst, ARG1);
+    }
+}
+
+/* ARG3 = live registers, ARG4 = update vector size, ARG5 = update vector. */
+void BeamGlobalAssembler::emit_update_map_assoc_shared() {
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<5>(erts_gc_update_map_assoc);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+void BeamModuleAssembler::emit_update_map_assoc(const ArgVal &Src,
+                                                const ArgVal &Dst,
+                                                const ArgVal &Live,
+                                                const ArgVal &Size,
+                                                const Span<ArgVal> &args) {
+    auto src_reg = load_source(Src, TMP1);
+
+    ASSERT(Size.getValue() == args.size());
+
+    embed_vararg_rodata(args, ARG5);
+
+    mov_arg(ArgVal(ArgVal::XReg, Live.getValue()), src_reg.reg);
+    mov_imm(ARG3, Live.getValue());
+    mov_imm(ARG4, args.size());
+
+    fragment_call(ga->get_update_map_assoc_shared());
+
+    mov_arg(Dst, ARG1);
+}
+
+/* ARG3 = live registers, ARG4 = update vector size, ARG5 = update vector.
+ *
+ * Result is returned in RET, error is indicated by ZF. */
+void BeamGlobalAssembler::emit_update_map_exact_guard_shared() {
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<5>(erts_gc_update_map_exact);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+    emit_leave_runtime_frame();
+
+    a.ret(a64::x30);
+}
+
+/* ARG3 = live registers, ARG4 = update vector size, ARG5 = update vector.
+ *
+ * Does not return on error. */
+void BeamGlobalAssembler::emit_update_map_exact_body_shared() {
+    Label error = a.newLabel();
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<5>(erts_gc_update_map_exact);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+    emit_leave_runtime_frame();
+
+    emit_branch_if_not_value(ARG1, error);
+    a.ret(a64::x30);
+
+    a.bind(error);
+    {
+        mov_imm(ARG4, 0);
+        a.b(labels[raise_exception]);
+    }
+}
+
+void BeamModuleAssembler::emit_update_map_exact(const ArgVal &Src,
+                                                const ArgVal &Fail,
+                                                const ArgVal &Dst,
+                                                const ArgVal &Live,
+                                                const ArgVal &Size,
+                                                const Span<ArgVal> &args) {
+    auto src_reg = load_source(Src, TMP1);
+
+    ASSERT(Size.getValue() == args.size());
+
+    embed_vararg_rodata(args, ARG5);
+
+    /* We _KNOW_ Src is a map */
+
+    mov_arg(ArgVal(ArgVal::XReg, Live.getValue()), src_reg.reg);
+    mov_imm(ARG3, Live.getValue());
+    mov_imm(ARG4, args.size());
+
+    if (Fail.getValue() != 0) {
+        fragment_call(ga->get_update_map_exact_guard_shared());
+        emit_branch_if_not_value(ARG1, resolve_beam_label(Fail, dispUnknown));
+    } else {
+        fragment_call(ga->get_update_map_exact_body_shared());
+    }
+
+    mov_arg(Dst, ARG1);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_msg.cpp b/erts/emulator/beam/jit/arm/instr_msg.cpp
new file mode 100644
index 0000000000..56ea4d67ce
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_msg.cpp
@@ -0,0 +1,385 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "bif.h"
+#include "code_ix.h"
+#include "erl_proc_sig_queue.h"
+#ifdef USE_VM_PROBES
+#    include "dtrace-wrapper.h"
+#endif
+}
+
+#ifdef ERTS_SUPPORT_OLD_RECV_MARK_INSTRS
+
+static void recv_mark(Process *p) {
+    /* inlined here... */
+    erts_msgq_recv_marker_insert_bind(p, erts_old_recv_marker_id);
+}
+
+static void recv_mark_set(Process *p) {
+    /* inlined here... */
+    erts_msgq_recv_marker_set_save(p, erts_old_recv_marker_id);
+}
+
+void BeamModuleAssembler::emit_i_recv_mark() {
+    /*
+     * OLD INSTRUCTION: This instruction is to be removed
+     *                  in OTP 26.
+     *
+     * Save the current end of message queue
+     */
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(recv_mark);
+
+    emit_leave_runtime();
+}
+
+void BeamModuleAssembler::emit_i_recv_set() {
+    /*
+     * OLD INSTRUCTION: This instruction is to be removed
+     *                  in OTP 26.
+     *
+     * If previously saved recv mark, set save pointer to it
+     */
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(recv_mark_set);
+
+    emit_leave_runtime();
+}
+
+#endif /* ERTS_SUPPORT_OLD_RECV_MARK_INSTRS */
+
+void BeamModuleAssembler::emit_recv_marker_reserve(const ArgVal &Dst) {
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(erts_msgq_recv_marker_insert);
+
+    emit_leave_runtime();
+
+    mov_arg(Dst, ARG1);
+}
+
+void BeamModuleAssembler::emit_recv_marker_bind(const ArgVal &Marker,
+                                                const ArgVal &Reference) {
+    mov_arg(ARG2, Marker);
+    mov_arg(ARG3, Reference);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<3>(erts_msgq_recv_marker_bind);
+
+    emit_leave_runtime();
+}
+
+void BeamModuleAssembler::emit_recv_marker_clear(const ArgVal &Reference) {
+    mov_arg(ARG2, Reference);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_msgq_recv_marker_clear);
+
+    emit_leave_runtime();
+}
+
+void BeamModuleAssembler::emit_recv_marker_use(const ArgVal &Reference) {
+    mov_arg(ARG2, Reference);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_msgq_recv_marker_set_save);
+
+    emit_leave_runtime();
+}
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+int erts_lc_proc_sig_receive_helper(Process *c_p,
+                                    int fcalls,
+                                    int neg_o_reds,
+                                    ErtsMessage **msgpp,
+                                    int *get_outp) {
+    int res;
+    /*
+     * erts_proc_sig_receive_helper() may temporarliy release
+     * its own main lock...
+     */
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    res = erts_proc_sig_receive_helper(c_p,
+                                       fcalls,
+                                       neg_o_reds,
+                                       msgpp,
+                                       get_outp);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    return res;
+}
+#endif
+
+void BeamGlobalAssembler::emit_i_loop_rec_shared() {
+    Label restart = a.newLabel(), peek_message = a.newLabel(),
+          schedule_out = a.newLabel(), check_is_distributed = a.newLabel(),
+          done = a.newLabel();
+
+    arm::Mem await_addr = TMP_MEM1q, message_ptr = TMP_MEM2q,
+             get_out = TMP_MEM3q;
+    arm::Mem flags = arm::Mem(c_p, offsetof(Process, flags));
+
+    a.mov(XREG1, a64::x30);
+
+    a.ldr(TMP1.w(), flags);
+    a.orr(TMP1, TMP1, imm(F_DELAY_GC));
+    a.str(TMP1.w(), flags);
+    a.str(ARG1, arm::Mem(c_p, offsetof(Process, i)));
+    a.str(ARG2, await_addr);
+
+    a.bind(restart);
+    {
+        a.tst(FCALLS, FCALLS);
+        a.cond_le().b(schedule_out);
+
+        /* !! FALL THROUGH !! */
+    }
+
+    comment("Peek next message");
+    a.bind(peek_message);
+    {
+        a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, sig_qs.save)));
+        a.ldr(ARG1, arm::Mem(TMP1));
+        a.cbnz(ARG1, check_is_distributed);
+        comment("Inner queue empty, fetch more from outer/middle queues");
+
+        emit_enter_runtime<Update::eReductions | Update::eStack |
+                           Update::eHeap>(0);
+
+        a.str(ZERO, message_ptr);
+        a.mov(ARG1, c_p);
+        a.mov(ARG2, FCALLS);
+        mov_imm(ARG3, 0);
+        lea(ARG4, message_ptr);
+        lea(ARG5, get_out);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+        runtime_call<5>(erts_lc_proc_sig_receive_helper);
+#else
+        runtime_call<5>(erts_proc_sig_receive_helper);
+#endif
+
+        /* erts_proc_sig_receive_helper merely inspects FCALLS, so we don't
+         * need to update it here.
+         *
+         * Also note that another process may have loaded new code and sent us
+         * a message to notify us about it, so we must update the active code
+         * index. */
+        emit_leave_runtime<Update::eStack | Update::eHeap | Update::eCodeIndex>(
+                0);
+
+        a.sub(FCALLS, FCALLS, ARG1);
+
+        /* Need to spill message_ptr to ARG1 as check_is_distributed uses it. */
+        a.ldr(ARG1, message_ptr);
+        a.cbnz(ARG1, check_is_distributed);
+
+        /* Did we receive a signal or run out of reds? */
+        a.ldr(TMP1.w(), get_out);
+        a.cbnz(TMP1, schedule_out);
+
+        /* The queue is empty and we're not yielding or exiting, so we'll jump
+         * to our wait/timeout instruction.
+         *
+         * Note that the message queue lock is still held in this case. */
+        a.ldr(TMP1.w(), flags);
+        a.and_(TMP1, TMP1, imm(~F_DELAY_GC));
+        a.str(TMP1.w(), flags);
+
+        a.ldr(TMP1, await_addr);
+        a.br(TMP1);
+    }
+
+    a.bind(schedule_out);
+    {
+        /* We either ran out of reductions or received an exit signal; schedule
+         * ourselves out. The yield address (`c_p->i`) was set on ingress. */
+        a.ldr(TMP1.w(), flags);
+        a.and_(TMP1, TMP1, imm(~F_DELAY_GC));
+        a.str(TMP1.w(), flags);
+        a.str(ZERO, arm::Mem(c_p, offsetof(Process, arity)));
+        a.str(ZERO, arm::Mem(c_p, offsetof(Process, current)));
+
+        a.b(labels[do_schedule]);
+    }
+
+    /*
+     * ARG1 now contains the pointer to a message.
+     */
+    comment("Check if message is distributed");
+    a.bind(check_is_distributed);
+    {
+        a.ldr(TMP1, arm::Mem(ARG1, offsetof(ErtsSignal, common.tag)));
+        emit_branch_if_value(TMP1, done);
+
+        sub(FCALLS, FCALLS, 10);
+
+        emit_enter_runtime(0);
+
+        a.mov(ARG2, ARG1);
+        a.mov(ARG1, c_p);
+        runtime_call<2>(beam_jit_decode_dist);
+
+        emit_leave_runtime(0);
+
+        a.cbz(ARG1, restart);
+
+        /* !! FALL THROUGH !! */
+    }
+
+    a.bind(done);
+    {
+        a.ldr(XREG0, arm::Mem(ARG1, offsetof(ErtsMessage, m[0])));
+        a.ret(XREG1);
+    }
+}
+
+void BeamModuleAssembler::emit_i_loop_rec(const ArgVal &Wait) {
+    Label entry = a.newLabel();
+
+    a.bind(entry);
+    a.adr(ARG1, entry);
+    a.ldr(ARG2, embed_constant(Wait, disp32K));
+    fragment_call(ga->get_i_loop_rec_shared());
+}
+
+void BeamModuleAssembler::emit_remove_message() {
+    /* HTOP and E are passed explicitly and only read from, so we don't need to
+     * swap them out. */
+    a.mov(ARG3, HTOP);
+    a.mov(ARG4, E);
+
+    emit_enter_runtime();
+
+    a.mov(ARG1, c_p);
+    a.mov(ARG2, FCALLS);
+    a.mov(ARG5, active_code_ix);
+    runtime_call<5>(beam_jit_remove_message);
+    a.mov(FCALLS, ARG1);
+
+    emit_leave_runtime();
+}
+
+void BeamModuleAssembler::emit_loop_rec_end(const ArgVal &Dest) {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(erts_msgq_set_save_next);
+
+    emit_leave_runtime(0);
+
+    a.sub(FCALLS, FCALLS, imm(1));
+    a.b(resolve_beam_label(Dest, disp128MB));
+}
+
+void BeamModuleAssembler::emit_wait_unlocked(const ArgVal &Dest) {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    a.ldr(ARG2, embed_constant(Dest, disp32K));
+    runtime_call<2>(beam_jit_wait_unlocked);
+
+    emit_leave_runtime(0);
+
+    a.b(resolve_fragment(ga->get_do_schedule(), disp128MB));
+}
+
+void BeamModuleAssembler::emit_wait_locked(const ArgVal &Dest) {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    a.ldr(ARG2, embed_constant(Dest, disp32K));
+    runtime_call<2>(beam_jit_wait_locked);
+
+    emit_leave_runtime(0);
+
+    a.b(resolve_fragment(ga->get_do_schedule(), disp128MB));
+}
+
+void BeamModuleAssembler::emit_wait_timeout_unlocked(const ArgVal &Src,
+                                                     const ArgVal &Dest) {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(beam_jit_take_receive_lock);
+
+    emit_leave_runtime(0);
+
+    emit_wait_timeout_locked(Src, Dest);
+}
+
+void BeamModuleAssembler::emit_wait_timeout_locked(const ArgVal &Src,
+                                                   const ArgVal &Dest) {
+    Label wait = a.newLabel(), next = a.newLabel();
+
+    mov_arg(ARG2, Src);
+
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    a.adr(ARG3, next);
+    runtime_call<3>(beam_jit_wait_timeout);
+
+    emit_leave_runtime(0);
+
+    ERTS_CT_ASSERT(RET_next < RET_wait && RET_wait < RET_badarg);
+    a.cmp(ARG1, imm(RET_wait));
+    a.cond_eq().b(wait);
+    a.cond_lt().b(next);
+
+    emit_raise_exception(currLabel, (ErtsCodeMFA *)nullptr);
+
+    a.bind(wait);
+    emit_wait_locked(Dest);
+
+    a.bind(next);
+}
+
+void BeamModuleAssembler::emit_timeout_locked() {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(beam_jit_timeout_locked);
+
+    emit_leave_runtime(0);
+}
+
+void BeamModuleAssembler::emit_timeout() {
+    emit_enter_runtime(0);
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(beam_jit_timeout);
+
+    emit_leave_runtime(0);
+}
diff --git a/erts/emulator/beam/jit/arm/instr_select.cpp b/erts/emulator/beam/jit/arm/instr_select.cpp
new file mode 100644
index 0000000000..bac2238be7
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_select.cpp
@@ -0,0 +1,479 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include <algorithm>
+#include <numeric>
+
+#include "beam_asm.hpp"
+
+using namespace asmjit;
+
+/* Request Support::isInt12 ! */
+template<typename T>
+bool isInt12(T value) {
+    typedef typename std::make_unsigned<T>::type U;
+    typedef typename std::make_signed<T>::type S;
+
+    return Support::isUInt12(U(value)) || Support::isUInt12(-S(value));
+}
+
+/* The cmp instruction for AArch accepts only a 12-bit immediate value.
+ * That means that to compare most atoms, the atom number to be compared
+ * must be loaded into a temporary register.
+ *
+ * We can use the immediate form of cmp for more values if we untag both
+ * the source value and the values to be compared.
+ *
+ * This function finds the `base` and `shift` that result in the most number
+ * of elements fitting in a 12-bit immediate. */
+static std::pair<UWord, int> plan_untag(const Span<ArgVal> &args) {
+    auto left = args.begin(), right = args.begin();
+    auto best_left = left, best_right = right;
+    int count, shift;
+
+    count = args.size() / 2;
+
+    ASSERT(left->isImmed() && (args.begin() + count)->isLabel());
+    ASSERT(is_small(left->getValue()) || is_atom(right->getValue()));
+
+    shift = is_small(left->getValue()) ? _TAG_IMMED1_SIZE : _TAG_IMMED2_SIZE;
+
+    while (right < (args.begin() + count)) {
+        auto distance = std::distance(left, right);
+        UWord left_value, mid_value, right_value;
+
+        left_value = left->getValue() >> shift;
+        mid_value = (left + distance / 2)->getValue() >> shift;
+        right_value = right->getValue() >> shift;
+
+        if (isInt12(left_value - mid_value) &&
+            isInt12(right_value - mid_value)) {
+            if (distance > std::distance(best_left, best_right)) {
+                best_right = right;
+                best_left = left;
+            }
+
+            right++;
+        } else {
+            left++;
+        }
+    }
+
+    auto distance = std::distance(best_left, best_right);
+
+    /* Skip everything if the best run is too short, untagging has its costs
+     * too. */
+    if (distance <= 6) {
+        return std::make_pair(0, 0);
+    }
+
+    /* Apply neither shift nor base if the best run doesn't need it: we're more
+     * likely to lose by rebasing/shifting. */
+    if (isInt12(best_left->getValue()) && isInt12(best_right->getValue())) {
+        return std::make_pair(0, 0);
+    }
+
+    /* Skip rebasing if the best run doesn't need it after shifting. */
+    if (isInt12(best_left->getValue() >> shift) &&
+        isInt12(best_right->getValue() >> shift)) {
+        return std::make_pair(0, shift);
+    }
+
+    auto mid_value = (best_left + distance / 2)->getValue();
+    return std::make_pair(mid_value, shift);
+}
+
+const std::vector<ArgVal> BeamModuleAssembler::emit_select_untag(
+        const Span<ArgVal> &args,
+        a64::Gp comparand,
+        Label fail,
+        UWord base,
+        int shift) {
+    ASSERT(base != 0 || shift > 0);
+
+    /* Emit code to test that the source value has the correct type and
+     * untag it. */
+    comment("(comparing untagged+rebased values)");
+    if (is_small(args.front().getValue())) {
+        a.and_(TMP1, comparand, imm(_TAG_IMMED1_MASK));
+        a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+    } else {
+        ASSERT(is_atom(args.front().getValue()));
+        a.and_(TMP1, comparand, imm(_TAG_IMMED2_MASK));
+        a.cmp(TMP1, imm(_TAG_IMMED2_ATOM));
+    }
+
+    a.cond_ne().b(resolve_label(fail, disp1MB));
+
+    if (shift != 0) {
+        a.lsr(ARG1, comparand, imm(shift));
+        base >>= shift;
+
+        comparand = ARG1;
+    }
+
+    std::vector<ArgVal> result(args.begin(), args.end());
+    int count = args.size() / 2;
+
+    if (base != 0) {
+        sub(ARG1, comparand, base);
+
+        /* The values will always be ordered differently after adjusting the
+         * base, so we have to sort them again.
+         *
+         * This is rather annoying because the labels and values can't be
+         * sorted together. Perhaps we should diverge from the other platforms,
+         * keeping them together just on ARM? */
+        std::vector<int> sorted_indexes(count);
+        std::iota(sorted_indexes.begin(), sorted_indexes.end(), 0);
+        std::sort(sorted_indexes.begin(),
+                  sorted_indexes.end(),
+                  [&](int lhs, int rhs) {
+                      auto lhs_value = (args[lhs].getValue() >> shift) - base;
+                      auto rhs_value = (args[rhs].getValue() >> shift) - base;
+                      return lhs_value < rhs_value;
+                  });
+
+        for (auto i = 0; i < count; i++) {
+            const auto &src_value = args[sorted_indexes[i]];
+            const auto &src_label = args[sorted_indexes[i] + count];
+            auto &dst_value = result[i];
+            auto &dst_label = result[i + count];
+
+            dst_value = ArgVal(ArgVal::Immediate,
+                               (src_value.getValue() >> shift) - base);
+            dst_label = src_label;
+        }
+    } else {
+        /* Fast-path for when a shift alone is enough, it won't affect the
+         * order. */
+        for (auto i = 0; i < count; i++) {
+            auto &dst_value = result[i];
+            auto &dst_label = result[i + count];
+
+            dst_value = ArgVal(ArgVal::Immediate, args[i].getValue() >> shift);
+            dst_label = args[i + count];
+        }
+    }
+
+    ASSERT(std::is_sorted(result.begin(),
+                          result.begin() + count,
+                          [](const ArgVal &lhs, const ArgVal &rhs) {
+                              return lhs.getValue() < rhs.getValue();
+                          }));
+
+    return result;
+}
+
+void BeamModuleAssembler::emit_linear_search(arm::Gp comparand,
+                                             Label fail,
+                                             const Span<ArgVal> &args) {
+    int count = args.size() / 2;
+
+    for (int i = 0; i < count; i++) {
+        const ArgVal &value = args[i];
+        const ArgVal &label = args[i + count];
+
+        if ((i % 128) == 0) {
+            /* Checking veneers on the first element is intentional. */
+            check_pending_stubs();
+        }
+
+        cmp_arg(comparand, value);
+        a.cond_eq().b(resolve_beam_label(label, disp1MB));
+    }
+
+    /* An invalid label means fallthrough to the next instruction. */
+    if (fail.isValid()) {
+        a.b(resolve_label(fail, disp128MB));
+    }
+}
+
+void BeamModuleAssembler::emit_i_select_tuple_arity(const ArgVal &Src,
+                                                    const ArgVal &Fail,
+                                                    const ArgVal &Size,
+                                                    const Span<ArgVal> &args) {
+    auto src = load_source(Src, TMP1);
+
+    emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg);
+
+    arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg);
+    a.ldur(TMP1, emit_boxed_val(boxed_ptr, 0));
+
+    ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0);
+    a.tst(TMP1, imm(_TAG_HEADER_MASK));
+    a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+
+    Label fail = rawLabels[Fail.getValue()];
+    emit_linear_search(TMP1, fail, args);
+}
+
+void BeamModuleAssembler::emit_i_select_val_lins(const ArgVal &Src,
+                                                 const ArgVal &Fail,
+                                                 const ArgVal &Size,
+                                                 const Span<ArgVal> &args) {
+    ASSERT(Size.getValue() == args.size());
+    Label fail, next;
+
+    /*
+     * To keep the code simpler, we will drop down a level and
+     * use rawLabels. That will allow us to use Label to represent
+     * a label present in the BEAM file or a label created here.
+     */
+
+    if (Fail.isLabel()) {
+        next = fail = rawLabels[Fail.getValue()];
+    } else {
+        /* Fail is [], meaning that if none of the values match,
+         * we should fall through to the next instruction.
+         *
+         * We set `next` to a label that will be located after the
+         * instructions for the linear search. That label is needed if
+         * values are untagged and a type test is emitted to skip the
+         * comparisons of the untagged values in case the type is
+         * wrong.
+         *
+         * We intentionally do not initialize the Label `fail`
+         * as an indication for emit_optimized_three_way_select() and
+         * emit_linear_search() that not branch is needed at the end
+         * of the linear search.
+         */
+        next = a.newLabel();
+    }
+
+    auto src = load_source(Src, ARG1);
+
+    auto plan = plan_untag(args);
+    auto base = plan.first;
+    auto shift = plan.second;
+
+    if (base == 0 && shift == 0) {
+        if (!emit_optimized_three_way_select(src.reg, fail, args)) {
+            emit_linear_search(src.reg, fail, args);
+        }
+    } else {
+        auto untagged = emit_select_untag(args, src.reg, next, base, shift);
+
+        if (!emit_optimized_three_way_select(ARG1, fail, untagged)) {
+            emit_linear_search(ARG1, fail, untagged);
+        }
+    }
+
+    if (!Fail.isLabel()) {
+        bind_veneer_target(next);
+    }
+}
+
+void BeamModuleAssembler::emit_i_select_val_bins(const ArgVal &Src,
+                                                 const ArgVal &Fail,
+                                                 const ArgVal &Size,
+                                                 const Span<ArgVal> &args) {
+    ASSERT(Size.getValue() == args.size());
+    int count = args.size() / 2;
+    Label fail;
+
+    /* See the comment in emit_i_select_val_lins() for an explanation
+     * why we use raw labels. */
+    if (Fail.isLabel()) {
+        fail = rawLabels[Fail.getValue()];
+    } else {
+        fail = a.newLabel();
+    }
+
+    comment("Binary search in table of %lu elements", count);
+
+    auto src = load_source(Src, ARG1);
+
+    auto plan = plan_untag(args);
+    auto base = plan.first;
+    auto shift = plan.second;
+
+    if (base == 0 && shift == 0) {
+        emit_binsearch_nodes(src.reg, 0, count - 1, fail, args);
+    } else {
+        auto untagged = emit_select_untag(args, src.reg, fail, base, shift);
+        emit_binsearch_nodes(ARG1, 0, count - 1, fail, untagged);
+    }
+
+    if (!Fail.isLabel()) {
+        bind_veneer_target(fail);
+    }
+}
+
+/*
+ * Emit code for a binary search through an interval Left <= Right of
+ * the i_select_val argument vector `args`.
+ */
+void BeamModuleAssembler::emit_binsearch_nodes(arm::Gp reg,
+                                               size_t Left,
+                                               size_t Right,
+                                               Label fail,
+                                               const Span<ArgVal> &args) {
+    ASSERT(Left <= Right);
+    ASSERT(Right < args.size() / 2);
+
+    size_t mid = (Left + Right) >> 1;
+    ArgVal midval(ArgVal::Immediate, args[mid].getValue());
+    int count = args.size() / 2;
+    size_t remaining = (Right - Left + 1);
+
+    if (remaining <= 10) {
+        /* Measurements on randomly generated select_val instructions
+           have shown that linear search is faster than binary search
+           when there are ten or less elements.
+        */
+        std::vector<ArgVal> shrunk;
+
+        comment("Linear search in [%lu..%lu], %lu elements",
+                Left,
+                Right,
+                remaining);
+
+        shrunk.reserve(remaining * 2);
+        shrunk.insert(shrunk.end(),
+                      args.begin() + Left,
+                      args.begin() + Left + remaining);
+        shrunk.insert(shrunk.end(),
+                      args.begin() + Left + count,
+                      args.begin() + count + Left + remaining);
+
+        emit_linear_search(reg, fail, shrunk);
+
+        return;
+    }
+
+    comment("Subtree [%lu..%lu], pivot %lu", Left, Right, mid);
+
+    check_pending_stubs();
+
+    cmp_arg(reg, midval);
+
+    auto &lbl = args[mid + count];
+
+    /* The search has failed if Left == Right, but that should never
+     * happen since we revert to a linear search when there are
+     * ten or less elements. */
+    ASSERT(Left != Right);
+    ASSERT(Left != mid);
+
+    a.cond_eq().b(resolve_beam_label(lbl, disp1MB));
+
+    Label right_tree = a.newLabel();
+    a.cond_hs().b(resolve_label(right_tree, disp1MB));
+
+    emit_binsearch_nodes(reg, Left, mid - 1, fail, args);
+
+    bind_veneer_target(right_tree);
+    emit_binsearch_nodes(reg, mid + 1, Right, fail, args);
+}
+
+void BeamModuleAssembler::emit_i_jump_on_val(const ArgVal &Src,
+                                             const ArgVal &Fail,
+                                             const ArgVal &Base,
+                                             const ArgVal &Size,
+                                             const Span<ArgVal> &args) {
+    Label fail;
+    auto src = load_source(Src, TMP1);
+
+    ASSERT(Size.getValue() == args.size());
+
+    a.and_(TMP3, src.reg, imm(_TAG_IMMED1_MASK));
+    a.cmp(TMP3, imm(_TAG_IMMED1_SMALL));
+
+    if (Fail.isLabel()) {
+        a.cond_ne().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        /* NIL means fallthrough to the next instruction. */
+        ASSERT(Fail.getType() == ArgVal::Immediate && Fail.getValue() == NIL);
+        fail = a.newLabel();
+        a.cond_ne().b(fail);
+    }
+
+    a.asr(TMP1, src.reg, imm(_TAG_IMMED1_SIZE));
+
+    if (Base.getValue() != 0) {
+        if (Support::isUInt12((Sint)Base.getValue())) {
+            a.sub(TMP1, TMP1, imm(Base.getValue()));
+        } else {
+            mov_imm(TMP3, Base.getValue());
+            a.sub(TMP1, TMP1, TMP3);
+        }
+    }
+
+    a.cmp(TMP1, imm(args.size()));
+    if (Fail.isLabel()) {
+        a.cond_hs().b(resolve_beam_label(Fail, disp1MB));
+    } else {
+        a.cond_hs().b(fail);
+    }
+
+    embed_vararg_rodata(args, TMP2);
+    a.ldr(TMP3, arm::Mem(TMP2, TMP1, arm::lsl(3)));
+    a.br(TMP3);
+
+    if (Fail.getType() == ArgVal::Immediate) {
+        a.bind(fail);
+    }
+}
+
+/*
+ * Attempt to optimize the case when a select_val has exactly two
+ * values which only differ by one bit and they both branch to the
+ * same label.
+ *
+ * The optimization makes use of the observation that (V == X || V ==
+ * Y) is equivalent to (V | (X ^ Y)) == (X | Y) when (X ^ Y) has only
+ * one bit set.
+ *
+ * Return true if the optimization was possible.
+ */
+bool BeamModuleAssembler::emit_optimized_three_way_select(
+        arm::Gp reg,
+        Label fail,
+        const Span<ArgVal> &args) {
+    if (args.size() != 4 || (args[2].getValue() != args[3].getValue()))
+        return false;
+
+    uint64_t x = args[0].getValue();
+    uint64_t y = args[1].getValue();
+    uint64_t combined = x | y;
+    uint64_t diff = x ^ y;
+    ArgVal val(ArgVal::Immediate, combined);
+
+    if ((diff & (diff - 1)) != 0) {
+        return false;
+    }
+
+    comment("(Src == 0x%x || Src == 0x%x) <=> (Src | 0x%x) == 0x%x",
+            x,
+            y,
+            diff,
+            combined);
+
+    a.orr(TMP1, reg, imm(diff));
+    cmp_arg(TMP1, val);
+    a.cond_eq().b(resolve_beam_label(args[2], disp1MB));
+
+    /* An invalid label means fallthrough to the next instruction. */
+    if (fail.isValid()) {
+        a.b(resolve_label(fail, disp128MB));
+    }
+    return true;
+}
diff --git a/erts/emulator/beam/jit/arm/instr_trace.cpp b/erts/emulator/beam/jit/arm/instr_trace.cpp
new file mode 100644
index 0000000000..ff32ebed31
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/instr_trace.cpp
@@ -0,0 +1,217 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "beam_common.h"
+#include "erl_bif_table.h"
+#include "beam_bp.h"
+};
+
+/* This function is jumped to from the export entry of a function.
+ *
+ * ARG1 = export entry */
+void BeamGlobalAssembler::emit_generic_bp_global() {
+    /* Enter an Erlang frame to make the stack consistent with local
+     * breakpoints. */
+    emit_enter_erlang_frame();
+
+    lea(ARG2, arm::Mem(ARG1, offsetof(Export, info)));
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    /* ARG2 is already set above */
+    load_x_reg_array(ARG3);
+    runtime_call<3>(erts_generic_breakpoint);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    /* This is technically a tail call so we must leave the current frame
+     * before jumping. Note that we might not leave the frame we entered
+     * earlier in this function, but one added by erts_generic_breakpoint. */
+    emit_leave_erlang_frame();
+    a.br(ARG1);
+}
+
+/* This function is called from the module header, which is in turn called from
+ * the prologue of the traced function. As such, the real return address is at
+ * SP+8 rather than LR (x30).
+ *
+ * See beam_asm.h for more details */
+void BeamGlobalAssembler::emit_generic_bp_local() {
+    a.ldr(ARG2, arm::Mem(a64::sp, 8));
+
+    /* Stash return address for later use in `debug_bp` */
+    a.str(ARG2, TMP_MEM1q);
+
+    /* Our actual return address is valid (and word-aligned), but it points
+     * just after the trampoline word so we'll need to skip that to find our
+     * ErtsCodeInfo. */
+    a.sub(ARG2, ARG2, imm(BEAM_ASM_BP_RETURN_OFFSET + sizeof(ErtsCodeInfo)));
+
+    emit_enter_runtime_frame();
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    /* ARG2 is already set above */
+    load_x_reg_array(ARG3);
+    runtime_call<3>(erts_generic_breakpoint);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.cmp(ARG1, imm(BeamOpCodeAddr(op_i_debug_breakpoint)));
+    a.cond_eq().b(labels[debug_bp]);
+
+    emit_leave_runtime_frame();
+    a.ret(a64::x30);
+}
+
+/* This function is called from the module header which is called from the
+ * prologue of the function to trace. See beam_asm.h for more details
+ *
+ * The only place that we can come to here is from generic_bp_local */
+void BeamGlobalAssembler::emit_debug_bp() {
+    Label error = a.newLabel();
+
+    /* Read and adjust the return address we saved in generic_bp_local. */
+    a.ldr(ARG2, TMP_MEM1q);
+    a.sub(ARG2, ARG2, imm(BEAM_ASM_BP_RETURN_OFFSET + sizeof(ErtsCodeMFA)));
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG3);
+    a.mov(ARG4, imm(am_breakpoint));
+    runtime_call<4>(call_error_handler);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>();
+
+    /* We skip two runtime frames (ours and the one entered in the module
+     * header) so that we can call the error handler's code instead of
+     * `call_nif_early`, if necessary. */
+    emit_leave_runtime_frame();
+    emit_leave_runtime_frame();
+
+    a.cbz(ARG1, error);
+
+    emit_leave_erlang_frame();
+    branch(emit_setup_export_call(ARG1));
+
+    a.bind(error);
+    {
+        a.ldr(ARG2, TMP_MEM1q);
+        mov_imm(ARG4, 0);
+
+        a.b(labels[raise_exception_shared]);
+    }
+}
+
+static void return_trace(Process *c_p,
+                         ErtsCodeMFA *mfa,
+                         Eterm val,
+                         ErtsTracer *tracer) {
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    erts_trace_return(c_p, mfa, val, tracer);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+}
+
+void BeamModuleAssembler::emit_return_trace() {
+    a.ldr(ARG2, getYRef(0));
+    a.mov(ARG3, XREG0);
+    lea(ARG4, getYRef(1));
+
+    ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 1);
+    emit_enter_runtime<Update::eStack | Update::eHeap>(1);
+
+    a.mov(ARG1, c_p);
+    runtime_call<4>(return_trace);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap>(1);
+
+    emit_deallocate(ArgVal(ArgVal::Word, 2));
+    emit_return();
+}
+
+void BeamModuleAssembler::emit_i_return_time_trace() {
+    /* Pass prev_info if present (is a CP), otherwise null. */
+    a.ldr(ARG2, getYRef(0));
+    mov_imm(ARG3, 0);
+
+    a.tst(ARG2, imm(_CPMASK));
+    a.sub(ARG2, ARG2, imm(sizeof(ErtsCodeInfo)));
+    a.csel(ARG2, ARG2, ARG3, arm::Cond::kEQ);
+
+    ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 1);
+    emit_enter_runtime<Update::eStack | Update::eHeap>(1);
+
+    a.mov(ARG1, c_p);
+    runtime_call<2>(erts_trace_time_return);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap>(1);
+
+    emit_deallocate(ArgVal(ArgVal::Word, 1));
+    emit_return();
+}
+
+void BeamModuleAssembler::emit_i_return_to_trace() {
+    ERTS_CT_ASSERT(ERTS_HIGHEST_CALLEE_SAVE_XREG >= 1);
+    emit_enter_runtime<Update::eStack | Update::eHeap>(1);
+
+    a.mov(ARG1, c_p);
+    runtime_call<1>(beam_jit_return_to_trace);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap>(1);
+
+    emit_deallocate(ArgVal(ArgVal::Word, 0));
+    emit_return();
+}
+
+void BeamModuleAssembler::emit_i_hibernate() {
+    Label error = a.newLabel();
+
+    emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(3);
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    runtime_call<2>(erts_hibernate);
+
+    emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs |
+                       Update::eReductions>(3);
+
+    a.cbz(ARG1, error);
+
+    a.ldr(TMP1.w(), arm::Mem(c_p, offsetof(Process, flags)));
+    a.and_(TMP1, TMP1, imm(~F_HIBERNATE_SCHED));
+    a.str(TMP1.w(), arm::Mem(c_p, offsetof(Process, flags)));
+    a.b(resolve_fragment(ga->get_do_schedule(), disp128MB));
+
+    a.bind(error);
+    emit_raise_exception(&BIF_TRAP_EXPORT(BIF_hibernate_3)->info.mfa);
+}
diff --git a/erts/emulator/beam/jit/arm/ops.tab b/erts/emulator/beam/jit/arm/ops.tab
new file mode 100644
index 0000000000..3fecfaafab
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/ops.tab
@@ -0,0 +1,1367 @@
+#
+# %CopyrightBegin%
+#
+# Copyright Ericsson AB 1997-2019. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# %CopyrightEnd%
+#
+
+#
+# Types that should never be used in specific operations.
+#
+
+FORBIDDEN_TYPES=hQ
+
+#
+# The instructions that follows are only known by the loader and the emulator.
+# They can be changed without recompiling old Beam files.
+#
+# Instructions starting with a "i_" prefix are instructions produced by
+# instruction transformations; thus, they never occur in BEAM files.
+#
+
+# The too_old_compiler/0 instruction is specially handled in beam_load.c
+# to produce a user-friendly message informing the user that the module
+# needs to be re-compiled with a modern compiler.
+
+too_old_compiler/0
+too_old_compiler | never() =>
+
+# In R9C and earlier, the loader used to insert special instructions inside
+# the module_info/0,1 functions. (In R10B and later, the compiler inserts
+# an explicit call to an undocumented BIF, so that no loader trickery is
+# necessary.) Since the instructions don't work correctly in R12B, simply
+# refuse to load the module.
+
+func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler
+func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler
+
+# The undocumented and unsupported guard BIF is_constant/1 was removed
+# in R13. The is_constant/2 operation is marked as obsolete in genop.tab,
+# so the loader will automatically generate a too_old_compiler message
+# it is used, but we need to handle the is_constant/1 BIF specially here.
+
+bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler
+
+# Since the constant pool was introduced in R12B, empty tuples ({})
+# are literals. Therefore we no longer need to allow put_tuple/2
+# with a tuple size of zero.
+
+put_tuple u==0 d => too_old_compiler
+
+#
+# All the other instructions.
+#
+
+%cold
+# An unaligned label. The address of an unaligned label must never be saved
+# on the stack or used in a context where it can be confused with an Erlang term.
+
+label L
+
+# An label aligned to a certain boundary. This is used in two cases:
+#
+# * When the label points to the start of a function, as the ErtsCodeInfo
+#   struct must be word-aligned.
+# * When the address is stored on the stack or otherwise needs to be properly
+#   tagged as a continuation pointer.
+aligned_label L t
+
+i_func_info I a a I
+int_code_end
+
+i_generic_breakpoint
+i_debug_breakpoint
+i_return_time_trace
+i_return_to_trace
+trace_jump W
+i_yield
+%hot
+
+return
+
+#
+# A tail call will not refer to the current function on error unless it's a
+# BIF, so we can omit the line instruction for non-BIFs.
+#
+
+move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \
+     move S X0 | call_ext_last Ar Func D
+move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \
+     move S X0 | call_ext_only Ar Func
+
+move S X0=x==0 | line Loc | call_last Ar Func D => \
+     move S X0 | call_last Ar Func D
+move S X0=x==0 | line Loc | call_only Ar Func => \
+     move S X0 | call_only Ar Func
+
+# The line number in int_func_start/5 can be NIL.
+func_line n => empty_func_line
+
+empty_func_line
+func_line I
+
+line n =>
+line I
+
+allocate t t?
+allocate_heap t I t?
+
+deallocate t
+
+init y
+
+trim t t
+
+test_heap I t?
+
+# Translate instructions generated by a compiler before OTP 24.
+allocate_zero Ns Live => allocate_heap_zero Ns u Live
+allocate_heap_zero Ns Nh Live => allocate_heap_zero(Ns, Nh, Live)
+
+init_yregs I *
+
+# Selecting values.
+
+# The size of the dispatch code for a jump table is at least 40 bytes
+# (8 instructions + one 64-bit word for the pointer to the
+# table). Therefore we shouldn't use a jump table if there are too few
+# values.
+
+select_val S Fail=fn Size=u Rest=* | use_jump_tab(Size, Rest, 6) => \
+  jump_tab(S, Fail, Size, Rest)
+is_integer Fail=f S | select_val S=s Fail=fn Size=u Rest=* | use_jump_tab(Size, Rest, 6) => \
+  jump_tab(S, Fail, Size, Rest)
+
+is_integer TypeFail=f S | select_val S=s Fail=fn Size=u Rest=* | \
+	   mixed_types(Size, Rest) => \
+  split_values(S, TypeFail, Fail, Size, Rest)
+
+select_val S Fail=fn Size=u Rest=* | mixed_types(Size, Rest) => \
+  split_values(S, Fail, Fail, Size, Rest)
+
+is_integer Fail=f S | select_val S=d Fail=fn Size=u Rest=* | \
+  fixed_size_values(Size, Rest) => select_val(S, Fail, Size, Rest)
+
+is_atom Fail=f S | select_val S=d Fail=fn Size=u Rest=* | \
+  fixed_size_values(Size, Rest) => select_val(S, Fail, Size, Rest)
+
+select_val S Fail=fn Size=u Rest=* | floats_or_bignums(Size, Rest) => \
+  select_literals(S, Fail, Size, Rest)
+
+select_val S Fail=fn Size=u Rest=* | fixed_size_values(Size, Rest) => \
+  select_val(S, Fail, Size, Rest)
+
+is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
+  select_tuple_arity(S, Fail, Size, Rest)
+
+select_tuple_arity S=d Fail=f Size=u Rest=* => \
+  select_tuple_arity(S, Fail, Size, Rest)
+
+i_select_val_bins s fn I *
+
+i_select_val_lins s fn I *
+
+i_select_tuple_arity S f? I *
+
+i_jump_on_val s fn W I *
+
+is_number f? s
+
+jump f
+
+#
+# List matching instructions. The combination of test for a nonempty list followed
+# by get_{list/hd/tl} are common, so we will optimize that.
+#
+is_nonempty_list Fail nqia => jump Fail
+
+is_nonempty_list f? S
+
+get_list S d d
+get_hd S d
+get_tl S d
+
+# Old-style catch.
+catch y H
+catch_end y
+
+# Try/catch.
+try Y F => catch Y F
+
+try_case y
+try_end y
+
+try_case_end s
+
+# Destructive set tuple element
+
+set_tuple_element s S P
+
+#
+# Get tuple element. Since this instruction is frequently used, we will try
+# to only fetch the pointer to the tuple once for a sequence of BEAM instructions
+# that fetch multiple elements from the same tuple.
+#
+
+current_tuple/1
+current_tuple/2
+
+is_tuple Fail=f Src | test_arity Fail Src Arity => \
+   i_is_tuple_of_arity Fail Src Arity | current_tuple Src
+
+test_arity Fail Src Arity => i_test_arity Fail Src Arity | current_tuple Src
+
+is_tuple NotTupleFail Tuple | is_tagged_tuple WrongRecordFail Tuple Arity Atom => \
+   i_is_tagged_tuple_ff NotTupleFail WrongRecordFail Tuple Arity Atom | current_tuple Tuple
+
+is_tagged_tuple Fail Tuple Arity Atom => \
+   i_is_tagged_tuple Fail Tuple Arity Atom | current_tuple Tuple
+
+is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src
+
+i_is_tuple_of_arity f? s A
+i_test_arity f? s A
+
+i_is_tagged_tuple f? s A a
+i_is_tagged_tuple_ff f? f? s A a
+
+i_is_tuple f? s
+
+# Generate instruction sequence for fetching the tuple element and remember
+# that we have a current tuple pointer.
+
+get_tuple_element Tuple Pos Dst => \
+  load_tuple_ptr Tuple | i_get_tuple_element Tuple Pos Dst | current_tuple Tuple
+current_tuple Tuple | get_tuple_element Tuple Pos Dst => \
+  i_get_tuple_element Tuple Pos Dst | current_tuple Tuple
+
+# Drop the current_tuple instruction if the tuple is overwritten.
+i_get_tuple_element Tuple Pos Tuple | current_tuple Tuple => i_get_tuple_element Tuple Pos Tuple
+
+# This is a current_tuple instruction instruction not followed by get_tuple_element.
+# Invalidate the current tuple pointer.
+
+current_tuple Tuple =>
+
+load_tuple_ptr s
+
+# If both positions and destinations are in consecutive memory, fetch and store
+# two words at once.
+i_get_tuple_element Tuple Pos1 Dst1 | current_tuple Tuple | \
+   get_tuple_element Tuple Pos2 Dst2 | consecutive_words(Pos1, Pos2) => \
+      get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2
+i_get_tuple_element Tuple Pos1 Dst1 | current_tuple Tuple | \
+   get_tuple_element Tuple Pos2 Dst2 | consecutive_words(Pos1, Pos2) => \
+      get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2
+
+# Drop the current_tuple instruction if the tuple is overwritten.
+current_tuple Tuple Tuple =>
+current_tuple Tuple Dst => current_tuple Tuple
+
+# The first operand will only be used in the debug-compiled runtime
+# system to verify that the register holding the tuple pointer agrees
+# with the source tuple operand.
+i_get_tuple_element s P S
+get_two_tuple_elements s P S S
+
+#
+# Expection rasing instructions. Infrequently executed.
+#
+
+%cold
+case_end s
+
+badmatch s
+
+if_end
+
+raise s s
+
+# Workaround the limitation that generators must always return at least one instruction.
+delete_me/0
+delete_me =>
+
+system_limit/1
+system_limit p => system_limit_body
+system_limit Fail=f => jump Fail
+
+system_limit_body
+
+%hot
+
+#
+# Optimize moves of consecutive memory addresses.
+#
+move Src=c Dst => i_move Src Dst
+move Src SrcDst | move SrcDst Dst => i_move Src SrcDst | move SrcDst Dst
+
+# Optimize two moves from X registers to Y registers when destination Y registers are consecutive.
+move S1=x D1=y | move S2=x D2=y | consecutive_words(D1, D2) => store_two_xregs S1 D1 S2 D2
+move S1=x D1=y | move S2=x D2=y | consecutive_words(D2, D1) => store_two_xregs S2 D2 S1 D1
+
+# Optimize two moves from Y registers to X registers when source Y registers are consecutive.
+move S1=y D1=x | move S2=y D2=x | consecutive_words(S1, S2) | distinct(D1, D2) => load_two_xregs S1 D1 S2 D2
+move S1=y D1=x | move S2=y D2=x | consecutive_words(S2, S1) | distinct(D1, D2) => load_two_xregs S2 D2 S1 D1
+
+# Optimize two moves of Y registers when destinations are consecutive.
+move S1=y D1=y | move S2=y D2=y | consecutive_words(D1, D2) => move_two_yregs S1 D1 S2 D2
+move S1=y D1=y | move S2=y D2=y | consecutive_words(D2, D1) => move_two_yregs S2 D2 S1 D1
+
+move Src Dst => i_move Src Dst
+
+#
+# Move instructions.
+#
+
+swap R1 R2 | swap R2 R3 => swap2 R1 R2 R3
+swap R1 R2 | swap R1 R3 => swap2 R2 R1 R3
+swap R1 R2 | swap R3 R2 => swap2 R1 R2 R3
+swap R1 R2 | swap R3 R1 => swap2 R2 R1 R3
+
+swap2 R1 R2 R3 | swap R3 R4 => swap3 R1 R2 R3 R4
+swap2 R1 R2 R3 | swap R4 R3 => swap3 R1 R2 R3 R4
+
+swap3 R1 R2 R3 R4 | swap R4 R5 => swap4 R1 R2 R3 R4 R5
+swap3 R1 R2 R3 R4 | swap R5 R4 => swap4 R1 R2 R3 R4 R5
+
+swap d d
+swap2 d d d
+swap3 d d d d
+swap4 d d d d d
+
+i_move s d
+store_two_xregs x y x y
+load_two_xregs y x y x
+move_two_yregs y y y y
+
+#
+# Receive operations. We conservatively align all labels before any
+# of the receive instructions.
+#
+# As the labels may be stored in the process structure, we must align them to
+# the nearest 4-byte boundary to ensure they're properly tagged as continuation
+# pointers.
+#
+
+label L | loop_rec Fail Reg => \
+    aligned_label L u=4 | loop_rec Fail Reg
+label L | wait_timeout Fail Src => \
+    aligned_label L u=4 | wait_timeout Fail Src
+label L | wait Fail => \
+    aligned_label L u=4 | wait Fail
+label L | timeout => \
+    aligned_label L u=4 | timeout
+
+loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail
+
+aligned_label L A | wait_timeout Fail Src | smp_already_locked(L) => \
+    aligned_label L A | wait_timeout_locked Src Fail
+wait_timeout Fail Src => wait_timeout_unlocked Src Fail
+
+aligned_label L A | wait Fail | smp_already_locked(L) => \
+    aligned_label L A | wait_locked Fail
+wait Fail => wait_unlocked Fail
+
+aligned_label L A | timeout | smp_already_locked(L) => \
+    aligned_label L A | timeout_locked
+
+remove_message
+timeout
+timeout_locked
+i_loop_rec f
+loop_rec_end f
+wait_locked f
+wait_unlocked f
+
+# Note that a timeout value must fit in 32 bits.
+wait_timeout_unlocked s f
+wait_timeout_locked s f
+
+send
+
+#
+# Optimized comparisons with one immediate/literal operand.
+#
+
+is_eq_exact Lbl S S =>
+is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C
+
+is_eq_exact Lbl R=xy n => is_nil Lbl R
+
+is_ne_exact Lbl S S => jump Lbl
+is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C
+
+is_eq_exact f? s s
+
+is_ne_exact f? s s
+
+is_lt f? s s
+is_ge f? s s
+
+is_eq Fail=f Const=c Reg=xy => is_eq Fail Reg Const
+is_eq f? s s
+
+is_ne Fail=f Const=c Reg=xy => is_ne Fail Reg Const
+is_ne f? s s
+
+#
+# Putting tuples.
+#
+# Code compiled with OTP 22 and later uses put_tuple2 to
+# to construct a tuple.
+#
+# Code compiled before OTP 22 uses put_tuple + one put instruction
+# per element. Translate to put_tuple2.
+#
+
+i_put_tuple/2
+put_tuple Arity Dst => i_put_tuple Dst u
+
+i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \
+  put S3 | put S4 | put S5 => \
+	    tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5)
+
+i_put_tuple Dst Arity Puts=* | put S => \
+	    tuple_append_put(Arity, Dst, Puts, S)
+
+i_put_tuple Dst Arity Puts=* => put_tuple2 Dst Arity Puts
+
+put_tuple2 S A *
+
+#
+# Putting lists.
+#
+
+put_list Hd1=y Tl Dst | put_list Hd2=y Dst Dst | consecutive_words(Hd1, Hd2) => \
+    put_list2 Hd1 Hd2 Tl Dst
+
+put_list s s d
+put_list2 s s s d
+
+#
+# Some more only used by the emulator
+#
+
+%cold
+normal_exit
+continue_exit
+call_bif W
+call_bif_mfa a a I
+call_nif W W W
+call_error_handler
+return_trace
+%hot
+
+#
+# Type tests. Note that the operands for most type tests are `s` to
+# ensure that literal operands will work. The BEAM compiler starting
+# from OTP 22 will never emit type tests with literal operands even if
+# all optimizations are turned off, but loading unoptimized code from
+# older releases and code generated by alternative code generators.
+#
+
+is_integer f? s
+is_list f? s
+is_atom f? s
+is_float f? s
+
+is_nil Fail=f n =>
+is_nil Fail=f qia => jump Fail
+is_nil f? S
+
+# XXX Deprecated.
+is_bitstr Fail Term => is_bitstring Fail Term
+
+is_binary f? s
+is_bitstring f? s
+
+is_reference f? s
+is_pid f? s
+is_port f? s
+
+is_boolean f? s
+
+is_function2 f? s s
+
+#################################################################
+# External function and bif calls.
+#################################################################
+
+# Expands into call_light_bif/2
+call_light_bif/1
+
+#
+# The load_nif/2 BIF is an instruction.
+#
+
+call_ext u==2 u$func:erlang:load_nif/2 => \
+    i_load_nif
+call_ext_last u==2 u$func:erlang:load_nif/2 D => \
+    i_load_nif | deallocate D | return
+call_ext_only u==2 u$func:erlang:load_nif/2 => \
+    i_load_nif | return
+
+%cold
+i_load_nif
+%hot
+
+#
+# The call_on_load_function/1 BIF is an instruction.
+#
+
+call_ext u==1 u$func:erlang:call_on_load_function/1 => \
+    i_call_on_load_function
+call_ext_last u==1 u$func:erlang:call_on_load_function/1 D => \
+    i_call_on_load_function | deallocate D | return
+call_ext_only u==1 u$func:erlang:call_on_load_function/1 => \
+    i_call_on_load_function | return
+
+%cold
+i_call_on_load_function
+%hot
+
+#
+# apply/2 is an instruction, not a BIF.
+#
+
+call_ext u==2 u$func:erlang:apply/2 => i_apply_fun
+call_ext_last u==2 u$func:erlang:apply/2 D => i_apply_fun_last D
+call_ext_only u==2 u$func:erlang:apply/2 => i_apply_fun_only
+
+#
+# The apply/3 BIF is an instruction.
+#
+
+call_ext u==3 u$func:erlang:apply/3 => i_apply
+call_ext_last u==3 u$func:erlang:apply/3 D => i_apply_last D
+call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only
+
+#
+# The yield/0 BIF is an instruction
+#
+
+call_ext u==0 u$func:erlang:yield/0 => i_yield
+call_ext_last u==0 u$func:erlang:yield/0 D => i_yield | deallocate D | return
+call_ext_only u==0 u$func:erlang:yield/0 => i_yield | return
+
+#
+# The hibernate/3 BIF is an instruction.
+#
+call_ext u==3 u$func:erlang:hibernate/3 => i_hibernate
+call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate
+call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate
+
+call_ext u==0 u$func:os:perf_counter/0 => \
+    i_perf_counter
+call_ext_last u==0 u$func:os:perf_counter/0 D => \
+    i_perf_counter | deallocate D | return
+call_ext_only u==0 u$func:os:perf_counter/0 => \
+    i_perf_counter | return
+
+#
+# BIFs like process_info/1,2 require up-to-date information about the current
+# emulator state, which the ordinary call_light_bif instruction doesn't save.
+#
+
+call_ext u Bif=u$is_bif | is_heavy_bif(Bif) => \
+    i_call_ext Bif
+call_ext_last u Bif=u$is_bif D | is_heavy_bif(Bif) => \
+    i_call_ext Bif | deallocate D | return
+call_ext_only Ar=u Bif=u$is_bif | is_heavy_bif(Bif) => \
+    allocate u Ar | i_call_ext Bif | deallocate u | return
+
+#
+# The general case for BIFs that have no special requirements.
+#
+
+call_ext u Bif=u$is_bif => \
+    call_light_bif Bif
+call_ext_last u Bif=u$is_bif D => \
+    call_light_bif Bif | deallocate D | return
+call_ext_only Ar=u Bif=u$is_bif => \
+    allocate u Ar | call_light_bif Bif | deallocate u | return
+
+#
+# Any remaining calls are calls to Erlang functions, not BIFs.
+# We rename the instructions to internal names.  This is necessary,
+# to avoid an end-less loop, because we want to call a few BIFs
+# with call instructions.
+#
+
+call_ext Ar Func        => i_call_ext Func
+call_ext_last Ar Func D => i_call_ext_last Func D
+call_ext_only Ar Func   => i_call_ext_only Func
+
+i_validate t
+
+i_apply
+i_apply_last t
+i_apply_only
+
+i_apply_fun
+i_apply_fun_last t
+i_apply_fun_only
+
+call_light_bif Bif => call_light_bif Bif Bif
+call_light_bif b e
+
+%cold
+
+i_hibernate
+i_perf_counter
+
+%hot
+
+#
+# Calls to non-building and guard BIFs.
+#
+
+bif0 u$bif:erlang:self/0 Dst=d => self Dst
+bif0 u$bif:erlang:node/0 Dst=d => node Dst
+
+bif1 Fail=f Bif=u$bif:erlang:hd/1 Src=n Dst => \
+    jump Fail
+bif1 Fail=f Bif=u$bif:erlang:hd/1 Src Dst => \
+    is_nonempty_list Fail Src | get_hd Src Dst
+bif1 Fail=p Bif=u$bif:erlang:hd/1 Src Dst => \
+    bif_hd Src Dst
+
+bif_hd s d
+
+bif1 Fail=f Bif=u$bif:erlang:tl/1 Src=n Dst => \
+    jump Fail
+bif1 Fail=f Bif=u$bif:erlang:tl/1 Src Dst => \
+    is_nonempty_list Fail Src | get_tl Src Dst
+bif1 Fail=p Bif=u$bif:erlang:tl/1 Src Dst => \
+    bif_tl Src Dst
+
+bif_tl s d
+
+bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => get(Src, Dst)
+
+bif2 Fail u$bif:erlang:element/2 S1 S2 Dst => bif_element Fail S1 S2 Dst
+bif_element j s s d
+
+bif2 Fail Bif=u$bif:erlang:and/2 Src1 Src2 Dst=d => bif_and Fail Src1 Src2 Dst
+bif_and j s s d
+
+bif2 Fail Bif=u$bif:erlang:or/2 Src1 Src2 Dst=d => bif_or Fail Src1 Src2 Dst
+bif_or j s s d
+
+bif1 Fail Bif=u$bif:erlang:not/1 Src=d Dst=d => bif_not Fail Src Dst
+bif_not j S d
+
+gc_bif1 Fail Live Bif=u$bif:erlang:bit_size/1 Src Dst=d => \
+    bif_bit_size Fail Src Dst
+bif_bit_size j s d
+
+gc_bif1 Fail Live Bif=u$bif:erlang:byte_size/1 Src Dst=d => \
+    bif_byte_size Fail Src Dst
+bif_byte_size j s d
+
+bif1 Fail Bif=u$bif:erlang:tuple_size/1 Src=d Dst=d => bif_tuple_size Fail Src Dst
+bif_tuple_size j S d
+
+bif1 Fail Bif S1 Dst | never_fails(Bif) => nofail_bif1 S1 Bif Dst
+bif2 Fail Bif S1 S2 Dst | never_fails(Bif) => nofail_bif2 S1 S2 Bif Dst
+
+bif1 Fail Bif S1 Dst    => i_bif1 S1 Fail Bif Dst
+bif2 Fail Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst
+
+nofail_bif2 S1=d S2 Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact S1 S2 Dst
+nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst
+
+i_get_hash c I d
+i_get s d
+
+self d
+
+node d
+
+nofail_bif1 s b d
+nofail_bif2 s s b d
+
+i_bif1 s j? b d
+i_bif2 s s j? b d
+i_bif3 s s s j? b d
+
+bif_is_eq_exact S s d
+bif_is_ne_exact S s d
+
+#
+# Internal calls.
+#
+
+call Ar Func        => i_call Func
+call_last Ar Func D => i_call_last Func D
+call_only Ar Func   => i_call_only Func
+
+i_call f
+i_call_last f t
+i_call_only f
+
+i_call_ext e
+i_call_ext_last e t
+i_call_ext_only e
+
+# Fun calls.
+
+call_fun Arity | deallocate D | return => i_call_fun_last Arity D
+call_fun Arity => i_call_fun Arity
+
+i_call_fun t
+i_call_fun_last t t
+
+#
+# A fun with an empty environment can be converted to a literal.
+# As a further optimization, the we try to move the fun to its
+# final destination directly.
+
+make_fun2 OldIndex=u => make_fun2(OldIndex)
+make_fun3 OldIndex=u Dst=d NumFree=u Env=* => make_fun3(OldIndex, Dst, NumFree, Env)
+
+%cold
+
+i_make_fun3 F S t t *
+
+# Psuedo-instruction for signalling lambda load errors. Never actually runs.
+i_lambda_error t
+
+%hot
+
+is_function f? S
+is_function Fail=f c => jump Fail
+
+# The start and end of a function.
+int_func_start/5
+int_func_end/2
+
+func_prologue/2
+
+int_func_start Func_Label Func_Line M F A | \
+  label Entry_Label | line Entry_Line => \
+    int_func_start Func_Label Func_Line M F A | \
+    func_prologue Entry_Label Entry_Line
+
+int_func_start Func_Label Func_Line M F A | \
+  label Entry_Label => \
+    int_func_start Func_Label Func_Line M F A | \
+    func_prologue Entry_Label n
+
+int_func_start Func_Label Func_Line M F A | \
+  func_prologue Entry_Label Entry_Line | \
+  is_mfa_bif(M, F, A) => \
+    i_flush_stubs | \
+    aligned_label Func_Label u=8 | \
+    func_line Func_Line | \
+    i_func_info Func_Label M F A | \
+      aligned_label Entry_Label u=8 | \
+      line Entry_Line | \
+      i_breakpoint_trampoline | \
+      call_bif_mfa M F A
+
+int_func_start Func_Label Func_Line M F A | \
+  func_prologue Entry_Label Entry_Line => \
+    i_flush_stubs | \
+    aligned_label Func_Label u=8 | \
+    func_line Func_Line | \
+    i_func_info Func_Label M F A | \
+      aligned_label Entry_Label u=8 | \
+      line Entry_Line | \
+      i_breakpoint_trampoline | \
+      i_test_yield
+
+int_func_end Func_Label Entry_Label => \
+    func_end(Func_Label, Entry_Label)
+
+# Handles yielding on function ingress (rather than on each call).
+i_test_yield
+
+# Ensures that the prior function is large enough to allow NIF patching.
+i_nif_padding
+
+# Flushes veneers prior to entering a new function so we don't have to worry
+# about them being emitted in the prologue or NIF padding.
+i_flush_stubs
+
+# Handles tracing, early NIF calls, and so on.
+i_breakpoint_trampoline
+
+# ================================================================
+# Bit syntax matching obsoleted in OTP 22.
+# ================================================================
+
+%cold
+bs_start_match2 Fail=f ica X Y D => jump Fail
+bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
+i_bs_start_match2 S f t t d
+
+bs_save2 Reg Index => bs_save(Reg, Index)
+i_bs_save2 x t
+
+bs_restore2 Reg Index => bs_restore(Reg, Index)
+i_bs_restore2 S t
+
+bs_context_to_binary S
+%warm
+
+# ================================================================
+# New bit syntax matching (R11B).
+# ================================================================
+
+%warm
+
+# Matching integers
+bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val
+
+i_bs_match_string S f W M
+
+# Fetching integers from binaries.
+bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \
+    get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
+
+i_bs_get_integer S f? t t s d
+i_bs_get_fixed_integer S f? t t t d
+
+# Fetching binaries from binaries.
+bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \
+			get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
+
+i_bs_get_binary2 S f t? s t d
+i_bs_get_binary_all2 S f? t t d
+
+# Fetching float from binaries.
+bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \
+		get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst)
+
+bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail
+
+i_bs_get_float2 S f? t s t d
+
+# Miscellanous
+
+bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \
+			skip_bits2(Fail, Ms, Sz, Unit, Flags)
+
+i_bs_skip_bits_imm2 f? S W
+i_bs_skip_bits2 S S f? t
+
+bs_test_tail2 f? S t
+
+bs_test_unit f? S t
+
+# Gets a bitstring from the tail of a context.
+bs_get_tail S d t
+
+# New bs_start_match variant for contexts with external position storage.
+#
+# bs_get/set_position is used to save positions into registers instead of
+# "slots" in the context itself, which lets us continue matching even after
+# we've passed it off to another function.
+
+bs_start_match4 a==am_no_fail Live=u Src=xy Ctx=d => \
+    bs_start_match3 p Src Live Ctx
+bs_start_match4 Fail=f Live=u Src=xy Ctx=d => \
+    bs_start_match3 Fail Src Live Ctx
+
+%if ARCH_64
+
+# This instruction nops on 64-bit platforms
+bs_start_match4 a==am_resume Live Same Same =>
+bs_start_match4 a==am_resume Live Ctx Dst => move Ctx Dst
+
+%else
+
+bs_start_match4 a==am_resume Live Ctx Dst => \
+    bs_start_match4 a=am_no_fail Live Ctx Dst
+
+%endif
+
+bs_start_match3 Fail=j ica Live Dst => jump Fail
+bs_start_match3 Fail Bin Live Dst => i_bs_start_match3 Bin Live Fail Dst
+
+i_bs_start_match3 S t j d
+
+# Match context position instructions. 64-bit assumes that all positions can
+# fit into an unsigned small.
+
+%if ARCH_64
+    bs_get_position Src Dst Live => i_bs_get_position Src Dst
+    i_bs_get_position S S
+    bs_set_position S S
+%else
+    bs_get_position S d t?
+    bs_set_position S S
+%endif
+
+#
+# Utf8/utf16/utf32 support. (R12B-5)
+#
+bs_get_utf8 Fail=f Ms=xy u u Dst=d => i_bs_get_utf8 Ms Fail Dst
+i_bs_get_utf8 S f? d
+
+bs_skip_utf8 Fail=f Ms=xy u u => i_bs_skip_utf8 Ms Fail
+i_bs_skip_utf8 S f?
+
+bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst
+bs_skip_utf16 Fail=f Ms=xy u Flags=u => i_bs_skip_utf16 Ms Fail Flags
+
+i_bs_get_utf16 S f? t d
+i_bs_skip_utf16 S f? t
+
+bs_get_utf32 Fail=f Ms=xy Live=u Flags=u Dst=d => \
+	bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \
+	i_bs_validate_unicode_retract Fail Dst Ms
+bs_skip_utf32 Fail=f Ms=xy Live=u Flags=u => \
+	bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \
+	i_bs_validate_unicode_retract Fail x Ms
+
+i_bs_validate_unicode_retract j s S
+%hot
+
+#
+# Constructing binaries
+#
+%warm
+
+bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
+
+bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init Sz Regs Dst
+
+bs_init2 Fail Sz=u Words Regs Flags Dst => \
+   i_bs_init_heap Sz Words Regs Dst
+
+bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \
+  i_bs_init_fail Sz Fail Regs Dst
+bs_init2 Fail Sz Words Regs Flags Dst => \
+  i_bs_init_fail_heap Sz Words Fail Regs Dst
+
+i_bs_init_fail S j? t? S
+
+i_bs_init_fail_heap s I j? t? S
+
+i_bs_init W t? S
+
+i_bs_init_heap W I t? S
+
+
+bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
+
+bs_init_bits Fail Sz=u Words=u==0 Regs Flags Dst => \
+    i_bs_init_bits Sz Regs Dst
+bs_init_bits Fail Sz=u Words Regs Flags Dst => \
+    i_bs_init_bits_heap Sz Words Regs Dst
+
+bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \
+  i_bs_init_bits_fail Sz Fail Regs Dst
+bs_init_bits Fail Sz Words Regs Flags Dst => \
+  i_bs_init_bits_fail_heap Sz Words Fail Regs Dst
+
+i_bs_init_bits_fail S j? t? S
+
+i_bs_init_bits_fail_heap s I j? t? S
+
+i_bs_init_bits W t? S
+i_bs_init_bits_heap W I t? S
+
+bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
+
+bs_add j? s s t? x
+
+bs_append Fail Size Extra Live Unit Bin Flags Dst => \
+  i_bs_append Fail Extra Live Unit Size Bin Dst
+
+bs_private_append Fail Size Unit Bin Flags Dst => \
+  i_bs_private_append Fail Unit Size Bin Dst
+
+i_bs_private_append Fail Unit Size Bin Dst=y => \
+  i_bs_private_append Fail Unit Size Bin x | move x Dst
+
+bs_init_writable
+
+i_bs_append j? I t? t s s S
+i_bs_private_append j? t s S x
+
+#
+# Storing integers into binaries.
+#
+
+bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \
+			put_integer(Fail, Sz, Unit, Flags, Src)
+
+i_new_bs_put_integer j? S t s
+i_new_bs_put_integer_imm s j? W t
+
+#
+# Utf8/utf16/utf32 support. (R12B-5)
+#
+
+bs_utf8_size j Src Dst=d => i_bs_utf8_size Src Dst
+bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst
+
+bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src
+
+bs_put_utf32 Fail=j Flags=u Src=s => \
+   i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src
+
+i_bs_utf8_size s x
+i_bs_utf16_size s x
+
+i_bs_put_utf8 j? s
+bs_put_utf16 j? t s
+
+i_bs_validate_unicode j? s
+
+#
+# Storing floats into binaries.
+#
+
+# Will fail. No need to keep the instruction, because bs_add or
+# bs_init* would already have raised an exception.
+bs_put_float Fail Sz=q Unit Flags Val =>
+
+bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \
+			put_float(Fail, Sz, Unit, Flags, Src)
+
+i_new_bs_put_float j? S t s
+i_new_bs_put_float_imm j? W t s
+
+#
+# Storing binaries into binaries.
+#
+
+bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \
+			put_binary(Fail, Sz, Unit, Flags, Src)
+
+i_new_bs_put_binary j? s t s
+i_new_bs_put_binary_imm j? W s
+i_new_bs_put_binary_all s j? t
+
+#
+# Warning: The i_bs_put_string and i_new_bs_put_string instructions
+# are specially treated in the loader.
+# Don't change the instruction format unless you change the loader too.
+#
+
+bs_put_string W M
+
+#
+# New floating point instructions (R8).
+#
+
+fadd p FR1 FR2 FR3 => i_fadd FR1 FR2 FR3
+fsub p FR1 FR2 FR3 => i_fsub FR1 FR2 FR3
+fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3
+fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3
+fnegate p FR1 FR2 => i_fnegate FR1 FR2
+
+fmove Arg=l Dst=d => fstore Arg Dst
+fmove Arg=dq Dst=l => fload Arg Dst
+
+fstore l d
+fload Sq l
+
+fconv s l
+
+i_fadd l l l
+i_fsub l l l
+i_fmul l l l
+i_fdiv l l l
+i_fnegate l l
+
+fclearerror =>
+fcheckerror p =>
+
+%hot
+
+#
+# New apply instructions in R10B.
+#
+
+apply t
+apply_last t t
+
+#
+# Handle compatibility with OTP 17 here.
+#
+
+i_put_map_assoc/4
+
+# We KNOW that in OTP 20 (actually OTP 18 and higher), a put_map_assoc instruction
+# is always preceded by an is_map test. That means that put_map_assoc can never
+# fail and does not need any failure label.
+
+put_map_assoc Fail Map Dst Live Size Rest=* | compiled_with_otp_20_or_higher() => \
+	      i_put_map_assoc Map Dst Live Size Rest
+
+# Translate the put_map_assoc instruction if the module was compiled by a compiler
+# before 20. This is only necessary if the OTP 17 compiler was used, but we
+# have no safe and relatively easy way to know whether OTP 18/19 was used.
+
+put_map_assoc Fail=p Map Dst Live Size Rest=* => \
+	      ensure_map Map | i_put_map_assoc Map Dst Live Size Rest
+put_map_assoc Fail=f Map Dst Live Size Rest=* => \
+	      is_map Fail Map | i_put_map_assoc Map Dst Live Size Rest
+
+ensure_map Lit=q | literal_is_map(Lit) =>
+
+%cold
+ensure_map s
+%hot
+
+#
+# Map instructions. First introduced in R17.
+#
+
+sorted_put_map_assoc/4
+i_put_map_assoc Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
+  sorted_put_map_assoc Map Dst Live Size Rest
+
+sorted_put_map_exact/5
+put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \
+  sorted_put_map_exact F Map Dst Live Size Rest
+
+sorted_put_map_assoc Map Dst Live Size Rest=* | is_empty_map(Map) => \
+   new_map Dst Live Size Rest
+sorted_put_map_assoc Src=s Dst Live Size Rest=* => \
+   update_map_assoc Src Dst Live Size Rest
+
+sorted_put_map_exact Fail Src Dst Live Size Rest=* => \
+   update_map_exact Src Fail Dst Live Size Rest
+
+new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \
+  new_small_map_lit(Dst, Live, Size, Rest)
+
+new_map d t I *
+i_new_small_map_lit d t q I *
+update_map_assoc s d t I *
+update_map_exact s j? d t I *
+
+is_map f? s
+
+## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements
+
+has_map_fields Fail Src Size Rest=* => \
+   has_map_fields(Fail, Src, Size, Rest)
+
+## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 }
+
+get_map_elements Fail Src Size=u==2 Rest=* => \
+    get_map_element(Fail, Src, Size, Rest)
+get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \
+   get_map_elements(Fail, Src, Size, Rest)
+
+i_get_map_elements f? s I *
+
+i_get_map_element_hash Fail Src=c Key Hash Dst => \
+    move Src x | i_get_map_element_hash Fail x Key Hash Dst
+i_get_map_element_hash f? S c I S
+
+i_get_map_element Fail Src=c Key Dst => \
+    move Src x | i_get_map_element Fail x Key Dst
+i_get_map_element f? S S S
+
+#
+# Arithmetic instructions.
+#
+
+gc_bif2 Fail Live u$bif:erlang:splus/2 Src1 Src2 Dst => \
+   i_plus Fail Live Src1 Src2 Dst
+
+gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \
+   i_unary_minus Fail Live Src Dst
+
+gc_bif2 Fail Live u$bif:erlang:sminus/2 Src1 Src2 Dst => \
+   i_minus Fail Live Src1 Src2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \
+  i_times Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \
+  i_m_div Fail Live S1 S2 Dst
+
+# Fused 'rem'/'div' pair.
+gc_bif2 Fail Live u$bif:erlang:rem/2 LHS RHS Remainder | \
+    gc_bif2 A B u$bif:erlang:intdiv/2 LHS RHS Quotient | \
+    distinct(LHS, Remainder) | \
+    distinct(RHS, Remainder) => \
+      i_rem_div Fail Live LHS RHS Remainder Quotient
+
+# As above but with a `line` in between
+gc_bif2 Fail Live u$bif:erlang:rem/2 LHS RHS Remainder | \
+    line Loc | \
+    gc_bif2 A B u$bif:erlang:intdiv/2 LHS RHS Quotient | \
+    distinct(LHS, Remainder) | \
+    distinct(RHS, Remainder) => \
+      i_rem_div Fail Live LHS RHS Remainder Quotient
+
+# Fused 'div'/'rem' pair
+gc_bif2 Fail Live u$bif:erlang:intdiv/2 LHS RHS Quotient | \
+    gc_bif2 A B u$bif:erlang:rem/2 LHS RHS Remainder | \
+    distinct(LHS, Quotient) | \
+    distinct(RHS, Quotient) => \
+      i_div_rem Fail Live LHS RHS Quotient Remainder
+
+# As above but with a `line` in between
+gc_bif2 Fail Live u$bif:erlang:intdiv/2 LHS RHS Quotient | \
+    line Loc | \
+    gc_bif2 A B u$bif:erlang:rem/2 LHS RHS Remainder | \
+    distinct(LHS, Quotient) | \
+    distinct(RHS, Quotient) => \
+      i_div_rem Fail Live LHS RHS Quotient Remainder
+
+gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \
+  i_int_div Fail Live S1 S2 Dst
+gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \
+  i_rem Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \
+  i_band Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \
+  i_bor Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \
+  i_bxor Fail Live S1 S2 Dst
+
+gc_bif1 Fail Live u$bif:erlang:bnot/1 Src Dst => \
+  i_bnot Fail Live Src Dst
+
+gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \
+  i_bsr Fail Live S1 S2 Dst
+
+gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \
+  i_bsl Fail Live S1 S2 Dst
+
+i_plus j I s s d
+i_unary_minus j I s d
+i_minus j I s s d
+i_times j I s s d
+
+i_m_div j I s s d
+
+i_rem_div j I s s d d
+i_div_rem j I s s d d
+i_int_div j I s s d
+i_rem j I s s d
+
+i_band j I s s d
+i_bor j I s s d
+i_bxor j I s s d
+
+i_bnot j I s d
+
+i_bsr j I s s d
+i_bsl j I s s d
+
+#
+# Old guard BIFs that creates heap fragments are no longer allowed.
+#
+bif1 Fail u$bif:erlang:length/1 s d => too_old_compiler
+bif1 Fail u$bif:erlang:size/1 s d => too_old_compiler
+bif1 Fail u$bif:erlang:abs/1 s d => too_old_compiler
+bif1 Fail u$bif:erlang:float/1 s d => too_old_compiler
+bif1 Fail u$bif:erlang:round/1 s d => too_old_compiler
+bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler
+
+#
+# Handle the length/1 guard BIF specially to make it trappable.
+#
+
+gc_bif1 Fail=j Live u$bif:erlang:length/1 Src Dst => \
+   i_length_setup Fail Live Src | i_length Fail Live Dst
+
+i_length_setup j? t s
+i_length j? t d
+
+#
+# Specialized guard BIFs.
+#
+
+gc_bif1 Fail Live Bif=u$bif:erlang:map_size/1 Src Dst=d => bif_map_size Fail Src Dst
+bif_map_size j s d
+
+#
+# Guard BIFs.
+#
+gc_bif1 Fail Live Bif Src Dst      => i_bif1 Src Fail Bif Dst
+gc_bif2 Fail Live Bif S1 S2 Dst    => i_bif2 S1 S2 Fail Bif Dst
+gc_bif3 Fail Live Bif S1 S2 S3 Dst => i_bif3 S1 S2 S3 Fail Bif Dst
+
+#
+# The following instruction is specially handled in beam_load.c
+# to produce a user-friendly message if an unsupported guard BIF is
+# encountered.
+#
+unsupported_guard_bif/3
+unsupported_guard_bif A B C | never() =>
+
+#
+# R13B03
+#
+on_load
+
+#
+# R14A.
+#
+# Superseded in OTP 24 by 'recv_marker_reserve' and friends.
+#
+
+recv_mark f => i_recv_mark
+i_recv_mark
+
+recv_set Fail | label Lbl | loop_rec Lf Reg => \
+   i_recv_set | label Lbl | loop_rec Lf Reg
+i_recv_set
+
+#
+# OTP 21.
+#
+
+build_stacktrace
+raw_raise
+
+#
+# Specialized move instructions. Since they don't require a second
+# instruction, we have intentionally placed them after any other
+# transformation rules that starts with a move instruction in order to
+# produce better code for the transformation engine.
+#
+
+move n D=y => init D
+
+#
+# OTP 24
+#
+
+recv_marker_reserve S
+recv_marker_bind S S
+recv_marker_clear S
+recv_marker_use S
+
+#
+# Mark all intentionally unused macros, predicates, and generators.
+#
+
+%unused pred.negation_is_small
+
+%unused gen_increment
+%unused gen.increment
+%unused gen.increment_from_minus
+%unused gen.plus_from_minus
+
+# Landing pad for fun calls/apply where we set up arguments and check errors
+i_lambda_trampoline W f W W
diff --git a/erts/emulator/beam/jit/arm/predicates.tab b/erts/emulator/beam/jit/arm/predicates.tab
new file mode 100644
index 0000000000..9cc3d49a47
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/predicates.tab
@@ -0,0 +1,107 @@
+// -*- c -*-
+//
+// %CopyrightBegin%
+//
+// Copyright Ericsson AB 2020. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// %CopyrightEnd%
+//
+
+pred.is_mfa_bif(M, F, A) {
+    Export *e;
+
+    ASSERT(M.type == TAG_a && F.type == TAG_a && A.type == TAG_u);
+    e = erts_active_export_entry(M.val, F.val, A.val);
+
+    if (e != NULL) {
+        return e->bif_number != -1;
+    }
+
+    return 0;
+}
+
+pred.never_fails(Bif) {
+    static Eterm nofail_bifs[] =
+	{am_Neqeq,
+	 am_Le,
+	 am_Neq,
+	 am_Eq,
+	 am_Le,
+	 am_Eqeq,
+	 am_Gt,
+	 am_Ge,
+	 am_is_atom,
+	 am_is_boolean,
+	 am_is_binary,
+	 am_is_bitstring,
+	 am_is_float,
+	 am_is_integer,
+	 am_is_list,
+	 am_is_map,
+	 am_is_number,
+	 am_is_pid,
+	 am_is_port,
+	 am_is_reference,
+	 am_is_tuple,
+	};
+
+    Uint index = Bif.val;
+
+    if (Bif.type == TAG_u && index < S->beam.imports.count) {
+	BeamFile_ImportEntry *entry = &S->beam.imports.entries[index];
+	int i;
+
+	if (entry->module != am_erlang) {
+	    return 0;
+	}
+
+	if (entry->function == am_is_function) {
+	    /* Note that is_function/2 may fail. */
+	    return entry->arity == 1;
+	}
+
+	for (i = 0; i < sizeof(nofail_bifs) / sizeof(nofail_bifs[0]); i++) {
+	    if (entry->function == nofail_bifs[i]) {
+		return 1;
+	    }
+	}
+    }
+    return 0;
+}
+
+pred.consecutive_words(A1, A2) {
+    return A1.type == A2.type && A1.val + 1 == A2.val;
+}
+pred.is_eq_exact_bif(Bif) {
+    Uint index = Bif.val;
+
+    if (Bif.type == TAG_u && index < S->beam.imports.count) {
+	BeamFile_ImportEntry *entry = &S->beam.imports.entries[index];
+
+	return entry->module == am_erlang && entry->function == am_Eq && entry->arity == 2;
+    }
+    return 0;
+}
+
+pred.is_ne_exact_bif(Bif) {
+    Uint index = Bif.val;
+
+    if (Bif.type == TAG_u && index < S->beam.imports.count) {
+	BeamFile_ImportEntry *entry = &S->beam.imports.entries[index];
+
+	return entry->module == am_erlang && entry->function == am_Neq && entry->arity == 2;
+    }
+    return 0;
+}
diff --git a/erts/emulator/beam/jit/arm/process_main.cpp b/erts/emulator/beam/jit/arm/process_main.cpp
new file mode 100644
index 0000000000..c9b56e26bf
--- /dev/null
+++ b/erts/emulator/beam/jit/arm/process_main.cpp
@@ -0,0 +1,314 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "bif.h"
+#include "beam_common.h"
+#include "code_ix.h"
+#include "export.h"
+}
+
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+static Process *erts_debug_schedule(ErtsSchedulerData *esdp,
+                                    Process *c_p,
+                                    int calls) {
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    c_p = erts_schedule(esdp, c_p, calls);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    return c_p;
+}
+#endif
+
+/* void process_main(ErtsSchedulerData *esdp); */
+void BeamGlobalAssembler::emit_process_main() {
+    Label context_switch_local = a.newLabel(),
+          context_switch_simplified_local = a.newLabel(),
+          do_schedule_local = a.newLabel(), schedule_next = a.newLabel();
+
+    const arm::Mem start_time_i =
+            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time_i));
+    const arm::Mem start_time =
+            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time));
+
+    /* Be kind to debuggers and `perf` by setting up a proper stack frame. */
+    a.stp(a64::x29, a64::x30, arm::Mem(a64::sp, -16).pre());
+
+    /* Allocate the register structure on the stack to allow computing the
+     * runtime stack address from it, greatly reducing the cost of stack
+     * swapping. */
+    a.mov(TMP1, a64::sp);
+    sub(TMP1, TMP1, sizeof(ErtsSchedulerRegisters) + ERTS_CACHE_LINE_SIZE);
+    a.and_(TMP1, TMP1, imm(~ERTS_CACHE_LINE_MASK));
+    a.mov(a64::sp, TMP1);
+    a.mov(a64::x29, a64::sp);
+
+    a.str(TMP1, arm::Mem(ARG1, offsetof(ErtsSchedulerData, registers)));
+
+    a.mov(scheduler_registers, a64::sp);
+
+    load_erl_bits_state(ARG1);
+    runtime_call<1>(erts_bits_init_state);
+
+    /* Save the initial SP of the thread so that we can verify that it
+     * doesn't grow. */
+#ifdef JIT_HARD_DEBUG
+    a.mov(TMP1, a64::sp);
+    a.str(TMP1, getInitialSPRef());
+#endif
+
+    a.str(a64::xzr, start_time_i);
+    a.str(a64::xzr, start_time);
+
+    mov_imm(c_p, 0);
+    mov_imm(FCALLS, 0);
+    mov_imm(ARG3, 0); /* Set reds_used for erts_schedule call */
+
+    a.b(schedule_next);
+
+    a.bind(do_schedule_local);
+    {
+        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
+        a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, def_arg_reg[5])));
+        a.sub(ARG3, TMP1, FCALLS);
+        a.b(schedule_next);
+    }
+
+    /*
+     * The *next* instruction pointer is provided in ARG3, and must be preceded
+     * by an ErtsCodeMFA.
+     */
+    a.bind(context_switch_local);
+    comment("Context switch, unknown arity/MFA");
+    {
+        Sint arity_offset = offsetof(ErtsCodeMFA, arity) - sizeof(ErtsCodeMFA);
+
+        a.ldur(TMP1, arm::Mem(ARG3, arity_offset));
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, arity)));
+
+        a.sub(TMP1, ARG3, imm((Uint)sizeof(ErtsCodeMFA)));
+        a.str(TMP1, arm::Mem(c_p, offsetof(Process, current)));
+
+        /* !! Fall through !! */
+    }
+
+    a.bind(context_switch_simplified_local);
+    comment("Context switch, known arity and MFA");
+    {
+        Label not_exiting = a.newLabel();
+
+#ifdef DEBUG
+        Label check_i = a.newLabel();
+        /* Check that ARG3 is set to a valid CP. */
+        a.tst(ARG3, imm(_CPMASK));
+        a.cond_eq().b(check_i);
+        a.udf(1);
+        a.bind(check_i);
+#endif
+
+        a.str(ARG3, arm::Mem(c_p, offsetof(Process, i)));
+        a.ldr(TMP1.w(), arm::Mem(c_p, offsetof(Process, state.value)));
+
+        a.tst(TMP1, imm(ERTS_PSFLG_EXITING));
+        a.cond_eq().b(not_exiting);
+        {
+            comment("Process exiting");
+
+            a.adr(TMP1, labels[process_exit]);
+            a.str(TMP1, arm::Mem(c_p, offsetof(Process, i)));
+            a.str(ZERO, arm::Mem(c_p, offsetof(Process, arity)));
+            a.str(ZERO, arm::Mem(c_p, offsetof(Process, current)));
+            a.b(do_schedule_local);
+        }
+
+        a.bind(not_exiting);
+
+        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
+        a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, def_arg_reg[5])));
+        a.sub(FCALLS, TMP1, FCALLS);
+
+        comment("Copy out X registers");
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        runtime_call<2>(copy_out_registers);
+
+        /* Restore reds_used from FCALLS */
+        a.mov(ARG3, FCALLS);
+
+        /* !! Fall through !! */
+    }
+
+    a.bind(schedule_next);
+    comment("schedule_next");
+
+    {
+        Label schedule = a.newLabel(), skip_long_schedule = a.newLabel();
+
+        /* ARG3 contains reds_used at this point */
+
+        a.ldr(TMP1, start_time);
+        a.cbz(TMP1, schedule);
+        {
+            a.mov(ARG1, c_p);
+            a.ldr(ARG2, start_time);
+
+            /* Spill reds_used in start_time slot */
+            a.str(ARG3, start_time);
+
+            a.ldr(ARG3, start_time_i);
+            runtime_call<3>(check_monitor_long_schedule);
+
+            /* Restore reds_used */
+            a.ldr(ARG3, start_time);
+        }
+
+        a.bind(schedule);
+        mov_imm(ARG1, 0);
+        a.mov(ARG2, c_p);
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+        runtime_call<3>(erts_debug_schedule);
+#else
+        runtime_call<3>(erts_schedule);
+#endif
+        a.mov(c_p, ARG1);
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+        lea(ARG1, erts_msacc_cache);
+        runtime_call<1>(erts_msacc_update_cache);
+#endif
+
+        a.str(ZERO, start_time);
+        mov_imm(ARG1, &erts_system_monitor_long_schedule);
+        a.ldr(TMP1, arm::Mem(ARG1));
+        a.cbz(TMP1, skip_long_schedule);
+
+        {
+            /* Enable long schedule test */
+            runtime_call<0>(erts_timestamp_millis);
+            a.str(ARG1, start_time);
+            a.ldr(TMP1, arm::Mem(c_p, offsetof(Process, i)));
+            a.str(TMP1, start_time_i);
+        }
+
+        a.bind(skip_long_schedule);
+        comment("skip_long_schedule");
+
+        /* Copy arguments */
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        runtime_call<2>(copy_in_registers);
+
+        /* Setup reduction counting */
+        a.ldr(FCALLS, arm::Mem(c_p, offsetof(Process, fcalls)));
+        a.str(FCALLS, arm::Mem(c_p, offsetof(Process, def_arg_reg[5])));
+
+#ifdef DEBUG
+        a.str(FCALLS, a64::Mem(c_p, offsetof(Process, debug_reds_in)));
+#endif
+
+        comment("check whether save calls is on");
+        a.mov(ARG1, c_p);
+        mov_imm(ARG2, ERTS_PSD_SAVED_CALLS_BUF);
+        runtime_call<2>(erts_psd_get);
+
+        /* Read the active code index, overriding it with
+         * ERTS_SAVE_CALLS_CODE_IX when save_calls is enabled (ARG1 != 0). */
+        mov_imm(TMP1, &the_active_code_index);
+        a.ldr(TMP1.w(), arm::Mem(TMP1));
+        a.mov(TMP2, imm(ERTS_SAVE_CALLS_CODE_IX));
+        a.cmp(ARG1, ZERO);
+        a.csel(active_code_ix, TMP1, TMP2, arm::Cond::kEQ);
+
+        /* Start executing the Erlang process. Note that reductions have
+         * already been set up above. */
+        emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+        /* Check if we are just returning from a dirty nif/bif call and if so we
+         * need to do a bit of cleaning up before continuing.
+         *
+         * This relies on `op_call_nif_WWW` / `op_call_bif_W` being encoded as
+         * UDF(opcode) followed by UDF(0), which we will never emit. */
+        a.ldr(ARG1, arm::Mem(c_p, offsetof(Process, i)));
+        a.ldr(TMP1, arm::Mem(ARG1));
+
+        ERTS_CT_ASSERT((op_call_nif_WWW & 0xFFFF0000) == 0);
+        a.cmp(TMP1, imm(op_call_nif_WWW));
+        a.cond_eq().b(labels[dispatch_nif]);
+
+        ERTS_CT_ASSERT((op_call_bif_W & 0xFFFF0000) == 0);
+        a.cmp(TMP1, imm(op_call_bif_W));
+        a.cond_eq().b(labels[dispatch_bif]);
+
+        a.br(ARG1);
+    }
+
+    /* Processes may jump to the exported entry points below, executing on the
+     * Erlang stack when entering. These are separate from the `_local` labels
+     * above as we don't want to worry about which stack we're on when the
+     * cases overlap. */
+
+    /* `ga->get_context_switch()`
+     *
+     * The *next* instruction pointer is provided in ARG3, and must be preceded
+     * by an ErtsCodeMFA.
+     *
+     * The X registers are expected to be in CPU registers.
+     */
+    a.bind(labels[context_switch]);
+    {
+        emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+        a.b(context_switch_local);
+    }
+
+    /* `ga->get_context_switch_simplified()`
+     *
+     * The next instruction pointer is provided in ARG3, which does not need to
+     * point past an ErtsCodeMFA as the process structure has already been
+     * updated.
+     *
+     * The X registers are expected to be in CPU registers.
+     */
+    a.bind(labels[context_switch_simplified]);
+    {
+        emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+        a.b(context_switch_simplified_local);
+    }
+
+    /* `ga->get_do_schedule()`
+     *
+     * `c_p->i` must be set prior to jumping here.
+     *
+     * The X registers are expected to be in CPU registers.
+     */
+    a.bind(labels[do_schedule]);
+    {
+        emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
+
+        a.b(do_schedule_local);
+    }
+}
diff --git a/erts/emulator/beam/jit/beam_asm.h b/erts/emulator/beam/jit/beam_asm.h
index 56bb7219f7..0b7260b063 100644
--- a/erts/emulator/beam/jit/beam_asm.h
+++ b/erts/emulator/beam/jit/beam_asm.h
@@ -27,6 +27,10 @@
 #    include "beam_file.h"
 #    include "beam_common.h"
 
+#    if defined(__APPLE__)
+#        include <libkern/OSCacheControl.h>
+#    endif
+
 /* Global configuration variables */
 #    ifdef HAVE_LINUX_PERF_SUPPORT
 #        define BEAMASM_PERF_DUMP (1 << 0)
@@ -77,8 +81,16 @@ char *beamasm_get_base(void *instance);
 /* Return current instruction offset, for line information. */
 size_t beamasm_get_offset(void *ba);
 
-// Number of bytes emitted at first label in order to support trace and nif load
-#    define BEAM_ASM_FUNC_PROLOGUE_SIZE 8
+void beamasm_flush_icache(const void *address, size_t size);
+
+/* Number of bytes emitted at first label in order to support trace and nif
+ * load. */
+#    if defined(__aarch64__)
+#        define BEAM_ASM_BP_RETURN_OFFSET 16
+#        define BEAM_ASM_FUNC_PROLOGUE_SIZE 16
+#    else
+#        define BEAM_ASM_FUNC_PROLOGUE_SIZE 8
+#    endif
 
 /*
  * The code below is used to deal with intercepting the execution of
@@ -88,7 +100,7 @@ size_t beamasm_get_offset(void *ba);
  * the first instruction in a function. In asm mode it is not as simple as
  * our code changes have to be to executing native code.
  *
- * The solution is as follows:
+ * On x86, the solution is as follows:
  *
  *   When emitting a function the first word (or function prologue) is:
  *      0x0: jmp 6
@@ -117,6 +129,12 @@ size_t beamasm_get_offset(void *ba);
  *   is updated. So if CALL_NIF_EARLY is set, then it is updated to be
  *   genericBPTramp + 0x10. If BP is set, it is updated to genericBPTramp + 0x20
  *   and the combination makes it be genericBPTramp + 0x30.
+ *
+ * The solution for AArch64 is similar but we move the flag to a register
+ * before jumping to `genericBPTramp`, where we branch on said flag value. This
+ * is necessary because the maximum jump distance is limited to 128MB and we
+ * need veneers to jump further than that, which are very annoying to deal with
+ * here.
  */
 
 enum erts_asm_bp_flag {
@@ -127,9 +145,61 @@ enum erts_asm_bp_flag {
             ERTS_ASM_BP_FLAG_CALL_NIF_EARLY | ERTS_ASM_BP_FLAG_BP
 };
 
-static inline void erts_asm_bp_set_flag(ErtsCodeInfo *ci,
+static inline enum erts_asm_bp_flag erts_asm_bp_get_flags(
+        const ErtsCodeInfo *ci_exec) {
+    enum erts_asm_bp_flag flag;
+
+#    if defined(__aarch64__)
+    Uint32 *exec_code = (Uint32 *)erts_codeinfo_to_code(ci_exec);
+
+    /* MOVZ instruction, flag lives in bits 5..21 */
+    const Uint32 flag_immed_shift = 5;
+    const Uint32 flag_immed_mask = ((1 << 16) - 1) << flag_immed_shift;
+
+    Uint32 set_flag = exec_code[2]; /* MOVZ x0, flag */
+
+    ASSERT((set_flag & ~flag_immed_mask) == 0xd2800000);
+    flag = (enum erts_asm_bp_flag)((set_flag & flag_immed_mask) >>
+                                   flag_immed_shift);
+
+#    else /* x86_64 */
+    byte *codebytes = (byte *)erts_codeinfo_to_code(ci_exec);
+    /* 0xEB = relative jmp, 0xE8 = relative call */
+    flag = (enum erts_asm_bp_flag)codebytes[2];
+#    endif
+
+    return flag;
+}
+
+static inline void erts_asm_bp_set_flag(ErtsCodeInfo *ci_rw,
+                                        const ErtsCodeInfo *ci_exec,
                                         enum erts_asm_bp_flag flag) {
-    BeamInstr volatile *code_ptr = (BeamInstr *)erts_codeinfo_to_code(ci);
+#    if defined(__aarch64__)
+    const Uint32 *exec_code = (Uint32 *)erts_codeinfo_to_code(ci_exec);
+    Uint32 volatile *rw_code = (Uint32 *)erts_codeinfo_to_code(ci_rw);
+
+    const Uint32 branch_target_shift = 0;
+    const Uint32 branch_target_mask = ((1 << 26) - 1) << branch_target_shift;
+    const Uint32 flag_immed_shift = 5;
+    const Uint32 flag_immed_mask = ((1 << 16) - 1) << flag_immed_shift;
+
+    Uint32 old_guard_branch = rw_code[1]; /* B next */
+    Uint32 old_set_flag = rw_code[2];     /* MOVZ x0, flag */
+
+    ASSERT((old_guard_branch & ~branch_target_mask) == 0x14000000);
+    ASSERT((old_set_flag & ~flag_immed_mask) == 0xd2800000);
+    (void)flag_immed_mask;
+
+    /* MOVZ instruction, flag lives in bits 5..21 */
+    rw_code[2] = old_set_flag | (flag << flag_immed_shift);
+
+    /* Reroute the initial branch instruction to the flag instruction. */
+    rw_code[1] = (old_guard_branch & ~branch_target_mask) |
+                 (1 << branch_target_shift);
+
+    beamasm_flush_icache(&exec_code[1], sizeof(Uint32[2]));
+#    else /* x86_64 */
+    BeamInstr volatile *code_ptr = (BeamInstr *)erts_codeinfo_to_code(ci_rw);
     BeamInstr code = *code_ptr;
     byte *codebytes = (byte *)&code;
     Uint32 *code32 = (Uint32 *)(codebytes + 4);
@@ -139,16 +209,50 @@ static inline void erts_asm_bp_set_flag(ErtsCodeInfo *ci,
     codebytes[2] |= flag;
     *code32 += flag * 16;
     code_ptr[0] = code;
-}
 
-static inline enum erts_asm_bp_flag erts_asm_bp_get_flags(ErtsCodeInfo *ci) {
-    byte *codebytes = (byte *)erts_codeinfo_to_code(ci);
-    return (enum erts_asm_bp_flag)codebytes[2];
+    (void)ci_exec;
+#    endif
 }
 
-static inline void erts_asm_bp_unset_flag(ErtsCodeInfo *ci,
+static inline void erts_asm_bp_unset_flag(ErtsCodeInfo *ci_rw,
+                                          const ErtsCodeInfo *ci_exec,
                                           enum erts_asm_bp_flag flag) {
-    BeamInstr volatile *code_ptr = (BeamInstr *)erts_codeinfo_to_code(ci);
+#    if defined(__aarch64__)
+    const Uint32 *exec_code = (Uint32 *)erts_codeinfo_to_code(ci_exec);
+    Uint32 volatile *rw_code = (Uint32 *)erts_codeinfo_to_code(ci_rw);
+
+    const Uint32 flag_immed_shift = 5;
+    const Uint32 flag_immed_mask = ((1 << 16) - 1) << flag_immed_shift;
+    const Uint32 branch_target_shift = 0;
+    const Uint32 branch_target_mask = ((1 << 26) - 1) << branch_target_shift;
+
+    Uint32 old_guard_branch = rw_code[1]; /* B next */
+    Uint32 old_set_flag = rw_code[2];     /* MOVZ x0, flag */
+    Uint32 new_set_flag;
+
+    ASSERT((old_guard_branch & ~branch_target_mask) == 0x14000000);
+    ASSERT((old_set_flag & ~flag_immed_mask) == 0xd2800000);
+
+    new_set_flag = old_set_flag & ~(flag << flag_immed_shift);
+    rw_code[2] = new_set_flag;
+
+    ERTS_CT_ASSERT(ERTS_ASM_BP_FLAG_NONE == 0);
+    if ((new_set_flag & flag_immed_mask) == 0) {
+        Uint32 new_guard_branch, new_target;
+
+        /* We've removed the last flag, route the branch instruction back
+         * past the prologue. */
+        new_target = (BEAM_ASM_FUNC_PROLOGUE_SIZE / sizeof(Uint32) - 1);
+
+        new_guard_branch = (old_guard_branch & ~branch_target_mask) |
+                           (new_target << branch_target_shift);
+        rw_code[1] = new_guard_branch;
+    }
+
+    beamasm_flush_icache(&exec_code[1], sizeof(Uint32[2]));
+
+#    else /* x86_64 */
+    BeamInstr volatile *code_ptr = (BeamInstr *)erts_codeinfo_to_code(ci_rw);
     BeamInstr code = *code_ptr;
     byte *codebytes = (byte *)&code;
     Uint32 *code32 = (Uint32 *)(codebytes + 4);
@@ -160,6 +264,9 @@ static inline void erts_asm_bp_unset_flag(ErtsCodeInfo *ci,
         codebytes[1] = 6;
     }
     code_ptr[0] = code;
+
+    (void)ci_exec;
+#    endif
 }
 
 #endif
diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp
index b8c1d073bf..043a4739db 100644
--- a/erts/emulator/beam/jit/beam_jit_common.cpp
+++ b/erts/emulator/beam/jit/beam_jit_common.cpp
@@ -121,6 +121,8 @@ void BeamAssembler::_codegen(JitAllocator *allocator,
                            code.codeSize(),
                            CodeHolder::kCopyPadSectionBuffer);
 
+    beamasm_flush_icache(*executable_ptr, code.codeSize());
+
 #ifdef DEBUG
     if (FileLogger *l = dynamic_cast<FileLogger *>(code.logger()))
         if (FILE *f = l->file())
diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp
index a93393f79f..019f241a7e 100644
--- a/erts/emulator/beam/jit/beam_jit_common.hpp
+++ b/erts/emulator/beam/jit/beam_jit_common.hpp
@@ -139,6 +139,16 @@ struct ArgVal : public BeamOpArg {
         }
     }
 
+    struct Hash {
+        size_t operator()(const ArgVal &key) const {
+            return ((size_t)key.getType() << 48) ^ (size_t)key.getValue();
+        }
+    };
+
+    bool operator==(const ArgVal &other) const {
+        return getType() == other.getType() && getValue() == other.getValue();
+    }
+
     template<typename T>
     ArgVal operator+(T val) const {
         return ArgVal(getType(), getValue() + val);
diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp
index aa25fe3352..8d804b5243 100644
--- a/erts/emulator/beam/jit/beam_jit_main.cpp
+++ b/erts/emulator/beam/jit/beam_jit_main.cpp
@@ -26,6 +26,12 @@ extern "C"
 #include "beam_common.h"
 #include "code_ix.h"
 #include "export.h"
+
+#if defined(__APPLE__)
+#    include <libkern/OSCacheControl.h>
+#elif defined(WIN32)
+#    include <windows.h>
+#endif
 }
 
 #ifdef ERLANG_FRAME_POINTERS
@@ -459,6 +465,25 @@ extern "C"
 #endif
     }
 
+    void beamasm_flush_icache(const void *address, size_t size) {
+#if defined(__aarch64__)
+#    if defined(WIN32)
+        FlushInstructionCache(GetCurrentProcess(), address, size);
+#    elif defined(__APPLE__)
+        sys_icache_invalidate((char *)address, size);
+#    elif defined(__GNUC__)
+        __builtin___clear_cache(&((char *)address)[0],
+                                &((char *)address)[size]);
+#    else
+#        error "Platform lacks implementation for clearing instruction cache." \
+                "Please report this bug."
+#    endif
+#else
+        (void)address;
+        (void)size;
+#endif
+    }
+
     void *beamasm_new_assembler(Eterm mod, int num_labels, int num_functions) {
         return new BeamModuleAssembler(bga, mod, num_labels, num_functions);
     }
diff --git a/erts/emulator/beam/predicates.tab b/erts/emulator/beam/predicates.tab
index 833542112b..7e82ddd18e 100644
--- a/erts/emulator/beam/predicates.tab
+++ b/erts/emulator/beam/predicates.tab
@@ -155,14 +155,16 @@ pred.literal_is_map(Lit) {
 
 // Predicate to test whether all of the given new small map keys are literals
 pred.is_small_map_literal_keys(Size, Rest) {
-    if (Size.val > MAP_SMALL_MAP_LIMIT) {
+    Uint pair_count = Size.val / 2;
+
+    if (pair_count > MAP_SMALL_MAP_LIMIT) {
         return 0;
     }
 
     /*
      * Operations with non-literals have always only one key.
      */
-    if (Size.val != 2) {
+    if (pair_count != 1) {
         return 1;
     }
 
diff --git a/erts/emulator/internal_doc/BeamAsm.md b/erts/emulator/internal_doc/BeamAsm.md
index 5934413788..39328da0c5 100644
--- a/erts/emulator/internal_doc/BeamAsm.md
+++ b/erts/emulator/internal_doc/BeamAsm.md
@@ -228,16 +228,17 @@ The `erts_writable_code_ptr` function can be used to get writable pointers,
 given a module instance:
 
     for (i = 0; i < n; ++i) {
-        ErtsCodeInfo* ci;
+        const ErtsCodeInfo* ci_exec;
+        ErtsCodeInfo* ci_rw;
         void *w_ptr;
 
-        w_ptr = erts_writable_code_ptr(&modp->curr,
-                                       code_hdr->functions[i]);
-        ci = (ErtsCodeInfo*)w_ptr;
+        ci_exec = code_hdr->functions[i];
+        w_ptr = erts_writable_code_ptr(&modp->curr, ci_exec);
+        ci_rw = (ErtsCodeInfo*)w_ptr;
 
-        uninstall_breakpoint(ci);
-        consolidate_bp_data(modp, ci, 1);
-        ASSERT(ci->u.gen_bp == NULL);
+        uninstall_breakpoint(ci_rw, ci_exec);
+        consolidate_bp_data(modp, ci_rw, 1);
+        ASSERT(ci_rw->u.gen_bp == NULL);
     }
 
 Without the module instance there's no reliable way to figure out the writable
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index e92c37749a..7d2992044d 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -50,6 +50,9 @@ my $code_model = 'unknown';
 my $jit = 'no';
 my %defs;                        # Defines (from command line).
 
+# Intentionally unused macros, predicates, and generators.
+my %intentionally_unused;
+
 # This is shift counts and mask for the packer.
 my $WHOLE_WORD = '';
 
@@ -452,6 +455,24 @@ while (<>) {
 	next;
     }
 
+    #
+    # Handle %unused.
+    #
+    if (/^\%unused\s+(\S+)/) {
+        my $name = $1;
+        error("Illegal predicate/generator name '$name'")
+            unless $name =~ /^[a-z][.\w\d_]*$/;
+        $intentionally_unused{$name} = 1;
+        next;
+    }
+
+    #
+    # Handle unrecognized % directives.
+    #
+    if (/^\%(\S+)/) {
+        error("Unrecognized directive '%$1'");
+    }
+
     #
     # Handle transformations.
     #
@@ -621,7 +642,8 @@ error("there were previous type errors") if $any_errors;
 #
 {
     my(@unused) = grep(!$c_code_used{$_}, keys %c_code);
-    foreach my $unused (@unused) {
+    @unused = grep(!$intentionally_unused{$_}, @unused);
+    foreach my $unused (sort @unused) {
         my(undef,$where) = @{$c_code{$unused}};
         warn "$where: $unused is unused\n";
     }
@@ -637,6 +659,13 @@ sub emulator_output {
     my $name;
     my $key;			# Loop variable.
 
+    foreach $key (sort keys %c_code) {
+        if ($intentionally_unused{$key}) {
+            delete $c_code{$key};
+        }
+#        print STDERR "$key: ", defined($c_code{$key}), "\n";
+    }
+
     # Initialize common information always needed.
     foreach $key (keys %specific_op) {
 	foreach (@{$specific_op{$key}}) {
-- 
2.26.2

Places

File 1120-jit-Implement-code-generation-for-ARM.patch of Package erlang (Revision f55041561f125384b28886aa21e55614)

Places