File 1117-erts-Refactor-fun-calls.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1117-erts-Refactor-fun-calls.patch of Package erlang (Revision 06790aed050a1354546a4c79936c324d)

Currently displaying revision 06790aed050a1354546a4c79936c324d , Show latest

From 0845ba82df3c7648c4e48404fef2eee991bb3a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 7 May 2021 08:31:47 +0200
Subject: [PATCH 17/20] erts: Refactor fun calls

---
 erts/emulator/Makefile.in                     |  15 +-
 erts/emulator/beam/beam_common.c              |  25 +-
 erts/emulator/beam/beam_common.h              |  14 +-
 erts/emulator/beam/beam_debug.c               |   4 +-
 erts/emulator/beam/beam_file.c                | 163 ++++----
 erts/emulator/beam/beam_load.c                |   6 +-
 erts/emulator/beam/beam_load.h                |   2 +-
 erts/emulator/beam/emu/beam_emu.c             |   8 +-
 erts/emulator/beam/emu/emu_load.c             |  72 ++--
 erts/emulator/beam/emu/instrs.tab             |  12 +-
 erts/emulator/beam/emu/ops.tab                |   9 +-
 erts/emulator/beam/emu/predicates.tab         |  12 +
 erts/emulator/beam/erl_bif_info.c             |   5 +-
 erts/emulator/beam/erl_fun.c                  | 108 ++---
 erts/emulator/beam/erl_fun.h                  |  27 +-
 erts/emulator/beam/generators.tab             |  23 +-
 erts/emulator/beam/jit/asm_load.c             |  26 +-
 erts/emulator/beam/jit/beam_asm.h             |   1 +
 erts/emulator/beam/jit/beam_jit_common.cpp    |  75 +++-
 erts/emulator/beam/jit/beam_jit_common.hpp    |   7 +
 erts/emulator/beam/jit/beam_jit_main.cpp      |   8 +
 erts/emulator/beam/jit/load.h                 |   8 +-
 erts/emulator/beam/jit/x86/beam_asm.hpp       |  31 +-
 .../emulator/beam/jit/x86/beam_asm_global.cpp |  13 +-
 .../emulator/beam/jit/x86/beam_asm_module.cpp |  13 +-
 erts/emulator/beam/jit/x86/generators.tab     |  47 +++
 erts/emulator/beam/jit/x86/instr_call.cpp     |  95 -----
 erts/emulator/beam/jit/x86/instr_common.cpp   |  34 --
 erts/emulator/beam/jit/x86/instr_fun.cpp      | 393 ++++++++++++++++++
 erts/emulator/beam/jit/x86/ops.tab            |  12 +-
 erts/emulator/beam/jit/x86/predicates.tab     |   6 -
 31 files changed, 868 insertions(+), 406 deletions(-)
 create mode 100644 erts/emulator/beam/jit/x86/instr_fun.cpp

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 5b67459c14..e4481ce669 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -1004,22 +1004,23 @@ COMMON_OBJS = \
 ASMJIT_OBJS = $(patsubst %.cpp,$(OBJDIR)/%.o,$(ASMJIT_CPP))
 
 JIT_OBJS = \
-        $(OBJDIR)/asm_load.o \
-        $(OBJDIR)/beam_jit_common.o \
-        $(OBJDIR)/beam_jit_main.o \
-        $(OBJDIR)/beam_jit_perf.o
+    $(OBJDIR)/asm_load.o \
+    $(OBJDIR)/beam_asm_global.o \
+    $(OBJDIR)/beam_asm_module.o \
+    $(OBJDIR)/beam_jit_common.o \
+    $(OBJDIR)/beam_jit_main.o \
+    $(OBJDIR)/beam_jit_perf.o \
+    $(OBJDIR)/process_main.o
 
 ifeq ($(JIT_ARCH), x86)
 JIT_OBJS += \
-	$(OBJDIR)/beam_asm_global.o \
-	$(OBJDIR)/beam_asm_module.o \
-	$(OBJDIR)/process_main.o \
 	$(OBJDIR)/instr_arith.o \
 	$(OBJDIR)/instr_bs.o \
 	$(OBJDIR)/instr_bif.o \
 	$(OBJDIR)/instr_call.o \
 	$(OBJDIR)/instr_common.o \
 	$(OBJDIR)/instr_float.o \
+	$(OBJDIR)/instr_fun.o \
 	$(OBJDIR)/instr_guard_bifs.o \
 	$(OBJDIR)/instr_map.o \
 	$(OBJDIR)/instr_msg.o \
diff --git a/erts/emulator/beam/beam_common.c b/erts/emulator/beam/beam_common.c
index 71bb96d42f..a73e70df83 100644
--- a/erts/emulator/beam/beam_common.c
+++ b/erts/emulator/beam/beam_common.c
@@ -530,7 +530,7 @@ handle_error(Process* c_p, ErtsCodePtr pc, Eterm* reg,
 	reg[3] = c_p->ftrace;
         if ((new_pc = next_catch(c_p, reg))) {
 
-#if defined(BEAMASM) && defined(NATIVE_ERLANG_STACK)
+#if defined(BEAMASM) && (defined(NATIVE_ERLANG_STACK) || defined(__aarch64__))
             /* In order to make use of native call and return
              * instructions, when beamasm uses the native stack it
              * doesn't include the CP in the current stack frame,
@@ -1692,8 +1692,7 @@ ErtsCodePtr
 call_fun(Process* p,    /* Current process. */
          int arity,     /* Number of arguments for Fun. */
          Eterm* reg,    /* Contents of registers. */
-         Eterm args,    /* THE_NON_VALUE or pre-built list of arguments. */
-         Export **epp)  /* Export entry, if any. */
+         Eterm args)    /* THE_NON_VALUE or pre-built list of arguments. */
 {
     Eterm fun = reg[arity];
     Eterm hdr;
@@ -1812,7 +1811,6 @@ call_fun(Process* p,    /* Current process. */
 		reg[1] = fun;
 		reg[2] = args;
 		reg[3] = NIL;
-                *epp = ep;
                 return ep->addresses[code_ix];
 	    }
 	}
@@ -1825,7 +1823,6 @@ call_fun(Process* p,    /* Current process. */
 
 	if (arity == actual_arity) {
             DTRACE_GLOBAL_CALL(p, &ep->info.mfa);
-            *epp = ep;
             return ep->addresses[erts_active_code_ix()];
 	} else {
 	    /*
@@ -1856,7 +1853,7 @@ call_fun(Process* p,    /* Current process. */
 }
 
 ErtsCodePtr
-apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg, Export **epp)
+apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg)
 {
     int arity;
     Eterm tmp;
@@ -1883,16 +1880,14 @@ apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg, Export **epp)
 	return NULL;
     }
     reg[arity] = fun;
-    return call_fun(p, arity, reg, args, epp);
+    return call_fun(p, arity, reg, args);
 }
 
 ErlFunThing*
-new_fun_thing(Process* p, ErlFunEntry* fe, int num_free)
+new_fun_thing(Process* p, ErlFunEntry* fe, int arity, int num_free)
 {
-    const ErtsCodeMFA *mfa;
     ErlFunThing* funp;
 
-    mfa = erts_code_to_codemfa(fe->address);
     funp = (ErlFunThing*) p->htop;
     p->htop += ERL_FUN_SIZE + num_free;
     erts_refc_inc(&fe->refc, 2);
@@ -1903,7 +1898,15 @@ new_fun_thing(Process* p, ErlFunEntry* fe, int num_free)
     funp->fe = fe;
     funp->num_free = num_free;
     funp->creator = p->common.id;
-    funp->arity = mfa->arity - num_free;
+    funp->arity = arity;
+
+#ifdef DEBUG
+    {
+        const ErtsCodeMFA *mfa = erts_get_fun_mfa(fe);
+        ASSERT(funp->arity == mfa->arity - num_free);
+        ASSERT(arity == fe->arity);
+    }
+#endif
 
     return funp;
 }
diff --git a/erts/emulator/beam/beam_common.h b/erts/emulator/beam/beam_common.h
index 6939a19b89..56bfec5f9f 100644
--- a/erts/emulator/beam/beam_common.h
+++ b/erts/emulator/beam/beam_common.h
@@ -255,13 +255,14 @@ Export* call_error_handler(Process* p, const ErtsCodeMFA* mfa,
 Export* fixed_apply(Process* p, Eterm* reg, Uint arity,
                     ErtsCodePtr I, Uint offs);
 Export* apply(Process* p, Eterm* reg, ErtsCodePtr I, Uint offs);
-ErtsCodePtr call_fun(Process* p, int arity, Eterm* reg,
-                     Eterm args, Export **epp);
-ErtsCodePtr apply_fun(Process* p, Eterm fun, Eterm args,
-                      Eterm* reg, Export **epp);
+ErtsCodePtr call_fun(Process* p, int arity, Eterm* reg, Eterm args);
+ErtsCodePtr apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg);
 Eterm new_fun(Process* p, Eterm* reg,
 		     ErlFunEntry* fe, int num_free);
-ErlFunThing* new_fun_thing(Process* p, ErlFunEntry* fe, int num_free);
+ErlFunThing* new_fun_thing(Process* p,
+                           ErlFunEntry* fe,
+                           int arity,
+                           int num_free);
 int is_function2(Eterm Term, Uint arity);
 Eterm erts_gc_new_map(Process* p, Eterm* reg, Uint live,
                       Uint n, const Eterm* data);
@@ -281,7 +282,6 @@ void copy_in_registers(Process *c_p, Eterm *reg);
 void check_monitor_long_schedule(Process *c_p, Uint64 start_time,
                                  ErtsCodePtr start_time_i);
 
-
 extern ErtsCodePtr beam_run_process;
 extern ErtsCodePtr beam_normal_exit;
 extern ErtsCodePtr beam_exit;
@@ -289,6 +289,8 @@ extern ErtsCodePtr beam_save_calls;
 extern ErtsCodePtr beam_bif_export_trap;
 extern ErtsCodePtr beam_export_trampoline;
 extern ErtsCodePtr beam_continue_exit;
+extern ErtsCodePtr beam_unloaded_fun;
+
 extern ErtsCodePtr beam_return_to_trace;   /* OpCode(i_return_to_trace) */
 extern ErtsCodePtr beam_return_trace;      /* OpCode(i_return_trace) */
 extern ErtsCodePtr beam_exception_trace;   /* OpCode(i_exception_trace) */
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index b790aeaff8..abe410f5c5 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -671,7 +671,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	case 'F':		/* Function definition */
 	    {
 		ErlFunEntry* fe = (ErlFunEntry *) *ap;
-		const ErtsCodeMFA *cmfa = erts_find_function_from_pc(fe->address);
+		const ErtsCodeMFA *cmfa = erts_get_fun_mfa(fe);
 		erts_print(to, to_arg, "fun(`%T`:`%T`/%bpu)", cmfa->module,
 			   cmfa->function, cmfa->arity);
 		ap++;
@@ -878,7 +878,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr)
 	    }
 	}
 	break;
-    case op_i_make_fun3_Fdt:
+    case op_i_make_fun3_Fdtt:
 	{
 	    int n = unpacked[-1];
 
diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c
index afb5b1206c..860e811f9d 100644
--- a/erts/emulator/beam/beam_file.c
+++ b/erts/emulator/beam/beam_file.c
@@ -57,7 +57,10 @@ struct BeamCodeReader__ {
     BeamFile *file;
 
     BeamOp *pending;
-    Uint first;
+
+    BeamOpArg current_func_label;
+    BeamOpArg current_entry_label;
+    int first;
 };
 
 typedef struct {
@@ -1465,82 +1468,101 @@ static int beamcodereader_read_next(BeamCodeReader *code_reader, BeamOp **out) {
     return 1;
 }
 
+static void synthesize_func_end(BeamCodeReader *code_reader) {
+    BeamOp *func_end;
+
+    func_end = beamopallocator_new_op(code_reader->allocator);
+    func_end->op = genop_int_func_end_2;
+    func_end->arity = 2;
+
+    ASSERT(code_reader->current_func_label.type == TAG_u);
+    func_end->a[0].val = code_reader->current_func_label.val;
+    func_end->a[0].type = TAG_f;
+
+    ASSERT(code_reader->current_entry_label.type == TAG_u);
+    func_end->a[1].val = code_reader->current_entry_label.val;
+    func_end->a[1].type = TAG_f;
+
+    func_end->next = code_reader->pending;
+    code_reader->pending = func_end;
+}
 
 int beamcodereader_next(BeamCodeReader *code_reader, BeamOp **out) {
     BeamOp *op;
 
     if (code_reader->pending) {
-        *out = code_reader->pending;
-        code_reader->pending = code_reader->pending->next;
+        op = code_reader->pending;
+        code_reader->pending = op->next;
+
+        *out = op;
         return 1;
     }
 
     LoadAssert(beamcodereader_read_next(code_reader, &op));
 
-    if (op->op != genop_label_1) {
-        *out = op;
-        return 1;
-    } else {
-        /*
-         * To simplify the rest of the loading process, attempt
-         * to synthesize int_func_start/5 and int_func_end/0
-         * instructions.
+    switch (op->op) {
+    case genop_label_1:
+        /* To simplify the rest of the loading process, attempt to synthesize
+         * int_func_start/5 and int_func_end/2 instructions.
          *
          * We look for the following instruction sequence to
          * find function boundaries: label Lbl | line Loc | func_info M F A.
          * (Where the line instruction is optional.)
          *
-         * So far we have seen a label/0 instruction. Put this
-         * instruction into the pending queue and decode the next
-         * instruction.
-         */
+         * So far we have seen a label/0 instruction. Put this instruction into
+         * the pending queue and decode the next instruction. */
         code_reader->pending = op;
         LoadAssert(beamcodereader_read_next(code_reader, &op->next));
         op = op->next;
 
-        /*
-         * If the current instruction is a line instruction, append it to
-         * the pending queue and decode the following instruction.
-         */
+        /* If the current instruction is a line instruction, append it to
+         * the pending queue and decode the following instruction. */
         if (op->op == genop_line_1) {
             LoadAssert(beamcodereader_read_next(code_reader, &op->next));
             op = op->next;
         }
 
-        /*
-         * If the current instruction is a func_info instruction, we
-         * have found a function boundary.
-         */
-        if (op->op == genop_func_info_3) {
+        /* The code must not end abruptly after a label. */
+        LoadAssert(op->op != genop_int_code_end_0);
+
+        /* If the current instruction is a func_info instruction, we
+         * have found a function boundary. */
+        if (ERTS_LIKELY(op->op != genop_func_info_3)) {
+            op->next = NULL;
+        } else {
+            BeamOpArg func_label, entry_label;
             BeamOp *func_start;
+            BeamOp *entry_op;
             BeamOp *next;
 
-            /*
-             * Prepare to walk through the queue of pending instructions.
-             */
+            /* The func_info/3 instruction must be followed by its entry
+             * label. */
+            LoadAssert(beamcodereader_read_next(code_reader, &entry_op));
+            LoadAssert(entry_op->op == genop_label_1);
+            entry_label = entry_op->a[0];
+            LoadAssert(entry_label.type == TAG_u);
+            entry_op->next = NULL;
+
+            /* Prepare to walk through the queue of pending instructions. */
             op = code_reader->pending;
-            ASSERT(op->op == genop_label_1);
 
-            /*
-             * Allocate the int_func_start/0 function.
-             */
-            func_start = beamopallocator_new_op(code_reader->allocator);
-            func_start->op = genop_int_func_start_5;
-            func_start->arity = 5;
+            /* Pick up the label from the first label/1 instruction. */
+            ASSERT(op->op == genop_label_1);
+            func_label = op->a[0];
+            LoadAssert(func_label.type == TAG_u);
 
-            /*
-             * Pick up the label from the label/1 instruction.
-             */
-            func_start->a[0] = op->a[0];
             next = op->next;
             beamopallocator_free_op(code_reader->allocator, op);
             op = next;
 
-            /*
-             * If the current instruction is a line/1 instruction,
-             * pick up the location from that instruction.
-             * Otherwise use NIL.
-             */
+            /* Allocate the int_func_start/0 function. */
+            func_start = beamopallocator_new_op(code_reader->allocator);
+            func_start->op = genop_int_func_start_5;
+            func_start->arity = 5;
+            func_start->a[0] = func_label;
+
+            /* If the current instruction is a line/1 instruction, pick up the
+             * location from that instruction. Otherwise use NIL. */
             func_start->a[1].type = TAG_n;
             if (op->op == genop_line_1) {
                 func_start->a[1] = op->a[0];
@@ -1549,45 +1571,46 @@ int beamcodereader_next(BeamCodeReader *code_reader, BeamOp **out) {
                 op = next;
             }
 
-            /*
-             * Pick up the MFA from the func_info/3 instruction.
-             */
+            /* Pick up the MFA from the func_info/3 instruction. */
             ASSERT(op->op == genop_func_info_3);
             func_start->a[2] = op->a[0];
             func_start->a[3] = op->a[1];
             func_start->a[4] = op->a[2];
             beamopallocator_free_op(code_reader->allocator, op);
 
-            /*
-             * Put the int_func_start/5 instruction into the pending
-             * queue.
-             */
+            /* Put the int_func_start/5 instruction into the pending queue,
+             * and link the entry label after it. */
             code_reader->pending = func_start;
-            op = func_start;
-
-            /*
-             * Unless this is the first function in the module,
-             * synthesize an int_func_end/0 instruction and prepend
-             * it to the pending queue.
-             */
-            if (code_reader->first) {
-                code_reader->first = 0;
-            } else {
-                BeamOp *func_end;
-                func_end = beamopallocator_new_op(code_reader->allocator);
-                func_end->op = genop_int_func_end_0;
-                func_end->arity = 0;
-                func_end->next = code_reader->pending;
-                code_reader->pending = func_end;
+            func_start->next = entry_op;
+
+            /* Unless this is the first function in the module, synthesize an
+             * int_func_end/2 instruction and prepend it to the pending
+             * queue. */
+            if (!code_reader->first) {
+                synthesize_func_end(code_reader);
             }
+
+            code_reader->current_func_label = func_label;
+            code_reader->current_entry_label = entry_label;
+            code_reader->first = 0;
+        }
+
+        /* At this point, there is at least one instruction in the pending
+         * queue, and the op variable points to the last instruction in the
+         * queue. */
+        return beamcodereader_next(code_reader, out);
+    case genop_int_code_end_0:
+        code_reader->pending = op;
+
+        if (!code_reader->first) {
+            synthesize_func_end(code_reader);
         }
 
-        /*
-         * At this point, there is at least one instruction in the pending
-         * queue. The op variable points to the last instruction in the queue.
-         */
         op->next = NULL;
         return beamcodereader_next(code_reader, out);
+    default:
+        *out = op;
+        return 1;
     }
 }
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 063b740c08..8d1f3822a2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -369,10 +369,12 @@ static int load_code(LoaderState* stp)
 
     int num_specific;
 
-    beam_load_prepare_emit(stp);
-
     op_reader = beamfile_get_code(&stp->beam, &stp->op_allocator);
 
+    if (!beam_load_prepare_emit(stp)) {
+        goto load_error;
+    }
+
     for (;;) {
     get_next_instr:
         if (!beamcodereader_next(op_reader, &last_op)) {
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 913574b5e5..e229efadfa 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -56,7 +56,7 @@ typedef struct LoaderState_ LoaderState;
 int beam_load_prepared_dtor(Binary *magic);
 void beam_load_prepared_free(Binary *magic);
 
-void beam_load_prepare_emit(LoaderState *stp);
+int beam_load_prepare_emit(LoaderState *stp);
 int beam_load_emit_op(LoaderState *stp, BeamOp *op);
 int beam_load_finish_emit(LoaderState *stp);
 
diff --git a/erts/emulator/beam/emu/beam_emu.c b/erts/emulator/beam/emu/beam_emu.c
index 37de5b0ef7..b16e23c61e 100644
--- a/erts/emulator/beam/emu/beam_emu.c
+++ b/erts/emulator/beam/emu/beam_emu.c
@@ -142,6 +142,12 @@ ErtsCodePtr beam_exception_trace;
 static BeamInstr beam_return_time_trace_[1];
 ErtsCodePtr beam_return_time_trace;
 
+/* The address field of every fun that has no loaded code will point to
+ * beam_unloaded_fun[]. The -1 in beam_unloaded_fun[0] will be interpreted
+ * as an illegal arity when attempting to call a fun. */
+static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0};
+ErtsCodePtr beam_unloaded_fun = &unloaded_fun_code[3];
+
 /*
  * All Beam instructions in numerical order.
  */
@@ -577,7 +583,7 @@ void process_main(ErtsSchedulerData *esdp)
  OpCase(label_L):
  OpCase(on_load):
  OpCase(line_I):
- OpCase(int_func_end):
+ OpCase(i_nif_padding):
     erts_exit(ERTS_ERROR_EXIT, "meta op\n");
 
     /*
diff --git a/erts/emulator/beam/emu/emu_load.c b/erts/emulator/beam/emu/emu_load.c
index 35937754d9..6ca9f28f4c 100644
--- a/erts/emulator/beam/emu/emu_load.c
+++ b/erts/emulator/beam/emu/emu_load.c
@@ -57,7 +57,7 @@ extern void check_allocated_block(Uint type, void *blk);
 
 static void init_label(Label* lp);
 
-void beam_load_prepare_emit(LoaderState *stp) {
+int beam_load_prepare_emit(LoaderState *stp) {
     BeamCodeHeader *hdr;
     int i;
 
@@ -136,6 +136,8 @@ void beam_load_prepare_emit(LoaderState *stp) {
                                     stp->beam.code.function_count *
                                         sizeof(unsigned int));
     }
+
+    return 1;
 }
 
 void beam_load_prepared_free(Binary* magic)
@@ -716,18 +718,6 @@ void beam_load_finalize_code(LoaderState* stp, struct erl_module_instance* inst_
     stp->code_hdr = NULL;
 }
 
-static int
-is_bif(Eterm mod, Eterm func, unsigned arity)
-{
-    Export *e = erts_active_export_entry(mod, func, arity);
-
-    if (e != NULL) {
-        return e->bif_number != -1;
-    }
-
-    return 0;
-}
-
 static void init_label(Label* lp)
 {
     lp->value = 0;
@@ -1356,38 +1346,30 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) {
 #endif
         }
         break;
-    case op_int_func_end:
-	{
-	    /*
-	     * Native function calls may be larger than their stubs, so
-	     * we'll need to make sure any potentially-native function stub
-	     * is padded with enough room.
-	     */
-	    int padding_required;
-
-	    ci--;		/* Get rid of the instruction */
-
-	    padding_required = stp->may_load_nif ||
-		is_bif(stp->module, stp->function, stp->arity);
-
-	    ASSERT(stp->last_func_start);
-	    if (padding_required) {
-		Sint pad = BEAM_NATIVE_MIN_FUNC_SZ - (ci - stp->last_func_start);
-		if (pad > 0) {
-		    ASSERT(pad < BEAM_NATIVE_MIN_FUNC_SZ);
-		    CodeNeed(pad);
-		    while (pad-- > 0) {
-			/*
-			 * Filling with actual instructions (instead
-			 * of zeroes) will look nicer in a disassembly
-			 * listing.
-			 */
-			code[ci++] = BeamOpCodeAddr(op_padding);
-		    }
-		}
-	    }
-	}
-	break;
+    case op_i_nif_padding:
+        {
+            /* Native function calls may be larger than their stubs, so we'll
+             * need to make sure any potentially-native function stub is padded
+             * with enough room. */
+            Sint pad;
+
+            ci -= 1;         /* Get rid of the instruction */
+
+            ASSERT(stp->last_func_start);
+            pad = BEAM_NATIVE_MIN_FUNC_SZ - (ci - stp->last_func_start);
+
+            if (pad > 0) {
+                ASSERT(pad < BEAM_NATIVE_MIN_FUNC_SZ);
+                CodeNeed(pad);
+
+                while (pad-- > 0) {
+                /* Filling with actual instructions (instead of zeroes) will
+                 * look nicer in a disassembly listing. */
+                    code[ci++] = BeamOpCodeAddr(op_padding);
+                }
+            }
+        }
+        break;
     case op_on_load:
         ci--;                /* Get rid of the instruction */
 
diff --git a/erts/emulator/beam/emu/instrs.tab b/erts/emulator/beam/emu/instrs.tab
index b05533c830..176dea5a10 100644
--- a/erts/emulator/beam/emu/instrs.tab
+++ b/erts/emulator/beam/emu/instrs.tab
@@ -310,10 +310,8 @@ apply_last(Arity, Deallocate) {
 }
 
 APPLY_FUN(Next) {
-    Export *ignored;
-
     HEAVY_SWAPOUT;
-    $Next = apply_fun(c_p, r(0), x(1), reg, &ignored);
+    $Next = apply_fun(c_p, r(0), x(1), reg);
     HEAVY_SWAPIN;
 
     if (ERTS_UNLIKELY(next == NULL)) {
@@ -350,10 +348,9 @@ i_apply_fun_only() {
 
 CALL_FUN(Fun, Next) {
     //| -no_next
-    Export *ignored;
 
     HEAVY_SWAPOUT;
-    $Next = call_fun(c_p, $Fun, reg, THE_NON_VALUE, &ignored);
+    $Next = call_fun(c_p, $Fun, reg, THE_NON_VALUE);
     HEAVY_SWAPIN;
 
     if (ERTS_UNLIKELY(next == NULL)) {
@@ -543,12 +540,13 @@ init3(Y1, Y2, Y3) {
     make_blank($Y3);
 }
 
-i_make_fun3(FunP, Dst, NumFree) {
+i_make_fun3(FunP, Dst, Arity, NumFree) {
     ErlFunThing* funp;
+    ErlFunEntry *fe = (ErlFunEntry *) $FunP;
     int i, num_free = $NumFree;
     //| -no_next
     SWAPOUT;
-    funp = new_fun_thing(c_p, (ErlFunEntry *) $FunP, num_free);
+    funp = new_fun_thing(c_p, fe, $Arity, num_free);
     SWAPIN;
     I = $NEXT_INSTRUCTION;
     for (i = 0; i < num_free; i++) {
diff --git a/erts/emulator/beam/emu/ops.tab b/erts/emulator/beam/emu/ops.tab
index 336c4d0283..a60f2075fe 100644
--- a/erts/emulator/beam/emu/ops.tab
+++ b/erts/emulator/beam/emu/ops.tab
@@ -85,8 +85,11 @@ int_func_start/5
 int_func_start Lbl Line M F A => label Lbl | i_func_info u M F A | line Line
 
 # The end of a function.
-int_func_end/0
-int_func_end
+int_func_end/2
+int_func_end Func Entry | needs_nif_padding() => i_nif_padding
+int_func_end Func Entry =>
+
+i_nif_padding
 
 # Instruction used for padding functions that use native code.
 %cold
@@ -1049,7 +1052,7 @@ make_fun3 OldIndex=u Dst=d NumFree=u Env=* => make_fun3(OldIndex, Dst, NumFree,
 
 %cold
 
-i_make_fun3 F d t *
+i_make_fun3 F d t t *
 
 # Psuedo-instruction for signalling lambda load errors. Never actually runs.
 i_lambda_error t
diff --git a/erts/emulator/beam/emu/predicates.tab b/erts/emulator/beam/emu/predicates.tab
index 4542d8bf0f..1230af5859 100644
--- a/erts/emulator/beam/emu/predicates.tab
+++ b/erts/emulator/beam/emu/predicates.tab
@@ -45,3 +45,15 @@ pred.independent_moves(Src1, Dst1, Src2, Dst2) {
         (Src2.type != Dst1.type || Src2.val != Dst1.val) &&
         (Dst1.type != Dst2.type ||Dst1.val != Dst2.val);
 }
+
+pred.needs_nif_padding() {
+    Export *e = erts_active_export_entry(S->module, S->function, S->arity);
+
+    if (e != NULL && e->bif_number != -1) {
+        return 1;
+    }
+
+    /* If the module may load a NIF all functions must be able to hold a NIF
+     * stub, so we'll pad to that size at the end of every function. */
+    return S->may_load_nif
+}
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index db299e053d..48b2ec705e 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3581,7 +3581,7 @@ fun_info_2(BIF_ALIST_2)
 	    break;
 	case am_name:
             {
-                const ErtsCodeMFA *mfa = erts_code_to_codemfa((funp->fe)->address);
+                const ErtsCodeMFA *mfa = erts_get_fun_mfa(funp->fe);
                 hp = HAlloc(p, 3);
                 val = mfa->function;
             }
@@ -3656,8 +3656,9 @@ fun_info_mfa_1(BIF_ALIST_1)
     if (is_fun(fun)) {
         const ErtsCodeMFA *mfa;
         ErlFunThing* funp;
+
         funp = (ErlFunThing *) fun_val(fun);
-        mfa = erts_code_to_codemfa((funp->fe)->address);
+        mfa = erts_get_fun_mfa(funp->fe);
 
         hp = HAlloc(p, 4);
         BIF_RET(TUPLE3(hp,
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 13141e93e1..4267cbed63 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -27,6 +27,7 @@
 #include "global.h"
 #include "erl_fun.h"
 #include "hash.h"
+#include "beam_common.h"
 
 static Hash erts_fun_table;
 
@@ -42,14 +43,6 @@ static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2);
 static ErlFunEntry* fun_alloc(ErlFunEntry* template);
 static void fun_free(ErlFunEntry* obj);
 
-/*
- * The address field of every fun that has no loaded code will point
- * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted
- * as an illegal arity when attempting to call a fun.
- */
-static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0};
-static ErtsCodePtr unloaded_fun = &unloaded_fun_code[3];
-
 void
 erts_init_fun_table(void)
 {
@@ -97,55 +90,46 @@ int erts_fun_table_sz(void)
 
 ErlFunEntry*
 erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
-		    const byte* uniq, int index, int arity)
+                    const byte* uniq, int index, int arity)
 {
     ErlFunEntry template;
     ErlFunEntry* fe;
     erts_aint_t refc;
 
+    /* All fields are copied from the template when inserting a new entry. */
     ASSERT(is_atom(mod));
+    template.old_index = old_index;
     template.old_uniq = old_uniq;
     template.index = index;
     template.module = mod;
+    template.arity = arity;
+    sys_memcpy(template.uniq, uniq, sizeof(template.uniq));
+
     erts_fun_write_lock();
     fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template);
-    sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq));
-    fe->old_index = old_index;
-    fe->arity = arity;
     refc = erts_refc_inctest(&fe->refc, 0);
-    if (refc < 2) /* New or pending delete */
-	erts_refc_inc(&fe->refc, 1);
+    if (refc < 2) {
+        /* New or pending delete */
+        erts_refc_inc(&fe->refc, 1);
+    }
     erts_fun_write_unlock();
+
     return fe;
 }
 
-ErlFunEntry*
-erts_get_fun_entry(Eterm mod, int uniq, int index)
-{
-    ErlFunEntry template;
-    ErlFunEntry *ret;
+const ErtsCodeMFA *erts_get_fun_mfa(const ErlFunEntry *fe) {
+    static const ErtsCodeMFA unloaded_mfa = {NIL, NIL, ERTS_UINT_MAX};
+    ErtsCodePtr address = fe->address;
 
-    ASSERT(is_atom(mod));
-    template.old_uniq = uniq;
-    template.index = index;
-    template.module = mod;
-    erts_fun_read_lock();
-    ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template);
-    if (ret) {
-	erts_aint_t refc = erts_refc_inctest(&ret->refc, 1);
-	if (refc < 2) /* Pending delete */
-	    erts_refc_inc(&ret->refc, 1);
+    if (address != beam_unloaded_fun) {
+        return erts_find_function_from_pc(address);
     }
-    erts_fun_read_unlock();
-    return ret;
-}
 
-int erts_is_fun_loaded(ErlFunEntry* fe) {
-    int res = fe->address != unloaded_fun;
-
-    ASSERT(res == (0 != ((const BeamInstr*)fe->address)[0]));
+    return &unloaded_mfa;
+}
 
-    return res;
+int erts_is_fun_loaded(const ErlFunEntry* fe) {
+    return fe->address != beam_unloaded_fun;
 }
 
 static void
@@ -164,13 +148,14 @@ erts_erase_fun_entry(ErlFunEntry* fe)
      */
     if (erts_refc_dectest(&fe->refc, -1) <= 0)
     {
-	if (fe->address != unloaded_fun)
-	    erts_exit(ERTS_ERROR_EXIT,
-		     "Internal error: "
-		     "Invalid reference count found on #Fun<%T.%d.%d>: "
-		     " About to erase fun still referred by code.\n",
-		     fe->module, fe->old_index, fe->old_uniq);
-	erts_erase_fun_entry_unlocked(fe);
+        if (erts_is_fun_loaded(fe)) {
+            erts_exit(ERTS_ERROR_EXIT,
+                "Internal error: "
+                "Invalid reference count found on #Fun<%T.%d.%d>: "
+                " About to erase fun still referred by code.\n",
+                fe->module, fe->old_index, fe->old_uniq);
+        }
+        erts_erase_fun_entry_unlocked(fe);
     }
     erts_fun_write_unlock();
 }
@@ -185,7 +170,7 @@ static void fun_purge_foreach(ErlFunEntry *fe, struct erl_module_instance* modp)
     if (ErtsInArea(fun_addr, mod_start, modp->code_length)) {
         fe->pend_purge_address = fe->address;
         ERTS_THR_WRITE_MEMORY_BARRIER;
-        fe->address = unloaded_fun;
+        fe->address = beam_unloaded_fun;
         erts_purge_state_add_fun(fe);
     }
 }
@@ -206,7 +191,7 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no)
     for (ix = 0; ix < no; ix++) {
         ErlFunEntry *fe = funs[ix];
 
-        if (fe->address == unloaded_fun) {
+        if (fe->address == beam_unloaded_fun) {
             fe->address = fe->pend_purge_address;
         }
     }
@@ -274,35 +259,26 @@ fun_hash(ErlFunEntry* obj)
 static int
 fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2)
 {
-    /*
-     * OTP 23: Use 'index' (instead of 'old_index') when comparing fun
-     * entries. In OTP 23, multiple make_fun2 instructions may refer to the
-     * the same 'index' (for the wrapper function generated for the
-     * 'fun F/A' syntax).
-     *
-     * This is safe when loading code compiled with OTP R15 and later,
-     * because since R15 (2011), the 'index' has been reliably equal
-     * to 'old_index'. The loader refuses to load modules compiled before
-     * OTP R15.
-     */
-
-    return !(obj1->module == obj2->module &&
-	     obj1->old_uniq == obj2->old_uniq &&
-	     obj1->index == obj2->index);
+    return !(obj1->old_index == obj2->old_index &&
+             obj1->old_uniq == obj2->old_uniq &&
+             obj1->module == obj2->module &&
+             obj1->index == obj2->index &&
+             obj1->arity == obj2->arity &&
+             !sys_memcmp(obj1->uniq, obj2->uniq, sizeof(obj1->uniq)));
 }
 
 static ErlFunEntry*
 fun_alloc(ErlFunEntry* template)
 {
     ErlFunEntry* obj = (ErlFunEntry *) erts_alloc(ERTS_ALC_T_FUN_ENTRY,
-						  sizeof(ErlFunEntry));
+                                                  sizeof(ErlFunEntry));
+
+    sys_memcpy(obj, template, sizeof(ErlFunEntry));
 
-    obj->old_uniq = template->old_uniq;
-    obj->index = template->index;
-    obj->module = template->module;
     erts_refc_init(&obj->refc, -1);
-    obj->address = unloaded_fun;
+    obj->address = beam_unloaded_fun;
     obj->pend_purge_address = NULL;
+
     return obj;
 }
 
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index e7768646e3..1f5e8669d7 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -26,18 +26,20 @@
 /*
  * Fun entry.
  */
-
 typedef struct erl_fun_entry {
-    HashBucket bucket;          /* MUST BE LOCATED AT TOP OF STRUCT!!! */
+    HashBucket bucket;
 
-    byte uniq[16];              /* MD5 for module. */
-    int index;                  /* New style index. */
-    int old_uniq;               /* Unique number (old_style) */
-    int old_index;              /* Old style index */
-    ErtsCodePtr address;        /* Pointer to code for fun */
+    ErtsCodePtr address;            /* Pointer to code for actual function */
 
-    Uint arity;                     /* The arity of the fun. */
+    /* These fields identify the function and must not be altered after fun
+     * creation. */
     Eterm module;                   /* Tagged atom for module. */
+    Uint arity;                     /* The arity of the fun. */
+    int index;                      /* New style index. */
+    byte uniq[16];                  /* MD5 for module. */
+    int old_uniq;                   /* Unique number (old_style) */
+    int old_index;                  /* Old style index */
+
     erts_refc_t refc;               /* Reference count: One for code + one for
                                      * each fun object in each process. */
     ErtsCodePtr pend_purge_address; /* Address during a pending purge */
@@ -67,12 +69,13 @@ void erts_init_fun_table(void);
 void erts_fun_info(fmtfn_t, void *);
 int erts_fun_table_sz(void);
 
-ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index);
-
+/* Finds or inserts a fun entry that matches the given signature. */
 ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
-				const byte* uniq, int index, int arity);
+                                 const byte* uniq, int index, int arity);
+
+const ErtsCodeMFA *erts_get_fun_mfa(const ErlFunEntry *fe);
 
-int erts_is_fun_loaded(ErlFunEntry* fe);
+int erts_is_fun_loaded(const ErlFunEntry* fe);
 
 void erts_erase_fun_entry(ErlFunEntry* fe);
 void erts_cleanup_funs(ErlFunThing* funp);
diff --git a/erts/emulator/beam/generators.tab b/erts/emulator/beam/generators.tab
index 520c688113..c564d8497f 100644
--- a/erts/emulator/beam/generators.tab
+++ b/erts/emulator/beam/generators.tab
@@ -351,6 +351,7 @@ gen.make_fun2(idx) {
 	int i;
         BeamFile_LambdaEntry *entry = &S->beam.lambdas.entries[idx.val];
 	unsigned num_free = entry->num_free;
+	unsigned arity = entry->arity;
 
 	$NewBeamOp(S, th);
 
@@ -361,17 +362,19 @@ gen.make_fun2(idx) {
 	th->a[1].val = num_free;
 	th->next = op;
 
-	$BeamOpNameArity(op, i_make_fun3, 3);
-	$BeamOpArity(op, 3 + num_free);
+	$BeamOpNameArity(op, i_make_fun3, 4);
+	$BeamOpArity(op, 4 + num_free);
 	op->a[0].type = TAG_u;
 	op->a[0].val = idx.val;
 	op->a[1].type = TAG_x;
 	op->a[1].val = 0;
 	op->a[2].type = TAG_u;
-	op->a[2].val = num_free;
+	op->a[2].val = arity - num_free;
+	op->a[3].type = TAG_u;
+	op->a[3].val = num_free;
 	for (i = 0; i < num_free; i++) {
-	    op->a[i+3].type = TAG_x;
-	    op->a[i+3].val = i;
+	    op->a[i+4].type = TAG_x;
+	    op->a[i+4].val = i;
 	}
 	return th;
     }
@@ -387,15 +390,17 @@ gen.make_fun3(idx, Dst, NumFree, Env) {
 	if (NumFree.val == entry->num_free) {
 	    int i;
 
-	    $BeamOpNameArity(op, i_make_fun3, 3);
-	    $BeamOpArity(op, 3 + NumFree.val);
+	    $BeamOpNameArity(op, i_make_fun3, 4);
+	    $BeamOpArity(op, 4 + NumFree.val);
 	    op->a[0].type = TAG_u;
 	    op->a[0].val = idx.val;
 	    op->a[1] = Dst;
 	    op->a[2].type = TAG_u;
-	    op->a[2].val = entry->num_free;
+	    op->a[2].val = entry->arity - entry->num_free;
+	    op->a[3].type = TAG_u;
+	    op->a[3].val = entry->num_free;
 	    for (i = 0; i < NumFree.val; i++) {
-		op->a[i+3] = Env[i];
+		op->a[i+4] = Env[i];
 	    }
 	    return op;
 	}
diff --git a/erts/emulator/beam/jit/asm_load.c b/erts/emulator/beam/jit/asm_load.c
index bcd8958d4b..46ea3ba9b7 100644
--- a/erts/emulator/beam/jit/asm_load.c
+++ b/erts/emulator/beam/jit/asm_load.c
@@ -36,9 +36,11 @@
 #    include <sanitizer/lsan_interface.h>
 #endif
 
+#define INVALID_LAMBDA_INDEX -1
+
 static void init_label(Label *lp);
 
-void beam_load_prepare_emit(LoaderState *stp) {
+int beam_load_prepare_emit(LoaderState *stp) {
     BeamCodeHeader *hdr;
     int i;
 
@@ -73,6 +75,23 @@ void beam_load_prepare_emit(LoaderState *stp) {
         init_label(&stp->labels[i]);
     }
 
+    for (i = 0; i < stp->beam.lambdas.count; i++) {
+        BeamFile_LambdaEntry *lambda = &stp->beam.lambdas.entries[i];
+
+        if (stp->labels[lambda->label].lambda_index == INVALID_LAMBDA_INDEX) {
+            stp->labels[lambda->label].lambda_index = i;
+        } else {
+            beam_load_report_error(__LINE__,
+                                   stp,
+                                   "lambda already defined for label %i. To "
+                                   "fix this, please recompile this module "
+                                   "with an OTP " ERLANG_OTP_RELEASE
+                                   " compiler.",
+                                   lambda->label);
+            return 0;
+        }
+    }
+
     stp->bif_imports =
             erts_alloc(ERTS_ALC_T_PREPARED_CODE,
                        stp->beam.imports.count * sizeof(BifEntry **));
@@ -111,10 +130,13 @@ void beam_load_prepare_emit(LoaderState *stp) {
                                     stp->beam.code.function_count *
                                             sizeof(unsigned int));
     }
+
+    return 1;
 }
 
 static void init_label(Label *lp) {
     sys_memset(lp, 0, sizeof(*lp));
+    lp->lambda_index = INVALID_LAMBDA_INDEX;
 }
 
 void beam_load_prepared_free(Binary *magic) {
@@ -924,7 +946,7 @@ void beam_load_finalize_code(LoaderState *stp,
                 erts_refc_dectest(&fun_entry->refc, 1);
             }
 
-            fun_entry->address = beamasm_get_code(stp->ba, lambda->label);
+            fun_entry->address = beamasm_get_lambda(stp->ba, i);
 
             beamasm_patch_lambda(stp->ba,
                                  stp->native_module_rw,
diff --git a/erts/emulator/beam/jit/beam_asm.h b/erts/emulator/beam/jit/beam_asm.h
index b82760603e..56bb7219f7 100644
--- a/erts/emulator/beam/jit/beam_asm.h
+++ b/erts/emulator/beam/jit/beam_asm.h
@@ -48,6 +48,7 @@ void beamasm_purge_module(const void *native_module_exec,
 void beamasm_delete_assembler(void *ba);
 int beamasm_emit(void *ba, unsigned specific_op, BeamOp *op);
 ErtsCodePtr beamasm_get_code(void *ba, int label);
+ErtsCodePtr beamasm_get_lambda(void *ba, int index);
 const byte *beamasm_get_rodata(void *ba, char *label);
 void beamasm_embed_rodata(void *ba,
                           const char *labelName,
diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp
index 6c9f069bbc..40fc99b94c 100644
--- a/erts/emulator/beam/jit/beam_jit_common.cpp
+++ b/erts/emulator/beam/jit/beam_jit_common.cpp
@@ -121,8 +121,6 @@ void BeamAssembler::_codegen(JitAllocator *allocator,
                            code.codeSize(),
                            CodeHolder::kCopyPadSectionBuffer);
 
-    beamasm_flush_icache(*executable_ptr, code.codeSize());
-
 #ifdef DEBUG
     if (FileLogger *l = dynamic_cast<FileLogger *>(code.logger()))
         if (FILE *f = l->file())
@@ -1071,3 +1069,76 @@ void beam_jit_return_to_trace(Process *c_p) {
         ERTS_REQ_PROC_MAIN_LOCK(c_p);
     }
 }
+
+Eterm beam_jit_build_argument_list(Process *c_p, const Eterm *regs, int arity) {
+    Eterm *hp;
+    Eterm res;
+
+    hp = HAlloc(c_p, arity * 2);
+    res = NIL;
+
+    for (int i = arity - 1; i >= 0; i--) {
+        res = CONS(hp, regs[i], res);
+        hp += 2;
+    }
+
+    return res;
+}
+
+Export *beam_jit_handle_unloaded_fun(Process *c_p,
+                                     Eterm *reg,
+                                     int arity,
+                                     Eterm fun_thing) {
+    ErtsCodeIndex code_ix = erts_active_code_ix();
+    Eterm module, args;
+    ErlFunThing *funp;
+    ErlFunEntry *fe;
+    Module *modp;
+    Export *ep;
+
+    funp = (ErlFunThing *)fun_val(fun_thing);
+    fe = funp->fe;
+    module = fe->module;
+
+    ERTS_THR_READ_MEMORY_BARRIER;
+
+    if (fe->pend_purge_address) {
+        /* The system is currently trying to purge the module containing this
+         * fun. Suspend the process and let it try again when the purge
+         * operation is done (may succeed or not). */
+        ep = erts_suspend_process_on_pending_purge_lambda(c_p, fe);
+    } else {
+        if ((modp = erts_get_module(module, code_ix)) != NULL &&
+            modp->curr.code_hdr != NULL) {
+            /* There is a module loaded, but obviously the fun is not defined
+             * in it. We must not call the error_handler (or we will get into
+             * an infinite loop). */
+            c_p->current = NULL;
+            c_p->freason = EXC_BADFUN;
+            c_p->fvalue = fun_thing;
+            return NULL;
+        }
+
+        /* No current code for this module. Call the error_handler module to
+         * attempt loading the module. */
+        ep = erts_find_function(erts_proc_get_error_handler(c_p),
+                                am_undefined_lambda,
+                                3,
+                                code_ix);
+        if (ERTS_UNLIKELY(ep == NULL)) {
+            /* No error handler, crash out. */
+            c_p->current = NULL;
+            c_p->freason = EXC_UNDEF;
+            return NULL;
+        }
+    }
+
+    args = beam_jit_build_argument_list(c_p, reg, arity);
+
+    reg[0] = module;
+    reg[1] = fun_thing;
+    reg[2] = args;
+    reg[3] = NIL;
+
+    return ep;
+}
diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp
index b91533ddbc..a93393f79f 100644
--- a/erts/emulator/beam/jit/beam_jit_common.hpp
+++ b/erts/emulator/beam/jit/beam_jit_common.hpp
@@ -303,4 +303,11 @@ void beam_jit_timeout_locked(Process *c_p);
 
 void beam_jit_return_to_trace(Process *c_p);
 
+Eterm beam_jit_build_argument_list(Process *c_p, const Eterm *regs, int arity);
+
+Export *beam_jit_handle_unloaded_fun(Process *c_p,
+                                     Eterm *reg,
+                                     int arity,
+                                     Eterm fun_thing);
+
 #endif
diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp
index a4d0bc9d40..de7f5f14c9 100644
--- a/erts/emulator/beam/jit/beam_jit_main.cpp
+++ b/erts/emulator/beam/jit/beam_jit_main.cpp
@@ -48,6 +48,7 @@ ErtsCodePtr beam_export_trampoline;
 ErtsCodePtr beam_bif_export_trap;
 ErtsCodePtr beam_continue_exit;
 ErtsCodePtr beam_save_calls;
+ErtsCodePtr beam_unloaded_fun;
 
 /* NOTE These should be the only variables containing trace instructions.
 **      Sometimes tests are for the instruction value, and sometimes
@@ -307,6 +308,8 @@ void beamasm_init() {
     beam_bif_export_trap = (ErtsCodePtr)bga->get_bif_export_trap();
 
     beam_exit = (ErtsCodePtr)bga->get_process_exit();
+
+    beam_unloaded_fun = (ErtsCodePtr)bga->get_unloaded_fun();
 }
 
 bool BeamAssembler::hasCpuFeature(uint32_t featureId) {
@@ -521,6 +524,11 @@ extern "C"
         return reinterpret_cast<ErtsCodePtr>(ba->getCode(label));
     }
 
+    ErtsCodePtr beamasm_get_lambda(void *instance, int index) {
+        BeamModuleAssembler *ba = static_cast<BeamModuleAssembler *>(instance);
+        return reinterpret_cast<ErtsCodePtr>(ba->getLambda(index));
+    }
+
     const byte *beamasm_get_rodata(void *instance, char *label) {
         BeamModuleAssembler *ba = static_cast<BeamModuleAssembler *>(instance);
         return reinterpret_cast<const byte *>(ba->getCode(label));
diff --git a/erts/emulator/beam/jit/load.h b/erts/emulator/beam/jit/load.h
index a3d099bb04..7bb36f0947 100644
--- a/erts/emulator/beam/jit/load.h
+++ b/erts/emulator/beam/jit/load.h
@@ -25,9 +25,11 @@
  * Type for a label.
  */
 typedef struct {
-    Uint value;            /* Value of label (0 if not known yet). */
-    Uint looprec_targeted; /* Non-zero if this label is the target of a
-                            * loop_rec instruction. */
+    Uint value;           /* Value of label (0 if not known yet). */
+    int looprec_targeted; /* Non-zero if this label is the target of a
+                           * loop_rec instruction. */
+    int lambda_index;     /* The lambda index of this label, or -1 if not
+                           * a target of a lambda. */
 } Label;
 
 /*
diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp
index 5e6c20dff2..a872064ba9 100644
--- a/erts/emulator/beam/jit/x86/beam_asm.hpp
+++ b/erts/emulator/beam/jit/x86/beam_asm.hpp
@@ -476,7 +476,7 @@ protected:
     }
 
     /* Explicitly position-independent absolute jump, for use in fragments that
-     * need to be memcpy'd for performance reasons (e.g. export entries) */
+     * need to be memcpy'd for performance reasons (e.g. NIF stubs) */
     template<typename T>
     void pic_jmp(T(*addr)) {
         a.mov(ARG6, imm(addr));
@@ -787,8 +787,12 @@ protected:
      * Generate the shortest instruction for setting a register to an immediate
      * value. May clear flags.
      */
-    void mov_imm(x86::Gp to, Uint value) {
-        if (value == 0) {
+    template<typename T>
+    void mov_imm(x86::Gp to, T value) {
+        static_assert(std::is_integral<T>::value || std::is_pointer<T>::value);
+        if (value) {
+            a.mov(to, imm(value));
+        } else {
             /*
              * Generate the shortest instruction to set the register to zero.
              *
@@ -801,11 +805,14 @@ protected:
              * Note: xor clears ZF and CF; mov does not change any flags.
              */
             a.xor_(to.r32(), to.r32());
-        } else {
-            a.mov(to, imm(value));
         }
     }
 
+    void mov_imm(x86::Gp to, std::nullptr_t value) {
+        (void)value;
+        mov_imm(to, 0);
+    }
+
 public:
     void embed_rodata(const char *labelName, const char *buff, size_t size);
     void embed_bss(const char *labelName, size_t size);
@@ -843,6 +850,7 @@ class BeamGlobalAssembler : public BeamAssembler {
 
     /* Please keep this in alphabetical order. */
 #define BEAM_GLOBAL_FUNCS(_)                                                   \
+    _(apply_fun_shared)                                                        \
     _(arith_compare_shared)                                                    \
     _(arith_eq_shared)                                                         \
     _(bif_nif_epilogue)                                                        \
@@ -869,6 +877,7 @@ class BeamGlobalAssembler : public BeamAssembler {
     _(generic_bp_local)                                                        \
     _(debug_bp)                                                                \
     _(fconv_shared)                                                            \
+    _(handle_call_fun_error)                                                   \
     _(handle_element_error)                                                    \
     _(handle_hd_error)                                                         \
     _(i_band_body_shared)                                                      \
@@ -908,6 +917,7 @@ class BeamGlobalAssembler : public BeamAssembler {
     _(times_guard_shared)                                                      \
     _(unary_minus_body_shared)                                                 \
     _(unary_minus_guard_shared)                                                \
+    _(unloaded_fun)                                                            \
     _(update_map_assoc_shared)                                                 \
     _(update_map_exact_guard_shared)                                           \
     _(update_map_exact_body_shared)
@@ -988,10 +998,10 @@ class BeamModuleAssembler : public BeamAssembler {
     typedef std::unordered_map<unsigned, struct patch_import> ImportMap;
     ImportMap imports;
 
-    /* Map of fun entry to patch labels */
+    /* Map of fun entry to trampoline labels and patches */
     struct patch_lambda {
         std::vector<struct patch> patches;
-        ErlFunEntry fe;
+        Label trampoline;
     };
     typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap;
     LambdaMap lambdas;
@@ -1055,6 +1065,8 @@ public:
     void codegen(char *buff, size_t len);
 
     ErtsCodePtr getCode(unsigned label);
+    ErtsCodePtr getLambda(unsigned index);
+
     void *getCode(Label label) {
         return BeamAssembler::getCode(label);
     }
@@ -1091,8 +1103,7 @@ protected:
     x86::Mem emit_variable_apply(bool includeI);
     x86::Mem emit_fixed_apply(const ArgVal &arity, bool includeI);
 
-    x86::Gp emit_call_fun(const ArgVal &Fun);
-    x86::Gp emit_apply_fun(void);
+    x86::Gp emit_call_fun(void);
 
     void emit_is_binary(Label Fail, x86::Gp Src, Label next, Label subbin);
 
@@ -1129,6 +1140,8 @@ protected:
     void emit_raise_exception();
     void emit_raise_exception(const ErtsCodeMFA *exp);
     void emit_raise_exception(Label I, const ErtsCodeMFA *exp);
+    void emit_raise_exception(x86::Gp I, const ErtsCodeMFA *exp);
+
     void emit_validate(const ArgVal &arity);
     void emit_bs_skip_bits(const ArgVal &Fail, const ArgVal &Ctx);
 
diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.cpp b/erts/emulator/beam/jit/x86/beam_asm_global.cpp
index 9f1a7a1ffc..fcea871c44 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_global.cpp
+++ b/erts/emulator/beam/jit/x86/beam_asm_global.cpp
@@ -230,7 +230,7 @@ void BeamModuleAssembler::emit_raise_exception() {
 }
 
 void BeamModuleAssembler::emit_raise_exception(const ErtsCodeMFA *exp) {
-    mov_imm(ARG4, (Uint)exp);
+    mov_imm(ARG4, exp);
     safe_fragment_call(ga->get_raise_exception());
 
     /*
@@ -245,7 +245,16 @@ void BeamModuleAssembler::emit_raise_exception(const ErtsCodeMFA *exp) {
 void BeamModuleAssembler::emit_raise_exception(Label I,
                                                const ErtsCodeMFA *exp) {
     a.lea(ARG2, x86::qword_ptr(I));
-    mov_imm(ARG4, (Uint)exp);
+    emit_raise_exception(ARG2, exp);
+}
+
+void BeamModuleAssembler::emit_raise_exception(x86::Gp I,
+                                               const ErtsCodeMFA *exp) {
+    if (I != ARG2) {
+        a.mov(ARG2, I);
+    }
+
+    mov_imm(ARG4, exp);
 
 #ifdef NATIVE_ERLANG_STACK
     /* The CP must be reserved for try/catch to work, so we'll fake a call with
diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp
index 8a044a7998..874e081b49 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp
+++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp
@@ -72,6 +72,11 @@ ErtsCodePtr BeamModuleAssembler::getCode(BeamLabel label) {
     return (ErtsCodePtr)getCode(labels[label]);
 }
 
+ErtsCodePtr BeamModuleAssembler::getLambda(unsigned index) {
+    const auto &lambda = lambdas[index];
+    return (ErtsCodePtr)getCode(lambda.trampoline);
+}
+
 BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga,
                                          Eterm mod,
                                          unsigned num_labels)
@@ -197,10 +202,6 @@ Label BeamModuleAssembler::embed_vararg_rodata(const Span<ArgVal> &args,
     return label;
 }
 
-static void i_emit_nyi(char *msg) {
-    erts_exit(ERTS_ERROR_EXIT, "NYI: %s\n", msg);
-}
-
 void BeamModuleAssembler::emit_i_nif_padding() {
     const size_t minimum_size = sizeof(UWord[BEAM_NATIVE_MIN_FUNC_SZ]);
     size_t prev_func_start, diff;
@@ -240,6 +241,10 @@ void BeamModuleAssembler::emit_i_breakpoint_trampoline() {
            BEAM_ASM_FUNC_PROLOGUE_SIZE);
 }
 
+static void i_emit_nyi(char *msg) {
+    erts_exit(ERTS_ERROR_EXIT, "NYI: %s\n", msg);
+}
+
 void BeamModuleAssembler::emit_nyi(const char *msg) {
     emit_enter_runtime();
 
diff --git a/erts/emulator/beam/jit/x86/generators.tab b/erts/emulator/beam/jit/x86/generators.tab
index ab755fde4c..8763e34a47 100644
--- a/erts/emulator/beam/jit/x86/generators.tab
+++ b/erts/emulator/beam/jit/x86/generators.tab
@@ -516,3 +516,50 @@ gen.allocate_heap_zero(Ns, Nh, Live) {
 
     return alloc;
 }
+
+gen.func_end(Func_Label, Entry_Label) {
+    BeamOp *op = NULL;
+
+    if (S->labels[Entry_Label.val].lambda_index != -1) {
+        int index = S->labels[Entry_Label.val].lambda_index;
+        BeamFile_LambdaEntry *lambda_entry;
+        BeamOp *lambda;
+
+        lambda_entry = &S->beam.lambdas.entries[index];
+
+        $NewBeamOp(S, lambda);
+        $BeamOpNameArity(lambda, i_lambda_trampoline, 4);
+
+        lambda->a[0].type = TAG_u;
+        lambda->a[0].val = index;
+        lambda->a[1] = Entry_Label;
+        lambda->a[2].type = TAG_u;
+        lambda->a[2].val = lambda_entry->arity;
+        lambda->a[3].type = TAG_u;
+        lambda->a[3].val = lambda_entry->num_free;
+
+        lambda->next = op;
+        op = lambda;
+    }
+
+    if (S->may_load_nif) {
+        BeamOp *padding;
+
+        $NewBeamOp(S, padding);
+        $BeamOpNameArity(padding, i_nif_padding, 0);
+
+        padding->next = op;
+        op = padding;
+    }
+
+    if (op == NULL) {
+        /* Workaround for the fact that we have to return an instruction from
+         * generators. The loader will remove this dummy instruction. */
+        $NewBeamOp(S, op);
+        $BeamOpNameArity(op, delete_me, 0);
+
+        op->next = NULL;
+    }
+
+    return op;
+}
diff --git a/erts/emulator/beam/jit/x86/instr_call.cpp b/erts/emulator/beam/jit/x86/instr_call.cpp
index b2de403153..466b184295 100644
--- a/erts/emulator/beam/jit/x86/instr_call.cpp
+++ b/erts/emulator/beam/jit/x86/instr_call.cpp
@@ -243,98 +243,3 @@ void BeamModuleAssembler::emit_apply_last(const ArgVal &Arity,
     emit_leave_frame();
     a.jmp(dest);
 }
-
-x86::Gp BeamModuleAssembler::emit_apply_fun() {
-    Label dispatch = a.newLabel();
-
-    emit_enter_runtime<Update::eStack | Update::eHeap>();
-
-    a.mov(ARG1, c_p);
-    a.mov(ARG2, getXRef(0));
-    a.mov(ARG3, getXRef(1));
-    load_x_reg_array(ARG4);
-    a.lea(ARG5, TMP_MEM1q);
-    runtime_call<5>(apply_fun);
-
-    emit_leave_runtime<Update::eStack | Update::eHeap>();
-
-    a.mov(ARG2, RET);
-    a.test(RET, RET);
-
-    /* Put the export entry, if any, into RET to follow the export calling
-     * convention in case we applied an external fun. See
-     * `emit_setup_export_call` for details. */
-    a.mov(RET, TMP_MEM1q);
-
-    a.short_().jne(dispatch);
-    emit_raise_exception();
-    a.bind(dispatch);
-
-    return ARG2;
-}
-
-void BeamModuleAssembler::emit_i_apply_fun() {
-    x86::Gp dest = emit_apply_fun();
-
-    ASSERT(dest != ARG1);
-    erlang_call(dest, ARG1);
-}
-
-void BeamModuleAssembler::emit_i_apply_fun_last(const ArgVal &Deallocate) {
-    emit_deallocate(Deallocate);
-    emit_i_apply_fun_only();
-}
-
-void BeamModuleAssembler::emit_i_apply_fun_only() {
-    x86::Gp dest = emit_apply_fun();
-
-    emit_leave_frame();
-    a.jmp(dest);
-}
-
-x86::Gp BeamModuleAssembler::emit_call_fun(const ArgVal &Fun) {
-    Label dispatch = a.newLabel();
-
-    mov_arg(ARG2, Fun);
-
-    emit_enter_runtime<Update::eStack | Update::eHeap>();
-
-    a.mov(ARG1, c_p);
-    load_x_reg_array(ARG3);
-    mov_imm(ARG4, THE_NON_VALUE);
-    a.lea(ARG5, TMP_MEM1q);
-    runtime_call<5>(call_fun);
-
-    emit_leave_runtime<Update::eStack | Update::eHeap>();
-
-    a.mov(ARG2, RET);
-    a.test(RET, RET);
-
-    /* Put the export entry, if any, into RET to follow the export calling
-     * convention in case we called an external fun. See
-     * `emit_setup_export_call` for details. */
-    a.mov(RET, TMP_MEM1q);
-
-    a.short_().jne(dispatch);
-    emit_raise_exception();
-    a.bind(dispatch);
-
-    return ARG2;
-}
-
-void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Fun) {
-    x86::Gp dest = emit_call_fun(Fun);
-
-    ASSERT(dest != ARG1);
-    erlang_call(dest, ARG1);
-}
-
-void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Fun,
-                                               const ArgVal &Deallocate) {
-    emit_deallocate(Deallocate);
-
-    x86::Gp dest = emit_call_fun(Fun);
-
-    emit_leave_frame();
-    a.jmp(dest);
-}
diff --git a/erts/emulator/beam/jit/x86/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp
index f1412b49e2..545bccbf44 100644
--- a/erts/emulator/beam/jit/x86/instr_common.cpp
+++ b/erts/emulator/beam/jit/x86/instr_common.cpp
@@ -257,40 +257,6 @@ void BeamModuleAssembler::emit_continue_exit() {
     abs_jmp(ga->get_do_schedule());
 }
 
-/* Psuedo-instruction for signalling lambda load errors. Never actually runs. */
-void BeamModuleAssembler::emit_i_lambda_error(const ArgVal &Dummy) {
-    a.hlt();
-}
-
-void BeamModuleAssembler::emit_i_make_fun3(const ArgVal &Fun,
-                                           const ArgVal &Dst,
-                                           const ArgVal &NumFree,
-                                           const Span<ArgVal> &env) {
-    size_t num_free = env.size();
-    ASSERT(NumFree.getValue() == num_free);
-
-    mov_arg(ARG3, NumFree);
-
-    emit_enter_runtime<Update::eHeap>();
-
-    a.mov(ARG1, c_p);
-    make_move_patch(ARG2, lambdas[Fun.getValue()].patches);
-    runtime_call<3>(new_fun_thing);
-
-    emit_leave_runtime<Update::eHeap>();
-
-    comment("Move fun environment");
-    for (unsigned i = 0; i < num_free; i++) {
-        mov_arg(x86::qword_ptr(RET,
-                               offsetof(ErlFunThing, env) + i * sizeof(Eterm)),
-                env[i]);
-    }
-
-    comment("Create boxed ptr");
-    a.or_(RETb, TAG_PRIMARY_BOXED);
-    mov_arg(Dst, RET);
-}
-
 void BeamModuleAssembler::emit_get_list(const x86::Gp src,
                                         const ArgVal &Hd,
                                         const ArgVal &Tl) {
diff --git a/erts/emulator/beam/jit/x86/instr_fun.cpp b/erts/emulator/beam/jit/x86/instr_fun.cpp
new file mode 100644
index 0000000000..73908d80b8
--- /dev/null
+++ b/erts/emulator/beam/jit/x86/instr_fun.cpp
@@ -0,0 +1,393 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2021-2021. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+/* Calls to functions that are being purged (but haven't finished) land here.
+ *
+ * ARG3 = arity
+ * ARG4 = fun thing
+ * ARG5 = current PC */
+void BeamGlobalAssembler::emit_unloaded_fun() {
+    Label error = a.newLabel();
+
+    emit_enter_frame();
+
+    a.mov(TMP_MEM1q, ARG5);
+
+    emit_enter_runtime<Update::eHeap | Update::eStack | Update::eReductions>();
+
+    a.mov(ARG1, c_p);
+    load_x_reg_array(ARG2);
+    /* ARG3 and ARG4 have already been set. */
+    runtime_call<4>(beam_jit_handle_unloaded_fun);
+
+    emit_leave_runtime<Update::eHeap | Update::eStack | Update::eReductions |
+                       Update::eCodeIndex>();
+
+    a.test(RET, RET);
+    a.jz(error);
+
+    emit_leave_frame();
+    a.jmp(emit_setup_export_call(RET));
+
+    a.bind(error);
+    {
+        /* The `raise_exception` fragment expects that the PC is on the
+         * stack. */
+        a.push(TMP_MEM1q);
+        mov_imm(ARG4, nullptr);
+        a.jmp(labels[raise_exception]);
+    }
+}
+
+/* Handles errors for `call_fun` and `i_lambda_trampoline`.
+ *
+ * ARG3 = arity
+ * ARG4 = fun thing
+ * ARG5 = current PC */
+void BeamGlobalAssembler::emit_handle_call_fun_error() {
+    Label bad_arity = a.newLabel(), bad_fun = a.newLabel();
+
+    emit_enter_frame();
+
+    emit_is_boxed(bad_fun, ARG4);
+
+    x86::Gp fun_thing = emit_ptr_val(RET, ARG4);
+    a.mov(RET, emit_boxed_val(fun_thing));
+    a.cmp(RET, imm(HEADER_EXPORT));
+    a.short_().je(bad_arity);
+    a.cmp(RET, imm(HEADER_FUN));
+    a.short_().je(bad_arity);
+
+    a.bind(bad_fun);
+    {
+        /* Not a fun: this is only reachable through `call_fun` */
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(EXC_BADFUN));
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), ARG4);
+
+        /* The `raise_exception` fragment expects that the PC is on the
+         * stack. */
+        a.push(ARG5);
+        mov_imm(ARG4, nullptr);
+        a.jmp(labels[raise_exception]);
+    }
+
+    a.bind(bad_arity);
+    {
+        /* Bad arity: this is reachable through `call_fun` when we have an
+         * export fun, and `i_lambda_trampoline` when we have a local one. */
+
+        /* Stash our fun and current PC. Note that we don't move the fun to
+         * {x,0} straight away as that would clobber the first argument. */
+        a.mov(TMP_MEM1q, ARG4);
+        a.mov(TMP_MEM2q, ARG5);
+
+        emit_enter_runtime<Update::eHeap | Update::eStack>();
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        /* ARG3 is already set. */
+        runtime_call<3>(beam_jit_build_argument_list);
+
+        emit_leave_runtime<Update::eHeap | Update::eStack>();
+
+        a.mov(ARG1, TMP_MEM1q);
+        a.mov(getXRef(0), ARG1);
+        a.mov(getXRef(1), RET);
+
+        /* Create the {Fun, Args} tuple. */
+        {
+            const int32_t bytes_needed = (3 + S_RESERVED) * sizeof(Eterm);
+            Label after_gc = a.newLabel();
+
+            a.lea(ARG3, x86::qword_ptr(HTOP, bytes_needed));
+            a.cmp(ARG3, E);
+            a.short_().jbe(after_gc);
+            {
+                mov_imm(ARG4, 2);
+                aligned_call(labels[garbage_collect]);
+            }
+            a.bind(after_gc);
+
+            a.mov(ARG1, getXRef(0));
+            a.mov(ARG2, getXRef(1));
+
+            a.mov(x86::qword_ptr(HTOP), imm(make_arityval(2)));
+            a.mov(x86::qword_ptr(HTOP, sizeof(Eterm[1])), ARG1);
+            a.mov(x86::qword_ptr(HTOP, sizeof(Eterm[2])), ARG2);
+
+            a.lea(ARG1, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED));
+            a.add(HTOP, imm(sizeof(Eterm[3])));
+        }
+
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)),
+              imm(EXC_BADARITY));
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), ARG1);
+
+        /* The `raise_exception` fragment expects that the PC is on the
+         * stack. */
+        a.push(TMP_MEM2q);
+        mov_imm(ARG4, nullptr);
+        a.jmp(labels[raise_exception]);
+    }
+}
+
+/* `call_fun` instructions land here to check arity and set up their
+ * environment before jumping to the actual implementation.
+ *
+ * Keep in mind that this runs in the limbo between caller and callee, so we
+ * must not enter a frame here.
+ *
+ * ARG3 = arity
+ * ARG4 = fun thing
+ * ARG5 = current PC */
+void BeamModuleAssembler::emit_i_lambda_trampoline(const ArgVal &Index,
+                                                   const ArgVal &Lbl,
+                                                   const ArgVal &Arity,
+                                                   const ArgVal &NumFree) {
+    const ssize_t effective_arity = Arity.getValue() - NumFree.getValue();
+    const ssize_t num_free = NumFree.getValue();
+    Label error = a.newLabel();
+    ssize_t i;
+
+    auto &lambda = lambdas[Index.getValue()];
+    lambda.trampoline = a.newLabel();
+    a.bind(lambda.trampoline);
+
+    a.cmp(ARG3, imm(effective_arity));
+    a.short_().jne(error);
+
+    emit_ptr_val(ARG4, ARG4);
+
+    for (i = 0; i < num_free - 1; i += 2) {
+        size_t offset = offsetof(ErlFunThing, env) + i * sizeof(Eterm);
+
+        a.movups(x86::xmm0, emit_boxed_val(ARG4, offset, sizeof(Eterm[2])));
+        a.movups(getXRef(i + effective_arity, sizeof(Eterm[2])), x86::xmm0);
+    }
+
+    if (i < num_free) {
+        size_t offset = offsetof(ErlFunThing, env) + i * sizeof(Eterm);
+
+        a.mov(RET, emit_boxed_val(ARG4, offset));
+        a.mov(getXRef(i + effective_arity), RET);
+    }
+
+    a.jmp(labels[Lbl.getValue()]);
+
+    a.bind(error);
+    abs_jmp(ga->get_handle_call_fun_error());
+}
+
+void BeamModuleAssembler::emit_i_make_fun3(const ArgVal &Fun,
+                                           const ArgVal &Dst,
+                                           const ArgVal &Arity,
+                                           const ArgVal &NumFree,
+                                           const Span<ArgVal> &env) {
+    size_t num_free = env.size();
+    ASSERT(NumFree.getValue() == num_free);
+
+    mov_arg(ARG3, Arity);
+    mov_arg(ARG4, NumFree);
+
+    emit_enter_runtime<Update::eHeap>();
+
+    a.mov(ARG1, c_p);
+    make_move_patch(ARG2, lambdas[Fun.getValue()].patches);
+    runtime_call<4>(new_fun_thing);
+
+    emit_leave_runtime<Update::eHeap>();
+
+    comment("Move fun environment");
+    for (unsigned i = 0; i < num_free; i++) {
+        mov_arg(x86::qword_ptr(RET,
+                               offsetof(ErlFunThing, env) + i * sizeof(Eterm)),
+                env[i]);
+    }
+
+    comment("Create boxed ptr");
+    a.or_(RETb, TAG_PRIMARY_BOXED);
+    mov_arg(Dst, RET);
+}
+
+/* Unwraps `apply_fun` so we can share the rest of the implementation with
+ * `call_fun`. */
+void BeamGlobalAssembler::emit_apply_fun_shared() {
+    Label finished = a.newLabel();
+
+    emit_enter_frame();
+
+    /* Put the arity and fun into the right registers for `call_fun`, and stash
+     * the argument list in ARG5 for the error path. We'll bump the arity as
+     * we go through the argument list. */
+    mov_imm(ARG3, 0);
+    a.mov(ARG4, getXRef(0));
+    a.mov(ARG5, getXRef(1));
+
+    {
+        Label unpack_next = a.newLabel(), malformed_list = a.newLabel(),
+              raise_error = a.newLabel();
+
+        auto x_register = getXRef(0);
+
+        ASSERT(x_register.shift() == 0);
+        x_register.setIndex(ARG3);
+        x_register.setShift(3);
+
+        a.mov(ARG1, ARG5);
+        a.bind(unpack_next);
+        {
+            a.cmp(ARG1d, imm(NIL));
+            a.short_().je(finished);
+
+            a.test(ARG1d, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST));
+            a.short_().jne(malformed_list);
+
+            emit_ptr_val(ARG1, ARG1);
+            a.mov(RET, getCARRef(ARG1));
+            a.mov(ARG1, getCDRRef(ARG1));
+            a.mov(x_register, RET);
+
+            a.inc(ARG3);
+
+            /* We bail at MAX_REG-1 rather than MAX_REG as the highest register
+             * is reserved for the loader. */
+            a.cmp(ARG3, imm(MAX_REG - 1));
+            a.jb(unpack_next);
+        }
+
+        a.mov(RET, imm(SYSTEM_LIMIT));
+        a.jmp(raise_error);
+
+        a.bind(malformed_list);
+        a.mov(RET, imm(BADARG));
+
+        a.bind(raise_error);
+        {
+            static const ErtsCodeMFA apply_mfa = {am_erlang, am_apply, 2};
+
+            a.mov(getXRef(0), ARG4);
+            a.mov(getXRef(1), ARG5);
+
+            a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), RET);
+            mov_imm(ARG4, &apply_mfa);
+
+            emit_leave_frame();
+            a.jmp(labels[raise_exception]);
+        }
+    }
+
+    a.bind(finished);
+
+    emit_leave_frame();
+    a.ret();
+}
+
+void BeamModuleAssembler::emit_i_apply_fun() {
+    safe_fragment_call(ga->get_apply_fun_shared());
+
+    x86::Gp dest = emit_call_fun();
+    ASSERT(dest != ARG6);
+    erlang_call(dest, ARG6);
+}
+
+void BeamModuleAssembler::emit_i_apply_fun_last(const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+    emit_i_apply_fun_only();
+}
+
+void BeamModuleAssembler::emit_i_apply_fun_only() {
+    safe_fragment_call(ga->get_apply_fun_shared());
+
+    x86::Gp dest = emit_call_fun();
+    emit_leave_frame();
+    a.jmp(dest);
+}
+
+/* Asssumes that:
+ *   ARG3 = arity
+ *   ARG4 = fun thing */
+x86::Gp BeamModuleAssembler::emit_call_fun() {
+    Label exported = a.newLabel(), next = a.newLabel();
+
+    /* Load the error fragment into ARG2 so we can CMOV ourselves there on
+     * error. */
+    a.mov(ARG2, ga->get_handle_call_fun_error());
+
+    /* The `handle_call_fun_error` and `i_lambda_trampoline` fragments expect
+     * current PC in ARG5. */
+    a.lea(ARG5, x86::qword_ptr(next));
+
+    /* As emit_is_boxed(), but explicitly sets ZF so we can rely on that for
+     * error checking in `next`. */
+    a.test(ARG4d, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED));
+    a.short_().jne(next);
+
+    x86::Gp fun_thing = emit_ptr_val(RET, ARG4);
+    a.cmp(emit_boxed_val(fun_thing), imm(HEADER_EXPORT));
+    a.short_().je(exported);
+    a.cmp(emit_boxed_val(fun_thing), imm(HEADER_FUN));
+    a.short_().jne(next);
+
+    a.mov(ARG1, emit_boxed_val(fun_thing, offsetof(ErlFunThing, fe)));
+    a.mov(ARG1, x86::qword_ptr(ARG1, offsetof(ErlFunEntry, address)));
+    a.short_().jmp(next);
+
+    a.bind(exported);
+    {
+        a.mov(RET, emit_boxed_val(fun_thing, sizeof(Eterm)));
+        a.mov(ARG1, emit_setup_export_call(RET));
+
+        a.cmp(x86::qword_ptr(RET, offsetof(Export, info.mfa.arity)), ARG3);
+    }
+
+    /* Assumes that ZF is set on success and clear on error, overwriting our
+     * destination with the error fragment's address. */
+    a.bind(next);
+    a.cmovne(ARG1, ARG2);
+
+    return ARG1;
+}
+
+void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Arity) {
+    mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue()));
+    mov_imm(ARG3, Arity.getValue());
+
+    x86::Gp dest = emit_call_fun();
+    erlang_call(dest, ARG6);
+}
+
+void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Arity,
+                                               const ArgVal &Deallocate) {
+    emit_deallocate(Deallocate);
+
+    mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue()));
+    mov_imm(ARG3, Arity.getValue());
+
+    x86::Gp dest = emit_call_fun();
+    emit_leave_frame();
+    a.jmp(dest);
+}
+
+/* Psuedo-instruction for signalling lambda load errors. Never actually runs. */
+void BeamModuleAssembler::emit_i_lambda_error(const ArgVal &Dummy) {
+    a.ud2();
+}
diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab
index d1306c2371..b2477df813 100644
--- a/erts/emulator/beam/jit/x86/ops.tab
+++ b/erts/emulator/beam/jit/x86/ops.tab
@@ -719,7 +719,7 @@ make_fun3 OldIndex=u Dst=d NumFree=u Env=* => make_fun3(OldIndex, Dst, NumFree,
 
 %cold
 
-i_make_fun3 F S t *
+i_make_fun3 F S t t *
 
 # Psuedo-instruction for signalling lambda load errors. Never actually runs.
 i_lambda_error t
@@ -731,7 +731,7 @@ is_function Fail=f c => jump Fail
 
 # The start and end of a function.
 int_func_start/5
-int_func_end/0
+int_func_end/2
 
 func_prologue/2
 
@@ -766,9 +766,8 @@ int_func_start Func_Label Func_Line M F A | \
       i_breakpoint_trampoline | \
       i_test_yield
 
-
-int_func_end | needs_nif_padding() => i_nif_padding
-int_func_end =>
+int_func_end Func_Label Entry_Label => \
+    func_end(Func_Label, Entry_Label)
 
 # Handles yielding on function ingress (rather than on each call).
 i_test_yield
@@ -776,6 +775,9 @@ i_test_yield
 # Ensures that the prior function is large enough to allow NIF patching.
 i_nif_padding
 
+# Landing pad for fun calls/apply where we set up arguments and check errors
+i_lambda_trampoline W f W W
+
 # Handles tracing, early NIF calls, and so on.
 i_breakpoint_trampoline
 
diff --git a/erts/emulator/beam/jit/x86/predicates.tab b/erts/emulator/beam/jit/x86/predicates.tab
index a4e00586f6..8b8aa58ca3 100644
--- a/erts/emulator/beam/jit/x86/predicates.tab
+++ b/erts/emulator/beam/jit/x86/predicates.tab
@@ -32,12 +32,6 @@ pred.is_mfa_bif(M, F, A) {
     return 0;
 }
 
-pred.needs_nif_padding() {
-    /* If the module may load a NIF all functions must be able to hold a NIF
-     * stub, so we'll pad to that size at the end of every function. */
-    return S->may_load_nif;
-}
-
 pred.never_fails(Bif) {
     static Eterm nofail_bifs[] =
 	{am_Neqeq,
-- 
2.26.2

Places

File 1117-erts-Refactor-fun-calls.patch of Package erlang (Revision 06790aed050a1354546a4c79936c324d)

Places