File 1104-jit-Break-out-architecture-agnostic-parts-... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1104-jit-Break-out-architecture-agnostic-parts-to-prepare.patch of Package erlang (Revision 0eb87917a75aae8f396ec39332087beb)

Currently displaying revision 0eb87917a75aae8f396ec39332087beb , Show latest

From e65733cd637e91d8bb38edb66aa0bddb148eb355 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Fri, 16 Apr 2021 08:51:50 +0200
Subject: [PATCH 04/20] jit: Break out architecture-agnostic parts to prepare
 for ARM JIT

---
 erts/emulator/Makefile.in                     |  26 +-
 ...{beam_jit_common.c => beam_jit_common.cpp} | 327 +++++++++++++++-
 erts/emulator/beam/jit/beam_jit_common.h      | 101 -----
 erts/emulator/beam/jit/beam_jit_common.hpp    |  87 ++++-
 .../{x86/beam_asm.cpp => beam_jit_main.cpp}   | 321 ----------------
 .../beam_asm_perf.cpp => beam_jit_perf.cpp}   |   2 +
 erts/emulator/beam/jit/x86/beam_asm.hpp       | 122 +-----
 .../emulator/beam/jit/x86/beam_asm_module.cpp | 168 +--------
 erts/emulator/beam/jit/x86/instr_trace.cpp    |  31 +-
 erts/emulator/beam/jit/x86/process_main.cpp   | 350 ++++++++++++++++++
 10 files changed, 792 insertions(+), 743 deletions(-)
 rename erts/emulator/beam/jit/{beam_jit_common.c => beam_jit_common.cpp} (70%)
 delete mode 100644 erts/emulator/beam/jit/beam_jit_common.h
 rename erts/emulator/beam/jit/{x86/beam_asm.cpp => beam_jit_main.cpp} (65%)
 rename erts/emulator/beam/jit/{x86/beam_asm_perf.cpp => beam_jit_perf.cpp} (99%)
 create mode 100644 erts/emulator/beam/jit/x86/process_main.cpp

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 63103b05f3..2ed8fe836c 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -906,14 +906,19 @@ ASMJIT_FLAGS=-DASMJIT_EMBED=1 -DASMJIT_NO_BUILDER=1 -DASMJIT_NO_DEPRECATED=1 -DA
 ASMJIT_PCH_OBJ=$(TTF_DIR)/asmjit/asmjit.hpp.gch
 ASMJIT_PCH_SRC=$(TTF_DIR)/asmjit/asmjit.hpp
 
-$(OBJDIR)/%.o: beam/jit/$(JIT_ARCH)/%.cpp beam/jit/$(JIT_ARCH)/beam_asm.hpp $(ASMJIT_PCH_OBJ)
-	$(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES)    			\
-          $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS))            	\
+$(OBJDIR)/%.o: beam/jit/%.cpp $(ASMJIT_PCH_OBJ)
+	$(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES)                            \
+          $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS))                \
+          -include $(ASMJIT_PCH_SRC) -c $< -o $@
+
+$(OBJDIR)/%.o: beam/jit/$(JIT_ARCH)/%.cpp beam/jit/$(JIT_ARCH)/beam_asm.hpp
+	$(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES)                            \
+          $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS))                \
           -include $(ASMJIT_PCH_SRC) -c $< -o $@
 
 $(OBJDIR)/asmjit/%.o: asmjit/%.cpp $(ASMJIT_PCH_OBJ) $(dir $@)
-	$(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES)    			\
-          $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS))            	\
+	$(V_CXX) $(ASMJIT_FLAGS) $(INCLUDES)                            \
+          $(subst -O2, $(GEN_OPT_FLGS), $(CXXFLAGS))                \
           -include $(ASMJIT_PCH_SRC) -c $< -o $@
 
 ## The dependency on erl_bif_info.c is in order to trigger a rebuild when
@@ -993,14 +998,15 @@ ASMJIT_OBJS = $(patsubst %.cpp,$(OBJDIR)/%.o,$(ASMJIT_CPP))
 
 JIT_OBJS = \
         $(OBJDIR)/asm_load.o \
-        $(OBJDIR)/beam_jit_common.o
+        $(OBJDIR)/beam_jit_common.o \
+        $(OBJDIR)/beam_jit_main.o \
+        $(OBJDIR)/beam_jit_perf.o
 
 ifeq ($(JIT_ARCH), x86)
 JIT_OBJS += \
-	$(OBJDIR)/beam_asm.o \
 	$(OBJDIR)/beam_asm_global.o \
 	$(OBJDIR)/beam_asm_module.o \
-	$(OBJDIR)/beam_asm_perf.o \
+	$(OBJDIR)/process_main.o \
 	$(OBJDIR)/instr_arith.o \
 	$(OBJDIR)/instr_bs.o \
 	$(OBJDIR)/instr_bif.o \
@@ -1389,11 +1395,11 @@ compdb:
 
 format-check:
 	clang-format --Werror --dry-run -i beam/jit/*.hpp beam/jit/*.c beam/jit/*.h \
-            beam/jit/*/*.cpp beam/jit/*/*.hpp
+            beam/jit/*.cpp beam/jit/*/*.cpp beam/jit/*/*.hpp
 
 format:
 	clang-format -i beam/jit/*.hpp beam/jit/*.c beam/jit/*.h \
-            beam/jit/*/*.cpp beam/jit/*/*.hpp
+            beam/jit/*.cpp beam/jit/*/*.cpp beam/jit/*/*.hpp
 
 ifneq ($(ERTS_SKIP_DEPEND),true)
 ifneq ($(MAKECMDGOALS),clean)
diff --git a/erts/emulator/beam/jit/beam_jit_common.c b/erts/emulator/beam/jit/beam_jit_common.cpp
similarity index 70%
rename from erts/emulator/beam/jit/beam_jit_common.c
rename to erts/emulator/beam/jit/beam_jit_common.cpp
index 9b56fb1ac1..6c9f069bbc 100644
--- a/erts/emulator/beam/jit/beam_jit_common.c
+++ b/erts/emulator/beam/jit/beam_jit_common.cpp
@@ -18,14 +18,14 @@
  * %CopyrightEnd%
  */
 
-#ifdef HAVE_CONFIG_H
-#    include "config.h"
-#endif
+#include "beam_jit_common.hpp"
+#include "beam_asm.hpp"
 
+extern "C"
+{
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
-#include "bif.h"
 #include "code_ix.h"
 #include "erl_proc_sig_queue.h"
 #include "erl_binary.h"
@@ -35,8 +35,297 @@
 #ifdef USE_VM_PROBES
 #    include "dtrace-wrapper.h"
 #endif
+}
+
+static std::string getAtom(Eterm atom) {
+    Atom *ap = atom_tab(atom_val(atom));
+    return std::string((char *)ap->name, ap->len);
+}
+
+BeamAssembler::BeamAssembler() : code() {
+    /* Setup with default code info */
+    Error err = code.init(hostEnvironment());
+    ERTS_ASSERT(!err && "Failed to init codeHolder");
+
+    err = code.newSection(&rodata, ".rodata", SIZE_MAX, Section::kFlagConst, 8);
+    ERTS_ASSERT(!err && "Failed to create .rodata section");
+
+    err = code.attach(&a);
+
+    ERTS_ASSERT(!err && "Failed to attach codeHolder");
+#ifdef DEBUG
+    a.addValidationOptions(BaseEmitter::kValidationOptionAssembler);
+#endif
+    a.addEncodingOptions(BaseEmitter::kEncodingOptionOptimizeForSize);
+    code.setErrorHandler(this);
+}
+
+BeamAssembler::BeamAssembler(const std::string &log) : BeamAssembler() {
+    if (erts_jit_asm_dump) {
+        setLogger(log + ".asm");
+    }
+}
+
+BeamAssembler::~BeamAssembler() {
+    if (logger.file()) {
+        fclose(logger.file());
+    }
+}
+
+void *BeamAssembler::getBaseAddress() {
+    ASSERT(code.hasBaseAddress());
+    return (void *)code.baseAddress();
+}
+
+size_t BeamAssembler::getOffset() {
+    return a.offset();
+}
+
+void BeamAssembler::_codegen(JitAllocator *allocator,
+                             const void **executable_ptr,
+                             void **writable_ptr) {
+    Error err = code.flatten();
+    ERTS_ASSERT(!err && "Could not flatten code");
+    err = code.resolveUnresolvedLinks();
+    ERTS_ASSERT(!err && "Could not resolve all links");
+
+    /* Verify that all labels are bound */
+#ifdef DEBUG
+    for (auto e : code.labelEntries()) {
+        if (!e->isBound()) {
+            if (e->hasName()) {
+                erts_exit(ERTS_ABORT_EXIT,
+                          "Label %d with name %s is not bound\n",
+                          e->id(),
+                          e->name());
+            } else {
+                erts_exit(ERTS_ABORT_EXIT, "Label %d is not bound\n", e->id());
+            }
+        }
+    }
+#endif
+
+    err = allocator->alloc(const_cast<void **>(executable_ptr),
+                           writable_ptr,
+                           code.codeSize() + 16);
+
+    if (err == ErrorCode::kErrorTooManyHandles) {
+        ERTS_ASSERT(!"Failed to allocate module code: "
+                     "out of file descriptors");
+    } else if (err) {
+        ERTS_ASSERT("Failed to allocate module code");
+    }
+
+    code.relocateToBase((uint64_t)*executable_ptr);
+    code.copyFlattenedData(*writable_ptr,
+                           code.codeSize(),
+                           CodeHolder::kCopyPadSectionBuffer);
+
+    beamasm_flush_icache(*executable_ptr, code.codeSize());
+
+#ifdef DEBUG
+    if (FileLogger *l = dynamic_cast<FileLogger *>(code.logger()))
+        if (FILE *f = l->file())
+            fprintf(f, "; CODE_SIZE: %zd\n", code.codeSize());
+#endif
+}
+
+void *BeamAssembler::getCode(Label label) {
+    ASSERT(label.isValid());
+    return (char *)getBaseAddress() + code.labelOffsetFromBase(label);
+}
+
+byte *BeamAssembler::getCode(char *labelName) {
+    return (byte *)getCode(code.labelByName(labelName, strlen(labelName)));
+}
+
+void BeamAssembler::handleError(Error err,
+                                const char *message,
+                                BaseEmitter *origin) {
+    comment(message);
+    fflush(logger.file());
+    ASSERT(0 && "Failed to encode instruction");
+}
+
+void BeamAssembler::embed_rodata(const char *labelName,
+                                 const char *buff,
+                                 size_t size) {
+    Label label = a.newNamedLabel(labelName);
+
+    a.section(rodata);
+    a.bind(label);
+    a.embed(buff, size);
+    a.section(code.textSection());
+}
+
+void BeamAssembler::embed_bss(const char *labelName, size_t size) {
+    Label label = a.newNamedLabel(labelName);
+
+    /* Reuse rodata section for now */
+    a.section(rodata);
+    a.bind(label);
+    embed_zeros(size);
+    a.section(code.textSection());
+}
+
+void BeamAssembler::embed_zeros(size_t size) {
+    static constexpr size_t buf_size = 16384;
+    static const char zeros[buf_size] = {};
+
+    while (size >= buf_size) {
+        a.embed(zeros, buf_size);
+        size -= buf_size;
+    }
+
+    if (size > 0) {
+        a.embed(zeros, size);
+    }
+}
+
+void BeamAssembler::setLogger(std::string log) {
+    FILE *f = fopen(log.data(), "w+");
+
+    /* FIXME: Don't crash when loading multiple modules with the same name.
+     *
+     * setLogger(nullptr) disables logging. */
+    if (f) {
+        setvbuf(f, NULL, _IONBF, 0);
+    }
+
+    setLogger(f);
+}
+
+void BeamAssembler::setLogger(FILE *log) {
+    logger.setFile(log);
+    logger.setIndentation(FormatOptions::kIndentationCode, 4);
+    code.setLogger(&logger);
+}
+
+void BeamModuleAssembler::codegen(JitAllocator *allocator,
+                                  const void **executable_ptr,
+                                  void **writable_ptr,
+                                  const BeamCodeHeader *in_hdr,
+                                  const BeamCodeHeader **out_exec_hdr,
+                                  BeamCodeHeader **out_rw_hdr) {
+    const BeamCodeHeader *code_hdr_exec;
+    BeamCodeHeader *code_hdr_rw;
+
+    codegen(allocator, executable_ptr, writable_ptr);
+
+    {
+        auto offset = code.labelOffsetFromBase(codeHeader);
+
+        auto base_exec = (const char *)(*executable_ptr);
+        code_hdr_exec = (const BeamCodeHeader *)&base_exec[offset];
+
+        auto base_rw = (const char *)(*writable_ptr);
+        code_hdr_rw = (BeamCodeHeader *)&base_rw[offset];
+    }
+
+    sys_memcpy(code_hdr_rw, in_hdr, sizeof(BeamCodeHeader));
+    code_hdr_rw->on_load = getOnLoad();
+
+    for (unsigned i = 0; i < functions.size(); i++) {
+        ErtsCodeInfo *ci = (ErtsCodeInfo *)getCode(functions[i]);
+        code_hdr_rw->functions[i] = ci;
+    }
+
+    char *module_end = (char *)code.baseAddress() + a.offset();
+    code_hdr_rw->functions[functions.size()] = (ErtsCodeInfo *)module_end;
+
+    *out_exec_hdr = code_hdr_exec;
+    *out_rw_hdr = code_hdr_rw;
+}
+
+void BeamModuleAssembler::codegen(JitAllocator *allocator,
+                                  const void **executable_ptr,
+                                  void **writable_ptr) {
+    _codegen(allocator, executable_ptr, writable_ptr);
+
+#if !(defined(WIN32) || defined(__APPLE__) || defined(__MACH__) ||             \
+      defined(__DARWIN__))
+    if (functions.size()) {
+        char *buff = (char *)erts_alloc(ERTS_ALC_T_TMP, 1024);
+        std::vector<AsmRange> ranges;
+        std::string name = getAtom(mod);
+        ranges.reserve(functions.size() + 2);
+
+        /* Push info about the header */
+        ranges.push_back({.start = (ErtsCodePtr)getBaseAddress(),
+                          .stop = getCode(functions[0]),
+                          .name = name + "::codeHeader"});
+
+        for (unsigned i = 0; i < functions.size(); i++) {
+            ErtsCodePtr start, stop;
+            const ErtsCodeInfo *ci;
+            int n;
+
+            start = getCode(functions[i]);
+            ci = (const ErtsCodeInfo *)start;
+
+            n = erts_snprintf(buff,
+                              1024,
+                              "%T:%T/%d",
+                              ci->mfa.module,
+                              ci->mfa.function,
+                              ci->mfa.arity);
+            stop = ((const char *)erts_codeinfo_to_code(ci)) +
+                   BEAM_ASM_FUNC_PROLOGUE_SIZE;
+
+            /* We use a different symbol for CodeInfo and the Prologue
+               in order for the perf disassembly to be better. */
+            std::string name(buff, n);
+            ranges.push_back({.start = start,
+                              .stop = stop,
+                              .name = name + "-CodeInfoPrologue"});
+
+            /* The actual code */
+            start = stop;
+            if (i + 1 < functions.size()) {
+                stop = getCode(functions[i + 1]);
+            } else {
+                stop = getCode(labels.size());
+            }
+
+            ranges.push_back({.start = start, .stop = stop, .name = name});
+        }
+
+        /* Push info about the footer */
+        ranges.push_back(
+                {.start = ranges.back().stop,
+                 .stop = (ErtsCodePtr)(code.baseAddress() + code.codeSize()),
+                 .name = name + "::codeFooter"});
+
+        update_gdb_jit_info(name, ranges);
+        beamasm_update_perf_info(name, ranges);
+        erts_free(ERTS_ALC_T_TMP, buff);
+    }
+#endif
+}
+
+void BeamModuleAssembler::codegen(char *buff, size_t len) {
+    code.flatten();
+    code.resolveUnresolvedLinks();
+    ERTS_ASSERT(code.codeSize() <= len);
+    code.relocateToBase((uint64_t)buff);
+    code.copyFlattenedData(buff,
+                           code.codeSize(),
+                           CodeHolder::kCopyPadSectionBuffer);
+}
 
-#include "beam_jit_common.h"
+BeamCodeHeader *BeamModuleAssembler::getCodeHeader() {
+    return (BeamCodeHeader *)getCode(codeHeader);
+}
+
+const ErtsCodeInfo *BeamModuleAssembler::getOnLoad() {
+    if (on_load.isValid()) {
+        return erts_code_to_codeinfo((ErtsCodePtr)getCode(on_load));
+    } else {
+        return 0;
+    }
+}
+
+/* ** */
 
 #if defined(DEBUG) && defined(JIT_HARD_DEBUG)
 void beam_jit_validate_term(Eterm term) {
@@ -754,3 +1043,31 @@ void beam_jit_timeout_locked(Process *c_p) {
     erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     beam_jit_timeout(c_p);
 }
+
+void beam_jit_return_to_trace(Process *c_p) {
+    if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) {
+        ErtsCodePtr return_to_address;
+        Uint *cpp;
+
+        cpp = (Uint *)c_p->stop;
+        ASSERT(is_CP(cpp[0]));
+
+        for (;;) {
+            erts_inspect_frame(cpp, &return_to_address);
+
+            if (BeamIsReturnTrace(return_to_address)) {
+                cpp += CP_SIZE + 2;
+            } else if (BeamIsReturnTimeTrace(return_to_address)) {
+                cpp += CP_SIZE + 1;
+            } else if (BeamIsReturnToTrace(return_to_address)) {
+                cpp += CP_SIZE;
+            } else {
+                break;
+            }
+        }
+
+        ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+        erts_trace_return_to(c_p, return_to_address);
+        ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    }
+}
diff --git a/erts/emulator/beam/jit/beam_jit_common.h b/erts/emulator/beam/jit/beam_jit_common.h
deleted file mode 100644
index b2da85d0b8..0000000000
--- a/erts/emulator/beam/jit/beam_jit_common.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2021. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#    include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
-
-#if defined(DEBUG) && defined(JIT_HARD_DEBUG)
-void beam_jit_validate_term(Eterm term);
-#endif
-
-typedef Eterm BeamJitNifF(struct enif_environment_t *, int argc, Eterm argv[]);
-enum beam_jit_nif_load_ret { RET_NIF_success, RET_NIF_error, RET_NIF_yield };
-
-Eterm beam_jit_call_bif(Process *c_p,
-                        Eterm *reg,
-                        ErtsCodePtr I,
-                        ErtsBifFunc vbf,
-                        Uint arity);
-Eterm beam_jit_call_nif(Process *c_p,
-                        ErtsCodePtr I,
-                        Eterm *reg,
-                        BeamJitNifF *fp,
-                        struct erl_module_nif *NifMod);
-enum beam_jit_nif_load_ret beam_jit_load_nif(Process *c_p,
-                                             ErtsCodePtr I,
-                                             Eterm *reg);
-
-Uint beam_jit_get_map_elements(Eterm map,
-                               Eterm *reg,
-                               Eterm *E,
-                               Uint n,
-                               Eterm *fs);
-
-void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size);
-void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B);
-Eterm beam_jit_bs_start_match2(Eterm context,
-                               Uint live,
-                               Uint slots,
-                               Process *c_p,
-                               Eterm *reg);
-Eterm beam_jit_bs_init(Process *c_p,
-                       Eterm *reg,
-                       ERL_BITS_DECLARE_STATEP,
-                       Eterm BsOp1,
-                       Eterm BsOp2,
-                       unsigned Live);
-Eterm beam_jit_bs_init_bits(Process *c_p,
-                            Eterm *reg,
-                            ERL_BITS_DECLARE_STATEP,
-                            Uint num_bits,
-                            Uint alloc,
-                            unsigned Live);
-Eterm beam_jit_bs_get_integer(Process *c_p,
-                              Eterm *reg,
-                              Eterm context,
-                              Uint flags,
-                              Uint size,
-                              Uint Live);
-
-ErtsMessage *beam_jit_decode_dist(Process *c_p, ErtsMessage *msgp);
-Sint beam_jit_remove_message(Process *c_p,
-                             Sint FCALLS,
-                             Eterm *HTOP,
-                             Eterm *E,
-                             Uint32 active_code_ix);
-void beam_jit_bs_context_to_binary(Eterm context);
-
-void beam_jit_take_receive_lock(Process *c_p);
-void beam_jit_wait_locked(Process *c_p, ErtsCodePtr cp);
-void beam_jit_wait_unlocked(Process *c_p, ErtsCodePtr cp);
-
-enum beam_jit_tmo_ret { RET_next = 0, RET_wait = 1, RET_badarg = 2 };
-
-enum beam_jit_tmo_ret beam_jit_wait_timeout(Process *c_p,
-                                            Eterm timeout_value,
-                                            ErtsCodePtr next);
-
-void beam_jit_timeout(Process *c_p);
-void beam_jit_timeout_locked(Process *c_p);
diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp
index f8f894c676..e3f5d10d17 100644
--- a/erts/emulator/beam/jit/beam_jit_common.hpp
+++ b/erts/emulator/beam/jit/beam_jit_common.hpp
@@ -18,6 +18,9 @@
  * %CopyrightEnd%
  */
 
+#ifndef __BEAM_JIT_COMMON_HPP__
+#define __BEAM_JIT_COMMON_HPP__
+
 #include <string>
 #include <vector>
 #include <unordered_map>
@@ -30,10 +33,10 @@ extern "C"
 #endif
 
 #include "sys.h"
+#include "bif.h"
 #include "erl_vm.h"
 #include "global.h"
-
-#include "beam_jit_common.h"
+#include "beam_file.h"
 }
 
 class ArgVal {
@@ -135,3 +138,83 @@ public:
         }
     };
 };
+
+/* ** */
+
+#if defined(DEBUG) && defined(JIT_HARD_DEBUG)
+void beam_jit_validate_term(Eterm term);
+#endif
+
+typedef Eterm BeamJitNifF(struct enif_environment_t *, int argc, Eterm argv[]);
+enum beam_jit_nif_load_ret { RET_NIF_success, RET_NIF_error, RET_NIF_yield };
+
+Eterm beam_jit_call_bif(Process *c_p,
+                        Eterm *reg,
+                        ErtsCodePtr I,
+                        ErtsBifFunc vbf,
+                        Uint arity);
+Eterm beam_jit_call_nif(Process *c_p,
+                        ErtsCodePtr I,
+                        Eterm *reg,
+                        BeamJitNifF *fp,
+                        struct erl_module_nif *NifMod);
+enum beam_jit_nif_load_ret beam_jit_load_nif(Process *c_p,
+                                             ErtsCodePtr I,
+                                             Eterm *reg);
+
+Uint beam_jit_get_map_elements(Eterm map,
+                               Eterm *reg,
+                               Eterm *E,
+                               Uint n,
+                               Eterm *fs);
+
+void beam_jit_bs_field_size_argument_error(Process *c_p, Eterm size);
+void beam_jit_bs_add_argument_error(Process *c_p, Eterm A, Eterm B);
+Eterm beam_jit_bs_start_match2(Eterm context,
+                               Uint live,
+                               Uint slots,
+                               Process *c_p,
+                               Eterm *reg);
+Eterm beam_jit_bs_init(Process *c_p,
+                       Eterm *reg,
+                       ERL_BITS_DECLARE_STATEP,
+                       Eterm BsOp1,
+                       Eterm BsOp2,
+                       unsigned Live);
+Eterm beam_jit_bs_init_bits(Process *c_p,
+                            Eterm *reg,
+                            ERL_BITS_DECLARE_STATEP,
+                            Uint num_bits,
+                            Uint alloc,
+                            unsigned Live);
+Eterm beam_jit_bs_get_integer(Process *c_p,
+                              Eterm *reg,
+                              Eterm context,
+                              Uint flags,
+                              Uint size,
+                              Uint Live);
+
+ErtsMessage *beam_jit_decode_dist(Process *c_p, ErtsMessage *msgp);
+Sint beam_jit_remove_message(Process *c_p,
+                             Sint FCALLS,
+                             Eterm *HTOP,
+                             Eterm *E,
+                             Uint32 active_code_ix);
+void beam_jit_bs_context_to_binary(Eterm context);
+
+void beam_jit_take_receive_lock(Process *c_p);
+void beam_jit_wait_locked(Process *c_p, ErtsCodePtr cp);
+void beam_jit_wait_unlocked(Process *c_p, ErtsCodePtr cp);
+
+enum beam_jit_tmo_ret { RET_next = 0, RET_wait = 1, RET_badarg = 2 };
+
+enum beam_jit_tmo_ret beam_jit_wait_timeout(Process *c_p,
+                                            Eterm timeout_value,
+                                            ErtsCodePtr next);
+
+void beam_jit_timeout(Process *c_p);
+void beam_jit_timeout_locked(Process *c_p);
+
+void beam_jit_return_to_trace(Process *c_p);
+
+#endif
diff --git a/erts/emulator/beam/jit/x86/beam_asm.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp
similarity index 65%
rename from erts/emulator/beam/jit/x86/beam_asm.cpp
rename to erts/emulator/beam/jit/beam_jit_main.cpp
index ff7ae5d774..15bd74a4e2 100644
--- a/erts/emulator/beam/jit/x86/beam_asm.cpp
+++ b/erts/emulator/beam/jit/beam_jit_main.cpp
@@ -317,322 +317,6 @@ void process_main(ErtsSchedulerData *esdp) {
     pmain(esdp);
 }
 
-#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
-static Process *erts_debug_schedule(ErtsSchedulerData *esdp,
-                                    Process *c_p,
-                                    int calls) {
-    PROCESS_MAIN_CHK_LOCKS(c_p);
-    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-    c_p = erts_schedule(esdp, c_p, calls);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-    ERTS_REQ_PROC_MAIN_LOCK(c_p);
-    PROCESS_MAIN_CHK_LOCKS(c_p);
-    return c_p;
-}
-#endif
-
-/* void process_main(ErtsSchedulerData *esdp); */
-void BeamGlobalAssembler::emit_process_main() {
-    Label context_switch_local = a.newLabel(),
-          context_switch_simplified_local = a.newLabel(),
-          do_schedule_local = a.newLabel(), schedule_next = a.newLabel();
-
-    const x86::Mem start_time_i =
-            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time_i));
-    const x86::Mem start_time =
-            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time));
-
-    /* Allocate the register structure on the stack to allow computing the
-     * runtime stack address from it, greatly reducing the cost of stack
-     * swapping. */
-    a.sub(x86::rsp, imm(sizeof(ErtsSchedulerRegisters) + ERTS_CACHE_LINE_SIZE));
-    a.and_(x86::rsp, imm(~ERTS_CACHE_LINE_MASK));
-
-    a.mov(x86::qword_ptr(ARG1, offsetof(ErtsSchedulerData, registers)),
-          x86::rsp);
-
-    /* Center `registers` at the base of x_reg_array so we can use negative
-     * 8-bit displacement to address the commonly used aux_regs, located at the
-     * start of the ErtsSchedulerRegisters struct. */
-    a.lea(registers,
-          x86::qword_ptr(x86::rsp,
-                         offsetof(ErtsSchedulerRegisters, x_reg_array.d)));
-
-    load_erl_bits_state(ARG1);
-    runtime_call<1>(erts_bits_init_state);
-
-#if defined(DEBUG) && defined(NATIVE_ERLANG_STACK)
-    /* Save stack bounds so they can be tested without clobbering anything. */
-    runtime_call<0>(erts_get_stacklimit);
-
-    a.mov(getSchedulerRegRef(
-                  offsetof(ErtsSchedulerRegisters, runtime_stack_end)),
-          RET);
-    a.mov(getSchedulerRegRef(
-                  offsetof(ErtsSchedulerRegisters, runtime_stack_start)),
-          x86::rsp);
-#elif !defined(NATIVE_ERLANG_STACK)
-    /* Save the initial SP of the thread so that we can verify that it
-     * doesn't grow. */
-#    ifdef JIT_HARD_DEBUG
-    a.mov(getInitialSPRef(), x86::rsp);
-#    endif
-
-    /* Manually do an `emit_enter_runtime` to match the `emit_leave_runtime`
-     * below. We avoid `emit_enter_runtime` because it may do additional
-     * assertions that may currently fail.
-     *
-     * IMPORTANT: We must ensure that this sequence leaves the stack
-     * aligned on a 16-byte boundary. */
-    a.mov(getRuntimeStackRef(), x86::rsp);
-    a.sub(x86::rsp, imm(15));
-    a.and_(x86::rsp, imm(-16));
-#endif
-
-    a.mov(start_time_i, imm(0));
-    a.mov(start_time, imm(0));
-
-    mov_imm(c_p, 0);
-    mov_imm(FCALLS, 0);
-    mov_imm(ARG3, 0); /* Set reds_used for erts_schedule call */
-
-    a.jmp(schedule_next);
-
-    a.bind(do_schedule_local);
-    {
-        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
-        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
-        a.sub(ARG3, FCALLS);
-
-        a.jmp(schedule_next);
-    }
-
-    a.bind(context_switch_local);
-    comment("Context switch, unknown arity/MFA");
-    {
-        Sint arity_offset = offsetof(ErtsCodeMFA, arity) - sizeof(ErtsCodeMFA);
-
-        a.mov(ARG1, x86::qword_ptr(ARG3, arity_offset));
-        a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), ARG1);
-
-        a.lea(ARG1, x86::qword_ptr(ARG3, -(Sint)sizeof(ErtsCodeMFA)));
-        a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), ARG1);
-
-        /* !! Fall through !! */
-    }
-
-    a.bind(context_switch_simplified_local);
-    comment("Context switch, known arity and MFA");
-    {
-        Label not_exiting = a.newLabel();
-
-#ifdef ERLANG_FRAME_POINTERS
-        /* Kill the current frame pointer to avoid confusing `perf` and similar
-         * tools. */
-        a.sub(frame_pointer, frame_pointer);
-#endif
-
-#ifdef DEBUG
-        Label check_i = a.newLabel();
-        /* Check that ARG3 is set to a valid CP. */
-        a.test(ARG3, imm(_CPMASK));
-        a.je(check_i);
-        a.ud2();
-        a.bind(check_i);
-#endif
-
-        a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG3);
-
-#if defined(JIT_HARD_DEBUG) && defined(ERLANG_FRAME_POINTERS)
-        a.mov(ARG1, c_p);
-        a.mov(ARG2, x86::qword_ptr(c_p, offsetof(Process, frame_pointer)));
-        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, stop)));
-
-        runtime_call<3>(erts_validate_stack);
-#endif
-
-#ifdef WIN32
-        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.value)));
-#else
-        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.counter)));
-#endif
-
-        a.test(ARG1d, imm(ERTS_PSFLG_EXITING));
-        a.short_().je(not_exiting);
-        {
-            comment("Process exiting");
-
-            /* We load the beam_exit from memory because it has not yet been set
-             * when the global assembler is created. */
-            a.mov(ARG1, imm(&beam_exit));
-            a.mov(ARG1, x86::qword_ptr(ARG1));
-            a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG1);
-            a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), imm(0));
-            a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), imm(0));
-            a.jmp(do_schedule_local);
-        }
-        a.bind(not_exiting);
-
-        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
-        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
-        a.sub(ARG3, FCALLS);
-
-        /* Spill reds_used to FCALLS as we no longer need that value */
-        a.mov(FCALLS, ARG3);
-
-        a.mov(ARG1, c_p);
-        load_x_reg_array(ARG2);
-        runtime_call<2>(copy_out_registers);
-
-        /* Restore reds_used from FCALLS */
-        a.mov(ARG3, FCALLS);
-
-        /* !! Fall through !! */
-    }
-
-    a.bind(schedule_next);
-    comment("schedule_next");
-    {
-        Label schedule = a.newLabel(), skip_long_schedule = a.newLabel();
-
-        /* ARG3 contains reds_used at this point */
-
-        a.cmp(start_time, imm(0));
-        a.short_().je(schedule);
-        {
-            a.mov(ARG1, c_p);
-            a.mov(ARG2, start_time);
-
-            /* Spill reds_used in start_time slot */
-            a.mov(start_time, ARG3);
-
-            a.mov(ARG3, start_time_i);
-            runtime_call<3>(check_monitor_long_schedule);
-
-            /* Restore reds_used */
-            a.mov(ARG3, start_time);
-        }
-        a.bind(schedule);
-
-#ifdef ERLANG_FRAME_POINTERS
-        if (erts_frame_layout == ERTS_FRAME_LAYOUT_FP_RA) {
-            /* Kill the current frame pointer so that misc jobs that execute
-             * during `erts_schedule` aren't attributed to the function we
-             * were scheduled out of. */
-            a.sub(frame_pointer, frame_pointer);
-        }
-#endif
-
-        mov_imm(ARG1, 0);
-        a.mov(ARG2, c_p);
-#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
-        runtime_call<3>(erts_debug_schedule);
-#else
-        runtime_call<3>(erts_schedule);
-#endif
-        a.mov(c_p, RET);
-
-#ifdef ERTS_MSACC_EXTENDED_STATES
-        a.lea(ARG1,
-              x86::qword_ptr(registers,
-                             offsetof(ErtsSchedulerRegisters,
-                                      aux_regs.d.erts_msacc_cache)));
-        runtime_call<1>(erts_msacc_update_cache);
-#endif
-
-        a.mov(ARG1, imm((UWord)&erts_system_monitor_long_schedule));
-        a.cmp(x86::qword_ptr(ARG1), imm(0));
-        a.mov(start_time, imm(0));
-        a.short_().je(skip_long_schedule);
-        {
-            /* Enable long schedule test */
-            runtime_call<0>(erts_timestamp_millis);
-            a.mov(start_time, RET);
-            a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
-            a.mov(start_time_i, RET);
-        }
-        a.bind(skip_long_schedule);
-
-        /* Copy arguments */
-        a.mov(ARG1, c_p);
-        load_x_reg_array(ARG2);
-        runtime_call<2>(copy_in_registers);
-
-        /* Setup reduction counting */
-        a.mov(FCALLS, x86::qword_ptr(c_p, offsetof(Process, fcalls)));
-        a.mov(x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])), FCALLS);
-
-#ifdef DEBUG
-        a.mov(x86::qword_ptr(c_p, offsetof(Process, debug_reds_in)), FCALLS);
-#endif
-
-        /* Check whether save calls is on */
-        a.mov(ARG1, c_p);
-        a.mov(ARG2, imm(ERTS_PSD_SAVED_CALLS_BUF));
-        runtime_call<2>(erts_psd_get);
-
-        /* Read the active code index, overriding it with
-         * ERTS_SAVE_CALLS_CODE_IX when save_calls is enabled (RET != 0). */
-        a.test(RET, RET);
-        a.mov(ARG1, imm(&the_active_code_index));
-        a.mov(ARG2, imm(ERTS_SAVE_CALLS_CODE_IX));
-        a.mov(active_code_ix.r32(), x86::dword_ptr(ARG1));
-        a.cmovnz(active_code_ix, ARG2);
-
-        /* Start executing the Erlang process. Note that reductions have
-         * already been set up above. */
-        emit_leave_runtime<Update::eStack | Update::eHeap>();
-
-        /* Check if we are just returning from a dirty nif/bif call and if so we
-         * need to do a bit of cleaning up before continuing. */
-        a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
-        a.cmp(x86::qword_ptr(RET), imm(op_call_nif_WWW));
-        a.je(labels[dispatch_nif]);
-        a.cmp(x86::qword_ptr(RET), imm(op_call_bif_W));
-        a.je(labels[dispatch_bif]);
-        a.jmp(RET);
-    }
-
-    /* Processes may jump to the exported entry points below, executing on the
-     * Erlang stack when entering. These are separate from the `_local` labels
-     * above as we don't want to worry about which stack we're on when the
-     * cases overlap. */
-
-    /* `ga->get_context_switch()`
-     *
-     * The *next* instruction pointer is provided in ARG3, and must be preceded
-     * by an ErtsCodeMFA. */
-    a.bind(labels[context_switch]);
-    {
-        emit_enter_runtime<Update::eStack | Update::eHeap>();
-
-        a.jmp(context_switch_local);
-    }
-
-    /* `ga->get_context_switch_simplified()`
-     *
-     * The next instruction pointer is provided in ARG3, which does not need to
-     * point past an ErtsCodeMFA as the process structure has already been
-     * updated. */
-    a.bind(labels[context_switch_simplified]);
-    {
-        emit_enter_runtime<Update::eStack | Update::eHeap>();
-
-        a.jmp(context_switch_simplified_local);
-    }
-
-    /* `ga->get_do_schedule()`
-     *
-     * `c_p->i` must be set prior to jumping here. */
-    a.bind(labels[do_schedule]);
-    {
-        emit_enter_runtime<Update::eStack | Update::eHeap>();
-
-        a.jmp(do_schedule_local);
-    }
-}
-
 enum jit_actions : uint32_t {
     JIT_NOACTION = 0,
     JIT_REGISTER_FN,
@@ -932,8 +616,3 @@ extern "C"
         ba->patchStrings(rw_base, string_table);
     }
 }
-
-const uint8_t *BeamAssembler::nops[3] = {nop1, nop2, nop3};
-const uint8_t BeamAssembler::nop1[1] = {0x90};
-const uint8_t BeamAssembler::nop2[2] = {0x66, 0x90};
-const uint8_t BeamAssembler::nop3[3] = {0x0F, 0x1F, 0x00};
diff --git a/erts/emulator/beam/jit/x86/beam_asm_perf.cpp b/erts/emulator/beam/jit/beam_jit_perf.cpp
similarity index 99%
rename from erts/emulator/beam/jit/x86/beam_asm_perf.cpp
rename to erts/emulator/beam/jit/beam_jit_perf.cpp
index 5f7c274195..5ed9415e9f 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_perf.cpp
+++ b/erts/emulator/beam/jit/beam_jit_perf.cpp
@@ -250,5 +250,7 @@ void beamasm_init_perf() {
 
 void beamasm_update_perf_info(std::string modulename,
                               std::vector<BeamAssembler::AsmRange> &ranges) {
+    (void)modulename;
+    (void)ranges;
 }
 #endif
diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp
index cebf9675b8..55a7f7f5b3 100644
--- a/erts/emulator/beam/jit/x86/beam_asm.hpp
+++ b/erts/emulator/beam/jit/x86/beam_asm.hpp
@@ -166,102 +166,23 @@ protected:
 public:
     static bool hasCpuFeature(uint32_t featureId);
 
-    BeamAssembler() : code() {
-        /* Setup with default code info */
-        Error err = code.init(hostEnvironment());
-        ERTS_ASSERT(!err && "Failed to init codeHolder");
+    BeamAssembler();
+    BeamAssembler(const std::string &log);
 
-        err = code.newSection(&rodata,
-                              ".rodata",
-                              SIZE_MAX,
-                              Section::kFlagConst,
-                              8);
-        ERTS_ASSERT(!err && "Failed to create .rodata section");
+    ~BeamAssembler();
 
-        err = code.attach(&a);
-
-        ERTS_ASSERT(!err && "Failed to attach codeHolder");
-#ifdef DEBUG
-        a.addValidationOptions(BaseEmitter::kValidationOptionAssembler);
-#endif
-        a.addEncodingOptions(BaseEmitter::kEncodingOptionOptimizeForSize);
-        code.setErrorHandler(this);
-    }
-
-    BeamAssembler(const std::string &log) : BeamAssembler() {
-        if (erts_jit_asm_dump) {
-            setLogger(log + ".asm");
-        }
-    }
-
-    ~BeamAssembler() {
-        if (logger.file())
-            fclose(logger.file());
-    }
-
-    void *getBaseAddress() {
-        ASSERT(code.hasBaseAddress());
-        return (void *)code.baseAddress();
-    }
-
-    size_t getOffset() {
-        return a.offset();
-    }
+    void *getBaseAddress();
+    size_t getOffset();
 
 protected:
     void _codegen(JitAllocator *allocator,
                   const void **executable_ptr,
-                  void **writable_ptr) {
-        Error err = code.flatten();
-        ERTS_ASSERT(!err && "Could not flatten code");
-        err = code.resolveUnresolvedLinks();
-        ERTS_ASSERT(!err && "Could not resolve all links");
-
-        /* Verify that all labels are bound */
-#ifdef DEBUG
-        for (auto e : code.labelEntries()) {
-            if (!e->isBound()) {
-                erts_exit(ERTS_ABORT_EXIT, "Label %s is not bound", e->name());
-            }
-        }
-#endif
-
-        err = allocator->alloc(const_cast<void **>(executable_ptr),
-                               writable_ptr,
-                               code.codeSize() + 16);
-
-        if (err == ErrorCode::kErrorTooManyHandles) {
-            ERTS_ASSERT(!"Failed to allocate module code: "
-                         "out of file descriptors");
-        } else if (err) {
-            ERTS_ASSERT("Failed to allocate module code");
-        }
+                  void **writable_ptr);
 
-        code.relocateToBase((uint64_t)*executable_ptr);
-        code.copyFlattenedData(*writable_ptr,
-                               code.codeSize(),
-                               CodeHolder::kCopyPadSectionBuffer);
-#ifdef DEBUG
-        if (FileLogger *l = dynamic_cast<FileLogger *>(code.logger()))
-            if (FILE *f = l->file())
-                fprintf(f, "; CODE_SIZE: %zd\n", code.codeSize());
-#endif
-    }
+    void *getCode(Label label);
+    byte *getCode(char *labelName);
 
-    void *getCode(Label label) {
-        ASSERT(label.isValid());
-        return (char *)getBaseAddress() + code.labelOffsetFromBase(label);
-    }
-
-    byte *getCode(char *labelName) {
-        return (byte *)getCode(code.labelByName(labelName, strlen(labelName)));
-    }
-
-    void handleError(Error err, const char *message, BaseEmitter *origin) {
-        comment(message);
-        fflush(logger.file());
-        ASSERT(0 && "Fault instruction encode");
-    }
+    void handleError(Error err, const char *message, BaseEmitter *origin);
 
     constexpr x86::Mem getRuntimeStackRef() const {
         int base = offsetof(ErtsSchedulerRegisters, aux_regs.d.runtime_stack);
@@ -888,27 +809,10 @@ protected:
 public:
     void embed_rodata(const char *labelName, const char *buff, size_t size);
     void embed_bss(const char *labelName, size_t size);
-
     void embed_zeros(size_t size);
 
-    void setLogger(std::string log) {
-        FILE *f = fopen(log.data(), "w+");
-
-        /* FIXME: Don't crash when loading multiple modules with the same name.
-         *
-         * setLogger(nullptr) disables logging. */
-        if (f) {
-            setvbuf(f, NULL, _IONBF, 0);
-        }
-
-        setLogger(f);
-    }
-
-    void setLogger(FILE *log) {
-        logger.setFile(log);
-        logger.setIndentation(FormatOptions::kIndentationCode, 4);
-        code.setLogger(&logger);
-    }
+    void setLogger(std::string log);
+    void setLogger(FILE *log);
 
     template<typename... Ts>
     void comment(const char *format, Ts... args) {
@@ -931,10 +835,6 @@ public:
 
     void update_gdb_jit_info(std::string modulename,
                              std::vector<AsmRange> &functions);
-
-    void embed(void *data, uint32_t size) {
-        a.embed((char *)data, size);
-    }
 };
 
 class BeamGlobalAssembler : public BeamAssembler {
diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp
index 29dc64af5e..442a8fd0f7 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp
+++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp
@@ -67,6 +67,11 @@ extern "C" void beamasm_dump_sizes() {
 }
 #endif
 
+ErtsCodePtr BeamModuleAssembler::getCode(BeamLabel label) {
+    ASSERT(label < labels.size() + 1);
+    return (ErtsCodePtr)getCode(labels[label]);
+}
+
 BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga,
                                          Eterm mod,
                                          unsigned num_labels)
@@ -149,46 +154,6 @@ BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga,
     }
 }
 
-ErtsCodePtr BeamModuleAssembler::getCode(unsigned label) {
-    ASSERT(label < labels.size() + 1);
-    return (ErtsCodePtr)getCode(labels[label]);
-}
-
-void BeamAssembler::embed_rodata(const char *labelName,
-                                 const char *buff,
-                                 size_t size) {
-    Label label = a.newNamedLabel(labelName);
-
-    a.section(rodata);
-    a.bind(label);
-    a.embed(buff, size);
-    a.section(code.textSection());
-}
-
-void BeamAssembler::embed_bss(const char *labelName, size_t size) {
-    Label label = a.newNamedLabel(labelName);
-
-    /* Reuse rodata section for now */
-    a.section(rodata);
-    a.bind(label);
-    embed_zeros(size);
-    a.section(code.textSection());
-}
-
-void BeamAssembler::embed_zeros(size_t size) {
-    static constexpr size_t buf_size = 16384;
-    static const char zeros[buf_size] = {};
-
-    while (size >= buf_size) {
-        a.embed(zeros, buf_size);
-        size -= buf_size;
-    }
-
-    if (size > 0) {
-        a.embed(zeros, size);
-    }
-}
-
 Label BeamModuleAssembler::embed_vararg_rodata(const std::vector<ArgVal> &args,
                                                int y_offset) {
     Label label = a.newLabel();
@@ -446,129 +411,6 @@ void BeamModuleAssembler::emit_call_error_handler() {
     emit_nyi("call_error_handler should never be called");
 }
 
-void BeamModuleAssembler::codegen(JitAllocator *allocator,
-                                  const void **executable_ptr,
-                                  void **writable_ptr,
-                                  const BeamCodeHeader *in_hdr,
-                                  const BeamCodeHeader **out_exec_hdr,
-                                  BeamCodeHeader **out_rw_hdr) {
-    const BeamCodeHeader *code_hdr_exec;
-    BeamCodeHeader *code_hdr_rw;
-
-    codegen(allocator, executable_ptr, writable_ptr);
-
-    {
-        auto offset = code.labelOffsetFromBase(codeHeader);
-
-        auto base_exec = (const char *)(*executable_ptr);
-        code_hdr_exec = (const BeamCodeHeader *)&base_exec[offset];
-
-        auto base_rw = (const char *)(*writable_ptr);
-        code_hdr_rw = (BeamCodeHeader *)&base_rw[offset];
-    }
-
-    sys_memcpy(code_hdr_rw, in_hdr, sizeof(BeamCodeHeader));
-    code_hdr_rw->on_load = getOnLoad();
-
-    for (unsigned i = 0; i < functions.size(); i++) {
-        ErtsCodeInfo *ci = (ErtsCodeInfo *)getCode(functions[i]);
-        code_hdr_rw->functions[i] = ci;
-    }
-
-    char *module_end = (char *)code.baseAddress() + a.offset();
-    code_hdr_rw->functions[functions.size()] = (ErtsCodeInfo *)module_end;
-
-    *out_exec_hdr = code_hdr_exec;
-    *out_rw_hdr = code_hdr_rw;
-}
-
-void BeamModuleAssembler::codegen(JitAllocator *allocator,
-                                  const void **executable_ptr,
-                                  void **writable_ptr) {
-    _codegen(allocator, executable_ptr, writable_ptr);
-
-#ifndef WIN32
-    if (functions.size()) {
-        char *buff = (char *)erts_alloc(ERTS_ALC_T_TMP, 1024);
-        std::vector<AsmRange> ranges;
-        std::string name = getAtom(mod);
-        ranges.reserve(functions.size() + 2);
-
-        /* Push info about the header */
-        ranges.push_back({.start = (ErtsCodePtr)getBaseAddress(),
-                          .stop = getCode(functions[0]),
-                          .name = name + "::codeHeader"});
-
-        for (unsigned i = 0; i < functions.size(); i++) {
-            ErtsCodePtr start, stop;
-            const ErtsCodeInfo *ci;
-            int n;
-
-            start = getCode(functions[i]);
-            ci = (const ErtsCodeInfo *)start;
-
-            n = erts_snprintf(buff,
-                              1024,
-                              "%T:%T/%d",
-                              ci->mfa.module,
-                              ci->mfa.function,
-                              ci->mfa.arity);
-            stop = ((const char *)erts_codeinfo_to_code(ci)) +
-                   BEAM_ASM_FUNC_PROLOGUE_SIZE;
-
-            /* We use a different symbol for CodeInfo and the Prologue
-               in order for the perf disassembly to be better. */
-            std::string name(buff, n);
-            ranges.push_back({.start = start,
-                              .stop = stop,
-                              .name = name + "-CodeInfoPrologue"});
-
-            /* The actual code */
-            start = stop;
-            if (i + 1 < functions.size()) {
-                stop = getCode(functions[i + 1]);
-            } else {
-                stop = getCode(labels.size());
-            }
-
-            ranges.push_back({.start = start, .stop = stop, .name = name});
-        }
-
-        /* Push info about the footer */
-        ranges.push_back(
-                {.start = ranges.back().stop,
-                 .stop = (ErtsCodePtr)(code.baseAddress() + code.codeSize()),
-                 .name = name + "::codeFooter"});
-
-        update_gdb_jit_info(name, ranges);
-        beamasm_update_perf_info(name, ranges);
-        erts_free(ERTS_ALC_T_TMP, buff);
-    }
-#endif
-}
-
-void BeamModuleAssembler::codegen(char *buff, size_t len) {
-    code.flatten();
-    code.resolveUnresolvedLinks();
-    ERTS_ASSERT(code.codeSize() <= len);
-    code.relocateToBase((uint64_t)buff);
-    code.copyFlattenedData(buff,
-                           code.codeSize(),
-                           CodeHolder::kCopyPadSectionBuffer);
-}
-
-BeamCodeHeader *BeamModuleAssembler::getCodeHeader() {
-    return (BeamCodeHeader *)getCode(codeHeader);
-}
-
-const ErtsCodeInfo *BeamModuleAssembler::getOnLoad() {
-    if (on_load.isValid()) {
-        return erts_code_to_codeinfo((ErtsCodePtr)getCode(on_load));
-    } else {
-        return 0;
-    }
-}
-
 unsigned BeamModuleAssembler::patchCatches(char *rw_base) {
     unsigned catch_no = BEAM_CATCHES_NIL;
 
diff --git a/erts/emulator/beam/jit/x86/instr_trace.cpp b/erts/emulator/beam/jit/x86/instr_trace.cpp
index f04f3493a8..f464152be9 100644
--- a/erts/emulator/beam/jit/x86/instr_trace.cpp
+++ b/erts/emulator/beam/jit/x86/instr_trace.cpp
@@ -199,46 +199,17 @@ void BeamModuleAssembler::emit_i_return_time_trace() {
     emit_return();
 }
 
-static void i_return_to_trace(Process *c_p) {
-    if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) {
-        ErtsCodePtr return_to_address;
-        Uint *cpp;
-
-        cpp = (Uint *)c_p->stop;
-        ASSERT(is_CP(cpp[0]));
-
-        for (;;) {
-            erts_inspect_frame(cpp, &return_to_address);
-
-            if (BeamIsReturnTrace(return_to_address)) {
-                cpp += CP_SIZE + 2;
-            } else if (BeamIsReturnTimeTrace(return_to_address)) {
-                cpp += CP_SIZE + 1;
-            } else if (BeamIsReturnToTrace(return_to_address)) {
-                cpp += CP_SIZE;
-            } else {
-                break;
-            }
-        }
-
-        ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
-        erts_trace_return_to(c_p, return_to_address);
-        ERTS_REQ_PROC_MAIN_LOCK(c_p);
-    }
-}
-
 void BeamModuleAssembler::emit_i_return_to_trace() {
     emit_enter_runtime<Update::eStack | Update::eHeap>();
 
     a.mov(ARG1, c_p);
-    runtime_call<1>(i_return_to_trace);
+    runtime_call<1>(beam_jit_return_to_trace);
 
     emit_leave_runtime<Update::eStack | Update::eHeap>();
 
     /* Remove the zero-sized stack frame. (Will actually do nothing if
      * the native stack is used.) */
     emit_deallocate(ArgVal(ArgVal::u, 0));
-
     emit_return();
 }
 
diff --git a/erts/emulator/beam/jit/x86/process_main.cpp b/erts/emulator/beam/jit/x86/process_main.cpp
new file mode 100644
index 0000000000..6016b47f10
--- /dev/null
+++ b/erts/emulator/beam/jit/x86/process_main.cpp
@@ -0,0 +1,350 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2020-2020. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "beam_asm.hpp"
+
+extern "C"
+{
+#include "bif.h"
+#include "beam_common.h"
+#include "code_ix.h"
+#include "export.h"
+}
+
+const uint8_t *BeamAssembler::nops[3] = {nop1, nop2, nop3};
+const uint8_t BeamAssembler::nop1[1] = {0x90};
+const uint8_t BeamAssembler::nop2[2] = {0x66, 0x90};
+const uint8_t BeamAssembler::nop3[3] = {0x0F, 0x1F, 0x00};
+
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+static Process *erts_debug_schedule(ErtsSchedulerData *esdp,
+                                    Process *c_p,
+                                    int calls) {
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    c_p = erts_schedule(esdp, c_p, calls);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+    ERTS_REQ_PROC_MAIN_LOCK(c_p);
+    PROCESS_MAIN_CHK_LOCKS(c_p);
+    return c_p;
+}
+#endif
+
+/* void process_main(ErtsSchedulerData *esdp); */
+void BeamGlobalAssembler::emit_process_main() {
+    Label context_switch_local = a.newLabel(),
+          context_switch_simplified_local = a.newLabel(),
+          do_schedule_local = a.newLabel(), schedule_next = a.newLabel();
+
+    const x86::Mem start_time_i =
+            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time_i));
+    const x86::Mem start_time =
+            getSchedulerRegRef(offsetof(ErtsSchedulerRegisters, start_time));
+
+    /* Allocate the register structure on the stack to allow computing the
+     * runtime stack address from it, greatly reducing the cost of stack
+     * swapping. */
+    a.sub(x86::rsp, imm(sizeof(ErtsSchedulerRegisters) + ERTS_CACHE_LINE_SIZE));
+    a.and_(x86::rsp, imm(~ERTS_CACHE_LINE_MASK));
+
+    a.mov(x86::qword_ptr(ARG1, offsetof(ErtsSchedulerData, registers)),
+          x86::rsp);
+
+    /* Center `registers` at the base of x_reg_array so we can use negative
+     * 8-bit displacement to address the commonly used aux_regs, located at the
+     * start of the ErtsSchedulerRegisters struct. */
+    a.lea(registers,
+          x86::qword_ptr(x86::rsp,
+                         offsetof(ErtsSchedulerRegisters, x_reg_array.d)));
+
+    load_erl_bits_state(ARG1);
+    runtime_call<1>(erts_bits_init_state);
+
+#if defined(DEBUG) && defined(NATIVE_ERLANG_STACK)
+    /* Save stack bounds so they can be tested without clobbering anything. */
+    runtime_call<0>(erts_get_stacklimit);
+
+    a.mov(getSchedulerRegRef(
+                  offsetof(ErtsSchedulerRegisters, runtime_stack_end)),
+          RET);
+    a.mov(getSchedulerRegRef(
+                  offsetof(ErtsSchedulerRegisters, runtime_stack_start)),
+          x86::rsp);
+#elif !defined(NATIVE_ERLANG_STACK)
+    /* Save the initial SP of the thread so that we can verify that it
+     * doesn't grow. */
+#    ifdef JIT_HARD_DEBUG
+    a.mov(getInitialSPRef(), x86::rsp);
+#    endif
+
+    /* Manually do an `emit_enter_runtime` to match the `emit_leave_runtime`
+     * below. We avoid `emit_enter_runtime` because it may do additional
+     * assertions that may currently fail.
+     *
+     * IMPORTANT: We must ensure that this sequence leaves the stack
+     * aligned on a 16-byte boundary. */
+    a.mov(getRuntimeStackRef(), x86::rsp);
+    a.sub(x86::rsp, imm(15));
+    a.and_(x86::rsp, imm(-16));
+#endif
+
+    a.mov(start_time_i, imm(0));
+    a.mov(start_time, imm(0));
+
+    mov_imm(c_p, 0);
+    mov_imm(FCALLS, 0);
+    mov_imm(ARG3, 0); /* Set reds_used for erts_schedule call */
+
+    a.jmp(schedule_next);
+
+    a.bind(do_schedule_local);
+    {
+        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
+        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
+        a.sub(ARG3, FCALLS);
+
+        a.jmp(schedule_next);
+    }
+
+    a.bind(context_switch_local);
+    comment("Context switch, unknown arity/MFA");
+    {
+        Sint arity_offset = offsetof(ErtsCodeMFA, arity) - sizeof(ErtsCodeMFA);
+
+        a.mov(ARG1, x86::qword_ptr(ARG3, arity_offset));
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), ARG1);
+
+        a.lea(ARG1, x86::qword_ptr(ARG3, -(Sint)sizeof(ErtsCodeMFA)));
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), ARG1);
+
+        /* !! Fall through !! */
+    }
+
+    a.bind(context_switch_simplified_local);
+    comment("Context switch, known arity and MFA");
+    {
+        Label not_exiting = a.newLabel();
+
+#ifdef ERLANG_FRAME_POINTERS
+        /* Kill the current frame pointer to avoid confusing `perf` and similar
+         * tools. */
+        a.sub(frame_pointer, frame_pointer);
+#endif
+
+#ifdef DEBUG
+        Label check_i = a.newLabel();
+        /* Check that ARG3 is set to a valid CP. */
+        a.test(ARG3, imm(_CPMASK));
+        a.je(check_i);
+        a.ud2();
+        a.bind(check_i);
+#endif
+
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG3);
+
+#if defined(JIT_HARD_DEBUG) && defined(ERLANG_FRAME_POINTERS)
+        a.mov(ARG1, c_p);
+        a.mov(ARG2, x86::qword_ptr(c_p, offsetof(Process, frame_pointer)));
+        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, stop)));
+
+        runtime_call<3>(erts_validate_stack);
+#endif
+
+#ifdef WIN32
+        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.value)));
+#else
+        a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.counter)));
+#endif
+
+        a.test(ARG1d, imm(ERTS_PSFLG_EXITING));
+        a.short_().je(not_exiting);
+        {
+            comment("Process exiting");
+
+            /* We load the beam_exit from memory because it has not yet been set
+             * when the global assembler is created. */
+            a.mov(ARG1, imm(&beam_exit));
+            a.mov(ARG1, x86::qword_ptr(ARG1));
+            a.mov(x86::qword_ptr(c_p, offsetof(Process, i)), ARG1);
+            a.mov(x86::qword_ptr(c_p, offsetof(Process, arity)), imm(0));
+            a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), imm(0));
+            a.jmp(do_schedule_local);
+        }
+        a.bind(not_exiting);
+
+        /* Figure out reds_used. def_arg_reg[5] = REDS_IN */
+        a.mov(ARG3, x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])));
+        a.sub(ARG3, FCALLS);
+
+        /* Spill reds_used to FCALLS as we no longer need that value */
+        a.mov(FCALLS, ARG3);
+
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        runtime_call<2>(copy_out_registers);
+
+        /* Restore reds_used from FCALLS */
+        a.mov(ARG3, FCALLS);
+
+        /* !! Fall through !! */
+    }
+
+    a.bind(schedule_next);
+    comment("schedule_next");
+    {
+        Label schedule = a.newLabel(), skip_long_schedule = a.newLabel();
+
+        /* ARG3 contains reds_used at this point */
+
+        a.cmp(start_time, imm(0));
+        a.short_().je(schedule);
+        {
+            a.mov(ARG1, c_p);
+            a.mov(ARG2, start_time);
+
+            /* Spill reds_used in start_time slot */
+            a.mov(start_time, ARG3);
+
+            a.mov(ARG3, start_time_i);
+            runtime_call<3>(check_monitor_long_schedule);
+
+            /* Restore reds_used */
+            a.mov(ARG3, start_time);
+        }
+        a.bind(schedule);
+
+#ifdef ERLANG_FRAME_POINTERS
+        if (erts_frame_layout == ERTS_FRAME_LAYOUT_FP_RA) {
+            /* Kill the current frame pointer so that misc jobs that execute
+             * during `erts_schedule` aren't attributed to the function we
+             * were scheduled out of. */
+            a.sub(frame_pointer, frame_pointer);
+        }
+#endif
+
+        mov_imm(ARG1, 0);
+        a.mov(ARG2, c_p);
+#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
+        runtime_call<3>(erts_debug_schedule);
+#else
+        runtime_call<3>(erts_schedule);
+#endif
+        a.mov(c_p, RET);
+
+#ifdef ERTS_MSACC_EXTENDED_STATES
+        a.lea(ARG1,
+              x86::qword_ptr(registers,
+                             offsetof(ErtsSchedulerRegisters,
+                                      aux_regs.d.erts_msacc_cache)));
+        runtime_call<1>(erts_msacc_update_cache);
+#endif
+
+        a.mov(ARG1, imm((UWord)&erts_system_monitor_long_schedule));
+        a.cmp(x86::qword_ptr(ARG1), imm(0));
+        a.mov(start_time, imm(0));
+        a.short_().je(skip_long_schedule);
+        {
+            /* Enable long schedule test */
+            runtime_call<0>(erts_timestamp_millis);
+            a.mov(start_time, RET);
+            a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
+            a.mov(start_time_i, RET);
+        }
+        a.bind(skip_long_schedule);
+
+        /* Copy arguments */
+        a.mov(ARG1, c_p);
+        load_x_reg_array(ARG2);
+        runtime_call<2>(copy_in_registers);
+
+        /* Setup reduction counting */
+        a.mov(FCALLS, x86::qword_ptr(c_p, offsetof(Process, fcalls)));
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, def_arg_reg[5])), FCALLS);
+
+#ifdef DEBUG
+        a.mov(x86::qword_ptr(c_p, offsetof(Process, debug_reds_in)), FCALLS);
+#endif
+
+        /* Check whether save calls is on */
+        a.mov(ARG1, c_p);
+        a.mov(ARG2, imm(ERTS_PSD_SAVED_CALLS_BUF));
+        runtime_call<2>(erts_psd_get);
+
+        /* Read the active code index, overriding it with
+         * ERTS_SAVE_CALLS_CODE_IX when save_calls is enabled (RET != 0). */
+        a.test(RET, RET);
+        a.mov(ARG1, imm(&the_active_code_index));
+        a.mov(ARG2, imm(ERTS_SAVE_CALLS_CODE_IX));
+        a.mov(active_code_ix.r32(), x86::dword_ptr(ARG1));
+        a.cmovnz(active_code_ix, ARG2);
+
+        /* Start executing the Erlang process. Note that reductions have
+         * already been set up above. */
+        emit_leave_runtime<Update::eStack | Update::eHeap>();
+
+        /* Check if we are just returning from a dirty nif/bif call and if so we
+         * need to do a bit of cleaning up before continuing. */
+        a.mov(RET, x86::qword_ptr(c_p, offsetof(Process, i)));
+        a.cmp(x86::qword_ptr(RET), imm(op_call_nif_WWW));
+        a.je(labels[dispatch_nif]);
+        a.cmp(x86::qword_ptr(RET), imm(op_call_bif_W));
+        a.je(labels[dispatch_bif]);
+        a.jmp(RET);
+    }
+
+    /* Processes may jump to the exported entry points below, executing on the
+     * Erlang stack when entering. These are separate from the `_local` labels
+     * above as we don't want to worry about which stack we're on when the
+     * cases overlap. */
+
+    /* `ga->get_context_switch()`
+     *
+     * The *next* instruction pointer is provided in ARG3, and must be preceded
+     * by an ErtsCodeMFA. */
+    a.bind(labels[context_switch]);
+    {
+        emit_enter_runtime<Update::eStack | Update::eHeap>();
+
+        a.jmp(context_switch_local);
+    }
+
+    /* `ga->get_context_switch_simplified()`
+     *
+     * The next instruction pointer is provided in ARG3, which does not need to
+     * point past an ErtsCodeMFA as the process structure has already been
+     * updated. */
+    a.bind(labels[context_switch_simplified]);
+    {
+        emit_enter_runtime<Update::eStack | Update::eHeap>();
+
+        a.jmp(context_switch_simplified_local);
+    }
+
+    /* `ga->get_do_schedule()`
+     *
+     * `c_p->i` must be set prior to jumping here. */
+    a.bind(labels[do_schedule]);
+    {
+        emit_enter_runtime<Update::eStack | Update::eHeap>();
+
+        a.jmp(do_schedule_local);
+    }
+}
-- 
2.26.2

Places

File 1104-jit-Break-out-architecture-agnostic-parts-to-prepare.patch of Package erlang (Revision 0eb87917a75aae8f396ec39332087beb)

Places