File 1791-Allocator-instances-for-dirty-schedulers.p... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1791-Allocator-instances-for-dirty-schedulers.patch of Package erlang (Revision de16d5bb9b1e73f72aa79d2ecdf361ff)

Currently displaying revision de16d5bb9b1e73f72aa79d2ecdf361ff , Show latest

From 651e4e09697141fa9d54024818233599973f1a3c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 27 Aug 2021 16:18:44 +0200
Subject: [PATCH] Allocator instances for dirty schedulers

---
 erts/doc/src/erts_alloc.xml            |  21 ++
 erts/emulator/beam/bif.tab             |   2 +
 erts/emulator/beam/erl_alloc.c         | 196 +++++++++-----
 erts/emulator/beam/erl_alloc.h         |  39 ++-
 erts/emulator/beam/erl_alloc_util.c    |  77 +++---
 erts/emulator/beam/erl_alloc_util.h    |   7 +-
 erts/emulator/beam/erl_bif_info.c      |   8 +-
 erts/emulator/beam/erl_bif_trace.c     |   1 +
 erts/emulator/beam/erl_db.c            |  22 +-
 erts/emulator/beam/erl_db.h            |   4 +-
 erts/emulator/beam/erl_gc.c            |   1 +
 erts/emulator/beam/erl_init.c          |  18 +-
 erts/emulator/beam/erl_msacc.c         |  15 +-
 erts/emulator/beam/erl_process.c       | 344 ++++++++++++++++---------
 erts/emulator/beam/erl_process.h       |  14 +-
 erts/emulator/beam/io.c                |   1 +
 erts/emulator/sys/common/erl_mseg.c    |  10 +-
 erts/emulator/sys/common/erl_mseg.h    |   4 +-
 erts/emulator/test/timer_bif_SUITE.erl |   2 +-
 erts/etc/common/erlexec.c              |   1 +
 erts/preloaded/ebin/erlang.beam        | Bin 130732 -> 130816 bytes
 erts/preloaded/ebin/erts_internal.beam | Bin 26432 -> 26596 bytes
 erts/preloaded/src/erlang.erl          |   4 +-
 erts/preloaded/src/erts_internal.erl   |   7 +
 lib/tools/src/instrument.erl           |   4 +-
 25 files changed, 517 insertions(+), 285 deletions(-)

diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml
index cd98123c5c..a5aff575f5 100644
--- a/erts/doc/src/erts_alloc.xml
+++ b/erts/doc/src/erts_alloc.xml
@@ -839,6 +839,27 @@
             condition if the user limit on the amount of locked memory is
             reached.</p>
         </item>
+        <tag><marker id="Mdai"/><c><![CDATA[+Mdai max|<amount>]]></c></tag>
+        <item>
+          <p>
+            Set amount of dirty allocator instances used. Defaults to
+            <c>0</c>. That is, by default no instances will be used.
+            The maximum amount of instances equals the amount of dirty
+            CPU schedulers on the system.
+          </p>
+          <p>
+            By default, each normal scheduler thread has its own allocator
+            instance for each allocator. All other threads in the system,
+            including dirty schedulers, share one instance for each allocator.
+            By enabling dirty allocator instances, dirty schedulers will
+            get and use their own set of allocator instances. Note that these
+            instances are not exclusive to each dirty scheduler, but instead
+            shared among dirty schedulers. The more instances used the less
+            risk of lock contention on these allocator instances. Memory
+            consumption do however increase with increased amount of dirty
+            allocator instances.
+          </p>
+        </item>
       </taglist>
     </section>
   </section>
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 9ba49376dd..9a1b96d847 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -199,6 +199,8 @@ bif erts_internal:ets_super_user/1
 
 bif erts_internal:spawn_request/4
 bif erts_internal:dist_spawn_request/4
+bif erts_internal:no_aux_work_threads/0
+
 bif erlang:spawn_request_abandon/1
 
 # Static native functions in erts_literal_area_collector
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 2cea68a817..125a5a240e 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -148,6 +148,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
                                  ERTS_ALC_T_AINFO_REQ)
 
 ErtsAlcType_t erts_fix_core_allocator_ix;
+erts_tsd_key_t erts_thr_alloc_data_key;
+
+Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_alloc_instances);
 
 struct au_init {
     int enable;
@@ -186,6 +189,7 @@ typedef struct {
 #endif
     int trim_threshold;
     int top_pad;
+    int dirty_alloc_insts;
     AlcUInit_t alloc_util;
     struct {
 	char *mtrace;
@@ -461,7 +465,7 @@ set_default_test_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
     ip->enable			= 0; /* Disabled by default */
-    ip->thr_spec		= -1 * erts_no_schedulers;
+    ip->thr_spec		= -1;
     ip->astrat			= ERTS_ALC_S_FIRSTFIT;
     ip->init.aoff.crr_order     = FF_AOFF;
     ip->init.aoff.blk_order     = FF_BF;
@@ -488,10 +492,10 @@ set_default_test_alloc_opts(struct au_init *ip)
 
 
 static void
-adjust_tpref(struct au_init *ip, int no_sched)
+adjust_tpref(struct au_init *ip, int no_sched, int no_dirty_inst)
 {
     if (ip->thr_spec) {
-	ip->thr_spec = no_sched;
+        ip->thr_spec = no_sched + no_dirty_inst;
 	ip->thr_spec *= -1; /* thread preferred */
 
 	/* If default ... */
@@ -607,6 +611,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #endif
 	ERTS_DEFAULT_TRIM_THRESHOLD,
 	ERTS_DEFAULT_TOP_PAD,
+	0, /* Default dirty alloc instances */
 	ERTS_DEFAULT_ALCU_INIT,
     };
     size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
@@ -644,6 +649,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     hdbg_init();
 #endif
 
+    erts_tsd_key_create(&erts_thr_alloc_data_key, "erts_alc_data_key");
+
     lock_all_physical_memory = 0;
 
     ncpu = eaiop->ncpu;
@@ -686,6 +693,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #endif
     }
 
+    erts_no_dirty_alloc_instances = init.dirty_alloc_insts;
 
     /* Make adjustments for carrier migration support */
     init.temp_alloc.init.util.acul = 0;
@@ -729,18 +737,19 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     /* Only temp_alloc can use thread specific interface */
     if (init.temp_alloc.thr_spec)
-	init.temp_alloc.thr_spec = erts_no_schedulers;
+        init.temp_alloc.thr_spec = erts_no_schedulers + init.dirty_alloc_insts;
 
     /* Others must use thread preferred interface */
-    adjust_tpref(&init.sl_alloc, erts_no_schedulers);
-    adjust_tpref(&init.std_alloc, erts_no_schedulers);
-    adjust_tpref(&init.ll_alloc, erts_no_schedulers);
-    adjust_tpref(&init.eheap_alloc, erts_no_schedulers);
-    adjust_tpref(&init.binary_alloc, erts_no_schedulers);
-    adjust_tpref(&init.ets_alloc, erts_no_schedulers);
-    adjust_tpref(&init.driver_alloc, erts_no_schedulers);
-    adjust_tpref(&init.fix_alloc, erts_no_schedulers);
-    adjust_tpref(&init.literal_alloc, erts_no_schedulers);
+    adjust_tpref(&init.sl_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.std_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.ll_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.eheap_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.binary_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.ets_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.driver_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.fix_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.literal_alloc, erts_no_schedulers, init.dirty_alloc_insts);
+    adjust_tpref(&init.test_alloc, erts_no_schedulers, init.dirty_alloc_insts);
 
 
     /*
@@ -756,6 +765,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     refuse_af_strategy(&init.driver_alloc);
     refuse_af_strategy(&init.fix_alloc);
     refuse_af_strategy(&init.literal_alloc);
+    refuse_af_strategy(&init.test_alloc);
 
     if (!init.temp_alloc.thr_spec)
 	refuse_af_strategy(&init.temp_alloc);
@@ -763,6 +773,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     erts_mtrace_pre_init();
 #if HAVE_ERTS_MSEG
     init.mseg.nos = erts_no_schedulers;
+    init.mseg.ndai = init.dirty_alloc_insts;
     erts_mseg_init(&init.mseg);
 #endif
 
@@ -1038,12 +1049,12 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 	    if (!as0)
 		continue;
 	    if (init->thr_spec < 0) {
-		init->init.util.ts = i == 0;
+		init->init.util.ts = (i == 0 || erts_no_schedulers < i);
 		init->init.util.tspec = 0;
 		init->init.util.tpref = -1*init->thr_spec + 1;
 	    }
 	    else {
-		if (i != 0)
+		if (0 < i && i <= erts_no_schedulers)
 		    init->init.util.ts = 0;
 		else {
 		    if (astrat == ERTS_ALC_S_AFIT)
@@ -1749,6 +1760,24 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		    }
 		    bad_param(param, param+2);
 		    break;
+		case 'd':
+                    if (has_prefix("ai", param+2)) {
+                        arg = get_value(argv[i]+5, argv, &i);
+                        if (sys_strcmp("max", arg) == 0)
+                            init->dirty_alloc_insts = (int) erts_no_dirty_cpu_schedulers;
+                        else {
+                            Sint tmp;
+                            char *rest;
+                            errno = 0;
+                            tmp = (Sint) ErtsStrToSint(arg, &rest, 10);
+                            if (errno != 0 || rest == arg || tmp < 0)
+                                bad_value(param, param+4, arg);
+                            init->dirty_alloc_insts = (int) tmp;
+                        }
+                        break;
+                    }
+                    bad_param(param, param+2);
+		    break;
 		case 'u':
 		    if (has_prefix("ycs", argv[i]+3)) {
 			init->alloc_util.ycs
@@ -1817,6 +1846,10 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 	    argv[j++] = argv[i];
     }
     *argc = j;
+
+    if (init->dirty_alloc_insts > erts_no_dirty_cpu_schedulers)
+        init->dirty_alloc_insts = (int) erts_no_dirty_cpu_schedulers;
+
 }
 
 static char *type_no_str(ErtsAlcType_t n)
@@ -1837,46 +1870,92 @@ void
 erts_alloc_register_scheduler(void *vesdp)
 {
     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
-    int ix = (int) esdp->no;
+    int ix;
     int aix;
+    int normal_sched;
 
-    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
+    ASSERT(esdp == erts_get_scheduler_data());
+
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+        ix = (int) esdp->no;
+        ASSERT(0 < ix && ix <= erts_no_schedulers);
+        normal_sched = !0;
+    }
+    else if (!erts_no_dirty_alloc_instances) {
+        ix = 0;
+        normal_sched = 0;
+    }
+    else {
+        ix = (int) esdp->dirty_no;
+        ASSERT(ix > 0);
+        ix = ((ix - 1) % erts_no_dirty_alloc_instances) + 1;
+        ix += erts_no_schedulers;
+        ASSERT(erts_no_schedulers < ix
+               && ix <= (erts_no_schedulers
+                         + erts_no_dirty_alloc_instances));
+        normal_sched = 0;
+    }
+
+    esdp->aux_work_data.alloc_data.delayed_dealloc_handler = normal_sched;
+    esdp->aux_work_data.alloc_data.alc_ix = ix;
     for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
 	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
-	esdp->alloc_data.deallctr[aix] = NULL;
-	esdp->alloc_data.pref_ix[aix] = -1;
-	if (tspec->enabled) {
-	    if (!tspec->dd)
-		esdp->alloc_data.pref_ix[aix] = ix;
-	    else {
-		Allctr_t *allctr = tspec->allctr[ix];
-		ASSERT(allctr);
-		esdp->alloc_data.deallctr[aix] = allctr;
-		esdp->alloc_data.pref_ix[aix] = ix;
-	    }
-	}
+	esdp->aux_work_data.alloc_data.deallctr[aix] = NULL;
+        if (!normal_sched)
+            continue;
+	/*
+	 * Delayed dealloc is handled by normal schedulers,
+	 * but not by dirty schedulers...
+	 */
+	if (tspec->enabled && tspec->dd) {
+	    Allctr_t *allctr = tspec->allctr[ix];
+	    ASSERT(allctr);
+            esdp->aux_work_data.alloc_data.deallctr[aix] = allctr;
+        }
     }
+    erts_tsd_set(erts_thr_alloc_data_key, (void *) &esdp->aux_work_data.alloc_data);
 }
 
 void
-erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
-					    int *need_thr_progress,
-					    ErtsThrPrgrVal *thr_prgr_p,
-					    int *more_work)
+erts_alloc_register_delayed_dealloc_handler_thread(ErtsThrAllocData *tadp, int ix)
 {
-    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
     int aix;
+
+    /*
+     * Should not be a scheduler of any kind and should not
+     * handle an 'ix' reserved for normal schedulers...
+     */
+    ASSERT(!erts_get_scheduler_data());
+    ASSERT(0 == ix /* Aux thread... */
+           || /* Handler thread for dirty scheds instances... */
+           (erts_no_schedulers < ix
+            && ix <= (erts_no_schedulers
+                      + erts_no_dirty_alloc_instances)));
+
+    tadp->delayed_dealloc_handler = !0;
+    tadp->alc_ix = ix;
     for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
-	Allctr_t *allctr;
-	if (esdp)
-	    allctr = esdp->alloc_data.deallctr[aix];
-	else {
-	    ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
-	    if (tspec->enabled && tspec->dd)
-		allctr = tspec->allctr[0];
-	    else
-		allctr = NULL;
-	}
+        ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
+        tadp->deallctr[aix] = NULL;
+        if (tspec->enabled && tspec->dd) {
+            Allctr_t *allctr = tspec->allctr[ix];
+            ASSERT(allctr);
+            tadp->deallctr[aix] = allctr;
+        }
+    }
+    erts_tsd_set(erts_thr_alloc_data_key, (void *) tadp);
+}
+
+void
+erts_alloc_handle_delayed_dealloc(ErtsThrAllocData *thr_alloc_data,
+				  int *need_thr_progress,
+				  ErtsThrPrgrVal *thr_prgr_p,
+				  int *more_work)
+{
+    int aix;
+    ASSERT(thr_alloc_data);
+    for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
+	Allctr_t *allctr = thr_alloc_data->deallctr[aix];
 	if (allctr) {
 	    erts_alcu_check_delayed_dealloc(allctr,
 					    1,
@@ -1912,7 +1991,7 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
     if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
 	&& erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
 	ErtsAllocatorThrSpec_t *tspec;
-	int ix = ERTS_ALC_GET_THR_IX();
+        int ix = erts_get_thr_alloc_ix();
 	tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
 
 	if (ix < tspec->size) {
@@ -3001,8 +3080,8 @@ static void
 reply_alloc_info(void *vair)
 {
     ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
-    Uint sched_id = erts_get_scheduler_id();
-    int global_instances = air->req_sched == sched_id;
+    int tix = erts_get_thr_alloc_ix();
+    int global_instances = air->req_sched == tix;
     ErtsProcLocks rp_locks;
     Process *rp = air->proc;
     Eterm ref_copy = NIL, ai_list, msg = NIL;
@@ -3028,7 +3107,7 @@ reply_alloc_info(void *vair)
 				  ? erts_alcu_sz_info
 				  : erts_alcu_info);
 
-    rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
+    rp_locks = air->req_sched == tix ? ERTS_PROC_LOCK_MAIN : 0;
 
     sz = 0;
     hpp = NULL;
@@ -3196,11 +3275,11 @@ reply_alloc_info(void *vair)
 	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
 #if HAVE_ERTS_MSEG
 		alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
-		ainfo = erts_mseg_info(sched_id, NULL, NULL,
+		ainfo = erts_mseg_info(tix, NULL, NULL,
                                        hpp != NULL, air->only_sz, hpp, szp);
 		ainfo = erts_bld_tuple(hpp, szp, 3,
 				       alloc_atom,
-				       make_small(sched_id),
+				       make_small(tix),
 				       ainfo);
 		ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
 #endif
@@ -3209,7 +3288,7 @@ reply_alloc_info(void *vair)
 		if (erts_allctrs_info[ai].thr_spec) {
 		    alloc_atom = erts_bld_atom(hpp, szp,
 					       (char *) ERTS_ALC_A2AD(ai));
-		    allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
+		    allctr = erts_allctr_thr_spec[ai].allctr[tix];
 		    ainfo = info_func(allctr, air->internal, hpp != NULL, NULL,
 				      NULL, hpp, szp);
 		    ai_list = erts_bld_cons(hpp, szp,
@@ -3217,7 +3296,7 @@ reply_alloc_info(void *vair)
 						hpp, szp,
 						3,
 						alloc_atom,
-						make_small(sched_id),
+						make_small(tix),
 						ainfo),
 					    ai_list);
 		}
@@ -3226,7 +3305,7 @@ reply_alloc_info(void *vair)
 	    msg = erts_bld_tuple(hpp, szp,
 				 3,
 				 ref_copy,
-				 make_small(sched_id),
+				 make_small(tix),
 				 ai_list);
 
 	}
@@ -3245,7 +3324,7 @@ reply_alloc_info(void *vair)
 
     erts_queue_message(rp, rp_locks, mp, msg, am_system);
 
-    if (air->req_sched == sched_id)
+    if (air->req_sched == tix)
 	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
  
     erts_proc_unlock(rp, rp_locks);
@@ -3325,13 +3404,14 @@ erts_request_alloc_info(struct process *c_p,
     air->allocs[airix] = ERTS_ALC_A_INVALID;
 
     erts_atomic32_init_nob(&air->refc,
-			       (erts_aint32_t) erts_no_schedulers);
+                           (erts_aint32_t) erts_no_aux_work_threads-1);
 
-    erts_proc_add_refc(c_p, (Sint) erts_no_schedulers);
+    erts_proc_add_refc(c_p, (Sint) erts_no_aux_work_threads-1);
 
-    if (erts_no_schedulers > 1)
+    if (erts_no_aux_work_threads > 2)
 	erts_schedule_multi_misc_aux_work(1,
-					  erts_no_schedulers,
+                                          1,
+					  erts_no_aux_work_threads-1,
 					  reply_alloc_info,
 					  (void *) air);
 
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 831e7ab0a7..dffcef7370 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -107,10 +107,9 @@ typedef struct {
 
 typedef struct {
     Allctr_t *deallctr[ERTS_ALC_A_MAX+1];
-    int pref_ix[ERTS_ALC_A_MAX+1];
-    int flist_ix[ERTS_ALC_A_MAX+1];
-    int pre_alc_ix;
-} ErtsSchedAllocData;
+    int delayed_dealloc_handler;
+    int alc_ix;
+} ErtsThrAllocData;
 
 void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop);
 void erts_alloc_late_init(void);
@@ -147,11 +146,14 @@ typedef struct {
     void *extra;
 } ErtsAllocatorFunctions_t;
 
+extern erts_tsd_key_t erts_thr_alloc_data_key;
 extern ErtsAllocatorFunctions_t
     ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]);
 extern ErtsAllocatorInfo_t
     ERTS_WRITE_UNLIKELY(erts_allctrs_info[ERTS_ALC_A_MAX+1]);
 
+extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_alloc_instances);
+
 typedef struct {
     int enabled;
     int dd;
@@ -175,10 +177,12 @@ void erts_allctr_wrapper_pre_lock(void);
 void erts_allctr_wrapper_pre_unlock(void);
 
 void erts_alloc_register_scheduler(void *vesdp);
-void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
-						 int *need_thr_progress,
-						 ErtsThrPrgrVal *thr_prgr_p,
-						 int *more_work);
+void erts_alloc_register_delayed_dealloc_handler_thread(ErtsThrAllocData *tadp,
+							int ix);
+void erts_alloc_handle_delayed_dealloc(ErtsThrAllocData *thr_alloc_data,
+				       int *need_thr_progress,
+				       ErtsThrPrgrVal *thr_prgr_p,
+				       int *more_work);
 erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
 
 __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)		
@@ -230,6 +234,8 @@ int erts_is_allctr_wrapper_prelocked(void);
 #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
 int erts_is_in_literal_range(void* ptr);
 #endif
+ErtsThrAllocData *erts_get_thr_alloc_data(void);
+int erts_get_thr_alloc_ix(void);
 
 #endif /* #if !ERTS_ALC_DO_INLINE */
 
@@ -321,6 +327,21 @@ int erts_is_allctr_wrapper_prelocked(void)
 	&& !!erts_tsd_get(erts_allctr_prelock_tsd_key);  /* by me  */
 }
 
+ERTS_ALC_INLINE
+ErtsThrAllocData *erts_get_thr_alloc_data(void)
+{
+    return (ErtsThrAllocData *) erts_tsd_get(erts_thr_alloc_data_key);
+}
+
+ERTS_ALC_INLINE
+int erts_get_thr_alloc_ix(void)
+{
+    ErtsThrAllocData *tadp = (ErtsThrAllocData *) erts_tsd_get(erts_thr_alloc_data_key);
+    if (!tadp)
+        return 0;
+    return tadp->alc_ix;
+}
+
 #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE
 
 ERTS_ALC_FORCE_INLINE
@@ -345,8 +366,6 @@ int erts_is_in_literal_range(void* ptr)
 
 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
 
-#define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())
-
 typedef void (*erts_alloc_verify_func_t)(Allctr_t *);
 
 erts_alloc_verify_func_t
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index ac3e9fe967..0c405a87b8 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1715,7 +1715,7 @@ get_pref_allctr(void *extra)
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
     int pref_ix;
 
-    pref_ix = ERTS_ALC_GET_THR_IX();
+    pref_ix = erts_get_thr_alloc_ix();
 
     ERTS_CT_ASSERT(sizeof(UWord) == sizeof(Allctr_t *));
     ASSERT(0 <= pref_ix && pref_ix < tspec->size);
@@ -6047,7 +6047,7 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
     Allctr_t *allctr;
     void *res;
 
-    ix = ERTS_ALC_GET_THR_IX();
+    ix = erts_get_thr_alloc_ix();
 
     ASSERT(0 <= ix && ix < tspec->size);
 
@@ -6174,7 +6174,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p)
     int ix;
     Allctr_t *allctr;
 
-    ix = ERTS_ALC_GET_THR_IX();
+    ix = erts_get_thr_alloc_ix();
 
     ASSERT(0 <= ix && ix < tspec->size);
 
@@ -6460,7 +6460,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
     Allctr_t *allctr;
     void *res;
 
-    ix = ERTS_ALC_GET_THR_IX();
+    ix = erts_get_thr_alloc_ix();
 
     ASSERT(0 <= ix && ix < tspec->size);
 
@@ -6497,7 +6497,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
     Allctr_t *allctr;
     void *res;
 
-    ix = ERTS_ALC_GET_THR_IX();
+    ix = erts_get_thr_alloc_ix();
 
     ASSERT(0 <= ix && ix < tspec->size);
 
@@ -7371,7 +7371,7 @@ static int blockscan_sweep_cpool(blockscan_t *state)
 }
 
 static int blockscan_get_specific_allocator(int allocator_num,
-                                            int sched_id,
+                                            int aux_work_tid,
                                             Allctr_t **out)
 {
     ErtsAllocatorInfo_t *ai;
@@ -7379,7 +7379,7 @@ static int blockscan_get_specific_allocator(int allocator_num,
 
     ASSERT(allocator_num >= ERTS_ALC_A_MIN &&
            allocator_num <= ERTS_ALC_A_MAX);
-    ASSERT(sched_id >= 0 && sched_id <= erts_no_schedulers);
+    ASSERT(0 <= aux_work_tid && aux_work_tid < erts_no_aux_work_threads);
 
     ai = &erts_allctrs_info[allocator_num];
 
@@ -7388,7 +7388,7 @@ static int blockscan_get_specific_allocator(int allocator_num,
     }
 
     if (!ai->thr_spec) {
-        if (sched_id != 0) {
+        if (aux_work_tid != 0) {
             /* Only thread-specific allocators can be scanned on a specific
              * scheduler. */
             return 0;
@@ -7399,9 +7399,9 @@ static int blockscan_get_specific_allocator(int allocator_num,
     } else {
         ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t*)ai->extra;
 
-        ASSERT(sched_id < tspec->size);
+        ASSERT(aux_work_tid < tspec->size);
 
-        allocator = tspec->allctr[sched_id];
+        allocator = tspec->allctr[aux_work_tid];
     }
 
     *out = allocator;
@@ -7411,14 +7411,9 @@ static int blockscan_get_specific_allocator(int allocator_num,
 
 static void blockscan_sched_trampoline(void *arg)
 {
-    ErtsAlcuBlockscanYieldData *yield;
-    ErtsSchedulerData *esdp;
-    blockscan_t *scanner;
-
-    esdp = erts_get_scheduler_data();
-    scanner = (blockscan_t*)arg;
-
-    yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan);
+    ErtsAuxWorkData *awdp = erts_get_aux_work_data();
+    ErtsAlcuBlockscanYieldData *yield = &awdp->yield.alcu_blockscan;
+    blockscan_t *scanner = (blockscan_t*)arg;
 
     ASSERT((yield->last == NULL) == (yield->current == NULL));
 
@@ -7435,20 +7430,14 @@ static void blockscan_sched_trampoline(void *arg)
     scanner->scanner_queue = NULL;
     yield->last = scanner;
 
-    erts_notify_new_aux_yield_work(esdp);
+    erts_more_yield_aux_work(awdp);
 }
 
 static void blockscan_dispatch(blockscan_t *scanner, Process *owner,
-                               Allctr_t *allocator, int sched_id)
+                               Allctr_t *allocator, int aux_work_tid)
 {
     ASSERT(erts_get_scheduler_id() != 0);
 
-    if (sched_id == 0) {
-        /* Global instances are always handled on the current scheduler. */
-        sched_id = ERTS_ALC_GET_THR_IX();
-        ASSERT(allocator->thread_safe);
-    }
-
     scanner->allocator = allocator;
     scanner->process = owner;
 
@@ -7464,22 +7453,20 @@ static void blockscan_dispatch(blockscan_t *scanner, Process *owner,
         scanner->next_op = blockscan_sweep_mbcs;
     }
 
-    /* Aux yield jobs can only be set up while running on the scheduler that
-     * services them, so we move there before continuing.
+    /* Aux yield jobs can only be set up while running on the aux work
+     * thread that services them, so we move there before continuing.
      *
-     * We can't drive the scan itself through this since the scheduler will
+     * We can't drive the scan itself through this since the aux work thread will
      * always finish *all* misc aux work in one go which makes it impossible to
      * yield. */
-    erts_schedule_misc_aux_work(sched_id, blockscan_sched_trampoline, scanner);
+    erts_schedule_misc_aux_work(aux_work_tid, blockscan_sched_trampoline, scanner);
 }
 
-int erts_handle_yielded_alcu_blockscan(ErtsSchedulerData *esdp,
-                                       ErtsAlcuBlockscanYieldData *yield)
+int erts_handle_yielded_alcu_blockscan(ErtsAuxWorkData *awdp)
 {
+    ErtsAlcuBlockscanYieldData *yield = &awdp->yield.alcu_blockscan;
     blockscan_t *scanner = yield->current;
 
-    (void)esdp;
-
     ASSERT((yield->last == NULL) == (yield->current == NULL));
 
     if (scanner) {
@@ -7508,14 +7495,10 @@ int erts_handle_yielded_alcu_blockscan(ErtsSchedulerData *esdp,
     return 0;
 }
 
-void erts_alcu_sched_spec_data_init(ErtsSchedulerData *esdp)
+void erts_alcu_blockscan_init(ErtsAuxWorkData *awdp)
 {
-    ErtsAlcuBlockscanYieldData *yield;
-
-    yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan);
-
-    yield->current = NULL;
-    yield->last = NULL;
+    awdp->yield.alcu_blockscan.current = NULL;
+    awdp->yield.alcu_blockscan.last = NULL;
 }
 
 /* ------------------------------------------------------------------------- */
@@ -7815,7 +7798,7 @@ static void gather_ahist_abort(void *arg)
 }
 
 int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num,
-                                      int sched_id, int hist_width,
+                                      int aux_work_tid, int hist_width,
                                       UWord hist_start, Eterm ref)
 {
     gather_ahist_t *gather_state;
@@ -7825,7 +7808,7 @@ int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num,
     ASSERT(is_internal_ref(ref));
 
     if (!blockscan_get_specific_allocator(allocator_num,
-                                          sched_id,
+                                          aux_work_tid,
                                           &allocator)) {
         return 0;
     }
@@ -7846,7 +7829,7 @@ int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num,
     gather_state->hist_slot_count = hist_width;
     gather_state->process = p;
 
-    blockscan_dispatch(scanner, p, allocator, sched_id);
+    blockscan_dispatch(scanner, p, allocator, aux_work_tid);
 
     return 1;
 }
@@ -8138,7 +8121,7 @@ static void gather_cinfo_abort(void *arg)
 }
 
 int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
-                                  int sched_id, int hist_width,
+                                  int aux_work_tid, int hist_width,
                                   UWord hist_start, Eterm ref)
 {
     gather_cinfo_t *gather_state;
@@ -8148,7 +8131,7 @@ int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
     ASSERT(is_internal_ref(ref));
 
     if (!blockscan_get_specific_allocator(allocator_num,
-                                          sched_id,
+                                          aux_work_tid,
                                           &allocator)) {
         return 0;
     }
@@ -8170,7 +8153,7 @@ int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
     gather_state->hist_slot_count = hist_width;
     gather_state->process = p;
 
-    blockscan_dispatch(scanner, p, allocator, sched_id);
+    blockscan_dispatch(scanner, p, allocator, aux_work_tid);
 
     return 1;
 }
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 45e5bfc211..9444a4e32b 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -259,9 +259,10 @@ typedef struct {
     struct alcu_blockscan *last;
 } ErtsAlcuBlockscanYieldData;
 
-int erts_handle_yielded_alcu_blockscan(struct ErtsSchedulerData_ *esdp,
-                                       ErtsAlcuBlockscanYieldData *yield);
-void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
+struct ErtsAuxWorkData_;
+
+int erts_handle_yielded_alcu_blockscan(struct ErtsAuxWorkData_ *awdp);
+void erts_alcu_blockscan_init(struct ErtsAuxWorkData_ *awdp);
 
 #endif /* !ERL_ALLOC_UTIL__ */
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 93ce1aace9..6c1347cfea 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -5122,7 +5122,7 @@ static BIF_RETTYPE
 gather_histograms_helper(Process * c_p, Eterm arg_tuple,
                          int gather(Process *, int, int, int, UWord, Eterm))
 {
-    SWord hist_start, hist_width, sched_id;
+    SWord hist_start, hist_width, aux_work_tid;
     int msg_count, alloc_num;
     Eterm *args;
 
@@ -5142,15 +5142,15 @@ gather_histograms_helper(Process * c_p, Eterm arg_tuple,
         BIF_ERROR(c_p, BADARG);
     }
 
-    sched_id = signed_val(args[2]);
+    aux_work_tid = signed_val(args[2]);
     hist_width = signed_val(args[3]);
     hist_start = signed_val(args[4]);
 
-    if (sched_id < 0 || sched_id > erts_no_schedulers) {
+    if (aux_work_tid < 0 || erts_no_aux_work_threads <= aux_work_tid) {
         BIF_ERROR(c_p, BADARG);
     }
 
-    msg_count = gather(c_p, alloc_num, sched_id, hist_width, hist_start, args[5]);
+    msg_count = gather(c_p, alloc_num, aux_work_tid, hist_width, hist_start, args[5]);
 
     BIF_RET(make_small(msg_count));
 }
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 8024687c5e..e01ad49f79 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -2359,6 +2359,7 @@ trace_delivered_1(BIF_ALIST_1)
                                    (erts_aint32_t) erts_no_schedulers);
         erts_proc_add_refc(BIF_P, 1);
         erts_schedule_multi_misc_aux_work(0,
+                                          1,
                                           erts_no_schedulers,
                                           reply_trace_delivered_all,
                                           (void *) tdarp);
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 06cf04f9ee..08fc410414 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -3197,12 +3197,21 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
 }
 
 int
-erts_handle_yielded_ets_all_request(ErtsSchedulerData *esdp,
-                                    ErtsEtsAllYieldData *eaydp)
+erts_handle_yielded_ets_all_request(ErtsAuxWorkData *awdp)
 {
-    int ix = (int) esdp->no - 1;
-    int yc = ERTS_ETS_ALL_TB_YCNT;
+    ErtsSchedulerData *esdp;
+    ErtsEtsAllYieldData *eaydp;
+    int ix, yc;
 
+    esdp = awdp->esdp;
+    /* only on normal scheduler threads... */
+    if (!esdp || esdp->type != ERTS_SCHED_NORMAL)
+        return 0;
+
+    eaydp = &awdp->yield.ets_all;
+    ix = (int) esdp->no - 1;
+    yc = ERTS_ETS_ALL_TB_YCNT;
+    
     while (1) {
         if (!eaydp->ongoing) {
             ErtsEtsAllReq *ongoing;
@@ -3249,7 +3258,7 @@ handle_ets_all_request(void *vreq)
             eayp->ongoing = req;
             eayp->hfrag = hf;
             eayp->tab = tb;
-            erts_notify_new_aux_yield_work(esdp);
+            erts_more_yield_aux_work(&esdp->aux_work_data);
         }
     }
     else {
@@ -3282,6 +3291,7 @@ BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0)
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
                                           handle_ets_all_request,
 					  (void *) req);
@@ -5353,7 +5363,7 @@ static void lcnt_update_db_locks_per_sched(void *enable) {
 }
 
 void erts_lcnt_update_db_locks(int enable) {
-    erts_schedule_multi_misc_aux_work(0, erts_no_schedulers,
+    erts_schedule_multi_misc_aux_work(0, 1, erts_no_schedulers,
         &lcnt_update_db_locks_per_sched, (void*)(UWord)enable);
 }
 #endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 2f6debe72a..a18be4d168 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -78,9 +78,7 @@ typedef struct {
 } DbTableRelease;
 
 struct ErtsSchedulerData_;
-int erts_handle_yielded_ets_all_request(struct ErtsSchedulerData_ *esdp,
-                                        ErtsEtsAllYieldData *eadp);
-
+int erts_handle_yielded_ets_all_request(ErtsAuxWorkData *awdp);
 void erts_ets_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
 
 /*
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d3d79959df..a147700bef 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -3384,6 +3384,7 @@ erts_gc_info_request(Process *c_p)
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
 					  reply_gc_info,
 					  (void *) gcirp);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 6ea1759cf8..bcaf702703 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -797,6 +797,7 @@ early_init(int *argc, char **argv) /*
     int dirty_cpu_scheds_pctg = 100;
     int dirty_cpu_scheds_onln_pctg = 100;
     int dirty_io_scheds;
+    int aux_threads = 1;
     int max_reader_groups;
     int reader_groups;
     int max_decentralized_counter_groups;
@@ -1201,11 +1202,12 @@ early_init(int *argc, char **argv) /*
     erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds;
     no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online;
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers = dirty_io_scheds;
-    erts_early_init_scheduling(no_schedulers);
+    erts_early_init_scheduling(no_schedulers + 1 + dirty_cpu_scheds);
 
     alloc_opts.ncpu = ncpu;
     erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes)
 						 -M flags. */
+    aux_threads += erts_no_dirty_alloc_instances;
     /* Require allocators */
 
     erts_init_check_io(argc, argv);
@@ -1215,7 +1217,7 @@ early_init(int *argc, char **argv) /*
      *
      * * Managed threads:
      * ** Scheduler threads (see erl_process.c)
-     * ** Aux thread (see erl_process.c)
+     * ** Aux threads (see erl_process.c)
      * ** Sys message dispatcher thread (see erl_trace.c)
      * ** IO Poll threads (see erl_check_io.c)
      *
@@ -1224,11 +1226,13 @@ early_init(int *argc, char **argv) /*
      * ** Dirty scheduler threads
      */
     erts_thr_progress_init(no_schedulers,
-			   no_schedulers+2+erts_no_poll_threads,
-			   erts_async_max_threads +
-			   erts_no_dirty_cpu_schedulers +
-			   erts_no_dirty_io_schedulers
-			   );
+			   (no_schedulers
+			    + aux_threads
+			    + 1
+			    + erts_no_poll_threads),
+			   (erts_async_max_threads
+			    + erts_no_dirty_cpu_schedulers
+			    + erts_no_dirty_io_schedulers));
     erts_thr_q_init();
     erts_init_utils();
     erts_early_init_cpu_topology(no_schedulers,
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index ed47cb0b83..89fdaf77cd 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -364,20 +364,17 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
     msaccrp->ref = STORE_NC(&hp, NULL, ref);
     msaccrp->req_sched = esdp->no;
 
-    *threads = erts_no_schedulers;
-    *threads += 1; /* aux thread */
+    *threads = erts_no_aux_work_threads;
 
     erts_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads);
 
     erts_proc_add_refc(c_p, *threads);
 
-    if (erts_no_schedulers > 1)
-	erts_schedule_multi_misc_aux_work(1,
-                                          erts_no_schedulers,
-                                          reply_msacc,
-                                          (void *) msaccrp);
-    /* aux thread */
-    erts_schedule_misc_aux_work(0, reply_msacc, (void *) msaccrp);
+    erts_schedule_multi_misc_aux_work(1,
+                                      0,
+                                      erts_no_aux_work_threads-1,
+                                      reply_msacc,
+                                      (void *) msaccrp);
 
     /* Manage unmanaged threads */
     switch (action) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 76a9232159..92586baa82 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -137,6 +137,7 @@ Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
+int ERTS_WRITE_UNLIKELY(erts_no_aux_work_threads);
 
 static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
 int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
@@ -171,7 +172,12 @@ sched_get_busy_wait_params(ErtsSchedulerData *esdp)
     return &sched_busy_wait_params[esdp->type];
 }
 
-static ErtsAuxWorkData *aux_thread_aux_work_data;
+typedef union {
+    ErtsAuxWorkData data;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAuxWorkData))];
+} ErtsAlignedAuxWorkData;
+
+static ErtsAlignedAuxWorkData *ERTS_WRITE_UNLIKELY(aligned_aux_work_data);
 
 #define ERTS_SCHDLR_SSPND_CHNG_NMSB		(((erts_aint32_t) 1) << 0)
 #define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
@@ -485,8 +491,8 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
 				 ERTS_ALC_T_PROC_LIST)
 
 #define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
-    (ASSERT(((int)-1) <= ((int) (IX))                 \
-            && ((int) (IX)) < ((int) erts_no_schedulers)),		\
+    (ASSERT((((int)-1) <= ((int) (IX)))					\
+            && (((int) (IX)) < (erts_no_aux_work_threads-1))),          \
      &aligned_sched_sleep_info[(IX)].ssi)
 #define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX)				\
     (ASSERT(0 <= ((int) (IX))					        \
@@ -555,7 +561,7 @@ static void print_function_from_pc(fmtfn_t to, void *to_arg, ErtsCodePtr x);
 static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg);
 
 static void aux_work_timeout(void *unused);
-static void aux_work_timeout_early_init(int no_schedulers);
+static void aux_work_timeout_early_init(int max_no_aux_work_threads);
 static void setup_aux_work_timer(ErtsSchedulerData *esdp);
 
 static int execute_sys_tasks(Process *c_p,
@@ -1393,6 +1399,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable,
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
 					  reply_sched_wall_time,
 					  (void *) swtrp);
@@ -1457,6 +1464,7 @@ Eterm erts_system_check_request(Process *c_p) {
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
 					  reply_system_check,
 					  (void *) scrp);
@@ -1814,11 +1822,9 @@ init_misc_aux_work(void)
     misc_aux_work_queues = 
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q,
 					   sizeof(erts_algnd_misc_aux_work_q_t)
-					   * (erts_no_schedulers+1));
-
-    ix = 0; /* aux_thread + schedulers */
+					   * erts_no_aux_work_threads);
 
-    for (; ix <= erts_no_schedulers; ix++) {
+    for (ix = 0; ix < erts_no_aux_work_threads; ix++) {
 	qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1);
 	erts_thr_q_initialize(&misc_aux_work_queues[ix].q, &qinit);
     }
@@ -1849,7 +1855,7 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp,
 		     erts_aint32_t aux_work,
 		     int waiting)
 {
-    ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q;
+    ErtsThrQ_t *q = &misc_aux_work_queues[awdp->aux_work_tid].q;
 
     unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_MISC);
     while (1) {
@@ -1877,23 +1883,36 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 
-    return misc_aux_work_clean(&misc_aux_work_queues[awdp->sched_id].q,
+    return misc_aux_work_clean(&misc_aux_work_queues[awdp->aux_work_tid].q,
 			       awdp,
 			       aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 }
 
+static ERTS_INLINE int
+get_aux_work_tid(void)
+{
+    /*
+     * All aux work threads are registered as delayed dealloc
+     * handlers, and their allocation index equals their tid.
+     */
+    ErtsThrAllocData *tadp = erts_get_thr_alloc_data();
+    if (!tadp || !tadp->delayed_dealloc_handler)
+        return -1; /* Not an aux work thread */
+    return tadp->alc_ix;
+}
 
 static ERTS_INLINE void
-schedule_misc_aux_work(int sched_id,
+schedule_misc_aux_work(int aux_work_tid,
 		       void (*func)(void *),
 		       void *arg)
 {
     ErtsThrQ_t *q;
     erts_misc_aux_work_t *mawp;
 
-    ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers);
+    ASSERT(0 <= aux_work_tid
+	   && aux_work_tid < erts_no_aux_work_threads);
 
-    q = &misc_aux_work_queues[sched_id].q;
+    q = &misc_aux_work_queues[aux_work_tid].q;
     mawp = misc_aux_work_alloc();
     mawp->func = func;
     mawp->arg = arg;
@@ -1901,37 +1920,53 @@ schedule_misc_aux_work(int sched_id,
 }
 
 void
-erts_schedule_misc_aux_work(int sched_id,
+erts_schedule_misc_aux_work(int aux_work_tid,
 			    void (*func)(void *),
 			    void *arg)
 {
-    schedule_misc_aux_work(sched_id, func, arg);
+    schedule_misc_aux_work(aux_work_tid, func, arg);
 }
 
 void
 erts_schedule_multi_misc_aux_work(int ignore_self,
-				  int max_sched,
+                                  int min_tid,
+				  int max_tid,
 				  void (*func)(void *),
 				  void *arg)
 {
-    int id, self = 0;
-
-    if (ignore_self) {
-        ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int tid, self;
 
-        /* ignore_self is meaningless on dirty schedulers since aux work can
-         * only run on normal schedulers, and their ids do not translate. */
-        if(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-            self = (int)esdp->no;
-        }
-    }
+    /*
+     * Threads handling misc aux work are:
+     * * Normal scheduler threads
+     * * Aux work threads
+     *
+     * Tids corresponds to threads as follows:
+     * * tid == 0
+     *     Standard aux thread. This thread is always
+     *     present on all systems.
+     * * 1 <= tid <= erts_no_schedulers
+     *      Normal scheduler threads. There are always at
+     *      least one normal scheduler. Tid equals scheduler
+     *      id.
+     * * erts_no_schedulers < tid < erts_no_aux_work_threads
+     *      Extra aux threads. Main purpose to handle
+     *      delayed dealloc for allocator instances for
+     *      dirty schedulers. May or may not exist. Maximum
+     *      amount of these threads equals amount of dirty
+     *      cpu schedulers.
+     */
 
-    ASSERT(0 < max_sched && max_sched <= erts_no_schedulers);
+    ASSERT(0 <= min_tid && min_tid < erts_no_aux_work_threads);
+    ASSERT(0 <= max_tid && max_tid < erts_no_aux_work_threads);
+    ASSERT(min_tid <= max_tid);
 
-    for (id = 1; id <= max_sched; id++) {
-	if (id == self)
+    self = ignore_self ? get_aux_work_tid() : -1;
+    
+    for (tid = min_tid; tid <= max_tid; tid++) {
+	if (tid == self)
 	    continue;
-	schedule_misc_aux_work(id, func, arg);
+	schedule_misc_aux_work(tid, func, arg);
    }
 }
 
@@ -2013,7 +2048,7 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
                                   | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC));
     aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
 		  | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC);
-    res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work);
+    res = erts_alloc_fix_alloc_shrink(awdp->aux_work_tid, aux_work);
     if (res) {
 	set_aux_work_flags(ssi, res);
 	aux_work |= res;
@@ -2060,10 +2095,10 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
 
     unset_aux_work_flags_mb(ssi, ERTS_SSI_AUX_WORK_DD);
     ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC);
-    erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
-						&need_thr_progress,
-						&wakeup,
-						&more_work);
+    erts_alloc_handle_delayed_dealloc(&awdp->alloc_data,
+				      &need_thr_progress,
+				      &wakeup,
+				      &more_work);
     ERTS_MSACC_POP_STATE_M_X();
     if (more_work) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
@@ -2103,10 +2138,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     need_thr_progress = 0;
     more_work = 0;
 
-    erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
-						&need_thr_progress,
-						&wakeup,
-						&more_work);
+    erts_alloc_handle_delayed_dealloc(&awdp->alloc_data,
+				      &need_thr_progress,
+				      &wakeup,
+				      &more_work);
     if (more_work) {
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -2347,13 +2382,24 @@ setup_thr_debug_wait_completed(void *vproc)
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsAuxWorkData *awdp;
     erts_aint32_t wait_flags, aux_work_flags;
-    awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data;
+
+    if (esdp)
+        awdp = &esdp->aux_work_data; /* A normal scheduler... */
+    else {
+        /* An aux thread... */
+        ErtsThrAllocData *tadp = erts_get_thr_alloc_data();
+        char *ptr;
+        ASSERT(tadp);
+        ptr = (char *) tadp;
+        ptr -= offsetof(ErtsAuxWorkData, alloc_data);
+        awdp = (ErtsAuxWorkData *) ptr;
+    }
 
     wait_flags = 0;
     aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
 
     if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) {
-	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
+	erts_alloc_fix_alloc_shrink(awdp->aux_work_tid, 0);
 	wait_flags |= (ERTS_SSI_AUX_WORK_DD
 		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR);
 	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
@@ -2383,19 +2429,15 @@ static void later_thr_debug_wait_completed(void *vlop)
     struct debug_lop *lop = vlop;
 
     if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 1) {
-        erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
-        count += 1; /* aux thread */
+        erts_aint32_t count = (erts_aint32_t) erts_no_aux_work_threads;
         erts_atomic32_set_nob(&debug_wait_completed_count, count);
 
-        /* scheduler threads */
+        /* All scheduler threads as well as all aux threads... */
         erts_schedule_multi_misc_aux_work(0,
-                                          erts_no_schedulers,
+                                          0,
+                                          erts_no_aux_work_threads-1,
                                           setup_thr_debug_wait_completed,
                                           lop->proc);
-        /* aux_thread */
-        erts_schedule_misc_aux_work(0,
-                                    setup_thr_debug_wait_completed,
-                                    lop->proc);
     }
     erts_free(ERTS_ALC_T_DEBUG, lop);
 }
@@ -2435,6 +2477,7 @@ erts_debug_wait_completed(Process *c_p, int flags)
 
         /* First flush later-ops on all scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
+                                          1,
 					  erts_no_schedulers,
 					  init_thr_debug_wait_completed,
 					  (void *) c_p);
@@ -2507,11 +2550,27 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 }
 
 void
-erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp)
+erts_more_yield_aux_work(ErtsAuxWorkData *awdp)
 {
-    ASSERT(esdp == erts_get_scheduler_data());
-    /* Always called by the scheduler itself... */
-    set_aux_work_flags_wakeup_nob(esdp->ssi, ERTS_SSI_AUX_WORK_YIELD);
+    /* Should *always* be called by the aux-work thread itself... */
+    ASSERT(awdp && awdp->aux_work_tid == get_aux_work_tid());
+    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_YIELD);
+}
+
+ErtsAuxWorkData *
+erts_get_aux_work_data(void)
+{
+    ErtsAuxWorkData *awdp;
+    int tid = get_aux_work_tid();
+    ASSERT(0 <= tid && tid < erts_no_aux_work_threads);
+    if (tid < 1)
+        awdp = tid == 0 ? &aligned_aux_work_data[0].data : NULL;
+    else if (tid <= erts_no_schedulers)
+        awdp = &erts_aligned_scheduler_data[tid-1].esd.aux_work_data;
+    else
+        awdp = &aligned_aux_work_data[tid - (int) erts_no_schedulers].data;
+    ASSERT(!awdp || awdp->aux_work_tid == tid);
+    return awdp;
 }
 
 static ERTS_INLINE erts_aint32_t
@@ -2531,10 +2590,8 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 
     /* Various yielding operations... */
 
-    yield |= erts_handle_yielded_ets_all_request(awdp->esdp,
-                                                 &awdp->yield.ets_all);
-    yield |= erts_handle_yielded_alcu_blockscan(awdp->esdp,
-                                                &awdp->yield.alcu_blockscan);
+    yield |= erts_handle_yielded_ets_all_request(awdp);
+    yield |= erts_handle_yielded_alcu_blockscan(awdp);
 
     /*
      * Other yielding operations...
@@ -2718,7 +2775,7 @@ start_aux_work_timer(ErtsSchedulerData *esdp)
 }
 
 static void
-aux_work_timeout_early_init(int no_schedulers)
+aux_work_timeout_early_init(int max_no_aux_work_threads)
 {
     int i;
     UWord p;
@@ -2729,7 +2786,8 @@ aux_work_timeout_early_init(int no_schedulers)
      */
 
     p = (UWord) malloc((sizeof(ErtsAuxWorkTmo)
-			+ sizeof(erts_atomic32_t)*(no_schedulers+1))
+			+ (sizeof(erts_atomic32_t)
+                           * max_no_aux_work_threads))
 		       + ERTS_CACHE_LINE_SIZE-1);
     if (!p) {
         ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
@@ -2744,7 +2802,7 @@ aux_work_timeout_early_init(int no_schedulers)
 #ifdef DEBUG
     erts_atomic32_init_nob(&aux_work_tmo->used, 0);
 #endif
-    for (i = 0; i <= no_schedulers; i++)
+    for (i = 0; i < max_no_aux_work_threads; i++)
 	erts_atomic32_init_nob(&aux_work_tmo->type[i], 0);
 }
 
@@ -3079,37 +3137,49 @@ thr_prgr_fin_wait(void *vssi)
 				      | ERTS_SSI_FLG_TSE_SLEEPING));
 }
 
-static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
+static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp,
+                               char *dawwp);
 
 void
-erts_aux_thread_poke()
+erts_aux_thread_poke(void)
 {
     erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1));
 }
 
 static void *
-aux_thread(void *unused)
+aux_thread(void *vix)
 {
-    ErtsAuxWorkData *awdp = aux_thread_aux_work_data;
-    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1);
+    int ix = (int) (Sint) vix;
+    int id = ix == 0 ? 1 : ix + 1 - erts_no_schedulers;
+    ErtsAuxWorkData *awdp = &aligned_aux_work_data[id-1].data;
+    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix-1);
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     ErtsThrPrgrData *tpd;
     int thr_prgr_active = 1;
     ERTS_MSACC_DECLARE_CACHE();
 
+    ASSERT(ix == 0
+           || (erts_no_schedulers < ix
+               && ix < erts_no_aux_work_threads));
+    ASSERT(0 < id && id <= 1 + erts_no_dirty_alloc_instances);
+    
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
-	char buf[] = "aux_thread";
-	erts_lc_set_thread_name(buf);
+        char buf[31];
+        erts_snprintf(&buf[0], 31, "aux-thread %d", id);
+        erts_lc_set_thread_name(&buf[0]);
     }
 #endif
 
-    erts_port_task_pre_alloc_init_thread();
+    if (ix == 0)
+        erts_port_task_pre_alloc_init_thread();
     ssi->event = erts_tse_fetch();
     erts_tse_return(ssi->event);
 
-    erts_msacc_init_thread("aux", 1, 1);
+    erts_msacc_init_thread("aux", id, 1);
+
+    erts_alloc_register_delayed_dealloc_handler_thread(&awdp->alloc_data, ix);
 
     callbacks.arg = (void *) ssi;
     callbacks.wakeup = thr_prgr_wakeup;
@@ -3120,13 +3190,16 @@ aux_thread(void *unused)
     tpd = erts_thr_progress_register_managed_thread(NULL, &callbacks, 1, 0);
     init_aux_work_data(awdp, NULL, NULL);
     awdp->ssi = ssi;
+    awdp->aux_work_tid = ix;
 
 #if ERTS_POLL_USE_FALLBACK
+    if (ix == 0) {
 #if ERTS_POLL_USE_SCHEDULER_POLLING
-    ssi->psi = erts_create_pollset_thread(-2, tpd);
+	ssi->psi = erts_create_pollset_thread(-2, tpd);
 #else
-    ssi->psi = erts_create_pollset_thread(-1, tpd);
+	ssi->psi = erts_create_pollset_thread(-1, tpd);
 #endif
+    }
 #endif
 
     sched_prep_spin_wait(ssi);
@@ -3149,7 +3222,7 @@ aux_thread(void *unused)
 	if (!aux_work) {
 
 #ifdef ERTS_BREAK_REQUESTED
-            if (ERTS_BREAK_REQUESTED)
+            if (ix == 0 && ERTS_BREAK_REQUESTED)
                 erts_do_break_handling();
 #endif
 
@@ -3157,40 +3230,43 @@ aux_thread(void *unused)
 		erts_thr_progress_active(tpd, thr_prgr_active = 0);
 
 #if ERTS_POLL_USE_FALLBACK
+            if (ix == 0) {
 
-	    flgs = sched_spin_wait(ssi, 0);
+                flgs = sched_spin_wait(ssi, 0);
 
-	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
-		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
-		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-                    erts_check_io(ssi->psi, ERTS_POLL_INF_TIMEOUT, 0);
-		}
+                if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+                    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                        ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+                        ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                        erts_check_io(ssi->psi, ERTS_POLL_INF_TIMEOUT, 0);
+                    }
+                }
             }
-#else
-            erts_thr_progress_prepare_wait(tpd);
-
-	    flgs = sched_spin_wait(ssi, 0);
+            else 
+#endif
+            {
+                erts_thr_progress_prepare_wait(tpd);
+		flgs = sched_spin_wait(ssi, 0);
 
-	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
-		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-                    int res;
-		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
-		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-                    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
-                    do {
-                        res = erts_tse_wait(ssi->event);
-                    } while (res == EINTR);
-                    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
-		}
-                erts_tse_return(ssi->event);
+                if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+                    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                        int res;
+                        ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+                        ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
+                        do {
+                            res = erts_tse_wait(ssi->event);
+                        } while (res == EINTR);
+                        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
+                    }
+                    erts_tse_return(ssi->event);
+                }
+                erts_thr_progress_finalize_wait(tpd);
             }
-            erts_thr_progress_finalize_wait(tpd);
-#endif
-	}
+        }
 
 	flgs = sched_prep_spin_wait(ssi);
     }
@@ -5609,11 +5685,11 @@ runq_supervisor(void *unused)
 
 
 void
-erts_early_init_scheduling(int no_schedulers)
+erts_early_init_scheduling(int max_no_aux_work_threads)
 {
     ErtsSchedType type;
 
-    aux_work_timeout_early_init(no_schedulers);
+    aux_work_timeout_early_init(max_no_aux_work_threads);
 
     for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) {
         erts_sched_set_wakeup_other_threshold(type, "medium");
@@ -5723,7 +5799,8 @@ erts_sched_set_wake_cleanup_threshold(char *str)
 }
 
 static void
-init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
+init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp,
+                   char *dawwp)
 {
     int id = 0;
     if (esdp) {
@@ -5746,7 +5823,7 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
         }
     }
 
-    awdp->sched_id = id;
+    awdp->aux_work_tid = id;
     awdp->esdp = esdp;
     awdp->ssi = esdp ? esdp->ssi : NULL;
     awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
@@ -5769,12 +5846,13 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     else {
 	int i;
 	awdp->delayed_wakeup.job = (ErtsDelayedAuxWorkWakeupJob *) dawwp;
-	dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*(erts_no_schedulers+1);
+	dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*erts_no_aux_work_threads;
 	awdp->delayed_wakeup.sched2jix = (int *) dawwp;
 	awdp->delayed_wakeup.jix = -1;
-	for (i = 0; i <= erts_no_schedulers; i++)
+	for (i = 0; i < erts_no_aux_work_threads; i++)
 	    awdp->delayed_wakeup.sched2jix[i] = -1;
     }
+    erts_alcu_blockscan_init(awdp);
     awdp->debug.wait_completed.flags = 0;
     awdp->debug.wait_completed.callback = NULL;
     awdp->debug.wait_completed.arg = NULL;
@@ -5896,7 +5974,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 		     int no_dirty_io_schedulers
 		     )
 {
-    int ix, n, no_ssi, tot_rqs;
+    int ix, n, tot_rqs;
     char *daww_ptr;
     size_t daww_sz;
     size_t size_runqs;
@@ -6019,13 +6097,19 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
     erts_no_total_schedulers += no_dirty_io_schedulers;
 
+    /* normal schedulers */
+    erts_no_aux_work_threads = n;
+    /* standard aux thread */
+    erts_no_aux_work_threads += 1;
+    /* extra aux threads */
+    erts_no_aux_work_threads += erts_no_dirty_alloc_instances;
+
     /* Create and initialize scheduler sleep info */
-    no_ssi = n + 1 /* aux thread */;
     aligned_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
-	    no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo));
-    for (ix = 0; ix < no_ssi; ix++) {
+	    erts_no_aux_work_threads*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < erts_no_aux_work_threads; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi;
 #if 0 /* no need to initialize these... */
 	ssi->next = NULL;
@@ -6067,7 +6151,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     /* Create and initialize scheduler specific data */
 
     daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob)
-					      + sizeof(int))*(n+1));
+					      + sizeof(int))*erts_no_aux_work_threads);
     daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 						  daww_sz*n);
 
@@ -6079,7 +6163,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix),
 			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz,
-			    NULL, 0);
+                            NULL, 0);
     }
 
     {
@@ -6109,7 +6193,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
 				ERTS_DIRTY_IO_RUNQ, NULL, 0,
-				&adsp[adspix++].dsp, ts);
+                                &adsp[adspix++].dsp, ts);
 	}
     }
 
@@ -6120,9 +6204,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
     debug_wait_completed_flags = 0;
 
-    aux_thread_aux_work_data =
+    aligned_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
-					   sizeof(ErtsAuxWorkData));
+					   sizeof(ErtsAlignedAuxWorkData)
+					   * (erts_no_dirty_alloc_instances + 1));
 
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
@@ -8487,7 +8572,6 @@ sched_thread_func(void *vesdp)
     esdp->ssi->psi = erts_create_pollset_thread(-1, NULL);
 #endif
 
-    erts_alloc_register_scheduler(vesdp);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
 	char buf[31];
@@ -8496,6 +8580,7 @@ sched_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
+    erts_alloc_register_scheduler(vesdp);
 #if HAVE_ERTS_MSEG
     erts_mseg_late_init();
 #endif
@@ -8512,7 +8597,6 @@ sched_thread_func(void *vesdp)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
 #endif
 
-    erts_alcu_sched_spec_data_init(esdp);
     erts_ets_sched_spec_data_init(esdp);
     erts_utils_sched_spec_data_init();
 
@@ -8560,6 +8644,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
+    erts_alloc_register_scheduler(vesdp);
     esdp->aux_work_data.async_ready.queue = NULL;
 
     erts_proc_lock_prepare_proc_lock_waiter();
@@ -8607,6 +8692,7 @@ sched_dirty_io_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
+    erts_alloc_register_scheduler(vesdp);
     esdp->aux_work_data.async_ready.queue = NULL;
 
     erts_proc_lock_prepare_proc_lock_waiter();
@@ -8685,11 +8771,19 @@ erts_start_schedulers(void)
 	}
     }
 
-    erts_snprintf(opts.name, sizeof(name), "aux");
+    ix = 0;
+    while (ix < erts_no_aux_work_threads) {
+	int id = ix == 0 ? 1 : ix + 1 - (int) erts_no_schedulers;
+	erts_snprintf(opts.name, sizeof(name), "%d_aux", id);
 
-    res = ethr_thr_create(&tid, aux_thread, NULL, &opts);
-    if (res != 0)
-	erts_exit(ERTS_ABORT_EXIT, "Failed to create aux thread, error = %d\n", res);
+	res = ethr_thr_create(&tid, aux_thread, (void *) (Sint) ix, &opts);
+	if (res != 0)
+	    erts_exit(ERTS_ABORT_EXIT, "Failed to create aux thread %d, error = %d\n", res);
+	if (ix == 0)
+	    ix = (int) (1 + erts_no_schedulers);
+	else
+	    ix++;
+    }
 
     block_poll_thread_data = (ErtsAlignedBlockPollThreadData *)
         erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BLOCK_PTHR_DATA,
@@ -8925,6 +9019,12 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1)
     BIF_RET(am_false);
 }
 
+BIF_RETTYPE
+erts_internal_no_aux_work_threads_0(BIF_ALIST_0)
+{
+    BIF_RET(make_small(erts_no_aux_work_threads));
+}
+
 static ERTS_INLINE void
 run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int incl_active_sched, int locked)
 {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 4777d9d1ce..37b1aa6ac3 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -106,6 +106,7 @@ extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
 extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
 extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
+extern int ERTS_WRITE_UNLIKELY(erts_no_aux_work_threads);
 extern int erts_sched_thread_suggested_stack_size;
 extern int erts_dcpu_sched_thread_suggested_stack_size;
 extern int erts_dio_sched_thread_suggested_stack_size;
@@ -558,8 +559,9 @@ typedef struct {
     erts_aint32_t aux_work;
 } ErtsDelayedAuxWorkWakeupJob;
 
-typedef struct {
-    int sched_id;
+typedef struct ErtsAuxWorkData_ {
+    int aux_work_tid;
+    ErtsThrAllocData alloc_data;
     ErtsSchedulerData *esdp;
     ErtsSchedulerSleepInfo *ssi;
     ErtsThrPrgrVal current_thr_prgr;
@@ -607,7 +609,8 @@ typedef struct {
 
 #define ERTS_SCHED_AUX_YIELD_DATA(ESDP, NAME) \
     (&(ESDP)->aux_work_data.yield.NAME)
-void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp);
+void erts_more_yield_aux_work(ErtsAuxWorkData *);
+ErtsAuxWorkData *erts_get_aux_work_data(void);
 
 typedef enum {
     ERTS_DIRTY_CPU_SCHEDULER,
@@ -707,8 +710,6 @@ struct ErtsSchedulerData_ {
     Uint64 unique;
     Uint64 ref;
 
-    ErtsSchedAllocData alloc_data;
-
     struct {
 	Uint64 out;
 	Uint64 in;
@@ -1938,7 +1939,8 @@ void erts_schedule_misc_aux_work(int sched_id,
 				 void (*func)(void *),
 				 void *arg);
 void erts_schedule_multi_misc_aux_work(int ignore_self,
-				       int max_sched,
+                                       int min_tid,
+				       int max_tid,
 				       void (*func)(void *),
 				       void *arg);
 erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index aff2db4260..c05c723d70 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4953,6 +4953,7 @@ erts_request_io_bytes(Process *c_p)
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
 					  reply_io_bytes,
 					  (void *) req);
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 030e5b00a7..7e0200bf34 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -193,7 +193,7 @@ static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr;
   (&aligned_mseg_allctr[(IX)].mseg_alloc)
 
 #define ERTS_MSEG_ALLCTR_SS() \
-  ERTS_MSEG_ALLCTR_IX((int) erts_get_scheduler_id())
+  ERTS_MSEG_ALLCTR_IX((int) erts_get_thr_alloc_ix())
 
 #define ERTS_MSEG_ALLCTR_OPT(OPT) \
   ((OPT)->sched_spec ? ERTS_MSEG_ALLCTR_SS() : ERTS_MSEG_ALLCTR_IX(0))
@@ -1391,8 +1391,10 @@ erts_mseg_init(ErtsMsegInit_t *init)
     int i;
     UWord x;
 
-    no_mseg_allocators = init->nos + 1;
-
+    no_mseg_allocators = 1; /* Global instance */
+    no_mseg_allocators += init->nos; /* Scheduler specific instances */
+    no_mseg_allocators += init->ndai; /* Dirty alloc instances */
+    
     x = (UWord) malloc(sizeof(ErtsAlgndMsegAllctr_t)
 		       *no_mseg_allocators
 		       + (ERTS_CACHE_LINE_SIZE-1));
@@ -1423,7 +1425,7 @@ erts_mseg_init(ErtsMsegInit_t *init)
 
 	ma->is_init_done = 0;
 
-	if (i != 0)
+	if (i != 0 && i <= init->nos)
 	    ma->is_thread_safe = 0;
 	else {
 	    ma->is_thread_safe = 1;
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index ea9060ddac..8d5e627664 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -59,6 +59,7 @@ typedef struct {
     Uint rmcbf;
     Uint mcs;
     Uint nos;
+    Uint ndai;
     ErtsMMapInit dflt_mmap;
     ErtsMMapInit literal_mmap;
 } ErtsMsegInit_t;
@@ -68,7 +69,8 @@ typedef struct {
     4*1024*1024,	/* amcbf: Absolute max cache bad fit	*/	\
     20,			/* rmcbf: Relative max cache bad fit	*/	\
     10,			/* mcs:   Max cache size		*/	\
-    1000,		/* cci:   Cache check interval		*/	\
+    0,                  /* nos:   Number of schedulers		*/	\
+    0,                  /* ndai:  Number of dirty alloc instances*/	\
     ERTS_MMAP_INIT_DEFAULT_INITER,					\
     ERTS_MMAP_INIT_LITERAL_INITER,                                      \
 }
diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl
index 6e5b672646..bd3446914c 100644
--- a/erts/emulator/test/timer_bif_SUITE.erl
+++ b/erts/emulator/test/timer_bif_SUITE.erl
@@ -897,7 +897,7 @@ mem_get() ->
     % Bif timer memory
     Ref = make_ref(),
     erlang:system_info({memory_internal, Ref, [fix_alloc]}),
-    mem_recv(erlang:system_info(schedulers), Ref, {0, 0}).
+    mem_recv(erts_internal:no_aux_work_threads()-1, Ref, {0, 0}).
 
 mem_recv(0, _Ref, AU) ->
     AU;
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index 06182c25ce..e5f8ee6f77 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -119,6 +119,7 @@ static char *plusM_other_switches[] = {
     "Ytt",
     "Iscs",
     "Xscs",
+    "dai",
     NULL
 };
 
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 649c67b1aa..660e71ffab 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -4167,7 +4167,7 @@ receive_emd(Ref, EMD, N) ->
     end.
 
 receive_emd(Ref) ->
-    receive_emd(Ref, #memory{}, erlang:system_info(schedulers)).
+    receive_emd(Ref, #memory{}, erts_internal:no_aux_work_threads()-1).
 
 aa_mem_data(#memory{total = Tot} = Mem,
 	    [{external_alloc, Sz} | Rest]) ->
@@ -4237,7 +4237,7 @@ get_alloc_info(Type, AList) when erlang:is_list(AList) ->
     Ref = erlang:make_ref(),
     erlang:system_info({Type, Ref, AList}),
     receive_allocator(Ref,
-		      erlang:system_info(schedulers),
+		      erts_internal:no_aux_work_threads()-1,
 		      mk_res_list(AList)).
 
 mk_res_list([]) ->
diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl
index 9925dbd3f9..866b97a094 100644
--- a/erts/preloaded/src/erts_internal.erl
+++ b/erts/preloaded/src/erts_internal.erl
@@ -113,6 +113,8 @@
 
 -export([prepare_loading/2, beamfile_chunk/2, beamfile_module_md5/1]).
 
+-export([no_aux_work_threads/0]).
+
 %%
 %% Await result of send to port
 %%
@@ -951,3 +953,8 @@ beamfile_chunk(_Bin, _Chunk) ->
 -spec beamfile_module_md5(binary()) -> binary() | undefined.
 beamfile_module_md5(_Bin) ->
     erlang:nif_error(undefined).
+
+-spec no_aux_work_threads() -> pos_integer().
+
+no_aux_work_threads() ->
+    erlang:nif_error(undefined).
diff --git a/lib/tools/src/instrument.erl b/lib/tools/src/instrument.erl
index 1d1edcbbea..31a9925e40 100644
--- a/lib/tools/src/instrument.erl
+++ b/lib/tools/src/instrument.erl
@@ -46,7 +46,7 @@ allocations() ->
 allocations(Options) ->
     Ref = make_ref(),
 
-    Defaults = #{ scheduler_ids => lists:seq(0, erlang:system_info(schedulers)),
+    Defaults = #{ scheduler_ids => lists:seq(0, erts_internal:no_aux_work_threads()-1),
                   allocator_types => erlang:system_info(alloc_util_allocators),
                   histogram_start => 128,
                   histogram_width => 18 },
@@ -118,7 +118,7 @@ carriers() ->
 carriers(Options) ->
     Ref = make_ref(),
 
-    Defaults = #{ scheduler_ids => lists:seq(0, erlang:system_info(schedulers)),
+    Defaults = #{ scheduler_ids => lists:seq(0, erts_internal:no_aux_work_threads()-1),
                   allocator_types => erlang:system_info(alloc_util_allocators),
                   histogram_start => 512,
                   histogram_width => 14 },
-- 
2.31.1

Places

File 1791-Allocator-instances-for-dirty-schedulers.patch of Package erlang (Revision de16d5bb9b1e73f72aa79d2ecdf361ff)

Places