File 4432-erts-Add-acful-abandon-carried-free-utiliz... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 4432-erts-Add-acful-abandon-carried-free-utilization-limi.patch of Package erlang (Revision e0e016f64b94e78db3440770f914218d)

Currently displaying revision e0e016f64b94e78db3440770f914218d , Show latest

From 9b0c7899680f894a87b8eee83f8ed07148e87d9c Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Mon, 17 Oct 2022 14:03:12 +0200
Subject: [PATCH 2/2] erts: Add acful, abandon carried free utilization limit

This limit can be used to let erts_alloc know that it should
use MADV_FREE on any pages within a carrier that are currently
unused.

Before this change we have tried to have this as default, but that
caused performance issues as carriers oscillated in and out of the
migration pool. So in 25 we decided to remove it. However, this caused
MemAvailable from /proc/memory to become significantly lower and thus
machines that have a tight memory budget need this option.

So the solution is to add a new config option where the user can set
at which utilization limit that free pages in a carrier should be placed
in MemAvailable.

The option is by default set to 0, to mimic the behaviour of OTP 25.
---
 erts/doc/src/erts_alloc.xml         | 23 ++++++++++++
 erts/emulator/beam/erl_alloc.c      | 52 +++++++++++++++++++++++++-
 erts/emulator/beam/erl_alloc_util.c | 58 +++++++++++++++++++----------
 erts/emulator/beam/erl_alloc_util.h |  5 +++
 erts/etc/common/erlexec.c           |  1 +
 5 files changed, 119 insertions(+), 20 deletions(-)

diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml
index b66f8803e2..da293f9c45 100644
--- a/erts/doc/src/erts_alloc.xml
+++ b/erts/doc/src/erts_alloc.xml
@@ -514,6 +514,29 @@
 	 See also <seecref marker="#M_acul"><c>acul</c></seecref>.
 </item>
 
+ <tag><marker id="M_acful"/><c><![CDATA[+M<S>acful <utilization>|de]]></c>
+ </tag>
+ <item>
+ Abandon carrier free utilization limit. When the utilization of
+ a carrier falls belows this limit erts_alloc instructs the OS that
+ unused memory in the carrier can be re-used for allocation by other OS
+ procesesses. On Unix this is done by calling <c>madvise(..., ..., MADV_FREE)</c>
+ on the unused memory region, on Windows it is done by calling
+ <c>VirtualAlloc(..., ..., MEM_RESET, PAGE_READWRITE)</c>. Defaults to 0
+ which means that no memory will be marked as re-usable by the OS.
+ A valid
+ <c><![CDATA[<utilization>]]></c> is an integer in the range
+ <c>[0, 100]</c> representing utilization in percent. If this value is
+ larger than the <c>acul</c> limit it will be lowered to the current
+ <c>acul</c> limit. If <c>de</c> (default
+ enabled) is passed instead of a <c><![CDATA[<utilization>]]></c>,
+ a recommended non-zero utilization value is used. The value
+ chosen depends on the allocator type and can be changed between
+ ERTS versions.
+ 
+	 See also <seecref marker="#M_acul"><c>acul</c></seecref>.
+ </item>
+
 <tag><marker id="M_as"/>
 <c><![CDATA[+M<S>as bf|aobf|aoff|aoffcbf|aoffcaobf|ageffcaoff|ageffcbf|ageffcaobf|gf|af]]></c></tag>
 <item>
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 154d838dfe..3ab9c9d1b6 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -79,6 +79,9 @@
 #define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC
 #define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC
 
+#define ERTS_ALC_DEFAULT_ENABLED_ACFUL ERTS_ALC_DEFAULT_ENABLED_ACUL - 20
+#define ERTS_ALC_DEFAULT_ENABLED_ACFUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC - 20
+#define ERTS_ALC_DEFAULT_ENABLED_ACFUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC - 20
 
 #ifdef DEBUG
 static Uint install_debug_functions(void);
@@ -310,6 +313,7 @@ set_default_literal_alloc_opts(struct au_init *ip)
 ip->init.util.rsbcmt	= 0;
 ip->init.util.rmbcmt	= 0;
 ip->init.util.acul		= 0;
+ ip->init.util.acful		= 0;
 
 #if defined(ARCH_32)
 # if HAVE_ERTS_MSEG
@@ -592,6 +596,9 @@ adjust_carrier_migration_support(struct au_init *auip)
 	 auip->init.aoff.blk_order = FF_BF;
 	}
 }
+ if (auip->init.util.acful > auip->init.util.acul) {
+ auip->init.util.acful = auip->init.util.acul;
+ }
 }
 
 void
@@ -691,6 +698,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
 /* Make adjustments for carrier migration support */
 init.temp_alloc.init.util.acul = 0;
+ init.temp_alloc.init.util.acful = 0;
 adjust_carrier_migration_support(&init.sl_alloc);
 adjust_carrier_migration_support(&init.std_alloc);
 adjust_carrier_migration_support(&init.ll_alloc);
@@ -727,6 +735,16 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.driver_alloc.init.util.acul = 0;
 	init.fix_alloc.init.util.acul = 0;
 init.literal_alloc.init.util.acul = 0;
+ init.temp_alloc.init.util.acful = 0;
+	init.sl_alloc.init.util.acful = 0;
+	init.std_alloc.init.util.acful = 0;
+	init.ll_alloc.init.util.acful = 0;
+	init.eheap_alloc.init.util.acful = 0;
+	init.binary_alloc.init.util.acful = 0;
+	init.ets_alloc.init.util.acful = 0;
+	init.driver_alloc.init.util.acful = 0;
+	init.fix_alloc.init.util.acful = 0;
+ init.literal_alloc.init.util.acful = 0;
 }
 
 /* Only temp_alloc can use thread specific interface */
@@ -1275,6 +1293,30 @@ get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip)
 return (Uint) tmp;
 }
 
+static Uint
+get_acful_value(struct au_init *auip, char *param_end, char** argv, int* ip)
+{
+ Sint tmp;
+ char *rest;
+ char *param = argv[*ip]+1;
+ char *value = get_value(param_end, argv, ip);
+ if (sys_strcmp(value, "de") == 0) {
+	switch (auip->init.util.alloc_no) {
+	case ERTS_ALC_A_LONG_LIVED:
+	 return ERTS_ALC_DEFAULT_ENABLED_ACFUL_LL_ALLOC;
+	case ERTS_ALC_A_EHEAP:
+	 return ERTS_ALC_DEFAULT_ENABLED_ACFUL_EHEAP_ALLOC;
+	default:
+	 return ERTS_ALC_DEFAULT_ENABLED_ACFUL;
+	}
+ }
+ errno = 0;
+ tmp = (Sint) ErtsStrToSint(value, &rest, 10);
+ if (errno != 0 || rest == value || tmp < 0 || 100 < tmp)
+	bad_value(param, param_end, value);
+ return (Uint) tmp;
+}
+
 static void
 handle_au_arg(struct au_init *auip,
 	 char* sub_param,
@@ -1304,6 +1346,10 @@ handle_au_arg(struct au_init *auip,
 value = get_amount_value(sub_param + 5, argv, ip);
 wp = &auip->init.util.acfml;
 }
+ else if (has_prefix("acful", sub_param)) {
+ value = get_acful_value(auip, sub_param + 5, argv, ip);
+ wp = &auip->init.util.acful;
+ }
 else
 goto bad_switch;
 
@@ -1370,8 +1416,10 @@ handle_au_arg(struct au_init *auip,
 bad_value(param, sub_param + 1, alg);
 }
 	 }
-	 if (!strategy_support_carrier_migration(auip))
+	 if (!strategy_support_carrier_migration(auip)) {
 		auip->init.util.acul = 0;
+		auip->init.util.acful = 0;
+ }
 	} else if (has_prefix("atags", sub_param)) {
 auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip);
 }
@@ -1503,6 +1551,7 @@ handle_au_arg(struct au_init *auip,
 	else if (res == 0) {
 	 auip->thr_spec = 0;
 	 auip->init.util.acul = 0;
+	 auip->init.util.acful = 0;
 	 break;
 	}
 	goto bad_switch;
@@ -1714,6 +1763,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 			 for (a = 0; a < aui_sz; a++) {
 				aui[a]->thr_spec = 0;
 				aui[a]->init.util.acul = 0;
+				aui[a]->init.util.acful = 0;
 				aui[a]->init.util.ramv = 0;
 				aui[a]->init.util.lmbcs = 5*1024*1024;
 			 }
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 7f3a9b420b..be5705c637 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1693,7 +1693,7 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr)
 }
 
 
-static UWord allctr_abandon_limit(Allctr_t *allctr);
+static UWord allctr_abandon_limit(Allctr_t *allctr, UWord);
 static void set_new_allctr_abandon_limit(Allctr_t*);
 static void abandon_carrier(Allctr_t*, Carrier_t*);
 static void poolify_my_carrier(Allctr_t*, Carrier_t*);
@@ -2251,7 +2251,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
 if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	return;
 
- ASSERT(allctr->cpool.abandon_limit == allctr_abandon_limit(allctr));
+ ASSERT(allctr->cpool.abandon_limit == allctr_abandon_limit(allctr, allctr->cpool.util_limit));
 ASSERT(erts_thr_progress_is_managed_thread());
 
 if (allctr->cpool.disable_abandon)
@@ -2655,6 +2655,9 @@ carrier_mem_discard_free_blocks(Allctr_t *allocator, Carrier_t *carrier)
 Block_t *block;
 int i;
 
+ if (carrier->cpool.total_blocks_size > carrier->cpool.discard_limit)
+ return;
+
 block = allocator->first_fblk_in_mbc(allocator, carrier);
 i = 0;
 
@@ -2723,7 +2726,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp
 	blk = PREV_BLK(blk);
 	(*allctr->unlink_free_block)(allctr, blk);
 
- if (discard) {
+ if (discard && crr->cpool.total_blocks_size <= crr->cpool.discard_limit) {
 mem_discard_coalesce(allctr, blk, &discard_region);
 }
 
@@ -2743,7 +2746,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp
 	 /* Coalesce with next block... */
 	 (*allctr->unlink_free_block)(allctr, nxt_blk);
 
- if (discard) {
+ if (discard && crr->cpool.total_blocks_size <= crr->cpool.discard_limit) {
 mem_discard_coalesce(allctr, nxt_blk, &discard_region);
 }
 
@@ -2783,7 +2786,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp
 	(*allctr->link_free_block)(allctr, blk);
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 
- if (discard) {
+ if (discard && crr->cpool.total_blocks_size <= crr->cpool.discard_limit) {
 mem_discard_finish(allctr, blk, &discard_region);
 }
 
@@ -3866,6 +3869,17 @@ static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr)
 erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix);
 }
 
+static ERTS_INLINE UWord
+cpoll_init_carrier_limit(Carrier_t *crr, UWord percent_limit) {
+ UWord csz = CARRIER_SZ(crr);
+ UWord limit = percent_limit*csz;
+ if (limit > csz)
+ limit /= 100;
+ else
+ limit = (csz/100)*percent_limit;
+ return limit;
+}
+
 static ERTS_INLINE void
 cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 {
@@ -3878,16 +3892,12 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 sys_memset(&crr->cpool.blocks_size, 0, sizeof(crr->cpool.blocks_size));
 sys_memset(&crr->cpool.blocks, 0, sizeof(crr->cpool.blocks));
 crr->cpool.total_blocks_size = 0;
- if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+ if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
 	crr->cpool.abandon_limit = 0;
- else {
-	UWord csz = CARRIER_SZ(crr);
-	UWord limit = csz*allctr->cpool.util_limit;
-	if (limit > csz)
-	 limit /= 100;
-	else
-	 limit = (csz/100)*allctr->cpool.util_limit;
-	crr->cpool.abandon_limit = limit;
+ crr->cpool.discard_limit = 0;
+ } else {
+	crr->cpool.abandon_limit = cpoll_init_carrier_limit(crr, allctr->cpool.util_limit);
+	crr->cpool.discard_limit = cpoll_init_carrier_limit(crr, allctr->cpool.free_util_limit);
 }
 crr->cpool.state = ERTS_MBC_IS_HOME;
 }
@@ -3895,7 +3905,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 
 
 static UWord
-allctr_abandon_limit(Allctr_t *allctr)
+allctr_abandon_limit(Allctr_t *allctr, UWord percent_limit)
 {
 UWord limit;
 UWord csz;
@@ -3906,11 +3916,11 @@ allctr_abandon_limit(Allctr_t *allctr)
 csz += allctr->mbcs.carriers[i].size;
 }
 
- limit = csz*allctr->cpool.util_limit;
+ limit = csz*percent_limit;
 if (limit > csz)
 	limit /= 100;
 else
-	limit = (csz/100)*allctr->cpool.util_limit;
+	limit = (csz/100)*percent_limit;
 
 return limit;
 }
@@ -3918,7 +3928,7 @@ allctr_abandon_limit(Allctr_t *allctr)
 static void ERTS_INLINE
 set_new_allctr_abandon_limit(Allctr_t *allctr)
 {
- allctr->cpool.abandon_limit = allctr_abandon_limit(allctr);
+ allctr->cpool.abandon_limit = allctr_abandon_limit(allctr, allctr->cpool.util_limit);
 }
 
 static void
@@ -4570,6 +4580,7 @@ static struct {
 Eterm smbcs;
 Eterm mbcgs;
 Eterm acul;
+ Eterm acful;
 Eterm acnl;
 Eterm acfml;
 Eterm cp;
@@ -4680,6 +4691,7 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(smbcs);
 	AM_INIT(mbcgs);
 	AM_INIT(acul);
+	AM_INIT(acful);
 AM_INIT(acnl);
 AM_INIT(acfml);
 AM_INIT(cp);
@@ -5439,7 +5451,7 @@ info_options(Allctr_t *allctr,
 	 Uint *szp)
 {
 Eterm res = THE_NON_VALUE;
- UWord acul, acnl, acfml;
+ UWord acul, acful, acnl, acfml;
 char *cp_str;
 Eterm cp_atom;
 
@@ -5454,6 +5466,7 @@ info_options(Allctr_t *allctr,
 }
 
 acul = allctr->cpool.util_limit;
+ acful = allctr->cpool.free_util_limit;
 acnl = allctr->cpool.in_pool_limit;
 acfml = allctr->cpool.fblk_min_limit;
 ASSERT(allctr->cpool.carrier_pool <= ERTS_ALC_A_MAX);
@@ -5502,6 +5515,7 @@ info_options(Allctr_t *allctr,
 		 "option smbcs: %beu\n"
 		 "option mbcgs: %beu\n"
 		 "option acul: %bpu\n"
+		 "option acful: %bpu\n"
 		 "option acnl: %bpu\n"
 		 "option acfml: %bpu\n"
 		 "option cp: %s\n",
@@ -5524,6 +5538,7 @@ info_options(Allctr_t *allctr,
 		 allctr->smallest_mbc_size,
 		 allctr->mbc_growth_stages,
 		 acul,
+		 acful,
 acnl,
 acfml,
 cp_str);
@@ -5543,6 +5558,9 @@ info_options(Allctr_t *allctr,
 	add_2tup(hpp, szp, &res,
 		 am.acul,
 		 bld_uint(hpp, szp, acul));
+ add_2tup(hpp, szp, &res,
+		 am.acful,
+		 bld_uint(hpp, szp, acful));
 	add_2tup(hpp, szp, &res,
 		 am.mbcgs,
 		 bld_uint(hpp, szp, allctr->mbc_growth_stages));
@@ -6795,6 +6813,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 ASSERT(allctr->next_fblk_in_mbc);
 
 allctr->cpool.util_limit = init->acul;
+ allctr->cpool.free_util_limit = init->acful;
 allctr->cpool.in_pool_limit = init->acnl;
 allctr->cpool.fblk_min_limit = init->acfml;
 allctr->cpool.carrier_pool = init->cp;
@@ -6808,6 +6827,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 }
 else {
 allctr->cpool.util_limit = 0;
+ allctr->cpool.free_util_limit = 0;
 allctr->cpool.in_pool_limit = 0;
 allctr->cpool.fblk_min_limit = 0;
 allctr->cpool.carrier_pool = -1;
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index d4f695443a..399fa2ccff 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -67,6 +67,7 @@ typedef struct {
 UWord smbcs;
 UWord mbcgs;
 UWord acul;
+ UWord acful;
 UWord acnl;
 UWord acfml;
 
@@ -127,6 +128,7 @@ typedef struct {
 1024*1024,		/* (bytes) smbcs: smallest mbc size */\
 10,			/* (amount) mbcgs: mbc growth stages */\
 0,			/* (%) acul: abandon carrier utilization limit */\
+ 0,			/* (%) acful: carrier free utilization limit */\
 1000,		/* (amount) acnl: abandoned carriers number limit */\
 0,			/* (bytes) acfml: abandoned carrier fblk min limit */\
 /* --- Data not options -------------------------------------------- */\
@@ -167,6 +169,7 @@ typedef struct {
 128*1024,		/* (bytes) smbcs: smallest mbc size */\
 10,			/* (amount) mbcgs: mbc growth stages */\
 0,			/* (%) acul: abandon carrier utilization limit */\
+ 0,			/* (%) acful: carrier free utilization limit */\
 1000,		/* (amount) acnl: abandoned carriers number limit */\
 0,			/* (bytes) acfml: abandoned carrier fblk min limit */\
 /* --- Data not options -------------------------------------------- */\
@@ -476,6 +479,7 @@ typedef struct {
 ErtsThrPrgrVal thr_prgr;
 erts_atomic_t max_size;
 UWord abandon_limit;
+ UWord discard_limit;
 UWord blocks[ERTS_ALC_A_COUNT];
 UWord blocks_size[ERTS_ALC_A_COUNT];
 UWord total_blocks_size;
@@ -697,6 +701,7 @@ struct Allctr_t_ {
 	int		disable_abandon;
 	int		check_limit_count;
 	UWord		util_limit; /* acul */
+	UWord		free_util_limit; /* acful */
 UWord in_pool_limit; /* acnl */
 UWord fblk_min_limit; /* acmfl */
 int carrier_pool; /* cp */
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index 9c3dc690ba..3e44ad2b77 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -77,6 +77,7 @@ static char *plusM_au_alloc_switches[] = {
 "acul",
 "acnl",
 "acfml",
+ "acful",
 "cp",
 "e",
 "t",
-- 
2.35.3

Places

File 4432-erts-Add-acful-abandon-carried-free-utilization-limi.patch of Package erlang (Revision e0e016f64b94e78db3440770f914218d)

Places