File 2771-erts-Fix-ets-memory-calculation-for-read_c... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 2771-erts-Fix-ets-memory-calculation-for-read_concurrency.patch of Package erlang (Revision ddfbbd0efffaf9570a604d2e0f074ad7)

Currently displaying revision ddfbbd0efffaf9570a604d2e0f074ad7 , Show latest

From 989c314afcf467258d4cb2e950441c4bd78d71c2 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Tue, 7 Dec 2021 09:38:26 +0100
Subject: [PATCH 1/3] erts: Fix ets memory calculation for read_concurrency

The amount of memory used for reader groups can be very
large on systems with many cores, so it needs to be part
of the ets table's memory utilization. We use that memory
to calculate how many tables can be created when testing.
---
 erts/emulator/beam/erl_db.c        | 11 +++++---
 erts/emulator/beam/erl_db_catree.c |  6 ++++-
 erts/emulator/beam/erl_db_hash.c   | 13 +++++++---
 erts/emulator/beam/erl_db_tree.c   |  2 ++
 erts/emulator/beam/erl_threads.h   |  6 +++++
 erts/include/internal/ethr_mutex.h |  1 +
 erts/lib_src/common/ethr_mutex.c   | 27 ++++++++++++++++++++
 lib/stdlib/test/ets_SUITE.erl      | 40 ++++++++++++++++++++++--------
 8 files changed, 87 insertions(+), 19 deletions(-)

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index c8300955cd..f043eed51f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -450,16 +450,18 @@ free_dbtable(void *vtb)
     erts_flxctr_add(&tb->common.counters,
                     ERTS_DB_TABLE_MEM_COUNTER_ID,
                     -((Sint)erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)));
-    ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
-           sizeof(DbTable) == DB_GET_APPROX_MEM_CONSUMED(tb));
-
-    ASSERT(is_immed(tb->common.heir_data));
 
     if (!DB_LOCK_FREE(tb)) {
+        ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&tb->common.rwlock), 0);
         erts_rwmtx_destroy(&tb->common.rwlock);
         erts_mtx_destroy(&tb->common.fixlock);
     }
 
+    ASSERT(is_immed(tb->common.heir_data));
+
+    ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
+           sizeof(DbTable) == DB_GET_APPROX_MEM_CONSUMED(tb));
+
     ASSERT(tb->common.btid);
     erts_bin_release(tb->common.btid);
 
@@ -644,6 +646,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock)
     if (!DB_LOCK_FREE(tb)) {
         erts_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab",
                             tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+        ERTS_DB_ALC_MEM_UPDATE_(tb, 0, erts_rwmtx_size(&tb->common.rwlock));
         erts_mtx_init(&tb->common.fixlock, "db_tab_fix",
                       tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
     }
diff --git a/erts/emulator/beam/erl_db_catree.c b/erts/emulator/beam/erl_db_catree.c
index e1304b831f..c2578fb521 100644
--- a/erts/emulator/beam/erl_db_catree.c
+++ b/erts/emulator/beam/erl_db_catree.c
@@ -1034,6 +1034,7 @@ static DbTableCATreeNode *create_base_node(DbTableCATree *tb,
                         "erl_db_catree_base_node",
                         NIL,
                         ERTS_LOCK_FLAGS_CATEGORY_DB);
+    ERTS_DB_ALC_MEM_UPDATE_((DbTable *) tb, 0, erts_rwmtx_size(&p->u.base.lock));
     BASE_NODE_STAT_SET(p, ((tb->common.status & DB_CATREE_FORCE_SPLIT)
                            ? INT_MAX : 0));
     p->u.base.is_valid = 1;
@@ -1092,7 +1093,7 @@ static void do_free_base_node(void* vptr)
 static void free_catree_base_node(DbTableCATree* tb, DbTableCATreeNode* p)
 {
     ASSERT(p->is_base_node);
-    ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof_base_node(), 0);
+    ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof_base_node() + erts_rwmtx_size(&p->u.base.lock), 0);
     do_free_base_node(p);
 }
 
@@ -1334,11 +1335,13 @@ static void join_catree(DbTableCATree *tb,
                           thiz,
                           &thiz->u.base.free_item,
                           sizeof_base_node());
+    ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&thiz->u.base.lock), 0);
     erts_schedule_db_free(&tb->common,
                           do_free_base_node,
                           neighbor,
                           &neighbor->u.base.free_item,
                           sizeof_base_node());
+    ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&neighbor->u.base.lock), 0);
 }
 
 static void split_catree(DbTableCATree *tb,
@@ -1383,6 +1386,7 @@ static void split_catree(DbTableCATree *tb,
                               base,
                               &base->u.base.free_item,
                               sizeof_base_node());
+        ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&base->u.base.lock), 0);
     }
 }
 
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index d3a29e5c04..105a5d76c3 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -375,8 +375,9 @@ void erl_db_hash_adapt_number_of_locks(DbTable* tb) {
                                                               (DbTable *) tb,
                                                               sizeof(DbTableHashFineLockSlot) * tbl->nlocks);
         for (i=0; i < tbl->nlocks; i++) {
-            erts_rwmtx_init_opt(&tbl->locks[i].u.lck_ctr.lck, &rwmtx_opt,
+            erts_rwmtx_init_opt(GET_LOCK(tbl, i), &rwmtx_opt,
                                 "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+            ERTS_DB_ALC_MEM_UPDATE_(tb, 0, erts_rwmtx_size(GET_LOCK(tbl,i)));
             tbl->locks[i].u.lck_ctr.nitems =
                 get_lock_nitems_form_prev_lock_array(i, tbl->nlocks, old_number_of_locks, old_locks);
             tbl->locks[i].u.lck_ctr.lck_stat = 0;
@@ -403,6 +404,7 @@ void erl_db_hash_adapt_number_of_locks(DbTable* tb) {
         erts_atomic_set_nob(&tbl->lock_array_resize_state, DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL);
         erts_rwmtx_rwunlock(&tb->common.rwlock);
         for (i = 0; i < old_number_of_locks; i++) {
+            ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&old_locks[i].u.lck_ctr.lck), 0);
             erts_rwmtx_destroy(&old_locks[i].u.lck_ctr.lck);
         }
         erts_db_free(ERTS_ALC_T_DB_SEG, tb, old_locks, sizeof(DbTableHashFineLockSlot) * old_number_of_locks);
@@ -1037,8 +1039,10 @@ int db_create_hash(Process *p, DbTable *tbl)
                                                              (DbTable *) tb,
                                                              sizeof(DbTableHashFineLockSlot) * tb->nlocks);
 	for (i=0; i<tb->nlocks; ++i) {
-            erts_rwmtx_init_opt(&tb->locks[i].u.lck_ctr.lck, &rwmtx_opt,
+            erts_rwmtx_init_opt(
+                GET_LOCK(tb,i), &rwmtx_opt,
                 "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+            ERTS_DB_ALC_MEM_UPDATE_(tb, 0, erts_rwmtx_size(GET_LOCK(tb,i)));
             tb->locks[i].u.lck_ctr.nitems = 0;
             tb->locks[i].u.lck_ctr.lck_stat = 0;
 	}
@@ -3065,14 +3069,15 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
     if (tb->locks != NULL) {
 	int i;
 	for (i=0; i<tb->nlocks; ++i) {
-	    erts_rwmtx_destroy(GET_LOCK(tb,i)); 
+            ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(GET_LOCK(tb,i)), 0);
+	    erts_rwmtx_destroy(GET_LOCK(tb,i));
 	}
 	erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb,
 		     (void*)tb->locks, tb->nlocks * sizeof(DbTableHashFineLockSlot));
 	tb->locks = NULL;
     }
     ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
-           ((sizeof(DbTable) +
+           ((sizeof(DbTable) + (!DB_LOCK_FREE(tb) ? erts_rwmtx_size(&tb->common.rwlock) : 0) +
              erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)) ==
             erts_flxctr_read_approx(&tb->common.counters,
                                     ERTS_DB_TABLE_MEM_COUNTER_ID)));
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index b2b1773211..8de4fc201b 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -2467,9 +2467,11 @@ static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds)
 	ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
                ((APPROX_MEM_CONSUMED(tb)
                  == (sizeof(DbTable) +
+                     (!DB_LOCK_FREE(tb) ? erts_rwmtx_size(&tb->common.rwlock) : 0) +
                      erts_flxctr_nr_of_allocated_bytes(&tb->common.counters))) ||
                 (APPROX_MEM_CONSUMED(tb)
                  == (sizeof(DbTable) +
+                     (!DB_LOCK_FREE(tb) ? erts_rwmtx_size(&tb->common.rwlock) : 0) +
                      sizeof(DbFixation) +
                      erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)))));
     }
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index e6f0300e7c..a73809f90a 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -445,6 +445,7 @@ ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
                                      char *name,
                                      Eterm extra,
                                      erts_lock_flags_t flags);
+ERTS_GLB_INLINE size_t erts_rwmtx_size(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line);
@@ -1883,6 +1884,11 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra,
     erts_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
 }
 
+ERTS_GLB_INLINE size_t
+erts_rwmtx_size(erts_rwmtx_t *rwmtx) {
+    return ethr_rwmutex_size(&rwmtx->rwmtx);
+}
+
 ERTS_GLB_INLINE void
 erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 {
diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
index 8ef3b1e40b..1c7808ce84 100644
--- a/erts/include/internal/ethr_mutex.h
+++ b/erts/include/internal/ethr_mutex.h
@@ -341,6 +341,7 @@ struct ethr_rwmutex_ {
 int ethr_rwmutex_set_reader_group(int);
 int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *);
 int ethr_rwmutex_init(ethr_rwmutex *);
+size_t ethr_rwmutex_size(ethr_rwmutex *);
 int ethr_rwmutex_destroy(ethr_rwmutex *);
 #if defined(ETHR_USE_OWN_RWMTX_IMPL__) \
     || !defined(ETHR_TRY_INLINE_FUNCS) \
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index d8d8cde8e8..69848dddff 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -2720,6 +2720,28 @@ ethr_rwmutex_init(ethr_rwmutex *rwmtx)
     return ethr_rwmutex_init_opt(rwmtx, NULL);
 }
 
+size_t
+ethr_rwmutex_size(ethr_rwmutex *rwmtx) {
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+        return sizeof(ethr_rwmtx_readers_array__) * (reader_groups_array_size + 1);
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+        return sizeof(ethr_rwmtx_readers_array__) * (main_threads_array_size + 1);
+    default:
+        return 0;
+    }
+}
+
 int
 ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
 {
@@ -3098,6 +3120,11 @@ ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
     return ethr_rwmutex_init(rwmtx);
 }
 
+size_t
+ethr_rwmutex_size(ethr_rwmutex *rwmtx) {
+    return 0;
+}
+
 int
 ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
 {
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 1470e3f8a7..2069f5f27c 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -3042,8 +3043,9 @@ write_concurrency(Config) when is_list(Config) ->
     YesTreeMem = ets:info(Yes7,memory),
     YesYesTreeMem = ets:info(Yes14,memory),
     NoTreeMem = ets:info(No4,memory),
-    io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p YesTreeMem=~p\n",[YesMem,NoHashMem,
-                                                                     NoTreeMem,YesTreeMem]),
+
+    io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p YesTreeMem=~p YesYesTreeMem=~p\n",
+              [YesMem,NoHashMem,NoTreeMem,YesTreeMem,YesYesTreeMem]),
 
     YesMem = ets:info(Yes2,memory),
     YesMem = ets:info(Yes3,memory),
@@ -3072,10 +3074,12 @@ write_concurrency(Config) when is_list(Config) ->
         true ->
             true = YesMem > NoHashMem,
             true = YesMem > NoTreeMem,
-            true = YesTreeMem < NoTreeMem,
+            true = YesTreeMem > NoTreeMem,
             true = YesYesTreeMem > YesTreeMem;
         _ ->
-            one_scheduler_only
+            YesMem = NoHashMem,
+            YesTreeMem = NoTreeMem,
+            YesYesTreeMem = YesTreeMem
     end,
 
     {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency,foo}])),
@@ -4384,12 +4388,28 @@ exit_many_many_tables_owner(Config) when is_list(Config) ->
     repeat_for_opts(fun(Opts) -> exit_many_many_tables_owner_do(Opts,FEData,Config) end).
 
 exit_many_many_tables_owner_do(Opts,FEData,Config) ->
-    verify_rescheduling_exit(Config, FEData, [named_table | Opts], true, 200, 5),
-    verify_rescheduling_exit(Config, FEData, Opts, false, 200, 5),
+
+    E = ets_new(tmp,Opts),
+    FEData(fun(Data) -> ets:insert(E, Data) end),
+    Mem = ets:info(E,memory) * erlang:system_info(wordsize),
+    ets:delete(E),
+
+    ct:log("Memory per table: ~p bytes",[Mem]),
+
+    Tables =
+        case erlang:system_info(wordsize) of
+            8 ->
+                200;
+            4 ->
+                lists:min([200,2_000_000_000 div (Mem * 5)])
+        end,
+
+    verify_rescheduling_exit(Config, FEData, [named_table | Opts], true, Tables, 5),
+    verify_rescheduling_exit(Config, FEData, Opts, false, Tables, 5),
     wait_for_test_procs(),
     EtsMem = etsmem(),
-    verify_rescheduling_exit(Config, FEData, Opts, true, 200, 5),
-    verify_rescheduling_exit(Config, FEData, [named_table | Opts], false, 200, 5),
+    verify_rescheduling_exit(Config, FEData, Opts, true, Tables, 5),
+    verify_rescheduling_exit(Config, FEData, [named_table | Opts], false, Tables, 5),
     verify_etsmem(EtsMem).
 
 
-- 
2.31.1

Places

File 2771-erts-Fix-ets-memory-calculation-for-read_concurrency.patch of Package erlang (Revision ddfbbd0efffaf9570a604d2e0f074ad7)

Places