File 3404-erts-Micro-optimize-ets-lock-adaption.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 3404-erts-Micro-optimize-ets-lock-adaption.patch of Package erlang (Revision 0c95bdcb413813ad538c04f6e313b575)

Currently displaying revision 0c95bdcb413813ad538c04f6e313b575 , Show latest

From 7751899402583c8b358d163466d2a4d7deacd8e6 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 3 Feb 2022 18:05:16 +0100
Subject: [PATCH 4/4] erts: Micro-optimize ets lock adaption

---
 erts/emulator/beam/erl_db.c      |  2 +-
 erts/emulator/beam/erl_db_hash.c | 89 ++++++++++++++++++--------------
 erts/emulator/beam/erl_db_hash.h | 11 +++-
 3 files changed, 61 insertions(+), 41 deletions(-)

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 861cc0d917..a070e0876f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -818,7 +818,7 @@ DbTable* db_get_table_aux(Process *p,
             return tb;
         }
 
-        erl_db_hash_adapt_number_of_locks(tb);
+        DB_HASH_ADAPT_NUMBER_OF_LOCKS(tb);
 	db_lock(tb, kind);
         if (name_lck)
             erts_rwmtx_runlock(name_lck);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index db8af4a8d3..e625c95313 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -291,17 +291,13 @@ static ERTS_INLINE int is_pseudo_deleted(HashDbTerm* p)
 
 static void calc_shrink_limit(DbTableHash* tb);
 
-void erl_db_hash_adapt_number_of_locks(DbTable* tb) {
+void db_hash_adapt_number_of_locks(DbTable* tb) {
     db_hash_lock_array_resize_state current_state;
     DbTableHash* tbl;
     int new_number_of_locks;
-    if(!IS_HASH_WITH_AUTO_TABLE(tb->common.type)) {
-        return;
-    }
-    current_state = erts_atomic_read_nob(&tb->hash.lock_array_resize_state);
-    if (current_state == DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL) {
-        return;
-    }
+
+    ASSERT(IS_HASH_WITH_AUTO_TABLE(tb->common.type));
+
     tbl = &tb->hash;
     erts_rwmtx_rwlock(&tb->common.rwlock);
     current_state = erts_atomic_read_nob(&tb->hash.lock_array_resize_state);
@@ -414,6 +410,40 @@ static ERTS_INLINE erts_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval)
     }
 }
 
+static void
+wlock_after_failed_trylock(DbTableHash* tb, DbTableHashLockAndCounter* lock)
+{
+    erts_rwmtx_rwlock(&lock->lck);
+    lock->lck_stat += LCK_AUTO_CONTENDED_STAT_CONTRIB;
+    if (lock->lck_stat > LCK_AUTO_GROW_LIMIT) {
+        /*
+         * Do not do any adaptation if the table is
+         * fixed as this can lead to missed slots when
+         * traversing over the table.
+         */
+        if (!IS_FIXED(tb)) {
+            if (tb->nlocks < LCK_AUTO_MAX_LOCKS &&
+                (DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL ==
+                 erts_atomic_read_nob(&tb->lock_array_resize_state))) {
+                /*
+                 * Trigger lock array increase later when we
+                 * can take the table lock
+                 */
+                erts_atomic_set_nob(&tb->lock_array_resize_state,
+                                    DB_HASH_LOCK_ARRAY_RESIZE_STATUS_GROW);
+            }
+            else {
+                /*
+                 * The lock statistics is kept if the table is
+                 * fixed as it is likely that we want to adapt
+                 * when the table is not fixed any more.
+                 */
+                lock->lck_stat = 0;
+            }
+        }
+    }
+}
+
 /* Fine grained write lock */
 static ERTS_INLINE
 DbTableHashLockAndCounter* WLOCK_HASH_GET_LCK_AND_CTR(DbTableHash* tb, HashValue hval)
@@ -425,37 +455,12 @@ DbTableHashLockAndCounter* WLOCK_HASH_GET_LCK_AND_CTR(DbTableHash* tb, HashValue
         if (tb->common.type & DB_FINE_LOCKED_AUTO) {
             DbTableHashLockAndCounter* lck_couter = GET_LOCK_AND_CTR(tb, hval);
             if (EBUSY == erts_rwmtx_tryrwlock(&lck_couter->lck)) {
-                erts_rwmtx_rwlock(&lck_couter->lck);
-                lck_couter->lck_stat += LCK_AUTO_CONTENDED_STAT_CONTRIB;
-                if (lck_couter->lck_stat > LCK_AUTO_GROW_LIMIT) {
-                    if (tb->nlocks < LCK_AUTO_MAX_LOCKS &&
-                        !IS_FIXED(tb) &&
-                        (DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL ==
-                         erts_atomic_read_nob(&tb->lock_array_resize_state))) {
-                        /*
-                          Trigger lock array increase later when we
-                          can take the table lock
-                        */
-                        erts_atomic_set_nob(&tb->lock_array_resize_state,
-                                            DB_HASH_LOCK_ARRAY_RESIZE_STATUS_GROW);
-                    } else if (!IS_FIXED(tb)) {
-                        /*
-                          Do not do any adaptation if the table is
-                          fixed as this can lead to missed slots when
-                          traversing over the table.
-
-                          The lock statistics is kept if the table is
-                          fixed as it is likely that we want to adapt
-                          when the table is not fixed any more.
-                        */
-                        lck_couter->lck_stat = 0;
-                    }
-                }
+                wlock_after_failed_trylock(tb, lck_couter);
             } else {
                 lck_couter->lck_stat += LCK_AUTO_UNCONTENDED_STAT_CONTRIB;
-                if (lck_couter->lck_stat < LCK_AUTO_SHRINK_LIMIT) {
+                if (lck_couter->lck_stat < LCK_AUTO_SHRINK_LIMIT
+                    && !IS_FIXED(tb)) {
                     if(tb->nlocks > LCK_AUTO_MIN_LOCKS &&
-                       !IS_FIXED(tb) &&
                        (DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL ==
                         erts_atomic_read_nob(&tb->lock_array_resize_state))) {
                         /*
@@ -464,7 +469,7 @@ DbTableHashLockAndCounter* WLOCK_HASH_GET_LCK_AND_CTR(DbTableHash* tb, HashValue
                         */
                         erts_atomic_set_nob(&tb->lock_array_resize_state,
                                             DB_HASH_LOCK_ARRAY_RESIZE_STATUS_SHRINK);
-                    } else if (!IS_FIXED(tb)) {
+                    } else {
                         lck_couter->lck_stat = 0;
                     }
                 }
@@ -531,6 +536,13 @@ static ERTS_INLINE void WUNLOCK_HASH_LCK_CTR(DbTableHashLockAndCounter* lck_ctr)
 static ERTS_INLINE Sint next_slot(DbTableHash* tb, Uint ix,
 				  erts_rwmtx_t** lck_ptr)
 {
+    /*
+     * To minimize locking ops, we jump to next bucket using same lock.
+     * In case of {write_concurrency,auto} this is safe as 'nlocks' does not
+     * change as long as table is fixed, which all single call select/match do.
+     * Unfixed next,prev and select/1 calls are also "safe" in the sence that
+     * we will seize correct locks as 'nlocks' will not change during the calls.
+     */
     ix += tb->nlocks;
     if (ix < NACTIVE(tb)) return ix;
     RUNLOCK_HASH(*lck_ptr);
@@ -1923,7 +1935,8 @@ static int match_traverse_continue(traverse_context_t* ctx,
     }
 
     lck = ctx->on_lock_hash(tb, slot_ix);
-    if (slot_ix >= NACTIVE(tb)) { /* Is this possible? */
+    if (slot_ix >= NACTIVE(tb)) {
+        /* Is this possible? Yes, for ets:select/1 without safe_fixtable */
         ctx->on_unlock_hash(lck);
         *ret = NIL;
         ret_value = DB_ERROR_BADPARAM;
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 1590afb931..6025cde555 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -95,8 +95,15 @@ typedef enum {
 } db_hash_lock_array_resize_state;
 
 /* To adapt number of locks if hash table with {write_concurrency, auto} */
-void erl_db_hash_adapt_number_of_locks(DbTable* tb);
-
+void db_hash_adapt_number_of_locks(DbTable* tb);
+#define  DB_HASH_ADAPT_NUMBER_OF_LOCKS(TB)                                   \
+    do {                                                                     \
+        if (IS_HASH_WITH_AUTO_TABLE(TB->common.type)                         \
+            && (erts_atomic_read_nob(&tb->hash.lock_array_resize_state)      \
+                != DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL)) {               \
+            db_hash_adapt_number_of_locks(tb);                               \
+        }                                                                    \
+    }while(0)
 /*
 ** Function prototypes, looks the same (except the suffix) for all 
 ** table types. The process is always an [in out] parameter.
-- 
2.34.1

Places

File 3404-erts-Micro-optimize-ets-lock-adaption.patch of Package erlang (Revision 0c95bdcb413813ad538c04f6e313b575)

Places