Move logic of adjust threshold to zmalloc module

Author: michalbiesek

redis.conf

@@ -1880,8 +1880,5 @@ dynamic-threshold-min 24
 # Maximum value of dynamic threshold
 dynamic-threshold-max 10000
 
-# DRAM/PMEM ratio period measured in miliseconds
-memory-ratio-check-period 100
-
 # Keep hashtable structure always on DRAM
 hashtable-on-dram yes

src/Makefile

@@ -225,7 +225,7 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o pmem.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o
 REDIS_CLI_NAME=redis-cli
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark

src/config.c

@@ -2229,7 +2229,6 @@ standardConfig configs[] = {
     createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ),
     createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
     createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves),
-    createIntConfig("memory-ratio-check-period", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.ratio_check_period, 100, INTEGER_CONFIG, NULL, NULL),
 
     /* Unsigned int configs */
     createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients),

src/pmem.c

@@ -1697,9 +1697,6 @@ void databasesCron(void) {
         }
     }
 
-    /* Adjust PMEM threshold. */
-    adjustPmemThresholdCycle();
-
     /* Defrag keys gradually. */
     activeDefragCycle();
 
@@ -2738,6 +2735,28 @@ void resetServerStats(void) {
     server.aof_delayed_fsync = 0;
 }
 
+/* Initialize the pmem threshold. */
+static void pmemThresholdInit(void) {
+    switch(server.memory_alloc_policy) {
+        case MEM_POLICY_ONLY_DRAM:
+            zmalloc_set_threshold(UINT_MAX);
+            break;
+        case MEM_POLICY_ONLY_PMEM:
+            zmalloc_set_threshold(0U);
+            break;
+        case MEM_POLICY_THRESHOLD:
+            zmalloc_set_threshold(server.static_threshold);
+            break;
+        case MEM_POLICY_RATIO:
+            zmalloc_set_threshold(server.initial_dynamic_threshold);
+            zmalloc_set_ratio(server.target_pmem_dram_ratio);
+            zmalloc_set_threshold_range(server.dynamic_threshold_min, server.dynamic_threshold_max);
+            break;
+        default:
+            serverAssert(NULL);
+    }
+}
+
 void initServer(void) {
     int j;
 

src/server.c

@@ -1347,7 +1347,6 @@ struct redisServer {
     unsigned int dynamic_threshold_max;       /* Maximum value of dynamic threshold */
     ratioDramPmemConfig dram_pmem_ratio;      /* DRAM/Persistent Memory ratio */
     double target_pmem_dram_ratio;            /* Target PMEM/DRAM ratio */
-    int ratio_check_period;                   /* Period of checking ratio in Cron*/
     int hashtable_on_dram;                    /* Keep hashtable always on DRAM */
     /* Blocked clients */
     unsigned int blocked_clients;   /* # of clients executing a blocking cmd.*/
@@ -2215,10 +2214,6 @@ uint64_t dictSdsHash(const void *key);
 int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2);
 void dictSdsDestructor(void *privdata, void *val);
 
-/* pmem.c - Handling Persistent Memory */
-void pmemThresholdInit(void);
-void adjustPmemThresholdCycle(void);
-
 /* Git SHA1 */
 char *redisGitSHA1(void);
 char *redisGitDirty(void);

src/server.h

@@ -29,6 +29,7 @@
  */
 
 #include <limits.h>
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
@@ -149,8 +150,61 @@ static void *zrealloc_pmem(void *ptr, size_t size) {
 static size_t pmem_threshold = UINT_MAX;
 static size_t used_memory = 0;
 static size_t used_pmem_memory = 0;
+static double pmem_dram_ratio = 0.0;
+static int use_ratio = 0;
+static unsigned int dynamic_min_threshold = 0;
+static unsigned int dynamic_max_threshold = 0;
+static int zmalloc_counter = 0;
+
 pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+#define THRESHOLD_STEP_NORMAL 0.05
+#define THRESHOLD_STEP_AGGRESIVE (THRESHOLD_STEP_NORMAL*5)
+#define THRESHOLD_UP(val, step)  ((size_t)ceil((1+(step))*val))
+#define THRESHOLD_DOWN(val, step) ((size_t)floor((1-(step))*val))
+
+static inline size_t absDiff(size_t a, size_t b) {
+    return a > b ? (a - b) : (b - a);
+}
+
+static void adjustPmemThresholdCycle(void) {
+    if (use_ratio) {
+        if (zmalloc_counter++ == 1000) {
+            zmalloc_counter = 0;
+            /* Difference between target ratio and current ratio in last checkpoint*/
+            static double ratio_diff_checkpoint;
+            /* PMEM and DRAM utilization in last checkpoint*/
+            static size_t total_memory_checkpoint;
+            size_t pmem_memory = zmalloc_used_pmem_memory();
+            size_t dram_memory = zmalloc_used_memory();
+            size_t total_memory_current = pmem_memory + dram_memory;
+            // do not modify threshold when change in memory usage is too small
+            if (absDiff(total_memory_checkpoint, total_memory_current) > 100) {
+                double current_ratio = (double)pmem_memory/dram_memory;
+                double current_ratio_diff = fabs(current_ratio - pmem_dram_ratio);
+                if (current_ratio_diff > 0.02) {
+                    //current ratio is worse than moment before
+                    double variableMultipler = current_ratio/pmem_dram_ratio;
+                    double step = (current_ratio_diff < ratio_diff_checkpoint) ?
+                                  variableMultipler*THRESHOLD_STEP_NORMAL : variableMultipler*THRESHOLD_STEP_AGGRESIVE;
+                    size_t threshold = zmalloc_get_threshold();
+                    if (pmem_dram_ratio < current_ratio) {
+                        size_t higher_threshold = THRESHOLD_UP(threshold,step);
+                        if (higher_threshold <= dynamic_max_threshold)
+                            zmalloc_set_threshold(higher_threshold);
+                    } else {
+                        size_t lower_threshold = THRESHOLD_DOWN(threshold,step);
+                        if (lower_threshold >= dynamic_min_threshold)
+                            zmalloc_set_threshold(lower_threshold);
+                    }
+                }
+                ratio_diff_checkpoint = current_ratio_diff;
+            }
+            total_memory_checkpoint = total_memory_current;
+        }
+    }
+}
+
 static void zmalloc_default_oom(size_t size) {
     fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",
         size);
@@ -263,6 +317,7 @@ static void *zrealloc_pmem(void *ptr, size_t size) {
 #endif
 
 void *zmalloc(size_t size) {
+    adjustPmemThresholdCycle();
     return (size < pmem_threshold) ? zmalloc_dram(size) : zmalloc_pmem(size);
 }
 
@@ -299,6 +354,7 @@ void *zcalloc_dram(size_t size) {
 }
 
 void *zcalloc(size_t size) {
+    adjustPmemThresholdCycle();
     return (size < pmem_threshold) ? zcalloc_dram(size) : zcalloc_pmem(size);
 }
 
@@ -418,6 +474,15 @@ void zmalloc_set_threshold(size_t threshold) {
     pmem_threshold = threshold;
 }
 
+void zmalloc_set_ratio(double ratio) {
+    use_ratio = 1;
+    pmem_dram_ratio = ratio;
+}
+void zmalloc_set_threshold_range(unsigned int min, unsigned int max) {
+    dynamic_min_threshold = min;
+    dynamic_max_threshold = max;
+}
+
 /* Get the RSS information in an OS-specific way.
  *
  * WARNING: the function zmalloc_get_rss() is not designed to be fast

src/zmalloc.c

@@ -104,6 +104,8 @@ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid);
 size_t zmalloc_get_memory_size(void);
 void zlibc_free(void *ptr);
 void zmalloc_set_threshold(size_t threshold);
+void zmalloc_set_ratio(double ratio);
+void zmalloc_set_threshold_range(unsigned int min, unsigned int max);
 size_t zmalloc_get_threshold(void);
 void *zmalloc_dram(size_t size);
 void *zcalloc_dram(size_t size);

src/zmalloc.h

@@ -103,7 +103,6 @@ start_server {tags {"introspection"}} {
             initial-dynamic-threshold
             dynamic-threshold-min
             dynamic-threshold-max
-            memory-ratio-check-period
             hashtable-on-dram
         }