Add maximum TTL setting and unit tests, some item refactoring

src/server/redis/data/cmd_list.c

@@ -23,7 +23,7 @@ _add_key(struct response *rsp, struct bstring *key)
 {
     struct element *reply = (struct element *)array_get(rsp->token, 0);
     struct item *it;
-    item_rstatus_t istatus;
+    item_rstatus_e istatus;
 
     it = item_get(key);
     if (it != NULL) {

src/server/twemcache/data/process.c

@@ -170,7 +170,7 @@ _process_delete(struct response *rsp, struct request *req)
 
 
 static void
-_error_rsp(struct response *rsp, item_rstatus_t status)
+_error_rsp(struct response *rsp, item_rstatus_e status)
 {
     INCR(process_metrics, process_ex);
 
@@ -204,7 +204,7 @@ _error_rsp(struct response *rsp, item_rstatus_t status)
  *   - PUT_PARTIAL
  */
 static put_rstatus_e
-_put(item_rstatus_t *istatus, struct request *req)
+_put(item_rstatus_e *istatus, struct request *req)
 {
     put_rstatus_e status;
     struct item *it = NULL;
@@ -248,7 +248,7 @@ static void
 _process_set(struct response *rsp, struct request *req)
 {
     put_rstatus_e status;
-    item_rstatus_t istatus;
+    item_rstatus_e istatus;
     struct item *it;
     struct bstring key;
 
@@ -278,7 +278,7 @@ static void
 _process_add(struct response *rsp, struct request *req)
 {
     put_rstatus_e status;
-    item_rstatus_t istatus;
+    item_rstatus_e istatus;
     struct item *it;
     struct bstring key;
 
@@ -314,7 +314,7 @@ static void
 _process_replace(struct response *rsp, struct request *req)
 {
     put_rstatus_e status;
-    item_rstatus_t istatus;
+    item_rstatus_e istatus;
     struct item *it = NULL;
     struct bstring key;
 
@@ -350,7 +350,7 @@ static void
 _process_cas(struct response *rsp, struct request *req)
 {
     put_rstatus_e status;
-    item_rstatus_t istatus;
+    item_rstatus_e istatus;
     struct item *it, *oit;
     struct bstring key;
 
@@ -391,11 +391,11 @@ _process_cas(struct response *rsp, struct request *req)
 /* get integer value of it */
 
 /* update item with integer value */
-static item_rstatus_t
+static item_rstatus_e
 _process_delta(struct response *rsp, struct item *it, struct request *req,
         struct bstring *key, bool incr)
 {
-    item_rstatus_t status;
+    item_rstatus_e status;
     uint32_t dataflag;
     uint64_t vint;
     struct bstring nval;
@@ -438,7 +438,7 @@ _process_delta(struct response *rsp, struct item *it, struct request *req,
 static void
 _process_incr(struct response *rsp, struct request *req)
 {
-    item_rstatus_t status;
+    item_rstatus_e status;
     struct bstring *key;
     struct item *it;
 
@@ -465,7 +465,7 @@ _process_incr(struct response *rsp, struct request *req)
 static void
 _process_decr(struct response *rsp, struct request *req)
 {
-    item_rstatus_t status;
+    item_rstatus_e status;
     struct bstring *key;
     struct item *it;
 
@@ -492,7 +492,7 @@ _process_decr(struct response *rsp, struct request *req)
 static void
 _process_append(struct response *rsp, struct request *req)
 {
-    item_rstatus_t status;
+    item_rstatus_e status;
     struct bstring *key;
     struct item *it;
 
@@ -522,7 +522,7 @@ _process_append(struct response *rsp, struct request *req)
 static void
 _process_prepend(struct response *rsp, struct request *req)
 {
-    item_rstatus_t status;
+    item_rstatus_e status;
     struct bstring *key;
     struct item *it;
 

src/storage/cuckoo/cuckoo.c

@@ -17,6 +17,7 @@
 uint64_t cas_val;
 bool cas_enabled = CUCKOO_ITEM_CAS;
 uint32_t cuckoo_policy = CUCKOO_POLICY;
+delta_time_i max_ttl = CUCKOO_MAX_TTL;
 
 cuckoo_metrics_st *cuckoo_metrics = NULL;
 static bool cuckoo_init; /* need to make sure memory has been pre-allocate */
@@ -270,6 +271,7 @@ cuckoo_setup(cuckoo_options_st *options, cuckoo_metrics_st *metrics)
         max_nitem = option_uint(&options->cuckoo_nitem);
         cuckoo_policy = option_uint(&options->cuckoo_policy);
         cas_enabled = option_bool(&options->cuckoo_item_cas);
+        max_ttl = option_uint(&options->cuckoo_max_ttl);
     }
 
     hash_size = item_size * max_nitem;

src/storage/cuckoo/cuckoo.h

@@ -16,14 +16,16 @@
 #define CUCKOO_ITEM_SIZE 64
 #define CUCKOO_NITEM 1024
 #define CUCKOO_POLICY CUCKOO_POLICY_RANDOM
+#define CUCKOO_MAX_TTL (30 * 24 * 60 * 60) /* 30 days */
 
 /*          name                type                default             description */
 #define CUCKOO_OPTION(ACTION)                                                                    \
     ACTION( cuckoo_displace,    OPTION_TYPE_UINT,   CUCKOO_DISPLACE,    "# displaces allowed"   )\
     ACTION( cuckoo_item_cas,    OPTION_TYPE_BOOL,   CUCKOO_ITEM_CAS,    "support cas in items"  )\
     ACTION( cuckoo_item_size,   OPTION_TYPE_UINT,   CUCKOO_ITEM_SIZE,   "item size (inclusive)" )\
     ACTION( cuckoo_nitem,       OPTION_TYPE_UINT,   CUCKOO_NITEM,       "# items allocated"     )\
-    ACTION( cuckoo_policy,      OPTION_TYPE_UINT,   CUCKOO_POLICY,      "evict policy"          )
+    ACTION( cuckoo_policy,      OPTION_TYPE_UINT,   CUCKOO_POLICY,      "evict policy"          )\
+    ACTION( cuckoo_max_ttl,     OPTION_TYPE_UINT,   CUCKOO_MAX_TTL,     "max ttl in seconds"    )
 
 typedef struct {
     CUCKOO_OPTION(OPTION_DECLARE)

src/storage/cuckoo/item.h

@@ -11,10 +11,11 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-
 extern bool cas_enabled;
 extern uint64_t cas_val; /* incr'ed before assignment, 0 is a special value */
 
+extern delta_time_i max_ttl;
+
 /**
  * val_type_t and struct val makes it easier to use one object to communicate
  * values between in-memory storage and other modules
@@ -72,14 +73,15 @@ struct item {
 
 #define KEY_MAXLEN 255
 #define CAS_VAL_MIN 1
+#define CAS_LEN (cas_enabled * sizeof(cas_val))
 #define MIN_ITEM_CHUNK_SIZE CC_ALIGN(sizeof(struct item) + 2, CC_ALIGNMENT)
 #define ITEM_HDR_SIZE sizeof(struct item)
 
 #define ITEM_CAS_POS(it) ((it)->data)
-#define ITEM_KEY_POS(it) ((it)->data + cas_enabled * sizeof(uint64_t))
+#define ITEM_KEY_POS(it) ((it)->data + CAS_LEN)
 #define ITEM_VAL_POS(it) (ITEM_KEY_POS(it) + (it)->klen)
 
-#define ITEM_OVERHEAD offsetof(struct item, data) + cas_enabled * sizeof(uint64_t)
+#define ITEM_OVERHEAD (offsetof(struct item, data) + CAS_LEN)
 
 static inline uint8_t
 item_klen(struct item *it)
@@ -232,7 +234,8 @@ item_value_update(struct item *it, struct val *val)
 static inline void
 item_update(struct item *it, struct val *val, proc_time_i expire)
 {
-    it->expire = expire;
+    proc_time_i expire_cap = time_delta2proc_sec(max_ttl);
+    it->expire = expire < expire_cap ? expire : expire_cap;
     item_value_update(it, val);
 }
 

src/storage/slab/item.c

@@ -5,6 +5,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+extern delta_time_i max_ttl;
 static proc_time_i flush_at = -1;
 
 static inline bool
@@ -53,7 +54,7 @@ _item_reset(struct item *it)
  *
  * On success we return the pointer to the allocated item.
  */
-static item_rstatus_t
+static item_rstatus_e
 _item_alloc(struct item **it_p, uint8_t klen, uint32_t vlen, uint8_t olen)
 {
     uint8_t id = slab_id(item_ntotal(klen, vlen, olen));
@@ -197,8 +198,10 @@ static void
 _item_define(struct item *it, const struct bstring *key, const struct bstring
         *val, uint8_t olen, proc_time_i expire_at)
 {
+    proc_time_i expire_cap = time_delta2proc_sec(max_ttl);
+
     it->create_at = time_proc_sec();
-    it->expire_at = expire_at;
+    it->expire_at = expire_at < expire_cap ? expire_at : expire_cap;
     item_set_cas(it);
     it->olen = olen;
     cc_memcpy(item_key(it), key->data, key->len);
@@ -209,11 +212,11 @@ _item_define(struct item *it, const struct bstring *key, const struct bstring
     it->vlen = (val == NULL) ? 0 : val->len;
 }
 
-item_rstatus_t
+item_rstatus_e
 item_reserve(struct item **it_p, const struct bstring *key, const struct bstring
         *val, uint32_t vlen, uint8_t olen, proc_time_i expire_at)
 {
-    item_rstatus_t status;
+    item_rstatus_e status;
     struct item *it;
 
     if ((status = _item_alloc(it_p, key->len, vlen, olen)) != ITEM_OK) {
@@ -250,11 +253,11 @@ item_backfill(struct item *it, const struct bstring *val)
             it, val->len, it->vlen);
 }
 
-item_rstatus_t
+item_rstatus_e
 item_annex(struct item *oit, const struct bstring *key, const struct bstring
         *val, bool append)
 {
-    item_rstatus_t status = ITEM_OK;
+    item_rstatus_e status = ITEM_OK;
     struct item *nit = NULL;
     uint8_t id;
     uint32_t ntotal = oit->vlen + val->len;

src/storage/slab/item.h

@@ -86,7 +86,7 @@ typedef enum item_rstatus {
     ITEM_ENOMEM,
     ITEM_ENAN, /* not a number */
     ITEM_EOTHER,
-} item_rstatus_t;
+} item_rstatus_e;
 
 extern bool use_cas;
 extern uint64_t cas_id;
@@ -160,7 +160,7 @@ item_data(struct item *it)
     return data;
 }
 
-static inline item_rstatus_t
+static inline item_rstatus_e
 item_atou64(uint64_t *vint, struct item *it)
 {
     rstatus_i status;
@@ -192,15 +192,15 @@ void item_insert(struct item *it, const struct bstring *key);
  * olen- optional data length, this can be used to reserve space for optional
  * data, e.g. flag in Memcached protocol) in payload, after cas.
  * */
-item_rstatus_t item_reserve(struct item **it_p, const struct bstring *key, const
+item_rstatus_e item_reserve(struct item **it_p, const struct bstring *key, const
         struct bstring *val, uint32_t vlen, uint8_t olen, proc_time_i expire_at);
 /* item_release is used for reserved item only (not linked) */
 void item_release(struct item **it_p);
 
 void item_backfill(struct item *it, const struct bstring *val);
 
 /* Append/prepend */
-item_rstatus_t item_annex(struct item *it, const struct bstring *key, const
+item_rstatus_e item_annex(struct item *it, const struct bstring *key, const
         struct bstring *val, bool append);
 
 

src/storage/slab/slab.c

@@ -46,6 +46,8 @@ bool use_cas = SLAB_USE_CAS;
 struct hash_table *hash_table = NULL;
 uint64_t cas_id;
 
+delta_time_i max_ttl = ITEM_MAX_TTL;
+
 static bool slab_init = false;
 slab_metrics_st *slab_metrics = NULL;
 
@@ -420,6 +422,7 @@ slab_setup(slab_options_st *options, slab_metrics_st *metrics)
         item_min = option_uint(&options->slab_item_min);
         item_max = option_uint(&options->slab_item_max);
         item_growth = option_fpn(&options->slab_item_growth);
+        max_ttl = option_uint(&options->slab_item_max_ttl);
         use_cas = option_bool(&options->slab_use_cas);
         hash_power = option_uint(&options->slab_hash_power);
     }

src/storage/slab/slab.h

@@ -14,8 +14,8 @@
 
 #define SLAB_MAGIC      0xdeadbeef
 #define SLAB_HDR_SIZE   offsetof(struct slab, data)
-#define SLAB_SIZE_MIN   ((size_t) 512)
-#define SLAB_SIZE_MAX   ((size_t) (128 * MiB))
+#define SLAB_SIZE_MIN   ((size_t)512)
+#define SLAB_SIZE_MAX   ((size_t)(128 * MiB))
 #define SLAB_SIZE       MiB
 #define SLAB_MEM        (64 * MiB)
 #define SLAB_PREALLOC   true
@@ -27,6 +27,7 @@
 #define ITEM_SIZE_MIN   44      /* 40 bytes item overhead */
 #define ITEM_SIZE_MAX   (SLAB_SIZE - SLAB_HDR_SIZE)
 #define ITEM_FACTOR     1.25
+#define ITEM_MAX_TTL    (30 * 24 * 60 * 60) /* 30 days */
 #define HASH_POWER      16
 
 /* Eviction options */
@@ -36,6 +37,7 @@
 #define EVICT_INVALID 4 /* go no further! */
 
 /* The defaults here are placeholder values for now */
+/* TODO: consider moving item options to item.[h|c] */
 /*          name                type                default         description */
 #define SLAB_OPTION(ACTION)                                                                          \
     ACTION( slab_size,          OPTION_TYPE_UINT,   SLAB_SIZE,      "Slab size"                     )\
@@ -47,8 +49,9 @@
     ACTION( slab_item_min,      OPTION_TYPE_UINT,   ITEM_SIZE_MIN,  "Minimum item size"             )\
     ACTION( slab_item_max,      OPTION_TYPE_UINT,   ITEM_SIZE_MAX,  "Maximum item size"             )\
     ACTION( slab_item_growth,   OPTION_TYPE_FPN,    ITEM_FACTOR,    "Slab class growth factor"      )\
+    ACTION( slab_item_max_ttl,  OPTION_TYPE_UINT,   ITEM_MAX_TTL,   "Max ttl in seconds"            )\
     ACTION( slab_use_cas,       OPTION_TYPE_BOOL,   SLAB_USE_CAS,   "Store CAS value in item"       )\
-    ACTION( slab_hash_power,    OPTION_TYPE_UINT,   HASH_POWER,     "Power for lookup hash table"  )
+    ACTION( slab_hash_power,    OPTION_TYPE_UINT,   HASH_POWER,     "Power for lookup hash table"   )
 
 typedef struct {
     SLAB_OPTION(OPTION_DECLARE)