【京东零售】浅析Redis大Key

userPin:{
      storeId:{门店下加车的所有商品基本信息},
      storeId:{门店下加车的所有商品基本信息},
      ......
}

$ redis-cli --bigkeys

# Scanning the entire keyspace to find biggest keys as well as
# average sizes per key type.  You can use -i 0.01 to sleep 0.01 sec
# per SCAN command (not usually needed).
-------- 第一部分start -------
[00.00%] Biggest string found so far 'key-419' with 3 bytes
[05.14%] Biggest list   found so far 'mylist' with 100004 items
[35.77%] Biggest string found so far 'counter:__rand_int__' with 6 bytes
[73.91%] Biggest hash   found so far 'myobject' with 3 fields

-------- 第一部分end -------

-------- summary -------

-------- 第二部分start -------
Sampled 506 keys in the keyspace!
Total key length in bytes is 3452 (avg len 6.82)

Biggest string found 'counter:__rand_int__' has 6 bytes
Biggest   list found 'mylist' has 100004 items
Biggest   hash found 'myobject' has 3 fields
-------- 第二部分end -------

-------- 第三部分start -------
504 strings with 1403 bytes (99.60% of keys, avg size 2.78)
1 lists with 100004 items (00.20% of keys, avg size 100004.00)
0 sets with 0 members (00.00% of keys, avg size 0.00)
1 hashs with 3 fields (00.20% of keys, avg size 3.00)
0 zsets with 0 members (00.00% of keys, avg size 0.00)
-------- 第三部分end -------

/* scan循环扫描 */
do {
    /* 计算完成的百分比情况 */
    pct = 100 * (double)sampled/total_keys;//这里记录下扫描的进度

    /* 获取一些键并指向键数组 */
    reply = sendScan(&it);//这里发送SCAN命令，结果保存在reply中
    keys  = reply->element[1];//keys来保存这次scan获取的所有键名，注意只是键名，每个键的数据类型是不知道的。
    ......

} while(it != 0);

/* 检索类型，然后检索大小*/
getKeyTypes(types_dict, keys, types);
getKeySizes(keys, types, sizes, memkeys, memkeys_samples);

/* Now update our stats */
for(i=0;i<keys->elements;i++) {
    ......//前面已解析

    //如果遍历到比记录值更大的key时
    if(type->biggest<sizes[i]) {
        /* Keep track of biggest key name for this type */
        if (type->biggest_key)
            sdsfree(type->biggest_key);
        //更新最大key的键名
        type->biggest_key = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
        if(!type->biggest_key) {
            fprintf(stderr, "Failed to allocate memory for key!n");
            exit(1);
        }

        //每当找到一个更大的key时则输出该key信息
        printf(
            "[%05.2f%%] Biggest %-6s found so far '%s' with %llu %sn",
            pct, type->name, type->biggest_key, sizes[i],
            !memkeys? type->sizeunit: "bytes");

        /* Keep track of the biggest size for this type */
        //更新最大key的大小
        type->biggest = sizes[i];
    }

    ......//前面已解析
}

/* 现在更新统计数据 */
for(i=0;i<keys->elements;i++) {
    typeinfo *type = types[i];
    /* 跳过在SCAN和TYPE之间消失的键 */
    if(!type)
        continue;

    //对每个key更新每种数据类型的统计信息
    type->totalsize += sizes[i];//某数据类型（如string）的总大小增加
    type->count++;//某数据类型的key数量增加
    totlen += keys->element[i]->len;//totlen不针对某个具体数据类型，将所有key的键名的长度进行统计，注意只统计键名长度。
    sampled++;//已经遍历的key数量

    ......//后续解析

    /* 更新整体进度 */
    if(sampled % 1000000 == 0) {
        printf("[%05.2f%%] Sampled %llu keys so farn", pct, sampled);
    }
}

/* We're done */
    printf("n-------- summary -------nn");
    if (force_cancel_loop) printf("[%05.2f%%] ", pct);
    printf("Sampled %llu keys in the keyspace!n", sampled);
    printf("Total key length in bytes is %llu (avg len %.2f)nn",
       totlen, totlen ? (double)totlen/sampled : 0);

/* Output the biggest keys we found, for types we did find */
    di = dictGetIterator(types_dict);
    while ((de = dictNext(di))) {
        typeinfo *type = dictGetVal(de);
        if(type->biggest_key) {
            printf("Biggest %6s found '%s' has %llu %sn", type->name, type->biggest_key,
               type->biggest, !memkeys? type->sizeunit: "bytes");
        }
    }
    dictReleaseIterator(di);

di = dictGetIterator(types_dict);
    while ((de = dictNext(di))) {
        typeinfo *type = dictGetVal(de);
        printf("%llu %ss with %llu %s (%05.2f%% of keys, avg size %.2f)n",
           type->count, type->name, type->totalsize, !memkeys? type->sizeunit: "bytes",
           sampled ? 100 * (double)type->count/sampled : 0,
           type->count ? (double)type->totalsize/type->count : 0);
    }
    dictReleaseIterator(di);

import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
public class JsonTest {
    @JsonProperty("u")
    private String userName;

    public String getUserName() {
        return userName;
    }
    public void setUserName(String userName) {
        this.userName = userName;
    }
    public static void main(String[] args) throws IOException {
        JsonTest output = new JsonTest();
        output.setUserName("京东到家");
        System.out.println(new ObjectMapper().writeValueAsString(output));

        String json = "{"u":"京东到家"}";
        JsonTest r1 = new ObjectMapper().readValue(json, JsonTest.class);
        System.out.println(r1.getUserName());
    }
}

{"u":"京东到家"}
京东到家

4.0 版本 < redis_version < 6.0 版本：引入lazy-free，手动开启lazy-free时，有4个选项可以控制，分别对应不同场景下，是否开启异步释放内存机制：

public void scanRedis(String cursor,String endCursor) {
        ReloadableJimClientFactory factory = new ReloadableJimClientFactory();
        String jimUrl = "jim://xxx/546";
        factory.setJimUrl(jimUrl);
        Cluster client = factory.getClient();
        ScanOptions.ScanOptionsBuilder scanOptions = ScanOptions.scanOptions();
        scanOptions.count(100);
 
        Boolean end = false;
        int k = 0;
        while (!end) {
            KeyScanResult< String > result = client.scan(cursor, scanOptions.build());
            for (String key :result.getResult()){
                if (client.ttl(key) == -1){
                    logger.info("永久key为:{}" , key);
                }
            }
            k++;
            cursor = result.getCursor();
            if (endCursor.equals(cursor)){
                break;
            }
        }
    }

void delCommand(client *c) {
    delGenericCommand(c,server.lazyfree_lazy_user_del);
}
void unlinkCommand(client *c) {
    delGenericCommand(c,1);
}

void delGenericCommand(client *c, int lazy) {
    int numdel = 0, j;
    // 遍历所有输入键
    for (j = 1; j < c->argc; j++) {
        // 先删除过期的键
        expireIfNeeded(c->db,c->argv[j],0);
        int deleted  = lazy ? dbAsyncDelete(c->db,c->argv[j]) :
                              dbSyncDelete(c->db,c->argv[j]);
        // 尝试删除键
        if (deleted) {
            // 删除键成功，发送通知
            signalModifiedKey(c,c->db,c->argv[j]);
            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[j],c->db->id);
            server.dirty++;
            // 成功删除才增加 deleted 计数器的值
            numdel++;
        }
    }
    // 返回被删除键的数量
    addReplyLongLong(c,numdel);
}

#define LAZYFREE_THRESHOLD 64

/* Free an object, if the object is huge enough, free it in async way. */
void freeObjAsync(robj *key, robj *obj, int dbid) {
    size_t free_effort = lazyfreeGetFreeEffort(key,obj,dbid);
    if (free_effort > LAZYFREE_THRESHOLD && obj->refcount == 1) {
        atomicIncr(lazyfree_objects,1);
        bioCreateLazyFreeJob(lazyfreeFreeObject,1,obj);
    } else {
        decrRefCount(obj);
    }
}

size_t lazyfreeGetFreeEffort(robj *key, robj *obj, int dbid) {
    if (obj->type == OBJ_LIST && obj->encoding == OBJ_ENCODING_QUICKLIST) {
        quicklist *ql = obj->ptr;
        return ql->len;
    } else if (obj->type == OBJ_SET && obj->encoding == OBJ_ENCODING_HT) {
        dict *ht = obj->ptr;
        return dictSize(ht);
    } else if (obj->type == OBJ_ZSET && obj->encoding == OBJ_ENCODING_SKIPLIST){
        zset *zs = obj->ptr;
        return zs->zsl->length;
    } else if (obj->type == OBJ_HASH && obj->encoding == OBJ_ENCODING_HT) {
        dict *ht = obj->ptr;
        return dictSize(ht);
    } else if (obj->type == OBJ_STREAM) {
        ...
        return effort;
    } else if (obj->type == OBJ_MODULE) {
        size_t effort = moduleGetFreeEffort(key, obj, dbid);
        /* If the module's free_effort returns 0, we will use asynchronous free
         * memory by default. */
        return effort == 0 ? ULONG_MAX : effort;
    } else {
        return 1; /* Everything else is a single allocation. */
    }
}

存储结构拆分成两种：
第一种：
    userPin：storeId的集合
第二种：
    userPin_storeId1:{门店下加车的所有商品基本信息}；
    userPin_storeId2:{门店下加车的所有商品基本信息}