feat: ValueManifest (#364)

netease-im · Jan 8, 2025 · 338157a · 338157a
1 parent c35a610
commit 338157a
Show file tree

Hide file tree

Showing 12 changed files with 447 additions and 58 deletions.
diff --git a/...im/camellia/redis/proxy/upstream/embedded/storage/constants/EmbeddedStorageConstants.java b/...im/camellia/redis/proxy/upstream/embedded/storage/constants/EmbeddedStorageConstants.java
@@ -10,7 +10,7 @@ public class EmbeddedStorageConstants {
     public static final int _64k = 64*1024;
     public static final int _256k = 256*1024;
     public static final int _1024k = 1024*1024;
-    public static final int bit_size = (int)(16*1024*1024*1024L / _64k);//16Gib
-    public static final long block_size = 128*1024*1024*1024L;//128Gib
+    public static final int key_manifest_bit_size = (int)(16*1024*1024*1024L / _64k);//16Gib
+    public static final long data_file_size = 192*1024*1024*1024L;//128Gib
     public static final int block_header_len = 4+2+1+4+4;
 }
diff --git a/.../com/netease/nim/camellia/redis/proxy/upstream/embedded/storage/key/slot/KeyManifest.java b/.../com/netease/nim/camellia/redis/proxy/upstream/embedded/storage/key/slot/KeyManifest.java
@@ -35,8 +35,8 @@ public class KeyManifest implements IKeyManifest {
     private final String fileName;
     private FileChannel fileChannel;
 
-    public KeyManifest(String fileName) {
-        this.fileName = fileName;
+    public KeyManifest(String dir) {
+        this.fileName = dir + "/key.manifest";
     }
 
     @Override
@@ -80,7 +80,7 @@ public void load() throws IOException {
                 if (fileId == 0) {
                     continue;
                 }
-                BitSet bits = fileBitsMap.computeIfAbsent(fileId, k -> new BitSet(EmbeddedStorageConstants.bit_size));
+                BitSet bits = fileBitsMap.computeIfAbsent(fileId, k -> new BitSet(EmbeddedStorageConstants.key_manifest_bit_size));
                 int bitsStart = (int)(offset / EmbeddedStorageConstants._64k);
                 int bitsEnd = (int)((offset + capacity) / EmbeddedStorageConstants._64k);
                 for (int index=bitsStart; index<bitsEnd; index++) {
@@ -123,7 +123,7 @@ public SlotInfo init(short slot) throws IOException {
             for (Map.Entry<Long, BitSet> entry : fileBitsMap.entrySet()) {
                 Long fileId = entry.getKey();
                 BitSet bits = entry.getValue();
-                for (int i = 0; i< EmbeddedStorageConstants.bit_size; i++) {
+                for (int i = 0; i< EmbeddedStorageConstants.key_manifest_bit_size; i++) {
                     boolean used = bits.get(i);
                     if (!used) {
                         update(slot, fileId, (long) i * EmbeddedStorageConstants._64k, EmbeddedStorageConstants._64k);
@@ -157,7 +157,7 @@ public SlotInfo expand(short slot) throws IOException {
             int bitsStart = (int)(offset / EmbeddedStorageConstants._64k);
             int bitsEnd = (int)((offset + capacity) / EmbeddedStorageConstants._64k);
             //直接顺延扩容
-            if (bitsStart + (bitsEnd - bitsStart) * 2 < EmbeddedStorageConstants.bit_size) {
+            if (bitsStart + (bitsEnd - bitsStart) * 2 < EmbeddedStorageConstants.key_manifest_bit_size) {
                 boolean directExpand = true;
                 for (int i=bitsEnd; i<bitsEnd + bitsStep; i++) {
                     boolean used = bits.get(i);
@@ -177,8 +177,8 @@ public SlotInfo expand(short slot) throws IOException {
                 }
             }
             //使用同一个文件的空闲区域，优先复用其他slot回收的区域
-            if (EmbeddedStorageConstants.bit_size - bits.cardinality() >= bitsStep*2) {
-                for (int i = 0; i< EmbeddedStorageConstants.bit_size-bitsStep*2; i++) {
+            if (EmbeddedStorageConstants.key_manifest_bit_size - bits.cardinality() >= bitsStep*2) {
+                for (int i = 0; i< EmbeddedStorageConstants.key_manifest_bit_size -bitsStep*2; i++) {
                     if (bits.get(i, bitsStep * 2).cardinality() == 0) {
                         //clear old
                         for (int j=bitsStart; j<bitsEnd; j++) {
@@ -209,7 +209,7 @@ private long initFileId() {
         while (fileBitsMap.containsKey(fileId)) {
             fileId ++;
         }
-        fileBitsMap.put(fileId, new BitSet(EmbeddedStorageConstants.bit_size));
+        fileBitsMap.put(fileId, new BitSet(EmbeddedStorageConstants.key_manifest_bit_size));
         return fileId;
     }
 

diff --git a/...com/netease/nim/camellia/redis/proxy/upstream/embedded/storage/value/block/BlockType.java b/...com/netease/nim/camellia/redis/proxy/upstream/embedded/storage/value/block/BlockType.java
@@ -11,7 +11,6 @@ public enum BlockType {
     _32k(2, 32*1024, 2),
     _256k(3, 256*1024, 4),
     _1024k(4, 1024*1024, 4),
-    no_align(5, -1, 4),
     ;
 
     private final int type;
@@ -36,6 +35,15 @@ public int getSizeLen() {
         return sizeLen;
     }
 
+    public static BlockType getByValue(int type) {
+        for (BlockType blockType : BlockType.values()) {
+            if (blockType.getType() == type) {
+                return blockType;
+            }
+        }
+        return null;
+    }
+
     public static BlockType fromData(byte[] data) {
         if (data.length + 4 + 2 < EmbeddedStorageConstants._4k) {
             return BlockType._4k;
@@ -46,7 +54,17 @@ public static BlockType fromData(byte[] data) {
         } else if (data.length + 4 + 4 < EmbeddedStorageConstants._1024k) {
             return BlockType._1024k;
         } else {
-            return BlockType.no_align;
+            throw new IllegalArgumentException("data too long");
         }
     }
+
+    public int valueManifestSize(long dataFileCapacity) {
+        int blockCount = (int) (dataFileCapacity / getBlockSize());
+        return blockCount / 8;
+    }
+
+    public int valueSlotManifestSize(long dataFileCapacity) {
+        int blockCount = (int) (dataFileCapacity / getBlockSize());
+        return blockCount * 2;
+    }
 }
diff --git a/...etease/nim/camellia/redis/proxy/upstream/embedded/storage/value/block/IValueManifest.java b/...etease/nim/camellia/redis/proxy/upstream/embedded/storage/value/block/IValueManifest.java
@@ -26,16 +26,26 @@ public interface IValueManifest {
      * block type
      * @param fileId fileId
      * @return block type
+     * @throws IOException exception
      */
     BlockType blockType(long fileId) throws IOException;
 
+
     /**
-     * block flush to disk
+     * flush block location to disk
      * @param slot slot
-     * @param location location
+     * @param blockLocation location
      * @throws IOException exception
      */
-    void flush(short slot, BlockLocation location) throws IOException;
+    void commit(short slot, BlockLocation blockLocation) throws IOException;
 
 
+    /**
+     * clear
+     * @param slot slot
+     * @param blockLocation location
+     * @throws IOException exception
+     */
+    void clear(short slot, BlockLocation blockLocation) throws IOException;
+
 }
diff --git a/...com/netease/nim/camellia/redis/proxy/upstream/embedded/storage/value/block/IndexFile.java b/...com/netease/nim/camellia/redis/proxy/upstream/embedded/storage/value/block/IndexFile.java
@@ -0,0 +1,21 @@
+package com.netease.nim.camellia.redis.proxy.upstream.embedded.storage.value.block;
+
+import java.io.File;
+
+/**
+ * Created by caojiajun on 2025/1/8
+ */
+public record IndexFile(long fileId, BlockType blockType) {
+
+    public static IndexFile parse(File file) {
+        String name = file.getName();
+        if (!name.endsWith(".index")) {
+            return null;
+        }
+        String[] split = name.split("\\.")[0].split("_");
+        long fileId = Long.parseLong(split[0]);
+        int type = Integer.parseInt(split[1]);
+        BlockType blockType = BlockType.getByValue(type);
+        return new IndexFile(fileId, blockType);
+    }
+}