turbo_persistence/constants.rs
1/// Values larger than this become blob files
2pub const MAX_MEDIUM_VALUE_SIZE: usize = 64 * 1024 * 1024;
3
4/// Values larger than this become separate value blocks
5// Note this must fit into 2 bytes length
6// Note that a medium value has 14 bytes of extra overhead compared to a small value.
7// Note that we want to benefit from better compression by merging small values together, so we can
8// avoid a compression dictionary. At ≥4kB block size, compression works well without a dictionary.
9// Note that medium values can be copied without decompression during compaction.
10pub const MAX_SMALL_VALUE_SIZE: usize = 4096;
11
12/// Maximum size for inline values stored directly in key blocks.
13/// Currently 8 bytes (break-even with the 8-byte indirection overhead).
14/// Can be increased up to 247 bytes (type 255 - 8) if desired.
15/// See static_sorted_file.rs for the static assertion enforcing this limit.
16pub const MAX_INLINE_VALUE_SIZE: usize = 8;
17
18/// Maximum number of entries per SST file
19pub const MAX_ENTRIES_PER_INITIAL_FILE: usize = 256 * 1024;
20
21/// Maximum number of entries per SST file
22pub const MAX_ENTRIES_PER_COMPACTED_FILE: usize = 1024 * 1024;
23
24/// Finish file when total amount of data exceeds this
25pub const DATA_THRESHOLD_PER_INITIAL_FILE: usize = 64 * 1024 * 1024;
26
27/// Finish file when total amount of data exceeds this
28pub const DATA_THRESHOLD_PER_COMPACTED_FILE: usize = 256 * 1024 * 1024;
29
30/// Reduction factor (as bit shift) for the size of the thread-local buffer as shift of
31/// MAX_ENTRIES_PER_INITIAL_FILE and DATA_THRESHOLD_PER_INITIAL_FILE.
32pub const THREAD_LOCAL_SIZE_SHIFT: usize = 7;
33
34/// The minimum bytes that should accumulate before emitting a small value block.
35/// Blocks are emitted once they reach this size, so actual block sizes range from
36/// MIN_SMALL_VALUE_BLOCK_SIZE to MIN_SMALL_VALUE_BLOCK_SIZE + MAX_SMALL_VALUE_SIZE.
37pub const MIN_SMALL_VALUE_BLOCK_SIZE: usize = 8 * 1024;
38
39/// Maximum number of value blocks per SST file.
40/// Must leave room for key blocks + index block within u16::MAX total blocks.
41/// Uses u16::MAX / 2 to account for the 50/50 merge-and-split at end of compaction,
42/// which can double the block count before splitting.
43pub const MAX_VALUE_BLOCK_COUNT: usize = u16::MAX as usize / 2;
44
45/// Maximum RAM bytes for key block cache
46pub const KEY_BLOCK_CACHE_SIZE: u64 = 400 * 1024 * 1024;
47pub const KEY_BLOCK_AVG_SIZE: usize = 16 * 1024;
48
49/// Maximum RAM bytes for value block cache
50pub const VALUE_BLOCK_CACHE_SIZE: u64 = 300 * 1024 * 1024;
51pub const VALUE_BLOCK_AVG_SIZE: usize = 132000;