public abstract class FileStore<C extends Chunk<C>>
extends java.lang.Object
| Modifier and Type | Class and Description |
|---|---|
class |
FileStore.PageSerializationManager |
| Modifier and Type | Field and Description |
|---|---|
protected C |
lastChunk
The newest chunk.
|
static java.lang.String |
META_ID_KEY
The key for the entry within "layout" map, which contains id of "meta" map.
|
static int |
PIPE_LENGTH |
protected java.util.concurrent.atomic.AtomicLong |
readBytes
The number of read bytes.
|
protected java.util.concurrent.atomic.AtomicLong |
readCount
The number of read operations.
|
protected boolean |
recoveryMode |
protected java.util.concurrent.locks.ReentrantLock |
saveChunkLock |
protected java.util.HashMap<java.lang.String,java.lang.Object> |
storeHeader |
protected java.util.concurrent.atomic.AtomicLong |
writeBytes
The number of written bytes.
|
protected java.util.concurrent.atomic.AtomicLong |
writeCount
The number of write operations.
|
| Modifier | Constructor and Description |
|---|---|
protected |
FileStore(java.util.Map<java.lang.String,java.lang.Object> config) |
| Modifier and Type | Method and Description |
|---|---|
void |
accountForRemovedPage(long pos,
long version,
boolean pinned,
int pageNo)
Remove a page.
|
protected abstract void |
adjustStoreToLastChunk()
Make persistent changes after lastChunk was reset
|
protected abstract void |
allocateChunkSpace(C chunk,
WriteBuffer buff)
Allocate logical space and assign position of the buffer within the store.
|
abstract void |
backup(java.util.zip.ZipOutputStream out) |
void |
bind(MVStore mvStore) |
void |
clear() |
void |
close() |
boolean |
compact(int targetFillRate,
int write)
Try to increase the fill rate by re-writing partially full chunks.
|
protected abstract void |
compactStore(int thresholdFillRate,
long maxCompactTime,
int maxWriteSize,
MVStore mvStore)
Compact store file, that is, compact blocks that have a low
fill rate, and move chunks next to each other.
|
void |
compactStore(long maxCompactTime) |
protected abstract C |
createChunk(int id) |
protected abstract C |
createChunk(java.util.Map<java.lang.String,java.lang.String> map) |
abstract C |
createChunk(java.lang.String s)
Build a Chunk from the given string.
|
boolean |
deregisterMapRoot(int mapId)
Performs final stage of map removal - delete root location info from the layout map.
|
protected C |
discoverChunk(long block)
Discover a valid chunk, searching file backwards from the given block
|
protected abstract void |
doHousekeeping(MVStore mvStore) |
void |
dropUnusedChunks() |
void |
executeFileStoreOperation(java.lang.Runnable operation) |
protected boolean |
findLastChunkWithCompleteValidChunkSet(java.util.Comparator<C> chunkComparator,
java.util.Map<java.lang.Long,C> validChunksByLocation,
boolean afterFullScan) |
protected abstract void |
freeChunkSpace(java.lang.Iterable<C> chunks)
Mark the space occupied by specified chunks as free.
|
int |
getAutoCommitDelay()
Get the auto-commit delay.
|
protected int |
getAutoCompactFillRate() |
int |
getCacheHitRatio() |
int |
getCacheSize()
Get the maximum cache size, in MB.
|
int |
getCacheSizeUsed()
Get the amount of memory used for caching, in MB.
|
protected java.util.concurrent.ConcurrentMap<java.lang.Integer,C> |
getChunks() |
int |
getChunksFillRate()
Get the current fill rate (percentage of used space in the file).
|
protected java.lang.Iterable<C> |
getChunksFromLayoutMap() |
long |
getCreationTime()
The time the store was created, in milliseconds since 1970.
|
int |
getDefaultRetentionTime()
Get the default retention time for this store in milliseconds.
|
java.lang.String |
getFileName()
Get the file name.
|
abstract int |
getFillRate() |
java.util.Map<java.lang.String,java.lang.String> |
getLayoutMap()
Get this store's layout map.
|
long |
getMaxPageSize() |
int |
getMetaMapId(java.util.function.IntSupplier nextIdSupplier) |
protected MVStore |
getMvStore() |
long |
getReadBytes()
Get the number of read bytes since this store was opened.
|
long |
getReadCount()
Get the number of read operations since this store was opened.
|
int |
getRetentionTime() |
protected java.util.Collection<C> |
getRewriteCandidates() |
long |
getRootPos(int mapId)
Get "position" of the root page for the specified map
|
java.util.Map<java.lang.String,java.lang.Object> |
getStoreHeader()
Get the store header.
|
int |
getTocCacheHitRatio() |
WriteBuffer |
getWriteBuffer()
Get a buffer for writing.
|
long |
getWriteCount()
Get the number of write operations since this store was opened.
|
boolean |
hasChangesSince(long lastStoredVersion)
Check whether there are any unsaved changes since specified version.
|
protected boolean |
hasPersistentData() |
protected void |
init(java.lang.String fileName,
boolean readOnly) |
protected void |
initializeCommonHeaderAttributes(long time) |
protected abstract void |
initializeStoreHeader(long time) |
protected boolean |
isIdle() |
boolean |
isKnownVersion(long version)
Check whether all data can be read from this version.
|
boolean |
isReadOnly() |
boolean |
isRegularMap(MVMap<?,?> map) |
boolean |
isSpaceReused() |
long |
lastChunkVersion() |
protected abstract void |
markUsed(long pos,
int length)
Mark the space as in use.
|
abstract FileStore<C> |
open(java.lang.String fileName,
boolean readOnly) |
abstract void |
open(java.lang.String fileName,
boolean readOnly,
char[] encryptionKey) |
void |
populateInfo(java.util.function.BiConsumer<java.lang.String,java.lang.String> consumer) |
protected void |
processCommonHeaderAttributes() |
protected C |
readChunkFooter(long block)
Try to read a chunk footer.
|
protected C |
readChunkHeaderAndFooter(long block,
int expectedId)
Read a chunk header and footer, and verify the stored data is consistent.
|
protected C |
readChunkHeaderOptionally(long block) |
protected C |
readChunkHeaderOptionally(long block,
int expectedId) |
abstract java.nio.ByteBuffer |
readFully(C chunk,
long pos,
int len)
Read data from the store.
|
protected java.nio.ByteBuffer |
readFully(java.nio.channels.FileChannel file,
long pos,
int len) |
protected abstract void |
readStoreHeader(boolean recoveryMode) |
protected void |
registerDeadChunk(C chunk) |
void |
releaseWriteBuffer(WriteBuffer buff)
Release a buffer for writing.
|
protected boolean |
rewriteChunks(int writeLimit,
int targetFillRate) |
void |
rollbackTo(long version) |
void |
saveChunkMetadataChanges(C chunk)
Store chunk's serialized metadata as an entry in a layout map.
|
void |
setAutoCommitDelay(int millis)
Set the maximum delay in milliseconds to auto-commit changes.
|
void |
setCacheSize(int mb)
Set the read cache size in MB.
|
protected void |
setLastChunk(C last) |
void |
setRetentionTime(int ms)
How long to retain old, persisted chunks, in milliseconds.
|
void |
setReuseSpace(boolean reuseSpace) |
protected void |
setSize(long size) |
abstract boolean |
shouldSaveNow(int unsavedMemory,
int autoCommitMemory)
Decision about autocommit is delegated to store
|
protected abstract void |
shrinkStoreIfPossible(int minPercent)
Shrink store if possible, and if at least a given percentage can be
saved.
|
long |
size()
Get the file size.
|
MVMap<java.lang.String,java.lang.String> |
start() |
void |
stop(long allowedCompactionTime) |
protected void |
store() |
void |
sync() |
protected abstract boolean |
validateFileLength(java.lang.String msg) |
protected abstract void |
writeChunk(C chunk,
WriteBuffer buffer)
Write buffer associated with chunk into store at chunk's allocated position
|
protected void |
writeCleanShutdown() |
protected abstract void |
writeCleanShutdownMark()
Performs final preparation before store is closed normally
|
protected abstract void |
writeFully(C chunk,
long pos,
java.nio.ByteBuffer src)
Write to the file.
|
public static final java.lang.String META_ID_KEY
protected final java.util.concurrent.atomic.AtomicLong readCount
protected final java.util.concurrent.atomic.AtomicLong readBytes
protected final java.util.concurrent.atomic.AtomicLong writeCount
protected final java.util.concurrent.atomic.AtomicLong writeBytes
protected volatile C extends Chunk<C> lastChunk
protected final java.util.concurrent.locks.ReentrantLock saveChunkLock
protected final java.util.HashMap<java.lang.String,java.lang.Object> storeHeader
protected final boolean recoveryMode
public static final int PIPE_LENGTH
protected FileStore(java.util.Map<java.lang.String,java.lang.Object> config)
public abstract void open(java.lang.String fileName,
boolean readOnly,
char[] encryptionKey)
protected final void init(java.lang.String fileName,
boolean readOnly)
public final void bind(MVStore mvStore)
public final void stop(long allowedCompactionTime)
public void close()
public final int getMetaMapId(java.util.function.IntSupplier nextIdSupplier)
public final java.util.Map<java.lang.String,java.lang.String> getLayoutMap()
The data in this map should not be modified (changing system data may corrupt the store).
The layout map contains the following entries:
chunk.{chunkId} = {chunk metadata}
root.{mapId} = {root position}
public final boolean isRegularMap(MVMap<?,?> map)
public final long getRootPos(int mapId)
mapId - to get root position forpublic final boolean deregisterMapRoot(int mapId)
mapId - to deregisterpublic final boolean hasChangesSince(long lastStoredVersion)
lastStoredVersion - version to take as a base for changespublic final long lastChunkVersion()
public final long getMaxPageSize()
public final int getRetentionTime()
public final void setRetentionTime(int ms)
The default value is 45000 (45 seconds) when using the default file
store. It is assumed that a file system and hard disk will flush all
write buffers within this time. Using a lower value might be dangerous,
unless the file system and hard disk flush the buffers earlier. To
manually flush the buffers, use
MVStore.getFile().force(true), however please note that
according to various tests this does not always work as expected
depending on the operating system and hardware.
The retention time needs to be long enough to allow reading old chunks while traversing over the entries of a map.
This setting is not persisted.
ms - how many milliseconds to retain old chunks (0 to overwrite them
as early as possible)public abstract boolean shouldSaveNow(int unsavedMemory,
int autoCommitMemory)
unsavedMemory - amount of unsaved memory, so farautoCommitMemory - configured limit on amount of unsaved memorypublic final int getAutoCommitDelay()
public final void setAutoCommitDelay(int millis)
To disable auto-commit, set the value to 0. In this case, changes are only committed when explicitly calling commit.
The default is 1000, meaning all changes are committed after at most one second.
millis - the maximum delaypublic final boolean isKnownVersion(long version)
version - the versionpublic final void rollbackTo(long version)
protected final void initializeCommonHeaderAttributes(long time)
protected final void processCommonHeaderAttributes()
protected final boolean hasPersistentData()
protected final boolean isIdle()
protected final void setLastChunk(C last)
protected final void registerDeadChunk(C chunk)
public final void dropUnusedChunks()
protected abstract void writeFully(C chunk, long pos, java.nio.ByteBuffer src)
chunk - to writepos - the write positionsrc - the source bufferpublic abstract java.nio.ByteBuffer readFully(C chunk, long pos, int len)
chunk - that owns data to be readpos - the read "position"len - the number of bytes to readprotected final java.nio.ByteBuffer readFully(java.nio.channels.FileChannel file,
long pos,
int len)
protected abstract void allocateChunkSpace(C chunk, WriteBuffer buff)
chunk - to allocate space forbuff - to allocate space forprotected abstract void writeChunk(C chunk, WriteBuffer buffer)
chunk - chunk to writebuffer - to writeprotected abstract void writeCleanShutdownMark()
protected abstract void adjustStoreToLastChunk()
public java.util.Map<java.lang.String,java.lang.Object> getStoreHeader()
protected abstract C createChunk(int id)
public abstract C createChunk(java.lang.String s)
s - the stringprotected abstract C createChunk(java.util.Map<java.lang.String,java.lang.String> map)
protected void writeCleanShutdown()
public void saveChunkMetadataChanges(C chunk)
chunk - to saveprotected abstract void freeChunkSpace(java.lang.Iterable<C> chunks)
chunks - chunks to be processedprotected abstract boolean validateFileLength(java.lang.String msg)
public boolean compact(int targetFillRate,
int write)
If the current fill rate is higher than the target fill rate, nothing is done.
Please note this method will not necessarily reduce the file size, as empty chunks are not overwritten.
Only data of open maps can be moved. For maps that are not open, the old chunk is still referenced. Therefore, it is recommended to open all maps before calling this method.
targetFillRate - the minimum percentage of live entrieswrite - the minimum number of bytes to writepublic void compactStore(long maxCompactTime)
protected abstract void compactStore(int thresholdFillRate,
long maxCompactTime,
int maxWriteSize,
MVStore mvStore)
thresholdFillRate - do not compact if store fill rate above this value (0-100)maxCompactTime - the maximum time in milliseconds to compactmaxWriteSize - the maximum amount of data to be written as part of this callmvStore - that owns this FileStoreprotected abstract void doHousekeeping(MVStore mvStore) throws java.lang.InterruptedException
java.lang.InterruptedExceptionpublic MVMap<java.lang.String,java.lang.String> start()
protected abstract void initializeStoreHeader(long time)
protected abstract void readStoreHeader(boolean recoveryMode)
protected final C discoverChunk(long block)
block - to start search from (found chunk footer should be no
further than block-1)protected final boolean findLastChunkWithCompleteValidChunkSet(java.util.Comparator<C> chunkComparator, java.util.Map<java.lang.Long,C> validChunksByLocation, boolean afterFullScan)
protected java.lang.Iterable<C> getChunksFromLayoutMap()
protected final C readChunkHeaderAndFooter(long block, int expectedId)
block - the blockexpectedId - of the chunkprotected final C readChunkHeaderOptionally(long block, int expectedId)
protected final C readChunkHeaderOptionally(long block)
protected final C readChunkFooter(long block)
block - the index of the next block after the chunkpublic WriteBuffer getWriteBuffer()
public void releaseWriteBuffer(WriteBuffer buff)
buff - the buffer than can be re-usedpublic long getCreationTime()
protected final int getAutoCompactFillRate()
public void sync()
public abstract int getFillRate()
protected abstract void shrinkStoreIfPossible(int minPercent)
minPercent - the minimum percentage to savepublic long size()
protected final void setSize(long size)
public long getWriteCount()
public long getReadCount()
public long getReadBytes()
public boolean isReadOnly()
public int getDefaultRetentionTime()
public void clear()
public java.lang.String getFileName()
protected final MVStore getMvStore()
protected abstract void markUsed(long pos,
int length)
pos - the position in byteslength - the number of bytespublic abstract void backup(java.util.zip.ZipOutputStream out)
throws java.io.IOException
java.io.IOExceptionprotected final java.util.concurrent.ConcurrentMap<java.lang.Integer,C> getChunks()
protected java.util.Collection<C> getRewriteCandidates()
public boolean isSpaceReused()
public void setReuseSpace(boolean reuseSpace)
protected final void store()
public int getChunksFillRate()
public int getCacheSize()
public int getCacheSizeUsed()
public void setCacheSize(int mb)
mb - the cache size in MB.public void populateInfo(java.util.function.BiConsumer<java.lang.String,java.lang.String> consumer)
public int getCacheHitRatio()
public int getTocCacheHitRatio()
protected boolean rewriteChunks(int writeLimit,
int targetFillRate)
public void executeFileStoreOperation(java.lang.Runnable operation)
public void accountForRemovedPage(long pos,
long version,
boolean pinned,
int pageNo)
pos - the position of the pageversion - at which page was removedpinned - whether page is considered pinnedpageNo - sequential page number within chunk