Package org.apache.hadoop.ozone.om

Class OmMetadataManagerImpl

java.lang.Object
org.apache.hadoop.ozone.om.OmMetadataManagerImpl
All Implemented Interfaces:
Closeable, AutoCloseable, org.apache.hadoop.hdds.utils.DBStoreHAManager, org.apache.hadoop.ozone.om.OMMetadataManager, org.apache.hadoop.ozone.om.S3SecretStore
public class OmMetadataManagerImpl extends Object implements org.apache.hadoop.ozone.om.OMMetadataManager, org.apache.hadoop.ozone.om.S3SecretStore, Closeable
Ozone metadata manager interface.

  • Constructor Details

    • OmMetadataManagerImpl

      public OmMetadataManagerImpl(org.apache.hadoop.hdds.conf.OzoneConfiguration conf, OzoneManager ozoneManager) throws IOException
      OmMetadataManagerImpl constructor.
      Parameters:
      conf - OzoneConfiguration
      ozoneManager - Points to parent OzoneManager. Can be null if not used (in some tests).
      Throws:
      IOException

    • OmMetadataManagerImpl

      protected OmMetadataManagerImpl()
      For subclass overriding.

    • OmMetadataManagerImpl

      protected OmMetadataManagerImpl(org.apache.hadoop.hdds.conf.OzoneConfiguration conf, File dir, String name) throws IOException
      Metadata constructor for checkpoints.
      Parameters:
      conf - - Ozone conf.
      dir - - Checkpoint parent directory.
      name - - Checkpoint directory name.
      Throws:
      IOException

  • Method Details

    • createCheckpointMetadataManager

      public static OmMetadataManagerImpl createCheckpointMetadataManager(org.apache.hadoop.hdds.conf.OzoneConfiguration conf, org.apache.hadoop.hdds.utils.db.DBCheckpoint checkpoint) throws IOException
      Throws:
      IOException

    • getOzoneManager

      public OzoneManager getOzoneManager()

    • getUserTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo> getUserTable()
      Specified by:
      getUserTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getDelegationTokenTable

      public org.apache.hadoop.hdds.utils.db.Table<org.apache.hadoop.ozone.security.OzoneTokenIdentifier,Long> getDelegationTokenTable()
      Specified by:
      getDelegationTokenTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getVolumeTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmVolumeArgs> getVolumeTable()
      Specified by:
      getVolumeTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getBucketTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmBucketInfo> getBucketTable()
      Specified by:
      getBucketTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getKeyTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmKeyInfo> getKeyTable(org.apache.hadoop.ozone.om.helpers.BucketLayout bucketLayout)
      Specified by:
      getKeyTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getFileTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmKeyInfo> getFileTable()
      Specified by:
      getFileTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getDeletedTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo> getDeletedTable()
      Specified by:
      getDeletedTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getDeletedDirTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmKeyInfo> getDeletedDirTable()
      Specified by:
      getDeletedDirTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOpenKeyTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmKeyInfo> getOpenKeyTable(org.apache.hadoop.ozone.om.helpers.BucketLayout bucketLayout)
      Specified by:
      getOpenKeyTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getPrefixTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmPrefixInfo> getPrefixTable()
      Specified by:
      getPrefixTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getDirectoryTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo> getDirectoryTable()
      Specified by:
      getDirectoryTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getMultipartInfoTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo> getMultipartInfoTable()
      Specified by:
      getMultipartInfoTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • start

      public void start(org.apache.hadoop.hdds.conf.OzoneConfiguration configuration) throws IOException
      Start metadata manager.
      Specified by:
      start in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • loadDB

      public static org.apache.hadoop.hdds.utils.db.DBStore loadDB(org.apache.hadoop.hdds.conf.OzoneConfiguration configuration, File metaDir, int maxOpenFiles) throws IOException
      Throws:
      IOException

    • initializeOmTables

      protected void initializeOmTables(org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType cacheType, boolean addCacheMetrics) throws IOException
      Initialize OM Tables.
      Throws:
      IOException

    • stop

      public void stop() throws IOException
      Stop metadata manager.
      Specified by:
      stop in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getStore

      public org.apache.hadoop.hdds.utils.db.DBStore getStore()
      Get metadata store.
      Specified by:
      getStore in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Returns:
      store - metadata store.

    • getVolumeKey

      public String getVolumeKey(String volume)
      Given a volume return the corresponding DB key.
      Specified by:
      getVolumeKey in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Parameters:
      volume - - Volume name

    • getUserKey

      public String getUserKey(String user)
      Given a user return the corresponding DB key.
      Specified by:
      getUserKey in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Parameters:
      user - - User name

    • getBucketKey

      public String getBucketKey(String volume, String bucket)
      Given a volume and bucket, return the corresponding DB key.
      Specified by:
      getBucketKey in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Parameters:
      volume - - Volume name
      bucket - - Bucket name

    • getBucketKeyPrefix

      public String getBucketKeyPrefix(String volume, String bucket)
      Specified by:
      getBucketKeyPrefix in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getBucketKeyPrefixFSO

      public String getBucketKeyPrefixFSO(String volume, String bucket) throws IOException
      Specified by:
      getBucketKeyPrefixFSO in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getVolumeBucketIdPairFSO

      public org.apache.hadoop.ozone.om.OMMetadataManager.VolumeBucketId getVolumeBucketIdPairFSO(String fsoKey) throws IOException
      Specified by:
      getVolumeBucketIdPairFSO in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getOzoneKey

      public String getOzoneKey(String volume, String bucket, String key)
      Specified by:
      getOzoneKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOzoneKeyFSO

      public String getOzoneKeyFSO(String volumeName, String bucketName, String keyPrefix) throws IOException
      Specified by:
      getOzoneKeyFSO in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getOzoneDirKey

      public String getOzoneDirKey(String volume, String bucket, String key)
      Specified by:
      getOzoneDirKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOpenKey

      public String getOpenKey(String volume, String bucket, String key, String clientId)
      Specified by:
      getOpenKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getMultipartKey

      public String getMultipartKey(String volume, String bucket, String key, String uploadId)
      Specified by:
      getMultipartKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getMultipartKeyFSO

      public String getMultipartKeyFSO(String volume, String bucket, String key, String uploadId) throws IOException
      Specified by:
      getMultipartKeyFSO in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getLock

      public org.apache.hadoop.ozone.om.lock.IOzoneManagerLock getLock()
      Returns the OzoneManagerLock used on Metadata DB.
      Specified by:
      getLock in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Returns:
      OzoneManagerLock

    • getHierarchicalLockManager

      public org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager getHierarchicalLockManager()
      Specified by:
      getHierarchicalLockManager in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOmEpoch

      public long getOmEpoch()
      Specified by:
      getOmEpoch in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • isVolumeEmpty

      public boolean isVolumeEmpty(String volume) throws IOException
      Given a volume, check if it is empty, i.e there are no buckets inside it. We iterate in the bucket table and see if there is any key that starts with the volume prefix. We actually look for /volume/, since if we don't have the trailing slash it is possible that we might match some other volume.

      For example, vol1 and vol122 might match, to avoid that we look for /vol1/

      Specified by:
      isVolumeEmpty in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Parameters:
      volume - - Volume name
      Returns:
      true if the volume is empty
      Throws:
      IOException

    • isBucketEmpty

      public boolean isBucketEmpty(String volume, String bucket) throws IOException
      Given a volume/bucket, check if it is empty, i.e there are no keys inside it. Prefix is /volume/bucket/, and we lookup the keyTable.
      Specified by:
      isBucketEmpty in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Parameters:
      volume - - Volume name
      bucket - - Bucket name
      Returns:
      true if the bucket is empty
      Throws:
      IOException

    • listBuckets

      public List<org.apache.hadoop.ozone.om.helpers.OmBucketInfo> listBuckets(String volumeName, String startBucket, String bucketPrefix, int maxNumOfBuckets, boolean hasSnapshot) throws IOException
      Specified by:
      listBuckets in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getBucketIterator

      public Iterator<Map.Entry<org.apache.hadoop.hdds.utils.db.cache.CacheKey<String>,org.apache.hadoop.hdds.utils.db.cache.CacheValue<org.apache.hadoop.ozone.om.helpers.OmBucketInfo>>> getBucketIterator()
      Specified by:
      getBucketIterator in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getKeyIterator

      public org.apache.hadoop.hdds.utils.db.TableIterator<String,? extends org.apache.hadoop.hdds.utils.db.Table.KeyValue<String,org.apache.hadoop.ozone.om.helpers.OmKeyInfo>> getKeyIterator() throws IOException
      Specified by:
      getKeyIterator in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • listOpenFiles

      public org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult listOpenFiles(org.apache.hadoop.ozone.om.helpers.BucketLayout bucketLayout, int maxKeys, String dbOpenKeyPrefix, boolean hasContToken, String dbContTokenPrefix) throws IOException
      Specified by:
      listOpenFiles in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • listKeys

      public org.apache.hadoop.ozone.om.helpers.ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException
      Specified by:
      listKeys in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getSnapshotInfo

      public org.apache.hadoop.ozone.om.helpers.SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, String snapshotName) throws IOException
      Specified by:
      getSnapshotInfo in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • listSnapshot

      public org.apache.hadoop.ozone.snapshot.ListSnapshotResponse listSnapshot(String volumeName, String bucketName, String snapshotPrefix, String prevSnapshot, int maxListResult) throws IOException
      Specified by:
      listSnapshot in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • listVolumes

      public List<org.apache.hadoop.ozone.om.helpers.OmVolumeArgs> listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException
      Specified by:
      listVolumes in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Parameters:
      userName - volume owner, null for listing all volumes.
      Throws:
      IOException

    • getExpiredOpenKeys

      public org.apache.hadoop.ozone.om.ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, int count, org.apache.hadoop.ozone.om.helpers.BucketLayout bucketLayout, Duration leaseThreshold) throws IOException
      Specified by:
      getExpiredOpenKeys in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getExpiredMultipartUploads

      public List<org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket> getExpiredMultipartUploads(Duration expireThreshold, int maxParts) throws IOException
      Specified by:
      getExpiredMultipartUploads in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • countRowsInTable

      public <KEY, VALUE> long countRowsInTable(org.apache.hadoop.hdds.utils.db.Table<KEY,VALUE> table) throws IOException
      Specified by:
      countRowsInTable in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • countEstimatedRowsInTable

      public <KEY, VALUE> long countEstimatedRowsInTable(org.apache.hadoop.hdds.utils.db.Table<KEY,VALUE> table) throws IOException
      Specified by:
      countEstimatedRowsInTable in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getMultipartUploadKeys

      public List<org.apache.hadoop.ozone.om.helpers.OmMultipartUpload> getMultipartUploadKeys(String volumeName, String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException
      Get the merged and paginated list of multipart upload keys from cache and DB. Only return at most maxUploads + 1 keys (lexicographical order).
      Specified by:
      getMultipartUploadKeys in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • storeSecret

      public void storeSecret(String kerberosId, org.apache.hadoop.ozone.om.helpers.S3SecretValue secret) throws IOException
      Specified by:
      storeSecret in interface org.apache.hadoop.ozone.om.S3SecretStore
      Throws:
      IOException

    • getSecret

      public org.apache.hadoop.ozone.om.helpers.S3SecretValue getSecret(String kerberosID) throws IOException
      Specified by:
      getSecret in interface org.apache.hadoop.ozone.om.S3SecretStore
      Throws:
      IOException

    • revokeSecret

      public void revokeSecret(String kerberosId) throws IOException
      Specified by:
      revokeSecret in interface org.apache.hadoop.ozone.om.S3SecretStore
      Throws:
      IOException

    • batcher

      public org.apache.hadoop.ozone.om.S3Batcher batcher()
      Specified by:
      batcher in interface org.apache.hadoop.ozone.om.S3SecretStore

    • getTransactionInfoTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.hdds.utils.TransactionInfo> getTransactionInfoTable()
      Specified by:
      getTransactionInfoTable in interface org.apache.hadoop.hdds.utils.DBStoreHAManager
      Specified by:
      getTransactionInfoTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getMetaTable

      public org.apache.hadoop.hdds.utils.db.Table<String,String> getMetaTable()
      Specified by:
      getMetaTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getTenantAccessIdTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo> getTenantAccessIdTable()
      Specified by:
      getTenantAccessIdTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getPrincipalToAccessIdsTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo> getPrincipalToAccessIdsTable()
      Specified by:
      getPrincipalToAccessIdsTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getTenantStateTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.OmDBTenantState> getTenantStateTable()
      Specified by:
      getTenantStateTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getSnapshotInfoTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.hadoop.ozone.om.helpers.SnapshotInfo> getSnapshotInfoTable()
      Specified by:
      getSnapshotInfoTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getSnapshotRenamedTable

      public org.apache.hadoop.hdds.utils.db.Table<String,String> getSnapshotRenamedTable()
      Specified by:
      getSnapshotRenamedTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getCompactionLogTable

      public org.apache.hadoop.hdds.utils.db.Table<String,org.apache.ozone.compaction.log.CompactionLogEntry> getCompactionLogTable()
      Specified by:
      getCompactionLogTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getSnapshotChainManager

      public SnapshotChainManager getSnapshotChainManager()
      Get Snapshot Chain Manager.
      Returns:
      SnapshotChainManager.

    • setStore

      protected void setStore(org.apache.hadoop.hdds.utils.db.DBStore store)
      Update store used by subclass.
      Parameters:
      store - DB store.

    • listTables

      public Map<String,org.apache.hadoop.hdds.utils.db.Table> listTables()
      Specified by:
      listTables in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getTable

      public org.apache.hadoop.hdds.utils.db.Table getTable(String tableName)
      Specified by:
      getTable in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • listTableNames

      public Set<String> listTableNames()
      Specified by:
      listTableNames in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOzonePathKey

      public String getOzonePathKey(long volumeId, long bucketId, long parentObjectId, String pathComponentName)
      Specified by:
      getOzonePathKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOzoneDeletePathKey

      public String getOzoneDeletePathKey(long objectId, String pathKey)
      Specified by:
      getOzoneDeletePathKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOzoneDeletePathDirKey

      public String getOzoneDeletePathDirKey(String ozoneDeletePath)
      Specified by:
      getOzoneDeletePathDirKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getOpenFileName

      public String getOpenFileName(long volumeId, long bucketId, long parentID, String fileName, String clientId)
      Specified by:
      getOpenFileName in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getRenameKey

      public String getRenameKey(String volumeName, String bucketName, long objectID)
      Specified by:
      getRenameKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • splitRenameKey

      public String[] splitRenameKey(String renameKey)
      Specified by:
      splitRenameKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getMultipartKey

      public String getMultipartKey(long volumeId, long bucketId, long parentID, String fileName, String uploadId)
      Specified by:
      getMultipartKey in interface org.apache.hadoop.ozone.om.OMMetadataManager

    • getBucketLayout

      public org.apache.hadoop.ozone.om.helpers.BucketLayout getBucketLayout()

    • getVolumeId

      public long getVolumeId(String volume) throws IOException
      Specified by:
      getVolumeId in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getBucketId

      public long getBucketId(String volume, String bucket) throws IOException
      Specified by:
      getBucketId in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • getBlocksForKeyDelete

      public List<org.apache.hadoop.ozone.common.BlockGroup> getBlocksForKeyDelete(String deletedKey) throws IOException
      Specified by:
      getBlocksForKeyDelete in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • containsIncompleteMPUs

      public boolean containsIncompleteMPUs(String volume, String bucket) throws IOException
      Specified by:
      containsIncompleteMPUs in interface org.apache.hadoop.ozone.om.OMMetadataManager
      Throws:
      IOException

    • close

      public void close() throws IOException
      Specified by:
      close in interface AutoCloseable
      Specified by:
      close in interface Closeable
      Throws:
      IOException