Inheritance diagram for ripple::NodeStore::DatabaseShardImp:

Collaboration diagram for ripple::NodeStore::DatabaseShardImp:

[legend]

Classes
struct	DatabaseImportStatus

Public Member Functions
	DatabaseShardImp ()=delete

	DatabaseShardImp (DatabaseShardImp const &)=delete

	DatabaseShardImp (DatabaseShardImp &&)=delete

DatabaseShardImp &	operator= (DatabaseShardImp const &)=delete

DatabaseShardImp &	operator= (DatabaseShardImp &&)=delete

	DatabaseShardImp (Application &app, Scheduler &scheduler, int readThreads, beast::Journal j)

	~DatabaseShardImp ()

bool	init () override
	Initialize the database. More...

std::optional< std::uint32_t >	prepareLedger (std::uint32_t validLedgerSeq) override
	Prepare to store a new ledger in the shard being acquired. More...

bool	prepareShards (std::vector< std::uint32_t > const &shardIndexes) override
	Prepare one or more shard indexes to be imported into the database. More...

void	removePreShard (std::uint32_t shardIndex) override
	Remove a previously prepared shard index for import. More...

std::string	getPreShards () override
	Get shard indexes being imported. More...

bool	importShard (std::uint32_t shardIndex, boost::filesystem::path const &srcDir) override
	Import a shard from the shard archive handler into the shard database. More...

std::shared_ptr< Ledger >	fetchLedger (uint256 const &hash, std::uint32_t ledgerSeq) override
	Fetch a ledger from the shard store. More...

void	setStored (std::shared_ptr< Ledger const > const &ledger) override
	Notifies the database that the given ledger has been fully acquired and stored. More...

std::unique_ptr< ShardInfo >	getShardInfo () const override
	Query information about shards held. More...

size_t	getNumTasks () const override
	Returns the number of queued tasks. More...

boost::filesystem::path const &	getRootDir () const override
	Returns the root database directory. More...

std::string	getName () const override
	Retrieve the name associated with this backend. More...

void	stop () override

void	importDatabase (Database &source) override
	Import the application local node store. More...

void	doImportDatabase ()

std::int32_t	getWriteLoad () const override
	Retrieve the estimated number of pending write operations. More...

bool	isSameDB (std::uint32_t s1, std::uint32_t s2) override

void	store (NodeObjectType type, Blob &&data, uint256 const &hash, std::uint32_t ledgerSeq) override
	Store the object. More...

void	sync () override

bool	storeLedger (std::shared_ptr< Ledger const > const &srcLedger) override
	Store a ledger from a different database. More...

void	sweep () override
	Remove expired entries from the positive and negative caches. More...

Json::Value	getDatabaseImportStatus () const override
	Returns a JSON object detailing the status of an ongoing database import if one is running, otherwise an error object. More...

Json::Value	startNodeToShard () override
	Initiates a NodeStore to ShardStore import and returns the result in a JSON object. More...

Json::Value	stopNodeToShard () override
	Terminates a NodeStore to ShardStore import and returns the result in a JSON object. More...

std::optional< std::uint32_t >	getDatabaseImportSequence () const override
	Returns the first ledger sequence of the shard currently being imported from the NodeStore. More...

bool	callForLedgerSQLByLedgerSeq (LedgerIndex ledgerSeq, std::function< bool(soci::session &session)> const &callback) override
	Invoke a callback on the SQLite db holding the corresponding ledger. More...

bool	callForLedgerSQLByShardIndex (std::uint32_t const shardIndex, std::function< bool(soci::session &session)> const &callback) override
	Invoke a callback on the ledger SQLite db for the corresponding shard. More...

bool	callForTransactionSQLByLedgerSeq (LedgerIndex ledgerSeq, std::function< bool(soci::session &session)> const &callback) override
	Invoke a callback on the transaction SQLite db for the corresponding ledger. More...

bool	callForTransactionSQLByShardIndex (std::uint32_t const shardIndex, std::function< bool(soci::session &session)> const &callback) override
	Invoke a callback on the transaction SQLite db for the corresponding shard. More...

bool	iterateLedgerSQLsForward (std::optional< std::uint32_t > minShardIndex, std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &callback) override
	iterateLedgerSQLsForward Checks out ledger databases for all shards in ascending order starting from given shard index until shard with the largest index visited or callback returned false. More...

bool	iterateTransactionSQLsForward (std::optional< std::uint32_t > minShardIndex, std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &callback) override
	iterateTransactionSQLsForward Checks out transaction databases for all shards in ascending order starting from given shard index until shard with the largest index visited or callback returned false. More...

bool	iterateLedgerSQLsBack (std::optional< std::uint32_t > maxShardIndex, std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &callback) override
	iterateLedgerSQLsBack Checks out ledger databases for all shards in descending order starting from given shard index until shard with the smallest index visited or callback returned false. More...

bool	iterateTransactionSQLsBack (std::optional< std::uint32_t > maxShardIndex, std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &callback) override
	iterateTransactionSQLsBack Checks out transaction databases for all shards in descending order starting from given shard index until shard with the smallest index visited or callback returned false. More...

std::shared_ptr< NodeObject >	fetchNodeObject (uint256 const &hash, std::uint32_t ledgerSeq=0, FetchType fetchType=FetchType::synchronous, bool duplicate=false)
	Fetch a node object. More...

virtual void	asyncFetch (uint256 const &hash, std::uint32_t ledgerSeq, std::function< void(std::shared_ptr< NodeObject > const &)> &&callback)
	Fetch an object without waiting. More...

std::uint64_t	getStoreCount () const
	Gather statistics pertaining to read and write activities. More...

std::uint32_t	getFetchTotalCount () const

std::uint32_t	getFetchHitCount () const

std::uint64_t	getStoreSize () const

std::uint32_t	getFetchSize () const

void	getCountsJson (Json::Value &obj)

int	fdRequired () const
	Returns the number of file descriptors the database expects to need. More...

bool	isStopping () const

std::uint32_t	ledgersPerShard () const noexcept

std::uint32_t	earliestLedgerSeq () const noexcept

std::uint32_t	earliestShardIndex () const noexcept

std::uint32_t	firstLedgerSeq (std::uint32_t shardIndex) const noexcept
	Calculates the first ledger sequence for a given shard index. More...

std::uint32_t	lastLedgerSeq (std::uint32_t shardIndex) const noexcept
	Calculates the last ledger sequence for a given shard index. More...

std::uint32_t	seqToShardIndex (std::uint32_t ledgerSeq) const noexcept
	Calculates the shard index for a given ledger sequence. More...

std::uint32_t	maxLedgers (std::uint32_t shardIndex) const noexcept
	Calculates the maximum ledgers for a given shard index. More...

Protected Member Functions
bool	storeLedger (Ledger const &srcLedger, std::shared_ptr< Backend > dstBackend)

void	storeStats (std::uint64_t count, std::uint64_t sz)

void	importInternal (Backend &dstBackend, Database &srcDB)

void	updateFetchMetrics (uint64_t fetches, uint64_t hits, uint64_t duration)

Protected Attributes
const beast::Journal	j_

Scheduler &	scheduler_

int	fdRequired_ {0}

std::atomic< std::uint32_t >	fetchHitCount_ {0}

std::atomic< std::uint32_t >	fetchSz_ {0}

const std::uint32_t	ledgersPerShard_

const std::uint32_t	earliestLedgerSeq_

const std::uint32_t	earliestShardIndex_

const int	requestBundle_

Private Types
enum	PathDesignation : uint8_t { PathDesignation::none, PathDesignation::historical }

Private Member Functions
bool	initConfig (std::lock_guard< std::mutex > const &)

std::shared_ptr< NodeObject >	fetchNodeObject (uint256 const &hash, std::uint32_t ledgerSeq, FetchReport &fetchReport, bool duplicate) override

void	for_each (std::function< void(std::shared_ptr< NodeObject >)> f) override
	Visit every object in the database This is usually called during import. More...

std::optional< std::uint32_t >	findAcquireIndex (std::uint32_t validLedgerSeq, std::lock_guard< std::mutex > const &)

void	finalizeShard (std::shared_ptr< Shard > &shard, bool writeSQLite, std::optional< uint256 > const &expectedHash)

void	updateFileStats ()

bool	sufficientStorage (std::uint32_t numShards, PathDesignation pathDesignation, std::lock_guard< std::mutex > const &) const

bool	setStoredInShard (std::shared_ptr< Shard > &shard, std::shared_ptr< Ledger const > const &ledger)

void	removeFailedShard (std::shared_ptr< Shard > &shard)

std::uint32_t	shardBoundaryIndex () const

std::uint32_t	numHistoricalShards (std::lock_guard< std::mutex > const &lock) const

void	relocateOutdatedShards (std::lock_guard< std::mutex > const &lock)

std::optional< PathDesignation >	prepareForNewShard (std::uint32_t shardIndex, std::uint32_t numHistoricalShards, std::lock_guard< std::mutex > const &lock)

boost::filesystem::path	chooseHistoricalPath (std::lock_guard< std::mutex > const &) const

bool	iterateShardsForward (std::optional< std::uint32_t > minShardIndex, std::function< bool(Shard &shard)> const &visit)
	iterateShardsForward Visits all shards starting from given in ascending order and calls given callback function to each of them passing shard as parameter. More...

bool	iterateShardsBack (std::optional< std::uint32_t > maxShardIndex, std::function< bool(Shard &shard)> const &visit)
	iterateShardsBack Visits all shards starting from given in descending order and calls given callback function to each of them passing shard as parameter. More...

bool	checkHistoricalPaths (std::lock_guard< std::mutex > const &) const

std::unique_ptr< ShardInfo >	getShardInfo (std::lock_guard< std::mutex > const &) const

void	updatePeers (std::lock_guard< std::mutex > const &lock) const

void	startDatabaseImportThread (std::lock_guard< std::mutex > const &)

virtual std::optional< Backend::Counters< std::uint64_t > >	getCounters () const
	Retrieve backend read and write stats. More...

void	threadEntry ()

Private Attributes
Application &	app_

std::mutex	mutex_

bool	init_ {false}

std::unique_ptr< nudb::context >	ctx_

TaskQueue	taskQueue_

std::map< std::uint32_t, std::shared_ptr< Shard > >	shards_

std::set< std::uint32_t >	preparedIndexes_

std::uint32_t	acquireIndex_ {0}

boost::filesystem::path	dir_

bool	canAdd_ {true}

std::string	backendName_

std::uint32_t	maxHistoricalShards_ {0}

std::vector< boost::filesystem::path >	historicalPaths_

std::uint64_t	fileSz_ {0}

std::uint64_t	avgShardFileSz_

const std::uint32_t	openFinalLimit_

std::optional< std::uint32_t >	latestShardIndex_

std::optional< std::uint32_t >	secondLatestShardIndex_

std::unique_ptr< DatabaseImportStatus >	databaseImportStatus_

std::thread	databaseImporter_

std::atomic_bool	haltDatabaseImport_ {false}

std::atomic< std::uint64_t >	storeCount_ {0}

std::atomic< std::uint64_t >	storeSz_ {0}

std::atomic< std::uint64_t >	fetchTotalCount_ {0}

std::atomic< std::uint64_t >	fetchDurationUs_ {0}

std::atomic< std::uint64_t >	storeDurationUs_ {0}

std::mutex	readLock_

std::condition_variable	readCondVar_

std::map< uint256, std::vector< std::pair< std::uint32_t, std::function< void(std::shared_ptr< NodeObject > const &)> > > >	read_

std::atomic< bool >	readStopping_ = false

std::atomic< int >	readThreads_ = 0

std::atomic< int >	runningThreads_ = 0

Static Private Attributes
static constexpr auto	databaseImportMarker_ = "database_import"

Detailed Description

Definition at line 32 of file DatabaseShardImp.h.

Member Enumeration Documentation

◆ PathDesignation

enum ripple::NodeStore::DatabaseShardImp::PathDesignation : uint8_t

strongprivate

Enumerator
none
historical

Definition at line 196 of file DatabaseShardImp.h.

Constructor & Destructor Documentation

◆ DatabaseShardImp() [1/4]

ripple::NodeStore::DatabaseShardImp::DatabaseShardImp ( )

delete

◆ DatabaseShardImp() [2/4]

ripple::NodeStore::DatabaseShardImp::DatabaseShardImp ( DatabaseShardImp const & )

delete

◆ DatabaseShardImp() [3/4]

ripple::NodeStore::DatabaseShardImp::DatabaseShardImp ( DatabaseShardImp && )

delete

◆ DatabaseShardImp() [4/4]

ripple::NodeStore::DatabaseShardImp::DatabaseShardImp	(	Application &	app,
		Scheduler &	scheduler,
		int	readThreads,
		beast::Journal	j
	)

Definition at line 46 of file DatabaseShardImp.cpp.

◆ ~DatabaseShardImp()

ripple::NodeStore::DatabaseShardImp::~DatabaseShardImp ( )

Definition at line 49 of file DatabaseShardImp.h.

Member Function Documentation

◆ operator=() [1/2]

DatabaseShardImp& ripple::NodeStore::DatabaseShardImp::operator= ( DatabaseShardImp const & )

delete

◆ operator=() [2/2]

DatabaseShardImp& ripple::NodeStore::DatabaseShardImp::operator= ( DatabaseShardImp && )

delete

◆ init()

bool ripple::NodeStore::DatabaseShardImp::init ( )

overridevirtual

Initialize the database.

Returns: true if the database initialized without error

Implements ripple::NodeStore::DatabaseShard.

Definition at line 70 of file DatabaseShardImp.cpp.

◆ prepareLedger()

std::optional< std::uint32_t > ripple::NodeStore::DatabaseShardImp::prepareLedger ( std::uint32_t validLedgerSeq )

overridevirtual

Prepare to store a new ledger in the shard being acquired.

Parameters

validLedgerSeq The sequence of the maximum valid ledgers

Returns: If a ledger should be fetched and stored, then returns the ledger sequence of the ledger to request. Otherwise returns std::nullopt. Some reasons this may return std::nullopt are: all shards are stored and full, max allowed disk space would be exceeded, or a ledger was recently requested and not enough time has passed between requests. @implNote adds a new writable shard if necessary

Implements ripple::NodeStore::DatabaseShard.

Definition at line 229 of file DatabaseShardImp.cpp.

◆ prepareShards()

bool ripple::NodeStore::DatabaseShardImp::prepareShards ( std::vector< std::uint32_t > const & shardIndexes )

overridevirtual

Prepare one or more shard indexes to be imported into the database.

Parameters

shardIndexes Shard indexes to be prepared for import

Returns: true if all shard indexes successfully prepared for import

Implements ripple::NodeStore::DatabaseShard.

Definition at line 299 of file DatabaseShardImp.cpp.

◆ removePreShard()

void ripple::NodeStore::DatabaseShardImp::removePreShard ( std::uint32_t shardIndex )

overridevirtual

Remove a previously prepared shard index for import.

Parameters

shardIndex Shard index to be removed from import

Implements ripple::NodeStore::DatabaseShard.

Definition at line 416 of file DatabaseShardImp.cpp.

◆ getPreShards()

std::string ripple::NodeStore::DatabaseShardImp::getPreShards ( )

overridevirtual

Get shard indexes being imported.

Returns: a string representing the shards prepared for import

Implements ripple::NodeStore::DatabaseShard.

Definition at line 426 of file DatabaseShardImp.cpp.

◆ importShard()

bool ripple::NodeStore::DatabaseShardImp::importShard	(	std::uint32_t	shardIndex,
		boost::filesystem::path const &	srcDir
	)

overridevirtual

Import a shard from the shard archive handler into the shard database.

This differs from 'importDatabase' which imports the contents of the NodeStore

Parameters

shardIndex	Shard index to import
srcDir	The directory to import from

Returns: true If the shard was successfully imported @implNote if successful, srcDir is moved to the database directory

Implements ripple::NodeStore::DatabaseShard.

Definition at line 444 of file DatabaseShardImp.cpp.

◆ fetchLedger()

std::shared_ptr< Ledger > ripple::NodeStore::DatabaseShardImp::fetchLedger	(	uint256 const &	hash,
		std::uint32_t	seq
	)

overridevirtual

Fetch a ledger from the shard store.

Parameters

hash	The key of the ledger to retrieve
seq	The sequence of the ledger

Returns: The ledger if found, nullptr otherwise

Implements ripple::NodeStore::DatabaseShard.

Definition at line 552 of file DatabaseShardImp.cpp.

◆ setStored()

void ripple::NodeStore::DatabaseShardImp::setStored ( std::shared_ptr< Ledger const > const & ledger )

overridevirtual

Notifies the database that the given ledger has been fully acquired and stored.

Parameters

ledger The stored ledger to be marked as complete

Implements ripple::NodeStore::DatabaseShard.

Definition at line 630 of file DatabaseShardImp.cpp.

◆ getShardInfo() [1/2]

std::unique_ptr< ShardInfo > ripple::NodeStore::DatabaseShardImp::getShardInfo ( ) const

overridevirtual

Query information about shards held.

Returns: Information about shards held by this node

Implements ripple::NodeStore::DatabaseShard.

Definition at line 692 of file DatabaseShardImp.cpp.

◆ getNumTasks()

size_t ripple::NodeStore::DatabaseShardImp::getNumTasks ( ) const

overridevirtual

Returns the number of queued tasks.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2194 of file DatabaseShardImp.cpp.

◆ getRootDir()

boost::filesystem::path const& ripple::NodeStore::DatabaseShardImp::getRootDir ( ) const

overridevirtual

Returns the root database directory.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 86 of file DatabaseShardImp.h.

◆ getName()

std::string ripple::NodeStore::DatabaseShardImp::getName ( ) const

overridevirtual

Retrieve the name associated with this backend.

This is used for diagnostics and may not reflect the actual path or paths used by the underlying backend.

Implements ripple::NodeStore::Database.

Definition at line 92 of file DatabaseShardImp.h.

◆ stop()

void ripple::NodeStore::DatabaseShardImp::stop ( )

overridevirtual

Reimplemented from ripple::NodeStore::Database.

Definition at line 699 of file DatabaseShardImp.cpp.

◆ importDatabase()

void ripple::NodeStore::DatabaseShardImp::importDatabase ( Database & source )

overridevirtual

Import the application local node store.

Parameters

source The application node store.

Implements ripple::NodeStore::Database.

Definition at line 765 of file DatabaseShardImp.cpp.

◆ doImportDatabase()

void ripple::NodeStore::DatabaseShardImp::doImportDatabase ( )

Definition at line 784 of file DatabaseShardImp.cpp.

◆ getWriteLoad()

std::int32_t ripple::NodeStore::DatabaseShardImp::getWriteLoad ( ) const

overridevirtual

Retrieve the estimated number of pending write operations.

This is used for diagnostics.

Implements ripple::NodeStore::Database.

Definition at line 1085 of file DatabaseShardImp.cpp.

◆ isSameDB()

bool ripple::NodeStore::DatabaseShardImp::isSameDB	(	std::uint32_t	s1,
		std::uint32_t	s2
	)

overridevirtual

Implements ripple::NodeStore::Database.

Definition at line 114 of file DatabaseShardImp.h.

◆ store()

void ripple::NodeStore::DatabaseShardImp::store	(	NodeObjectType	type,
		Blob &&	data,
		uint256 const &	hash,
		std::uint32_t	ledgerSeq
	)

overridevirtual

Store the object.

The caller's Blob parameter is overwritten.

Parameters

type	The type of object.
data	The payload of the object. The caller's variable is overwritten.
hash	The 256-bit hash of the payload data.
ledgerSeq	The sequence of the ledger the object belongs to.

Returns: true if the object was stored?

Implements ripple::NodeStore::Database.

Definition at line 1102 of file DatabaseShardImp.cpp.

◆ sync()

void ripple::NodeStore::DatabaseShardImp::sync ( )

overridevirtual

Implements ripple::NodeStore::Database.

Definition at line 127 of file DatabaseShardImp.h.

◆ storeLedger() [1/2]

bool ripple::NodeStore::DatabaseShardImp::storeLedger ( std::shared_ptr< Ledger const > const & srcLedger )

overridevirtual

Store a ledger from a different database.

Parameters

srcLedger The ledger to store.

Returns: true if the operation was successful

Implements ripple::NodeStore::Database.

Definition at line 1136 of file DatabaseShardImp.cpp.

◆ sweep()

void ripple::NodeStore::DatabaseShardImp::sweep ( )

overridevirtual

Remove expired entries from the positive and negative caches.

Implements ripple::NodeStore::Database.

Definition at line 1171 of file DatabaseShardImp.cpp.

◆ getDatabaseImportStatus()

Json::Value ripple::NodeStore::DatabaseShardImp::getDatabaseImportStatus ( ) const

overridevirtual

Returns a JSON object detailing the status of an ongoing database import if one is running, otherwise an error object.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 1223 of file DatabaseShardImp.cpp.

◆ startNodeToShard()

Json::Value ripple::NodeStore::DatabaseShardImp::startNodeToShard ( )

overridevirtual

Initiates a NodeStore to ShardStore import and returns the result in a JSON object.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 1252 of file DatabaseShardImp.cpp.

◆ stopNodeToShard()

Json::Value ripple::NodeStore::DatabaseShardImp::stopNodeToShard ( )

overridevirtual

Terminates a NodeStore to ShardStore import and returns the result in a JSON object.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 1275 of file DatabaseShardImp.cpp.

◆ getDatabaseImportSequence()

std::optional< std::uint32_t > ripple::NodeStore::DatabaseShardImp::getDatabaseImportSequence ( ) const

overridevirtual

Returns the first ledger sequence of the shard currently being imported from the NodeStore.

Returns: The ledger sequence or an unseated value if no import is running

Implements ripple::NodeStore::DatabaseShard.

Definition at line 1297 of file DatabaseShardImp.cpp.

◆ callForLedgerSQLByLedgerSeq()

bool ripple::NodeStore::DatabaseShardImp::callForLedgerSQLByLedgerSeq	(	LedgerIndex	ledgerSeq,
		std::function< bool(soci::session &session)> const &	callback
	)

overridevirtual

Invoke a callback on the SQLite db holding the corresponding ledger.

Returns: Value returned by callback function.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2023 of file DatabaseShardImp.cpp.

◆ callForLedgerSQLByShardIndex()

bool ripple::NodeStore::DatabaseShardImp::callForLedgerSQLByShardIndex	(	std::uint32_t const	shardIndex,
		std::function< bool(soci::session &session)> const &	callback
	)

overridevirtual

Invoke a callback on the ledger SQLite db for the corresponding shard.

Returns: Value returned by callback function.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2038 of file DatabaseShardImp.cpp.

◆ callForTransactionSQLByLedgerSeq()

bool ripple::NodeStore::DatabaseShardImp::callForTransactionSQLByLedgerSeq	(	LedgerIndex	ledgerSeq,
		std::function< bool(soci::session &session)> const &	callback
	)

overridevirtual

Invoke a callback on the transaction SQLite db for the corresponding ledger.

Returns: Value returned by callback function.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2052 of file DatabaseShardImp.cpp.

◆ callForTransactionSQLByShardIndex()

bool ripple::NodeStore::DatabaseShardImp::callForTransactionSQLByShardIndex	(	std::uint32_t const	shardIndex,
		std::function< bool(soci::session &session)> const &	callback
	)

overridevirtual

Invoke a callback on the transaction SQLite db for the corresponding shard.

Returns: Value returned by callback function.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2061 of file DatabaseShardImp.cpp.

◆ iterateLedgerSQLsForward()

bool ripple::NodeStore::DatabaseShardImp::iterateLedgerSQLsForward	(	std::optional< std::uint32_t >	minShardIndex,
		std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &	callback
	)

overridevirtual

iterateLedgerSQLsForward Checks out ledger databases for all shards in ascending order starting from given shard index until shard with the largest index visited or callback returned false.

For each visited shard calls given callback function passing shard index and session with the database to it.

Parameters

minShardIndex	Start shard index to visit or none if all shards should be visited.
callback	Callback function to call.

Returns: True if each callback function returns true, false otherwise.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2103 of file DatabaseShardImp.cpp.

◆ iterateTransactionSQLsForward()

bool ripple::NodeStore::DatabaseShardImp::iterateTransactionSQLsForward	(	std::optional< std::uint32_t >	minShardIndex,
		std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &	callback
	)

overridevirtual

iterateTransactionSQLsForward Checks out transaction databases for all shards in ascending order starting from given shard index until shard with the largest index visited or callback returned false.

For each visited shard calls given callback function passing shard index and session with the database to it.

Parameters

minShardIndex	Start shard index to visit or none if all shards should be visited.
callback	Callback function to call.

Returns: True if each callback function returns true, false otherwise.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2115 of file DatabaseShardImp.cpp.

◆ iterateLedgerSQLsBack()

bool ripple::NodeStore::DatabaseShardImp::iterateLedgerSQLsBack	(	std::optional< std::uint32_t >	maxShardIndex,
		std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &	callback
	)

overridevirtual

iterateLedgerSQLsBack Checks out ledger databases for all shards in descending order starting from given shard index until shard with the smallest index visited or callback returned false.

For each visited shard calls given callback function passing shard index and session with the database to it.

Parameters

maxShardIndex	Start shard index to visit or none if all shards should be visited.
callback	Callback function to call.

Returns: True if each callback function returns true, false otherwise.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2156 of file DatabaseShardImp.cpp.

◆ iterateTransactionSQLsBack()

bool ripple::NodeStore::DatabaseShardImp::iterateTransactionSQLsBack	(	std::optional< std::uint32_t >	maxShardIndex,
		std::function< bool(soci::session &session, std::uint32_t shardIndex)> const &	callback
	)

overridevirtual

iterateTransactionSQLsBack Checks out transaction databases for all shards in descending order starting from given shard index until shard with the smallest index visited or callback returned false.

For each visited shard calls given callback function passing shard index and session with the database to it.

Parameters

maxShardIndex	Start shard index to visit or none if all shards should be visited.
callback	Callback function to call.

Returns: True if each callback function returns true, false otherwise.

Implements ripple::NodeStore::DatabaseShard.

Definition at line 2167 of file DatabaseShardImp.cpp.

◆ initConfig()

bool ripple::NodeStore::DatabaseShardImp::initConfig ( std::lock_guard< std::mutex > const & )

private

Definition at line 1308 of file DatabaseShardImp.cpp.

◆ fetchNodeObject() [1/2]

std::shared_ptr< NodeObject > ripple::NodeStore::DatabaseShardImp::fetchNodeObject	(	uint256 const &	hash,
		std::uint32_t	ledgerSeq,
		FetchReport &	fetchReport,
		bool	duplicate
	)

overrideprivatevirtual

Implements ripple::NodeStore::Database.

Definition at line 1382 of file DatabaseShardImp.cpp.

◆ for_each()

void ripple::NodeStore::DatabaseShardImp::for_each ( std::function< void(std::shared_ptr< NodeObject >)> f )

overrideprivatevirtual

Visit every object in the database This is usually called during import.

Note: This routine will not be called concurrently with itself or other methods.

See also: import

Implements ripple::NodeStore::Database.

Definition at line 310 of file DatabaseShardImp.h.

◆ findAcquireIndex()

std::optional< std::uint32_t > ripple::NodeStore::DatabaseShardImp::findAcquireIndex	(	std::uint32_t	validLedgerSeq,
		std::lock_guard< std::mutex > const &
	)

private

Definition at line 1402 of file DatabaseShardImp.cpp.

◆ finalizeShard()

void ripple::NodeStore::DatabaseShardImp::finalizeShard	(	std::shared_ptr< Shard > &	shard,
		bool	writeSQLite,
		std::optional< uint256 > const &	expectedHash
	)

private

Definition at line 1467 of file DatabaseShardImp.cpp.

◆ updateFileStats()

void ripple::NodeStore::DatabaseShardImp::updateFileStats ( )

private

Definition at line 1541 of file DatabaseShardImp.cpp.

◆ sufficientStorage()

bool ripple::NodeStore::DatabaseShardImp::sufficientStorage	(	std::uint32_t	numShards,
		PathDesignation	pathDesignation,
		std::lock_guard< std::mutex > const &
	)		const

private

Definition at line 1602 of file DatabaseShardImp.cpp.

◆ setStoredInShard()

bool ripple::NodeStore::DatabaseShardImp::setStoredInShard	(	std::shared_ptr< Shard > &	shard,
		std::shared_ptr< Ledger const > const &	ledger
	)

private

Definition at line 1657 of file DatabaseShardImp.cpp.

◆ removeFailedShard()

void ripple::NodeStore::DatabaseShardImp::removeFailedShard ( std::shared_ptr< Shard > & shard )

private

Definition at line 1690 of file DatabaseShardImp.cpp.

◆ shardBoundaryIndex()

std::uint32_t ripple::NodeStore::DatabaseShardImp::shardBoundaryIndex ( ) const

private

Definition at line 1714 of file DatabaseShardImp.cpp.

◆ numHistoricalShards()

std::uint32_t ripple::NodeStore::DatabaseShardImp::numHistoricalShards ( std::lock_guard< std::mutex > const & lock ) const

private

Definition at line 1730 of file DatabaseShardImp.cpp.

◆ relocateOutdatedShards()

void ripple::NodeStore::DatabaseShardImp::relocateOutdatedShards ( std::lock_guard< std::mutex > const & lock )

private

Definition at line 1741 of file DatabaseShardImp.cpp.

◆ prepareForNewShard()

auto ripple::NodeStore::DatabaseShardImp::prepareForNewShard	(	std::uint32_t	shardIndex,
		std::uint32_t	numHistoricalShards,
		std::lock_guard< std::mutex > const &	lock
	)

private

Definition at line 1878 of file DatabaseShardImp.cpp.

◆ chooseHistoricalPath()

boost::filesystem::path ripple::NodeStore::DatabaseShardImp::chooseHistoricalPath ( std::lock_guard< std::mutex > const & ) const

private

Definition at line 1909 of file DatabaseShardImp.cpp.

◆ iterateShardsForward()

bool ripple::NodeStore::DatabaseShardImp::iterateShardsForward	(	std::optional< std::uint32_t >	minShardIndex,
		std::function< bool(Shard &shard)> const &	visit
	)

private

iterateShardsForward Visits all shards starting from given in ascending order and calls given callback function to each of them passing shard as parameter.

Parameters

minShardIndex	Start shard index to visit or none if all shards should be visited.
visit	Callback function to call.

Returns: True if each callback function returned true, false otherwise.

Definition at line 2075 of file DatabaseShardImp.cpp.

◆ iterateShardsBack()

bool ripple::NodeStore::DatabaseShardImp::iterateShardsBack	(	std::optional< std::uint32_t >	maxShardIndex,
		std::function< bool(Shard &shard)> const &	visit
	)

private

iterateShardsBack Visits all shards starting from given in descending order and calls given callback function to each of them passing shard as parameter.

Parameters

maxShardIndex	Start shard index to visit or none if all shards should be visited.
visit	Callback function to call.

Returns: True if each callback function returned true, false otherwise.

Definition at line 2127 of file DatabaseShardImp.cpp.

◆ checkHistoricalPaths()

bool ripple::NodeStore::DatabaseShardImp::checkHistoricalPaths ( std::lock_guard< std::mutex > const & ) const

private

Definition at line 1942 of file DatabaseShardImp.cpp.

◆ getShardInfo() [2/2]

std::unique_ptr< ShardInfo > ripple::NodeStore::DatabaseShardImp::getShardInfo ( std::lock_guard< std::mutex > const & ) const

private

Definition at line 2178 of file DatabaseShardImp.cpp.

◆ updatePeers()

void ripple::NodeStore::DatabaseShardImp::updatePeers ( std::lock_guard< std::mutex > const & lock ) const

private

Definition at line 2201 of file DatabaseShardImp.cpp.

◆ startDatabaseImportThread()

void ripple::NodeStore::DatabaseShardImp::startDatabaseImportThread ( std::lock_guard< std::mutex > const & )

private

Definition at line 2213 of file DatabaseShardImp.cpp.

◆ fetchNodeObject() [2/2]

std::shared_ptr< NodeObject > ripple::NodeStore::Database::fetchNodeObject	(	uint256 const &	hash,
		std::uint32_t	ledgerSeq = `0`,
		FetchType	fetchType = `FetchType::synchronous`,
		bool	duplicate = `false`
	)

inherited

Fetch a node object.

If the object is known to be not in the database, isn't found in the database during the fetch, or failed to load correctly during the fetch, nullptr is returned.

Note: This can be called concurrently.

Parameters

hash	The key of the object to retrieve.
ledgerSeq	The sequence of the ledger where the object is stored.
fetchType	the type of fetch, synchronous or asynchronous.

Returns: The object, or nullptr if it couldn't be retrieved.

Definition at line 252 of file Database.cpp.

◆ asyncFetch()

void ripple::NodeStore::Database::asyncFetch	(	uint256 const &	hash,
		std::uint32_t	ledgerSeq,
		std::function< void(std::shared_ptr< NodeObject > const &)> &&	callback
	)

virtualinherited

Fetch an object without waiting.

If I/O is required to determine whether or not the object is present, false is returned. Otherwise, true is returned and object is set to refer to the object, or nullptr if the object is not present. If I/O is required, the I/O is scheduled and true is returned

Note: This can be called concurrently.

Parameters

hash	The key of the object to retrieve
ledgerSeq	The sequence of the ledger where the object is stored, used by the shard store.
callback	Callback function when read completes

Reimplemented in ripple::NodeStore::DatabaseNodeImp.

Definition at line 198 of file Database.cpp.

◆ storeLedger() [2/2]

bool ripple::NodeStore::Database::storeLedger	(	Ledger const &	srcLedger,
		std::shared_ptr< Backend >	dstBackend
	)

protectedinherited

Definition at line 279 of file Database.cpp.

◆ getStoreCount()

std::uint64_t ripple::NodeStore::Database::getStoreCount ( ) const

inherited

Gather statistics pertaining to read and write activities.

Parameters

obj	Json object reference into which to place counters.

Definition at line 182 of file Database.h.

◆ getFetchTotalCount()

std::uint32_t ripple::NodeStore::Database::getFetchTotalCount ( ) const

inherited

Definition at line 188 of file Database.h.

◆ getFetchHitCount()

std::uint32_t ripple::NodeStore::Database::getFetchHitCount ( ) const

inherited

Definition at line 194 of file Database.h.

◆ getStoreSize()

std::uint64_t ripple::NodeStore::Database::getStoreSize ( ) const

inherited

Definition at line 200 of file Database.h.

◆ getFetchSize()

std::uint32_t ripple::NodeStore::Database::getFetchSize ( ) const

inherited

Definition at line 206 of file Database.h.

◆ getCountsJson()

void ripple::NodeStore::Database::getCountsJson ( Json::Value & obj )

inherited

Definition at line 378 of file Database.cpp.

◆ fdRequired()

int ripple::NodeStore::Database::fdRequired ( ) const

inherited

Returns the number of file descriptors the database expects to need.

Definition at line 216 of file Database.h.

◆ isStopping()

bool ripple::NodeStore::Database::isStopping ( ) const

inherited

Definition at line 146 of file Database.cpp.

◆ ledgersPerShard()

std::uint32_t ripple::NodeStore::Database::ledgersPerShard ( ) const

noexceptinherited

Returns: The maximum number of ledgers stored in a shard

Definition at line 230 of file Database.h.

◆ earliestLedgerSeq()

std::uint32_t ripple::NodeStore::Database::earliestLedgerSeq ( ) const

noexceptinherited

Returns: The earliest ledger sequence allowed

Definition at line 238 of file Database.h.

◆ earliestShardIndex()

std::uint32_t ripple::NodeStore::Database::earliestShardIndex ( ) const

noexceptinherited

Returns: The earliest shard index

Definition at line 246 of file Database.h.

◆ firstLedgerSeq()

std::uint32_t ripple::NodeStore::Database::firstLedgerSeq ( std::uint32_t shardIndex ) const

noexceptinherited

Calculates the first ledger sequence for a given shard index.

Parameters

shardIndex The shard index considered

Returns: The first ledger sequence pertaining to the shard index

Definition at line 257 of file Database.h.

◆ lastLedgerSeq()

std::uint32_t ripple::NodeStore::Database::lastLedgerSeq ( std::uint32_t shardIndex ) const

noexceptinherited

Calculates the last ledger sequence for a given shard index.

Parameters

shardIndex The shard index considered

Returns: The last ledger sequence pertaining to the shard index

Definition at line 271 of file Database.h.

◆ seqToShardIndex()

std::uint32_t ripple::NodeStore::Database::seqToShardIndex ( std::uint32_t ledgerSeq ) const

noexceptinherited

Calculates the shard index for a given ledger sequence.

Parameters

ledgerSeq ledger sequence

Returns: The shard index of the ledger sequence

Definition at line 283 of file Database.h.

◆ maxLedgers()

std::uint32_t ripple::NodeStore::Database::maxLedgers ( std::uint32_t shardIndex ) const

noexceptinherited

Calculates the maximum ledgers for a given shard index.

Parameters

shardIndex The shard index considered

Returns: The maximum ledgers pertaining to the shard index

Note: The earliest shard may store less if the earliest ledger sequence truncates its beginning

Definition at line 152 of file Database.cpp.

◆ storeStats()

void ripple::NodeStore::Database::storeStats	(	std::uint64_t	count,
		std::uint64_t	sz
	)

protectedinherited

Definition at line 333 of file Database.h.

◆ importInternal()

void ripple::NodeStore::Database::importInternal	(	Backend &	dstBackend,
		Database &	srcDB
	)

protectedinherited

Definition at line 213 of file Database.cpp.

◆ updateFetchMetrics()

void ripple::NodeStore::Database::updateFetchMetrics	(	uint64_t	fetches,
		uint64_t	hits,
		uint64_t	duration
	)

protectedinherited

Definition at line 349 of file Database.h.

◆ getCounters()

virtual std::optional<Backend::Counters<std::uint64_t> > ripple::NodeStore::Database::getCounters ( ) const

privatevirtualinherited

Retrieve backend read and write stats.

Note: The Counters struct is specific to and only used by CassandraBackend.

Reimplemented in ripple::NodeStore::DatabaseNodeImp.

Definition at line 401 of file Database.h.

◆ threadEntry()

void ripple::NodeStore::Database::threadEntry ( )

privateinherited

Member Data Documentation

◆ app_

Application& ripple::NodeStore::DatabaseShardImp::app_

private

Definition at line 232 of file DatabaseShardImp.h.

◆ mutex_

std::mutex ripple::NodeStore::DatabaseShardImp::mutex_

mutableprivate

Definition at line 233 of file DatabaseShardImp.h.

◆ init_

bool ripple::NodeStore::DatabaseShardImp::init_ {false}

private

Definition at line 234 of file DatabaseShardImp.h.

◆ ctx_

std::unique_ptr<nudb::context> ripple::NodeStore::DatabaseShardImp::ctx_

private

Definition at line 237 of file DatabaseShardImp.h.

◆ taskQueue_

TaskQueue ripple::NodeStore::DatabaseShardImp::taskQueue_

private

Definition at line 240 of file DatabaseShardImp.h.

◆ shards_

std::map<std::uint32_t, std::shared_ptr<Shard> > ripple::NodeStore::DatabaseShardImp::shards_

private

Definition at line 243 of file DatabaseShardImp.h.

◆ preparedIndexes_

std::set<std::uint32_t> ripple::NodeStore::DatabaseShardImp::preparedIndexes_

private

Definition at line 246 of file DatabaseShardImp.h.

◆ acquireIndex_

std::uint32_t ripple::NodeStore::DatabaseShardImp::acquireIndex_ {0}

private

Definition at line 249 of file DatabaseShardImp.h.

◆ dir_

boost::filesystem::path ripple::NodeStore::DatabaseShardImp::dir_

private

Definition at line 252 of file DatabaseShardImp.h.

◆ canAdd_

bool ripple::NodeStore::DatabaseShardImp::canAdd_ {true}

private

Definition at line 255 of file DatabaseShardImp.h.

◆ backendName_

std::string ripple::NodeStore::DatabaseShardImp::backendName_

private

Definition at line 258 of file DatabaseShardImp.h.

◆ maxHistoricalShards_

std::uint32_t ripple::NodeStore::DatabaseShardImp::maxHistoricalShards_ {0}

private

Definition at line 261 of file DatabaseShardImp.h.

◆ historicalPaths_

std::vector<boost::filesystem::path> ripple::NodeStore::DatabaseShardImp::historicalPaths_

private

Definition at line 264 of file DatabaseShardImp.h.

◆ fileSz_

std::uint64_t ripple::NodeStore::DatabaseShardImp::fileSz_ {0}

private

Definition at line 267 of file DatabaseShardImp.h.

◆ avgShardFileSz_

std::uint64_t ripple::NodeStore::DatabaseShardImp::avgShardFileSz_

private

Definition at line 270 of file DatabaseShardImp.h.

◆ openFinalLimit_

const std::uint32_t ripple::NodeStore::DatabaseShardImp::openFinalLimit_

private

Definition at line 273 of file DatabaseShardImp.h.

◆ databaseImportMarker_

constexpr auto ripple::NodeStore::DatabaseShardImp::databaseImportMarker_ = "database_import"

staticconstexprprivate

Definition at line 276 of file DatabaseShardImp.h.

◆ latestShardIndex_

std::optional<std::uint32_t> ripple::NodeStore::DatabaseShardImp::latestShardIndex_

private

Definition at line 285 of file DatabaseShardImp.h.

◆ secondLatestShardIndex_

std::optional<std::uint32_t> ripple::NodeStore::DatabaseShardImp::secondLatestShardIndex_

private

Definition at line 286 of file DatabaseShardImp.h.

◆ databaseImportStatus_

std::unique_ptr<DatabaseImportStatus> ripple::NodeStore::DatabaseShardImp::databaseImportStatus_

private

Definition at line 289 of file DatabaseShardImp.h.

◆ databaseImporter_

std::thread ripple::NodeStore::DatabaseShardImp::databaseImporter_

private

Definition at line 292 of file DatabaseShardImp.h.

◆ haltDatabaseImport_

std::atomic_bool ripple::NodeStore::DatabaseShardImp::haltDatabaseImport_ {false}

private

Definition at line 295 of file DatabaseShardImp.h.

◆ j_

const beast::Journal ripple::NodeStore::Database::j_

protectedinherited

Definition at line 301 of file Database.h.

◆ scheduler_

Scheduler& ripple::NodeStore::Database::scheduler_

protectedinherited

Definition at line 302 of file Database.h.

◆ fdRequired_

int ripple::NodeStore::Database::fdRequired_ {0}

protectedinherited

Definition at line 303 of file Database.h.

◆ fetchHitCount_

std::atomic<std::uint32_t> ripple::NodeStore::Database::fetchHitCount_ {0}

protectedinherited

Definition at line 305 of file Database.h.

◆ fetchSz_

std::atomic<std::uint32_t> ripple::NodeStore::Database::fetchSz_ {0}

protectedinherited

Definition at line 306 of file Database.h.

◆ ledgersPerShard_

const std::uint32_t ripple::NodeStore::Database::ledgersPerShard_

protectedinherited

Definition at line 314 of file Database.h.

◆ earliestLedgerSeq_

const std::uint32_t ripple::NodeStore::Database::earliestLedgerSeq_

protectedinherited

Definition at line 322 of file Database.h.

◆ earliestShardIndex_

const std::uint32_t ripple::NodeStore::Database::earliestShardIndex_

protectedinherited

Definition at line 325 of file Database.h.

◆ requestBundle_

const int ripple::NodeStore::Database::requestBundle_

protectedinherited

Definition at line 330 of file Database.h.

◆ storeCount_

std::atomic<std::uint64_t> ripple::NodeStore::Database::storeCount_ {0}

privateinherited

Definition at line 357 of file Database.h.

◆ storeSz_

std::atomic<std::uint64_t> ripple::NodeStore::Database::storeSz_ {0}

privateinherited

Definition at line 358 of file Database.h.

◆ fetchTotalCount_

std::atomic<std::uint64_t> ripple::NodeStore::Database::fetchTotalCount_ {0}

privateinherited

Definition at line 359 of file Database.h.

◆ fetchDurationUs_

std::atomic<std::uint64_t> ripple::NodeStore::Database::fetchDurationUs_ {0}

privateinherited

Definition at line 360 of file Database.h.

◆ storeDurationUs_

std::atomic<std::uint64_t> ripple::NodeStore::Database::storeDurationUs_ {0}

privateinherited

Definition at line 361 of file Database.h.

◆ readLock_

std::mutex ripple::NodeStore::Database::readLock_

mutableprivateinherited

Definition at line 363 of file Database.h.

◆ readCondVar_

std::condition_variable ripple::NodeStore::Database::readCondVar_

privateinherited

Definition at line 364 of file Database.h.

◆ read_

std::map< uint256, std::vector<std::pair< std::uint32_t, std::function<void(std::shared_ptr<NodeObject> const&)> > > > ripple::NodeStore::Database::read_

privateinherited

Definition at line 372 of file Database.h.

◆ readStopping_

std::atomic<bool> ripple::NodeStore::Database::readStopping_ = false

privateinherited

Definition at line 374 of file Database.h.

◆ readThreads_

std::atomic<int> ripple::NodeStore::Database::readThreads_ = 0

privateinherited

Definition at line 375 of file Database.h.

◆ runningThreads_

std::atomic<int> ripple::NodeStore::Database::runningThreads_ = 0

privateinherited

Definition at line 376 of file Database.h.

Classes

Public Member Functions

Protected Member Functions

Protected Attributes

Private Types

Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Member Enumeration Documentation

◆ PathDesignation

Constructor & Destructor Documentation

◆ DatabaseShardImp() [1/4]

◆ DatabaseShardImp() [2/4]

◆ DatabaseShardImp() [3/4]

◆ DatabaseShardImp() [4/4]

◆ ~DatabaseShardImp()

Member Function Documentation

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ init()

◆ prepareLedger()

◆ prepareShards()

◆ removePreShard()

◆ getPreShards()

◆ importShard()

◆ fetchLedger()

◆ setStored()

◆ getShardInfo() [1/2]

◆ getNumTasks()

◆ getRootDir()

◆ getName()

◆ stop()

◆ importDatabase()

◆ doImportDatabase()

◆ getWriteLoad()

◆ isSameDB()

◆ store()

◆ sync()

◆ storeLedger() [1/2]

◆ sweep()

◆ getDatabaseImportStatus()

◆ startNodeToShard()

◆ stopNodeToShard()

◆ getDatabaseImportSequence()

◆ callForLedgerSQLByLedgerSeq()

◆ callForLedgerSQLByShardIndex()

◆ callForTransactionSQLByLedgerSeq()

◆ callForTransactionSQLByShardIndex()

◆ iterateLedgerSQLsForward()

◆ iterateTransactionSQLsForward()

◆ iterateLedgerSQLsBack()

◆ iterateTransactionSQLsBack()

◆ initConfig()

◆ fetchNodeObject() [1/2]

◆ for_each()

◆ findAcquireIndex()

◆ finalizeShard()

◆ updateFileStats()

◆ sufficientStorage()

◆ setStoredInShard()

◆ removeFailedShard()

◆ shardBoundaryIndex()

◆ numHistoricalShards()

◆ relocateOutdatedShards()

◆ prepareForNewShard()

◆ chooseHistoricalPath()

◆ iterateShardsForward()

◆ iterateShardsBack()

◆ checkHistoricalPaths()

◆ getShardInfo() [2/2]

◆ updatePeers()

◆ startDatabaseImportThread()

◆ fetchNodeObject() [2/2]

◆ asyncFetch()

◆ storeLedger() [2/2]

◆ getStoreCount()

◆ getFetchTotalCount()

◆ getFetchHitCount()

◆ getStoreSize()