rippled
Database_test.cpp
1 //------------------------------------------------------------------------------
2 /*
3  This file is part of rippled: https://github.com/ripple/rippled
4  Copyright (c) 2012, 2013 Ripple Labs Inc.
5 
6  Permission to use, copy, modify, and/or distribute this software for any
7  purpose with or without fee is hereby granted, provided that the above
8  copyright notice and this permission notice appear in all copies.
9 
10  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18 //==============================================================================
19 
20 #include <ripple/beast/utility/temp_dir.h>
21 #include <ripple/core/DatabaseCon.h>
22 #include <ripple/nodestore/DummyScheduler.h>
23 #include <ripple/nodestore/Manager.h>
24 #include <test/jtx.h>
25 #include <test/jtx/CheckMessageLogs.h>
26 #include <test/jtx/envconfig.h>
27 #include <test/nodestore/TestBase.h>
28 #include <test/unit_test/SuiteJournal.h>
29 
30 namespace ripple {
31 
32 namespace NodeStore {
33 
34 class Database_test : public TestBase
35 {
37 
38 public:
39  Database_test() : journal_("Database_test", *this)
40  {
41  }
42 
43  void
45  {
46  testcase("Config");
47 
48  using namespace ripple::test;
49  using namespace ripple::test::jtx;
50 
51  auto const integrityWarning =
52  "reducing the data integrity guarantees from the "
53  "default [sqlite] behavior is not recommended for "
54  "nodes storing large amounts of history, because of the "
55  "difficulty inherent in rebuilding corrupted data.";
56  {
57  // defaults
58  Env env(*this);
59 
60  auto const s = setup_DatabaseCon(env.app().config());
61 
62  if (BEAST_EXPECT(s.globalPragma->size() == 3))
63  {
64  BEAST_EXPECT(
65  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
66  BEAST_EXPECT(
67  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
68  BEAST_EXPECT(
69  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
70  }
71  }
72  {
73  // High safety level
75 
76  bool found = false;
77  Env env = [&]() {
78  auto p = test::jtx::envconfig();
79  {
80  auto& section = p->section("sqlite");
81  section.set("safety_level", "high");
82  }
83  p->LEDGER_HISTORY = 100'000'000;
84 
85  return Env(
86  *this,
87  std::move(p),
88  std::make_unique<CheckMessageLogs>(
89  integrityWarning, &found),
91  }();
92 
93  BEAST_EXPECT(!found);
94  auto const s = setup_DatabaseCon(env.app().config());
95  if (BEAST_EXPECT(s.globalPragma->size() == 3))
96  {
97  BEAST_EXPECT(
98  s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
99  BEAST_EXPECT(
100  s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
101  BEAST_EXPECT(
102  s.globalPragma->at(2) == "PRAGMA temp_store=file;");
103  }
104  }
105  {
106  // Low safety level
108 
109  bool found = false;
110  Env env = [&]() {
111  auto p = test::jtx::envconfig();
112  {
113  auto& section = p->section("sqlite");
114  section.set("safety_level", "low");
115  }
116  p->LEDGER_HISTORY = 100'000'000;
117 
118  return Env(
119  *this,
120  std::move(p),
121  std::make_unique<CheckMessageLogs>(
122  integrityWarning, &found),
124  }();
125 
126  BEAST_EXPECT(found);
127  auto const s = setup_DatabaseCon(env.app().config());
128  if (BEAST_EXPECT(s.globalPragma->size() == 3))
129  {
130  BEAST_EXPECT(
131  s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
132  BEAST_EXPECT(
133  s.globalPragma->at(1) == "PRAGMA synchronous=off;");
134  BEAST_EXPECT(
135  s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
136  }
137  }
138  {
139  // Override individual settings
141 
142  bool found = false;
143  Env env = [&]() {
144  auto p = test::jtx::envconfig();
145  {
146  auto& section = p->section("sqlite");
147  section.set("journal_mode", "off");
148  section.set("synchronous", "extra");
149  section.set("temp_store", "default");
150  }
151 
152  return Env(
153  *this,
154  std::move(p),
155  std::make_unique<CheckMessageLogs>(
156  integrityWarning, &found),
158  }();
159 
160  // No warning, even though higher risk settings were used because
161  // LEDGER_HISTORY is small
162  BEAST_EXPECT(!found);
163  auto const s = setup_DatabaseCon(env.app().config());
164  if (BEAST_EXPECT(s.globalPragma->size() == 3))
165  {
166  BEAST_EXPECT(
167  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
168  BEAST_EXPECT(
169  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
170  BEAST_EXPECT(
171  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
172  }
173  }
174  {
175  // Override individual settings with large history
177 
178  bool found = false;
179  Env env = [&]() {
180  auto p = test::jtx::envconfig();
181  {
182  auto& section = p->section("sqlite");
183  section.set("journal_mode", "off");
184  section.set("synchronous", "extra");
185  section.set("temp_store", "default");
186  }
187  p->LEDGER_HISTORY = 50'000'000;
188 
189  return Env(
190  *this,
191  std::move(p),
192  std::make_unique<CheckMessageLogs>(
193  integrityWarning, &found),
195  }();
196 
197  // No warning, even though higher risk settings were used because
198  // LEDGER_HISTORY is small
199  BEAST_EXPECT(found);
200  auto const s = setup_DatabaseCon(env.app().config());
201  if (BEAST_EXPECT(s.globalPragma->size() == 3))
202  {
203  BEAST_EXPECT(
204  s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
205  BEAST_EXPECT(
206  s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
207  BEAST_EXPECT(
208  s.globalPragma->at(2) == "PRAGMA temp_store=default;");
209  }
210  }
211  {
212  // Error: Mix safety_level and individual settings
214  auto const expected =
215  "Failed to initialize SQL databases: "
216  "Configuration file may not define both \"safety_level\" and "
217  "\"journal_mode\"";
218  bool found = false;
219 
220  auto p = test::jtx::envconfig();
221  {
222  auto& section = p->section("sqlite");
223  section.set("safety_level", "low");
224  section.set("journal_mode", "off");
225  section.set("synchronous", "extra");
226  section.set("temp_store", "default");
227  }
228 
229  try
230  {
231  Env env(
232  *this,
233  std::move(p),
234  std::make_unique<CheckMessageLogs>(expected, &found),
236  fail();
237  }
238  catch (...)
239  {
240  BEAST_EXPECT(found);
241  }
242  }
243  {
244  // Error: Mix safety_level and one setting (gotta catch 'em all)
246  auto const expected =
247  "Failed to initialize SQL databases: Configuration file may "
248  "not define both \"safety_level\" and \"journal_mode\"";
249  bool found = false;
250 
251  auto p = test::jtx::envconfig();
252  {
253  auto& section = p->section("sqlite");
254  section.set("safety_level", "high");
255  section.set("journal_mode", "off");
256  }
257 
258  try
259  {
260  Env env(
261  *this,
262  std::move(p),
263  std::make_unique<CheckMessageLogs>(expected, &found),
265  fail();
266  }
267  catch (...)
268  {
269  BEAST_EXPECT(found);
270  }
271  }
272  {
273  // Error: Mix safety_level and one setting (gotta catch 'em all)
275  auto const expected =
276  "Failed to initialize SQL databases: Configuration file may "
277  "not define both \"safety_level\" and \"synchronous\"";
278  bool found = false;
279 
280  auto p = test::jtx::envconfig();
281  {
282  auto& section = p->section("sqlite");
283  section.set("safety_level", "low");
284  section.set("synchronous", "extra");
285  }
286 
287  try
288  {
289  Env env(
290  *this,
291  std::move(p),
292  std::make_unique<CheckMessageLogs>(expected, &found),
294  fail();
295  }
296  catch (...)
297  {
298  BEAST_EXPECT(found);
299  }
300  }
301  {
302  // Error: Mix safety_level and one setting (gotta catch 'em all)
304  auto const expected =
305  "Failed to initialize SQL databases: Configuration file may "
306  "not define both \"safety_level\" and \"temp_store\"";
307  bool found = false;
308 
309  auto p = test::jtx::envconfig();
310  {
311  auto& section = p->section("sqlite");
312  section.set("safety_level", "high");
313  section.set("temp_store", "default");
314  }
315 
316  try
317  {
318  Env env(
319  *this,
320  std::move(p),
321  std::make_unique<CheckMessageLogs>(expected, &found),
323  fail();
324  }
325  catch (...)
326  {
327  BEAST_EXPECT(found);
328  }
329  }
330  {
331  // Error: Invalid value
333  auto const expected =
334  "Failed to initialize SQL databases: Invalid safety_level "
335  "value: slow";
336  bool found = false;
337 
338  auto p = test::jtx::envconfig();
339  {
340  auto& section = p->section("sqlite");
341  section.set("safety_level", "slow");
342  }
343 
344  try
345  {
346  Env env(
347  *this,
348  std::move(p),
349  std::make_unique<CheckMessageLogs>(expected, &found),
351  fail();
352  }
353  catch (...)
354  {
355  BEAST_EXPECT(found);
356  }
357  }
358  {
359  // Error: Invalid value
361  auto const expected =
362  "Failed to initialize SQL databases: Invalid journal_mode "
363  "value: fast";
364  bool found = false;
365 
366  auto p = test::jtx::envconfig();
367  {
368  auto& section = p->section("sqlite");
369  section.set("journal_mode", "fast");
370  }
371 
372  try
373  {
374  Env env(
375  *this,
376  std::move(p),
377  std::make_unique<CheckMessageLogs>(expected, &found),
379  fail();
380  }
381  catch (...)
382  {
383  BEAST_EXPECT(found);
384  }
385  }
386  {
387  // Error: Invalid value
389  auto const expected =
390  "Failed to initialize SQL databases: Invalid synchronous "
391  "value: instant";
392  bool found = false;
393 
394  auto p = test::jtx::envconfig();
395  {
396  auto& section = p->section("sqlite");
397  section.set("synchronous", "instant");
398  }
399 
400  try
401  {
402  Env env(
403  *this,
404  std::move(p),
405  std::make_unique<CheckMessageLogs>(expected, &found),
407  fail();
408  }
409  catch (...)
410  {
411  BEAST_EXPECT(found);
412  }
413  }
414  {
415  // Error: Invalid value
417  auto const expected =
418  "Failed to initialize SQL databases: Invalid temp_store "
419  "value: network";
420  bool found = false;
421 
422  auto p = test::jtx::envconfig();
423  {
424  auto& section = p->section("sqlite");
425  section.set("temp_store", "network");
426  }
427 
428  try
429  {
430  Env env(
431  *this,
432  std::move(p),
433  std::make_unique<CheckMessageLogs>(expected, &found),
435  fail();
436  }
437  catch (...)
438  {
439  BEAST_EXPECT(found);
440  }
441  }
442  }
443 
444  //--------------------------------------------------------------------------
445 
446  void
448  std::string const& destBackendType,
449  std::string const& srcBackendType,
450  std::int64_t seedValue)
451  {
452  DummyScheduler scheduler;
453 
454  beast::temp_dir node_db;
455  Section srcParams;
456  srcParams.set("type", srcBackendType);
457  srcParams.set("path", node_db.path());
458 
459  // Create a batch
460  auto batch = createPredictableBatch(numObjectsToTest, seedValue);
461 
462  // Write to source db
463  {
465  megabytes(4), scheduler, 2, srcParams, journal_);
466  storeBatch(*src, batch);
467  }
468 
469  Batch copy;
470 
471  {
472  // Re-open the db
474  megabytes(4), scheduler, 2, srcParams, journal_);
475 
476  // Set up the destination database
477  beast::temp_dir dest_db;
478  Section destParams;
479  destParams.set("type", destBackendType);
480  destParams.set("path", dest_db.path());
481 
483  megabytes(4), scheduler, 2, destParams, journal_);
484 
485  testcase(
486  "import into '" + destBackendType + "' from '" +
487  srcBackendType + "'");
488 
489  // Do the import
490  dest->importDatabase(*src);
491 
492  // Get the results of the import
493  fetchCopyOfBatch(*dest, &copy, batch);
494  }
495 
496  // Canonicalize the source and destination batches
497  std::sort(batch.begin(), batch.end(), LessThan{});
498  std::sort(copy.begin(), copy.end(), LessThan{});
499  BEAST_EXPECT(areBatchesEqual(batch, copy));
500  }
501 
502  //--------------------------------------------------------------------------
503 
504  void
506  std::string const& type,
507  bool const testPersistence,
508  std::int64_t const seedValue,
509  int numObjsToTest = 2000)
510  {
511  DummyScheduler scheduler;
512 
513  std::string s = "NodeStore backend '" + type + "'";
514 
515  testcase(s);
516 
517  beast::temp_dir node_db;
518  Section nodeParams;
519  nodeParams.set("type", type);
520  nodeParams.set("path", node_db.path());
521 
522  beast::xor_shift_engine rng(seedValue);
523 
524  // Create a batch
525  auto batch = createPredictableBatch(numObjsToTest, rng());
526 
527  {
528  // Open the database
530  megabytes(4), scheduler, 2, nodeParams, journal_);
531 
532  // Write the batch
533  storeBatch(*db, batch);
534 
535  {
536  // Read it back in
537  Batch copy;
538  fetchCopyOfBatch(*db, &copy, batch);
539  BEAST_EXPECT(areBatchesEqual(batch, copy));
540  }
541 
542  {
543  // Reorder and read the copy again
544  std::shuffle(batch.begin(), batch.end(), rng);
545  Batch copy;
546  fetchCopyOfBatch(*db, &copy, batch);
547  BEAST_EXPECT(areBatchesEqual(batch, copy));
548  }
549  }
550 
551  if (testPersistence)
552  {
553  // Re-open the database without the ephemeral DB
555  megabytes(4), scheduler, 2, nodeParams, journal_);
556 
557  // Read it back in
558  Batch copy;
559  fetchCopyOfBatch(*db, &copy, batch);
560 
561  // Canonicalize the source and destination batches
562  std::sort(batch.begin(), batch.end(), LessThan{});
563  std::sort(copy.begin(), copy.end(), LessThan{});
564  BEAST_EXPECT(areBatchesEqual(batch, copy));
565  }
566 
567  if (type == "memory")
568  {
569  // Verify default earliest ledger sequence
570  {
573  megabytes(4), scheduler, 2, nodeParams, journal_);
574  BEAST_EXPECT(
575  db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
576  }
577 
578  // Set an invalid earliest ledger sequence
579  try
580  {
581  nodeParams.set("earliest_seq", "0");
584  megabytes(4), scheduler, 2, nodeParams, journal_);
585  }
586  catch (std::runtime_error const& e)
587  {
588  BEAST_EXPECT(
589  std::strcmp(e.what(), "Invalid earliest_seq") == 0);
590  }
591 
592  {
593  // Set a valid earliest ledger sequence
594  nodeParams.set("earliest_seq", "1");
597  megabytes(4), scheduler, 2, nodeParams, journal_);
598 
599  // Verify database uses the earliest ledger sequence setting
600  BEAST_EXPECT(db->earliestLedgerSeq() == 1);
601  }
602 
603  // Create another database that attempts to set the value again
604  try
605  {
606  // Set to default earliest ledger sequence
607  nodeParams.set(
608  "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
611  megabytes(4), scheduler, 2, nodeParams, journal_);
612  }
613  catch (std::runtime_error const& e)
614  {
615  BEAST_EXPECT(
616  std::strcmp(e.what(), "earliest_seq set more than once") ==
617  0);
618  }
619 
620  // Verify default ledgers per shard
621  {
624  megabytes(4), scheduler, 2, nodeParams, journal_);
625  BEAST_EXPECT(
626  db->ledgersPerShard() == DEFAULT_LEDGERS_PER_SHARD);
627  }
628 
629  // Set an invalid ledgers per shard
630  try
631  {
632  nodeParams.set("ledgers_per_shard", "100");
635  megabytes(4), scheduler, 2, nodeParams, journal_);
636  }
637  catch (std::runtime_error const& e)
638  {
639  BEAST_EXPECT(
640  std::strcmp(e.what(), "Invalid ledgers_per_shard") == 0);
641  }
642 
643  // Set a valid ledgers per shard
644  nodeParams.set("ledgers_per_shard", "256");
646  megabytes(4), scheduler, 2, nodeParams, journal_);
647 
648  // Verify database uses the ledgers per shard
649  BEAST_EXPECT(db->ledgersPerShard() == 256);
650  }
651  }
652 
653  //--------------------------------------------------------------------------
654 
655  void
656  run() override
657  {
658  std::int64_t const seedValue = 50;
659 
660  testConfig();
661 
662  testNodeStore("memory", false, seedValue);
663 
664  // Persistent backend tests
665  {
666  testNodeStore("nudb", true, seedValue);
667 
668 #if RIPPLE_ROCKSDB_AVAILABLE
669  testNodeStore("rocksdb", true, seedValue);
670 #endif
671  }
672 
673  // Import tests
674  {
675  testImport("nudb", "nudb", seedValue);
676 
677 #if RIPPLE_ROCKSDB_AVAILABLE
678  testImport("rocksdb", "rocksdb", seedValue);
679 #endif
680 
681 #if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
682  testImport("sqlite", "sqlite", seedValue);
683 #endif
684  }
685  }
686 };
687 
688 BEAST_DEFINE_TESTSUITE(Database, NodeStore, ripple);
689 
690 } // namespace NodeStore
691 } // namespace ripple
ripple::NodeStore::DummyScheduler
Simple NodeStore Scheduler that just peforms the tasks synchronously.
Definition: DummyScheduler.h:29
ripple::Section
Holds a collection of configuration values.
Definition: BasicConfig.h:42
std::strcmp
T strcmp(T... args)
ripple::DatabaseCon::Setup::globalPragma
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition: DatabaseCon.h:104
ripple::DEFAULT_LEDGERS_PER_SHARD
static constexpr std::uint32_t DEFAULT_LEDGERS_PER_SHARD
The number of ledgers in a shard.
Definition: SystemParameters.h:76
ripple::NodeStore::TestBase
Definition: TestBase.h:68
std::string
STL class.
ripple::NodeStore::Database_test::Database_test
Database_test()
Definition: Database_test.cpp:39
ripple::setup_DatabaseCon
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
Definition: DatabaseCon.cpp:106
std::vector< std::shared_ptr< NodeObject > >
ripple::NodeStore::LessThan
Binary function that satisfies the strict-weak-ordering requirement.
Definition: TestBase.h:44
ripple::NodeStore::Database_test::testNodeStore
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Definition: Database_test.cpp:505
ripple::NodeStore::Database_test::testImport
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
Definition: Database_test.cpp:447
ripple::test::jtx::Env::app
Application & app()
Definition: Env.h:241
ripple::test::jtx::envconfig
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:49
ripple::NodeStore::Database_test::run
void run() override
Definition: Database_test.cpp:656
std::sort
T sort(T... args)
std::unique_ptr::reset
T reset(T... args)
ripple::NodeStore::TestBase::areBatchesEqual
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition: TestBase.h:120
ripple::NodeStore::BEAST_DEFINE_TESTSUITE
BEAST_DEFINE_TESTSUITE(Backend, ripple_core, ripple)
ripple::Application::config
virtual Config & config()=0
ripple::megabytes
constexpr auto megabytes(T value) noexcept
Definition: ByteUtilities.h:34
ripple::NodeStore::TestBase::fetchCopyOfBatch
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition: TestBase.h:155
std::to_string
T to_string(T... args)
std::runtime_error
STL class.
std::int64_t
ripple::test::SuiteJournal
Definition: SuiteJournal.h:88
beast::temp_dir::path
std::string path() const
Get the native path for the temporary directory.
Definition: temp_dir.h:66
ripple::NodeStore::TestBase::numObjectsToTest
static const int numObjectsToTest
Definition: TestBase.h:75
ripple::test::jtx
Definition: Check_test.cpp:26
ripple::NodeStore::TestBase::createPredictableBatch
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition: TestBase.h:80
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
ripple::test
Definition: LedgerDeltaAcquire.h:35
ripple::NodeStore::Database_test::journal_
test::SuiteJournal journal_
Definition: Database_test.cpp:36
ripple::Section::set
void set(std::string const &key, std::string const &value)
Set a key/value pair.
Definition: BasicConfig.cpp:32
beast::severities::kWarning
@ kWarning
Definition: Journal.h:37
ripple::XRP_LEDGER_EARLIEST_SEQ
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
Definition: SystemParameters.h:69
beast::detail::xor_shift_engine
Definition: xor_shift_engine.h:32
ripple::NodeStore::TestBase::storeBatch
void storeBatch(Backend &backend, Batch const &batch)
Definition: TestBase.h:145
ripple::NodeStore::Manager::instance
static Manager & instance()
Returns the instance of the manager singleton.
Definition: ManagerImp.cpp:120
ripple::NodeStore::Database_test::testConfig
void testConfig()
Definition: Database_test.cpp:44
ripple::NodeStore::Manager::make_Database
virtual std::unique_ptr< Database > make_Database(std::size_t burstSize, Scheduler &scheduler, int readThreads, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
std::unique_ptr
STL class.
std::shuffle
T shuffle(T... args)
ripple::test::jtx::Env
A transaction testing environment.
Definition: Env.h:116
beast::temp_dir
RAII temporary directory.
Definition: temp_dir.h:33
std::runtime_error::what
T what(T... args)
ripple::NodeStore::Database_test
Definition: Database_test.cpp:34