rippled
ReportingETL_test.cpp
1 
2 //------------------------------------------------------------------------------
3 /*
4  This file is part of rippled: https://github.com/ripple/rippled
5  Copyright (c) 2020 Ripple Labs Inc.
6 
7  Permission to use, copy, modify, and/or distribute this software for any
8  purpose with or without fee is hereby granted, provided that the above
9  copyright notice and this permission notice appear in all copies.
10 
11  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 */
19 //==============================================================================
20 
21 #include <ripple/app/ledger/LedgerMaster.h>
22 #include <ripple/app/reporting/P2pProxy.h>
23 #include <ripple/beast/unit_test.h>
24 #include <ripple/rpc/impl/Tuning.h>
25 
26 #include <test/jtx.h>
27 #include <test/jtx/Env.h>
28 #include <test/jtx/envconfig.h>
29 #include <test/rpc/GRPCTestClientBase.h>
30 
31 namespace ripple {
32 namespace test {
33 
34 class ReportingETL_test : public beast::unit_test::suite
35 {
36  // gRPC stuff
38  {
39  public:
40  org::xrpl::rpc::v1::GetLedgerRequest request;
41  org::xrpl::rpc::v1::GetLedgerResponse reply;
42 
43  explicit GrpcLedgerClient(std::string const& port)
44  : GRPCTestClientBase(port)
45  {
46  }
47 
48  void
50  {
51  status = stub_->GetLedger(&context, request, &reply);
52  }
53  };
54  void
56  {
57  testcase("GetLedger");
58  using namespace test::jtx;
60  std::string grpcPort = *(*config)["port_grpc"].get<std::string>("port");
61  Env env(*this, std::move(config));
62 
63  env.close();
64 
65  auto ledger = env.app().getLedgerMaster().getLedgerBySeq(3);
66 
67  BEAST_EXPECT(env.current()->info().seq == 4);
68 
69  auto grpcLedger = [&grpcPort](
70  auto sequence,
71  bool transactions,
72  bool expand,
73  bool get_objects,
74  bool get_object_neighbors) {
75  GrpcLedgerClient grpcClient{grpcPort};
76 
77  grpcClient.request.mutable_ledger()->set_sequence(sequence);
78  grpcClient.request.set_transactions(transactions);
79  grpcClient.request.set_expand(expand);
80  grpcClient.request.set_get_objects(get_objects);
81  grpcClient.request.set_get_object_neighbors(get_object_neighbors);
82 
83  grpcClient.GetLedger();
84  return std::make_pair(grpcClient.status, grpcClient.reply);
85  };
86 
87  {
88  auto [status, reply] = grpcLedger(3, false, false, false, false);
89 
90  BEAST_EXPECT(status.ok());
91  BEAST_EXPECT(reply.validated());
92  BEAST_EXPECT(!reply.has_hashes_list());
93  BEAST_EXPECT(!reply.has_transactions_list());
94  BEAST_EXPECT(!reply.skiplist_included());
95  BEAST_EXPECT(reply.ledger_objects().objects_size() == 0);
96 
97  Serializer s;
98  addRaw(ledger->info(), s, true);
99  BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header()));
100  }
101 
102  Account const alice{"alice"};
103  Account const bob{"bob"};
104  env.fund(XRP(10000), alice);
105  env.fund(XRP(10000), bob);
106  env.close();
107 
108  ledger = env.app().getLedgerMaster().getLedgerBySeq(4);
109 
110  std::vector<uint256> hashes;
113  for (auto& [sttx, meta] : ledger->txs)
114  {
115  hashes.push_back(sttx->getTransactionID());
116  transactions.push_back(sttx);
117  metas.push_back(meta);
118  }
119 
120  Serializer s;
121  addRaw(ledger->info(), s, true);
122 
123  {
124  auto [status, reply] = grpcLedger(4, true, false, false, false);
125  BEAST_EXPECT(status.ok());
126  BEAST_EXPECT(reply.validated());
127  BEAST_EXPECT(reply.has_hashes_list());
128  BEAST_EXPECT(reply.hashes_list().hashes_size() == hashes.size());
129  BEAST_EXPECT(
130  uint256::fromVoid(reply.hashes_list().hashes(0).data()) ==
131  hashes[0]);
132  BEAST_EXPECT(
133  uint256::fromVoid(reply.hashes_list().hashes(1).data()) ==
134  hashes[1]);
135  BEAST_EXPECT(
136  uint256::fromVoid(reply.hashes_list().hashes(2).data()) ==
137  hashes[2]);
138  BEAST_EXPECT(
139  uint256::fromVoid(reply.hashes_list().hashes(3).data()) ==
140  hashes[3]);
141 
142  BEAST_EXPECT(!reply.has_transactions_list());
143  BEAST_EXPECT(!reply.skiplist_included());
144  BEAST_EXPECT(reply.ledger_objects().objects_size() == 0);
145 
146  BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header()));
147  }
148 
149  {
150  auto [status, reply] = grpcLedger(4, true, true, false, false);
151 
152  BEAST_EXPECT(status.ok());
153  BEAST_EXPECT(reply.validated());
154  BEAST_EXPECT(!reply.has_hashes_list());
155 
156  BEAST_EXPECT(reply.has_transactions_list());
157  BEAST_EXPECT(reply.transactions_list().transactions_size() == 4);
158 
159  BEAST_EXPECT(
160  makeSlice(reply.transactions_list()
161  .transactions(0)
162  .transaction_blob()) ==
163  transactions[0]->getSerializer().slice());
164 
165  BEAST_EXPECT(
166  makeSlice(reply.transactions_list()
167  .transactions(0)
168  .metadata_blob()) ==
169  metas[0]->getSerializer().slice());
170 
171  BEAST_EXPECT(
172  makeSlice(reply.transactions_list()
173  .transactions(1)
174  .transaction_blob()) ==
175  transactions[1]->getSerializer().slice());
176 
177  BEAST_EXPECT(
178  makeSlice(reply.transactions_list()
179  .transactions(1)
180  .metadata_blob()) ==
181  metas[1]->getSerializer().slice());
182 
183  BEAST_EXPECT(
184  makeSlice(reply.transactions_list()
185  .transactions(2)
186  .transaction_blob()) ==
187  transactions[2]->getSerializer().slice());
188 
189  BEAST_EXPECT(
190  makeSlice(reply.transactions_list()
191  .transactions(2)
192  .metadata_blob()) ==
193  metas[2]->getSerializer().slice());
194 
195  BEAST_EXPECT(
196  makeSlice(reply.transactions_list()
197  .transactions(3)
198  .transaction_blob()) ==
199  transactions[3]->getSerializer().slice());
200 
201  BEAST_EXPECT(
202  makeSlice(reply.transactions_list()
203  .transactions(3)
204  .metadata_blob()) ==
205  metas[3]->getSerializer().slice());
206 
207  BEAST_EXPECT(!reply.skiplist_included());
208  BEAST_EXPECT(reply.ledger_objects().objects_size() == 0);
209 
210  BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header()));
211  }
212 
213  {
214  auto [status, reply] = grpcLedger(4, true, true, true, false);
215 
216  BEAST_EXPECT(status.ok());
217  BEAST_EXPECT(reply.validated());
218  BEAST_EXPECT(!reply.has_hashes_list());
219 
220  BEAST_EXPECT(reply.has_transactions_list());
221  BEAST_EXPECT(reply.transactions_list().transactions_size() == 4);
222 
223  BEAST_EXPECT(
224  makeSlice(reply.transactions_list()
225  .transactions(0)
226  .transaction_blob()) ==
227  transactions[0]->getSerializer().slice());
228 
229  BEAST_EXPECT(
230  makeSlice(reply.transactions_list()
231  .transactions(0)
232  .metadata_blob()) ==
233  metas[0]->getSerializer().slice());
234 
235  BEAST_EXPECT(
236  makeSlice(reply.transactions_list()
237  .transactions(1)
238  .transaction_blob()) ==
239  transactions[1]->getSerializer().slice());
240 
241  BEAST_EXPECT(
242  makeSlice(reply.transactions_list()
243  .transactions(1)
244  .metadata_blob()) ==
245  metas[1]->getSerializer().slice());
246 
247  BEAST_EXPECT(
248  makeSlice(reply.transactions_list()
249  .transactions(2)
250  .transaction_blob()) ==
251  transactions[2]->getSerializer().slice());
252 
253  BEAST_EXPECT(
254  makeSlice(reply.transactions_list()
255  .transactions(2)
256  .metadata_blob()) ==
257  metas[2]->getSerializer().slice());
258 
259  BEAST_EXPECT(
260  makeSlice(reply.transactions_list()
261  .transactions(3)
262  .transaction_blob()) ==
263  transactions[3]->getSerializer().slice());
264 
265  BEAST_EXPECT(
266  makeSlice(reply.transactions_list()
267  .transactions(3)
268  .metadata_blob()) ==
269  metas[3]->getSerializer().slice());
270  BEAST_EXPECT(reply.skiplist_included());
271 
272  BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header()));
273 
274  auto parent = env.app().getLedgerMaster().getLedgerBySeq(3);
275 
276  SHAMap::Delta differences;
277 
278  int maxDifferences = std::numeric_limits<int>::max();
279 
280  bool res = parent->stateMap().compare(
281  ledger->stateMap(), differences, maxDifferences);
282  BEAST_EXPECT(res);
283 
284  size_t idx = 0;
285  for (auto& [k, v] : differences)
286  {
287  BEAST_EXPECT(
288  k ==
290  reply.ledger_objects().objects(idx).key().data()));
291  if (v.second)
292  {
293  BEAST_EXPECT(
294  v.second->slice() ==
295  makeSlice(reply.ledger_objects().objects(idx).data()));
296  }
297  ++idx;
298  }
299  }
300  {
301  auto [status, reply] = grpcLedger(4, true, true, true, true);
302 
303  BEAST_EXPECT(status.ok());
304  BEAST_EXPECT(reply.validated());
305  BEAST_EXPECT(!reply.has_hashes_list());
306  BEAST_EXPECT(reply.object_neighbors_included());
307 
308  BEAST_EXPECT(reply.has_transactions_list());
309  BEAST_EXPECT(reply.transactions_list().transactions_size() == 4);
310 
311  BEAST_EXPECT(
312  makeSlice(reply.transactions_list()
313  .transactions(0)
314  .transaction_blob()) ==
315  transactions[0]->getSerializer().slice());
316 
317  BEAST_EXPECT(
318  makeSlice(reply.transactions_list()
319  .transactions(0)
320  .metadata_blob()) ==
321  metas[0]->getSerializer().slice());
322 
323  BEAST_EXPECT(
324  makeSlice(reply.transactions_list()
325  .transactions(1)
326  .transaction_blob()) ==
327  transactions[1]->getSerializer().slice());
328 
329  BEAST_EXPECT(
330  makeSlice(reply.transactions_list()
331  .transactions(1)
332  .metadata_blob()) ==
333  metas[1]->getSerializer().slice());
334 
335  BEAST_EXPECT(
336  makeSlice(reply.transactions_list()
337  .transactions(2)
338  .transaction_blob()) ==
339  transactions[2]->getSerializer().slice());
340 
341  BEAST_EXPECT(
342  makeSlice(reply.transactions_list()
343  .transactions(2)
344  .metadata_blob()) ==
345  metas[2]->getSerializer().slice());
346 
347  BEAST_EXPECT(
348  makeSlice(reply.transactions_list()
349  .transactions(3)
350  .transaction_blob()) ==
351  transactions[3]->getSerializer().slice());
352 
353  BEAST_EXPECT(
354  makeSlice(reply.transactions_list()
355  .transactions(3)
356  .metadata_blob()) ==
357  metas[3]->getSerializer().slice());
358  BEAST_EXPECT(reply.skiplist_included());
359 
360  BEAST_EXPECT(s.slice() == makeSlice(reply.ledger_header()));
361 
362  auto parent = env.app().getLedgerMaster().getLedgerBySeq(3);
363 
364  SHAMap::Delta differences;
365 
366  int maxDifferences = std::numeric_limits<int>::max();
367 
368  bool res = parent->stateMap().compare(
369  ledger->stateMap(), differences, maxDifferences);
370  BEAST_EXPECT(res);
371 
372  size_t idx = 0;
373 
374  for (auto& [k, v] : differences)
375  {
376  auto obj = reply.ledger_objects().objects(idx);
377  BEAST_EXPECT(k == uint256::fromVoid(obj.key().data()));
378  if (v.second)
379  {
380  BEAST_EXPECT(v.second->slice() == makeSlice(obj.data()));
381  }
382  else
383  BEAST_EXPECT(obj.data().size() == 0);
384 
385  if (!(v.first && v.second))
386  {
387  auto succ = ledger->stateMap().upper_bound(k);
388  auto pred = ledger->stateMap().lower_bound(k);
389 
390  if (succ != ledger->stateMap().end())
391  BEAST_EXPECT(
392  succ->key() ==
393  uint256::fromVoid(obj.successor().data()));
394  else
395  BEAST_EXPECT(obj.successor().size() == 0);
396  if (pred != ledger->stateMap().end())
397  BEAST_EXPECT(
398  pred->key() ==
399  uint256::fromVoid(obj.predecessor().data()));
400  else
401  BEAST_EXPECT(obj.predecessor().size() == 0);
402  }
403  ++idx;
404  }
405  }
406 
407  // Delete an account
408 
409  env(noop(alice));
410 
411  std::uint32_t const ledgerCount{
412  env.current()->seq() + 257 - env.seq(alice)};
413 
414  for (std::uint32_t i = 0; i < ledgerCount; ++i)
415  env.close();
416 
417  auto const acctDelFee{drops(env.current()->fees().increment)};
418  env(acctdelete(alice, bob), fee(acctDelFee));
419  env.close();
420 
421  {
422  auto [status, reply] =
423  grpcLedger(env.closed()->seq(), true, true, true, true);
424 
425  BEAST_EXPECT(status.ok());
426  BEAST_EXPECT(reply.validated());
427  auto base =
428  env.app().getLedgerMaster().getLedgerBySeq(env.closed()->seq());
429 
430  auto parent = env.app().getLedgerMaster().getLedgerBySeq(
431  env.closed()->seq() - 1);
432 
433  SHAMap::Delta differences;
434 
435  int maxDifferences = std::numeric_limits<int>::max();
436 
437  bool res = parent->stateMap().compare(
438  base->stateMap(), differences, maxDifferences);
439  BEAST_EXPECT(res);
440 
441  size_t idx = 0;
442  for (auto& [k, v] : differences)
443  {
444  auto obj = reply.ledger_objects().objects(idx);
445  BEAST_EXPECT(k == uint256::fromVoid(obj.key().data()));
446  if (v.second)
447  {
448  BEAST_EXPECT(
449  v.second->slice() ==
450  makeSlice(reply.ledger_objects().objects(idx).data()));
451  }
452  else
453  BEAST_EXPECT(obj.data().size() == 0);
454  if (!(v.first && v.second))
455  {
456  auto succ = base->stateMap().upper_bound(k);
457  auto pred = base->stateMap().lower_bound(k);
458 
459  if (succ != base->stateMap().end())
460  BEAST_EXPECT(
461  succ->key() ==
462  uint256::fromVoid(obj.successor().data()));
463  else
464  BEAST_EXPECT(obj.successor().size() == 0);
465  if (pred != base->stateMap().end())
466  BEAST_EXPECT(
467  pred->key() ==
468  uint256::fromVoid(obj.predecessor().data()));
469  else
470  BEAST_EXPECT(obj.predecessor().size() == 0);
471  }
472 
473  ++idx;
474  }
475  }
476  }
477 
478  // gRPC stuff
480  {
481  public:
482  org::xrpl::rpc::v1::GetLedgerDataRequest request;
483  org::xrpl::rpc::v1::GetLedgerDataResponse reply;
484 
485  explicit GrpcLedgerDataClient(std::string const& port)
486  : GRPCTestClientBase(port)
487  {
488  }
489 
490  void
492  {
493  status = stub_->GetLedgerData(&context, request, &reply);
494  }
495  };
496  void
498  {
499  testcase("GetLedgerData");
500  using namespace test::jtx;
502  std::string grpcPort = *(*config)["port_grpc"].get<std::string>("port");
503  Env env(*this, std::move(config));
504  auto grpcLedgerData = [&grpcPort](
505  auto sequence, std::string marker = "") {
506  GrpcLedgerDataClient grpcClient{grpcPort};
507 
508  grpcClient.request.mutable_ledger()->set_sequence(sequence);
509  if (marker.size())
510  {
511  grpcClient.request.set_marker(marker);
512  }
513 
514  grpcClient.GetLedgerData();
515  return std::make_pair(grpcClient.status, grpcClient.reply);
516  };
517 
518  Account const alice{"alice"};
519  env.fund(XRP(100000), alice);
520 
521  int num_accounts = 10;
522 
523  for (auto i = 0; i < num_accounts; i++)
524  {
525  Account const bob{std::string("bob") + std::to_string(i)};
526  env.fund(XRP(1000), bob);
527  }
528  env.close();
529 
530  {
531  auto [status, reply] = grpcLedgerData(env.closed()->seq());
532  BEAST_EXPECT(status.ok());
533 
534  BEAST_EXPECT(
535  reply.ledger_objects().objects_size() == num_accounts + 4);
536  BEAST_EXPECT(reply.marker().size() == 0);
537  auto ledger = env.closed();
538  size_t idx = 0;
539  for (auto& sle : ledger->sles)
540  {
541  BEAST_EXPECT(
542  sle->getSerializer().slice() ==
543  makeSlice(reply.ledger_objects().objects(idx).data()));
544  ++idx;
545  }
546  }
547 
548  {
549  auto [status, reply] =
550  grpcLedgerData(env.closed()->seq(), "bad marker");
551  BEAST_EXPECT(!status.ok());
552  BEAST_EXPECT(
553  status.error_code() == grpc::StatusCode::INVALID_ARGUMENT);
554  }
555 
556  num_accounts = 3000;
557 
558  for (auto i = 0; i < num_accounts; i++)
559  {
560  Account const cat{std::string("cat") + std::to_string(i)};
561  env.fund(XRP(1000), cat);
562  if (i % 100 == 0)
563  env.close();
564  }
565  env.close();
566 
567  {
568  auto [status, reply] = grpcLedgerData(env.closed()->seq());
569  BEAST_EXPECT(status.ok());
570 
571  int maxLimit = RPC::Tuning::pageLength(true);
572  BEAST_EXPECT(reply.ledger_objects().objects_size() == maxLimit);
573  BEAST_EXPECT(reply.marker().size() != 0);
574 
575  auto [status2, reply2] =
576  grpcLedgerData(env.closed()->seq(), reply.marker());
577  BEAST_EXPECT(status2.ok());
578  BEAST_EXPECT(reply2.marker().size() == 0);
579 
580  auto ledger = env.closed();
581  size_t idx = 0;
582  for (auto& sle : ledger->sles)
583  {
584  auto& obj = idx < maxLimit
585  ? reply.ledger_objects().objects(idx)
586  : reply2.ledger_objects().objects(idx - maxLimit);
587 
588  BEAST_EXPECT(
589  sle->getSerializer().slice() == makeSlice(obj.data()));
590  ++idx;
591  }
592  BEAST_EXPECT(
593  idx ==
594  reply.ledger_objects().objects_size() +
595  reply2.ledger_objects().objects_size());
596  }
597  }
598 
599  // gRPC stuff
601  {
602  public:
603  org::xrpl::rpc::v1::GetLedgerDiffRequest request;
604  org::xrpl::rpc::v1::GetLedgerDiffResponse reply;
605 
606  explicit GrpcLedgerDiffClient(std::string const& port)
607  : GRPCTestClientBase(port)
608  {
609  }
610 
611  void
613  {
614  status = stub_->GetLedgerDiff(&context, request, &reply);
615  }
616  };
617 
618  void
620  {
621  testcase("GetLedgerDiff");
622  using namespace test::jtx;
624  std::string grpcPort = *(*config)["port_grpc"].get<std::string>("port");
625  Env env(*this, std::move(config));
626 
627  auto grpcLedgerDiff = [&grpcPort](
628  auto baseSequence, auto desiredSequence) {
629  GrpcLedgerDiffClient grpcClient{grpcPort};
630 
631  grpcClient.request.mutable_base_ledger()->set_sequence(
632  baseSequence);
633  grpcClient.request.mutable_desired_ledger()->set_sequence(
634  desiredSequence);
635  grpcClient.request.set_include_blobs(true);
636 
637  grpcClient.GetLedgerDiff();
638  return std::make_pair(grpcClient.status, grpcClient.reply);
639  };
640 
641  int num_accounts = 20;
642  for (auto i = 0; i < num_accounts; i++)
643  {
644  Account const cat{std::string("cat") + std::to_string(i)};
645  env.fund(XRP(1000), cat);
646  if (i % 2 == 0)
647  env.close();
648  }
649  env.close();
650 
651  auto compareDiffs = [&](auto baseSequence, auto desiredSequence) {
652  auto [status, reply] =
653  grpcLedgerDiff(baseSequence, desiredSequence);
654 
655  BEAST_EXPECT(status.ok());
656  auto desired =
657  env.app().getLedgerMaster().getLedgerBySeq(desiredSequence);
658 
659  auto base =
660  env.app().getLedgerMaster().getLedgerBySeq(baseSequence);
661 
662  SHAMap::Delta differences;
663 
664  int maxDifferences = std::numeric_limits<int>::max();
665 
666  bool res = base->stateMap().compare(
667  desired->stateMap(), differences, maxDifferences);
668  if (!BEAST_EXPECT(res))
669  return false;
670 
671  size_t idx = 0;
672  for (auto& [k, v] : differences)
673  {
674  if (!BEAST_EXPECT(
675  k ==
677  reply.ledger_objects().objects(idx).key().data())))
678  return false;
679  if (v.second)
680  {
681  if (!BEAST_EXPECT(
682  v.second->slice() ==
683  makeSlice(
684  reply.ledger_objects().objects(idx).data())))
685  return false;
686  }
687 
688  ++idx;
689  }
690  return true;
691  };
692 
693  // Adjacent ledgers
694  BEAST_EXPECT(
695  compareDiffs(env.closed()->seq() - 1, env.closed()->seq()));
696 
697  // Adjacent ledgers further in the past
698  BEAST_EXPECT(
699  compareDiffs(env.closed()->seq() - 3, env.closed()->seq() - 2));
700 
701  // Non-adjacent ledgers
702  BEAST_EXPECT(
703  compareDiffs(env.closed()->seq() - 5, env.closed()->seq() - 1));
704 
705  // Adjacent ledgers but in reverse order
706  BEAST_EXPECT(
707  compareDiffs(env.closed()->seq(), env.closed()->seq() - 1));
708 
709  // Non-adjacent ledgers in reverse order
710  BEAST_EXPECT(
711  compareDiffs(env.closed()->seq() - 1, env.closed()->seq() - 5));
712  }
713 
714  // gRPC stuff
716  {
717  public:
718  org::xrpl::rpc::v1::GetLedgerEntryRequest request;
719  org::xrpl::rpc::v1::GetLedgerEntryResponse reply;
720 
721  explicit GrpcLedgerEntryClient(std::string const& port)
722  : GRPCTestClientBase(port)
723  {
724  }
725 
726  void
728  {
729  status = stub_->GetLedgerEntry(&context, request, &reply);
730  }
731  };
732 
733  void
735  {
736  testcase("GetLedgerDiff");
737  using namespace test::jtx;
739  std::string grpcPort = *(*config)["port_grpc"].get<std::string>("port");
740  Env env(*this, std::move(config));
741 
742  auto grpcLedgerEntry = [&grpcPort](auto sequence, auto key) {
743  GrpcLedgerEntryClient grpcClient{grpcPort};
744 
745  grpcClient.request.mutable_ledger()->set_sequence(sequence);
746  grpcClient.request.set_key(key.data(), key.size());
747 
748  grpcClient.GetLedgerEntry();
749  return std::make_pair(grpcClient.status, grpcClient.reply);
750  };
751 
752  Account const alice{"alice"};
753  env.fund(XRP(1000), alice);
754  env.close();
755 
756  for (auto& sle : env.closed()->sles)
757  {
758  auto [status, reply] =
759  grpcLedgerEntry(env.closed()->seq(), sle->key());
760 
761  BEAST_EXPECT(status.ok());
762 
763  BEAST_EXPECT(
764  uint256::fromVoid(reply.ledger_object().key().data()) ==
765  sle->key());
766  BEAST_EXPECT(
767  makeSlice(reply.ledger_object().data()) ==
768  sle->getSerializer().slice());
769  }
770  }
771 
772  void
774  {
775  testcase("NeedCurrentOrClosed");
776 
777  {
778  org::xrpl::rpc::v1::GetLedgerRequest request;
779  request.mutable_ledger()->set_sequence(1);
780  BEAST_EXPECT(!needCurrentOrClosed(request));
781  request.mutable_ledger()->set_hash("");
782  BEAST_EXPECT(!needCurrentOrClosed(request));
783  request.mutable_ledger()->set_shortcut(
784  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
785  BEAST_EXPECT(!needCurrentOrClosed(request));
786  request.mutable_ledger()->set_shortcut(
787  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_UNSPECIFIED);
788  BEAST_EXPECT(!needCurrentOrClosed(request));
789  request.mutable_ledger()->set_shortcut(
790  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT);
791  BEAST_EXPECT(needCurrentOrClosed(request));
792  request.mutable_ledger()->set_shortcut(
793  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CLOSED);
794  BEAST_EXPECT(needCurrentOrClosed(request));
795  }
796 
797  {
798  org::xrpl::rpc::v1::GetLedgerDataRequest request;
799  request.mutable_ledger()->set_sequence(1);
800  BEAST_EXPECT(!needCurrentOrClosed(request));
801  request.mutable_ledger()->set_hash("");
802  BEAST_EXPECT(!needCurrentOrClosed(request));
803  request.mutable_ledger()->set_shortcut(
804  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
805  BEAST_EXPECT(!needCurrentOrClosed(request));
806  request.mutable_ledger()->set_shortcut(
807  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_UNSPECIFIED);
808  BEAST_EXPECT(!needCurrentOrClosed(request));
809  request.mutable_ledger()->set_shortcut(
810  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT);
811  BEAST_EXPECT(needCurrentOrClosed(request));
812  request.mutable_ledger()->set_shortcut(
813  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CLOSED);
814  BEAST_EXPECT(needCurrentOrClosed(request));
815  }
816 
817  {
818  org::xrpl::rpc::v1::GetLedgerEntryRequest request;
819  request.mutable_ledger()->set_sequence(1);
820  BEAST_EXPECT(!needCurrentOrClosed(request));
821  request.mutable_ledger()->set_hash("");
822  BEAST_EXPECT(!needCurrentOrClosed(request));
823  request.mutable_ledger()->set_shortcut(
824  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
825  BEAST_EXPECT(!needCurrentOrClosed(request));
826  request.mutable_ledger()->set_shortcut(
827  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_UNSPECIFIED);
828  BEAST_EXPECT(!needCurrentOrClosed(request));
829  request.mutable_ledger()->set_shortcut(
830  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT);
831  BEAST_EXPECT(needCurrentOrClosed(request));
832  request.mutable_ledger()->set_shortcut(
833  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CLOSED);
834  BEAST_EXPECT(needCurrentOrClosed(request));
835  }
836 
837  {
838  org::xrpl::rpc::v1::GetLedgerDiffRequest request;
839 
840  // set desired ledger, so desired ledger does not need current or
841  // closed
842  request.mutable_base_ledger()->set_shortcut(
843  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
844 
845  request.mutable_base_ledger()->set_sequence(1);
846  BEAST_EXPECT(!needCurrentOrClosed(request));
847  request.mutable_base_ledger()->set_hash("");
848  BEAST_EXPECT(!needCurrentOrClosed(request));
849  request.mutable_base_ledger()->set_shortcut(
850  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
851  BEAST_EXPECT(!needCurrentOrClosed(request));
852  request.mutable_base_ledger()->set_shortcut(
853  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_UNSPECIFIED);
854  BEAST_EXPECT(!needCurrentOrClosed(request));
855  request.mutable_base_ledger()->set_shortcut(
856  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT);
857  BEAST_EXPECT(needCurrentOrClosed(request));
858  request.mutable_base_ledger()->set_shortcut(
859  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CLOSED);
860  BEAST_EXPECT(needCurrentOrClosed(request));
861 
862  // reset base ledger, so base ledger doesn't need current or closed
863  request.mutable_base_ledger()->set_shortcut(
864  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
865 
866  request.mutable_desired_ledger()->set_sequence(1);
867  BEAST_EXPECT(!needCurrentOrClosed(request));
868  request.mutable_desired_ledger()->set_hash("");
869  BEAST_EXPECT(!needCurrentOrClosed(request));
870  request.mutable_desired_ledger()->set_shortcut(
871  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED);
872  BEAST_EXPECT(!needCurrentOrClosed(request));
873  request.mutable_desired_ledger()->set_shortcut(
874  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_UNSPECIFIED);
875  BEAST_EXPECT(!needCurrentOrClosed(request));
876  request.mutable_desired_ledger()->set_shortcut(
877  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT);
878  BEAST_EXPECT(needCurrentOrClosed(request));
879  request.mutable_desired_ledger()->set_shortcut(
880  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CLOSED);
881  BEAST_EXPECT(needCurrentOrClosed(request));
882 
883  // both base and desired need current or closed
884  request.mutable_base_ledger()->set_shortcut(
885  org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT);
886  BEAST_EXPECT(needCurrentOrClosed(request));
887  }
888  }
889 
890  void
892  {
893  testcase("SecureGateway");
894  using namespace test::jtx;
895  {
898  std::string grpcPort =
899  *(*config)["port_grpc"].get<std::string>("port");
900  Env env(*this, std::move(config));
901 
902  env.close();
903 
904  auto ledger = env.app().getLedgerMaster().getLedgerBySeq(3);
905 
906  BEAST_EXPECT(env.current()->info().seq == 4);
907 
908  auto grpcLedger = [&grpcPort](
909  auto sequence,
910  std::string const& clientIp,
911  std::string const& user) {
912  GrpcLedgerClient grpcClient{grpcPort};
913 
914  grpcClient.request.mutable_ledger()->set_sequence(sequence);
915  grpcClient.request.set_client_ip(clientIp);
916  grpcClient.request.set_user(user);
917 
918  grpcClient.GetLedger();
919  return std::make_pair(grpcClient.status, grpcClient.reply);
920  };
921 
922  {
923  auto [status, reply] =
924  grpcLedger(env.current()->info().seq, "", "");
925  BEAST_EXPECT(!reply.is_unlimited());
926  BEAST_EXPECT(status.ok());
927  }
928  {
929  auto [status, reply] =
930  grpcLedger(env.current()->info().seq, "", "ETL");
931  BEAST_EXPECT(reply.is_unlimited());
932  BEAST_EXPECT(status.ok());
933  }
934  {
935  auto [status, reply] =
936  grpcLedger(env.current()->info().seq, "", "Reporting");
937  BEAST_EXPECT(reply.is_unlimited());
938  BEAST_EXPECT(status.ok());
939  }
940  {
941  auto [status, reply] =
942  grpcLedger(env.current()->info().seq, "127.0.0.1", "ETL");
943  BEAST_EXPECT(!reply.is_unlimited());
944  BEAST_EXPECT(status.ok());
945  }
946  {
947  auto [status, reply] =
948  grpcLedger(env.current()->info().seq, "127.0.0.1", "");
949  BEAST_EXPECT(!reply.is_unlimited());
950  BEAST_EXPECT(status.ok());
951  }
952  }
953 
954  {
955  std::string secureGatewayIp = "44.124.234.79";
956  std::unique_ptr<Config> config =
957  envconfig(addGrpcConfigWithSecureGateway, secureGatewayIp);
958  std::string grpcPort =
959  *(*config)["port_grpc"].get<std::string>("port");
960  Env env(*this, std::move(config));
961 
962  env.close();
963 
964  auto ledger = env.app().getLedgerMaster().getLedgerBySeq(3);
965 
966  BEAST_EXPECT(env.current()->info().seq == 4);
967 
968  auto grpcLedger = [&grpcPort](
969  auto sequence,
970  std::string const& clientIp,
971  std::string const& user) {
972  GrpcLedgerClient grpcClient{grpcPort};
973 
974  grpcClient.request.mutable_ledger()->set_sequence(sequence);
975  grpcClient.request.set_client_ip(clientIp);
976  grpcClient.request.set_user(user);
977 
978  grpcClient.GetLedger();
979  return std::make_pair(grpcClient.status, grpcClient.reply);
980  };
981 
982  {
983  auto [status, reply] =
984  grpcLedger(env.current()->info().seq, "", "");
985  BEAST_EXPECT(!reply.is_unlimited());
986  BEAST_EXPECT(status.ok());
987  }
988  {
989  auto [status, reply] =
990  grpcLedger(env.current()->info().seq, "", "ETL");
991  BEAST_EXPECT(!reply.is_unlimited());
992  BEAST_EXPECT(status.ok());
993  }
994  {
995  auto [status, reply] = grpcLedger(
996  env.current()->info().seq, secureGatewayIp, "ETL");
997  BEAST_EXPECT(!reply.is_unlimited());
998  BEAST_EXPECT(status.ok());
999  }
1000  {
1001  auto [status, reply] =
1002  grpcLedger(env.current()->info().seq, secureGatewayIp, "");
1003  BEAST_EXPECT(!reply.is_unlimited());
1004  BEAST_EXPECT(status.ok());
1005  }
1006  }
1007 
1008  {
1011  std::string grpcPort =
1012  *(*config)["port_grpc"].get<std::string>("port");
1013  Env env(*this, std::move(config));
1014 
1015  env.close();
1016 
1017  auto ledger = env.app().getLedgerMaster().getLedgerBySeq(3);
1018 
1019  BEAST_EXPECT(env.current()->info().seq == 4);
1020  auto grpcLedgerData = [&grpcPort](
1021  auto sequence,
1022  std::string const& clientIp,
1023  std::string const& user) {
1024  GrpcLedgerDataClient grpcClient{grpcPort};
1025 
1026  grpcClient.request.mutable_ledger()->set_sequence(sequence);
1027  grpcClient.request.set_client_ip(clientIp);
1028  grpcClient.request.set_user(user);
1029 
1030  grpcClient.GetLedgerData();
1031  return std::make_pair(grpcClient.status, grpcClient.reply);
1032  };
1033  {
1034  auto [status, reply] =
1035  grpcLedgerData(env.current()->info().seq, "", "");
1036  BEAST_EXPECT(!reply.is_unlimited());
1037  BEAST_EXPECT(status.ok());
1038  }
1039  {
1040  auto [status, reply] =
1041  grpcLedgerData(env.current()->info().seq, "", "ETL");
1042  BEAST_EXPECT(reply.is_unlimited());
1043  BEAST_EXPECT(status.ok());
1044  }
1045  {
1046  auto [status, reply] =
1047  grpcLedgerData(env.current()->info().seq, "", "Reporting");
1048  BEAST_EXPECT(reply.is_unlimited());
1049  BEAST_EXPECT(status.ok());
1050  }
1051  {
1052  auto [status, reply] = grpcLedgerData(
1053  env.current()->info().seq, "127.0.0.1", "ETL");
1054  BEAST_EXPECT(!reply.is_unlimited());
1055  BEAST_EXPECT(status.ok());
1056  }
1057  {
1058  auto [status, reply] =
1059  grpcLedgerData(env.current()->info().seq, "127.0.0.1", "");
1060  BEAST_EXPECT(!reply.is_unlimited());
1061  BEAST_EXPECT(status.ok());
1062  }
1063  }
1064  {
1065  std::string secureGatewayIp = "44.124.234.79";
1066  std::unique_ptr<Config> config =
1067  envconfig(addGrpcConfigWithSecureGateway, secureGatewayIp);
1068  std::string grpcPort =
1069  *(*config)["port_grpc"].get<std::string>("port");
1070  Env env(*this, std::move(config));
1071 
1072  env.close();
1073 
1074  auto ledger = env.app().getLedgerMaster().getLedgerBySeq(3);
1075 
1076  BEAST_EXPECT(env.current()->info().seq == 4);
1077 
1078  auto grpcLedgerData = [&grpcPort](
1079  auto sequence,
1080  std::string const& clientIp,
1081  std::string const& user) {
1082  GrpcLedgerDataClient grpcClient{grpcPort};
1083 
1084  grpcClient.request.mutable_ledger()->set_sequence(sequence);
1085  grpcClient.request.set_client_ip(clientIp);
1086  grpcClient.request.set_user(user);
1087 
1088  grpcClient.GetLedgerData();
1089  return std::make_pair(grpcClient.status, grpcClient.reply);
1090  };
1091 
1092  {
1093  auto [status, reply] =
1094  grpcLedgerData(env.current()->info().seq, "", "");
1095  BEAST_EXPECT(!reply.is_unlimited());
1096  BEAST_EXPECT(status.ok());
1097  }
1098  {
1099  auto [status, reply] =
1100  grpcLedgerData(env.current()->info().seq, "", "ETL");
1101  BEAST_EXPECT(!reply.is_unlimited());
1102  BEAST_EXPECT(status.ok());
1103  }
1104  {
1105  auto [status, reply] = grpcLedgerData(
1106  env.current()->info().seq, secureGatewayIp, "ETL");
1107  BEAST_EXPECT(!reply.is_unlimited());
1108  BEAST_EXPECT(status.ok());
1109  }
1110  {
1111  auto [status, reply] = grpcLedgerData(
1112  env.current()->info().seq, secureGatewayIp, "");
1113  BEAST_EXPECT(!reply.is_unlimited());
1114  BEAST_EXPECT(status.ok());
1115  }
1116  }
1117  }
1118 
1119 public:
1120  void
1121  run() override
1122  {
1123  testGetLedger();
1124 
1126 
1128 
1130 
1132 
1134  }
1135 };
1136 
1138 
1139 } // namespace test
1140 } // namespace ripple
ripple::test::ReportingETL_test::GrpcLedgerDiffClient::request
org::xrpl::rpc::v1::GetLedgerDiffRequest request
Definition: ReportingETL_test.cpp:603
ripple::test::jtx::noop
Json::Value noop(Account const &account)
The null transaction.
Definition: noop.h:31
ripple::test::ReportingETL_test::run
void run() override
Definition: ReportingETL_test.cpp:1121
ripple::test::jtx::XRP
const XRP_t XRP
Converts to XRP Issue or STAmount.
Definition: amount.cpp:105
ripple::makeSlice
std::enable_if_t< std::is_same< T, char >::value||std::is_same< T, unsigned char >::value, Slice > makeSlice(std::array< T, N > const &a)
Definition: Slice.h:241
std::string
STL class.
ripple::test::jtx::drops
PrettyAmount drops(Integer i)
Returns an XRP PrettyAmount, which is trivially convertible to STAmount.
Definition: amount.h:241
ripple::test::ReportingETL_test::testGetLedgerDiff
void testGetLedgerDiff()
Definition: ReportingETL_test.cpp:619
ripple::test::GRPCTestClientBase::stub_
std::unique_ptr< org::xrpl::rpc::v1::XRPLedgerAPIService::Stub > stub_
Definition: GRPCTestClientBase.h:44
ripple::addRaw
void addRaw(LedgerInfo const &info, Serializer &s, bool includeHash)
Definition: View.cpp:162
std::vector
STL class.
ripple::test::ReportingETL_test::GrpcLedgerClient
Definition: ReportingETL_test.cpp:37
std::vector::size
T size(T... args)
ripple::test::jtx::addGrpcConfig
std::unique_ptr< Config > addGrpcConfig(std::unique_ptr< Config >)
add a grpc address and port to config
Definition: envconfig.cpp:140
ripple::test::ReportingETL_test::GrpcLedgerDataClient
Definition: ReportingETL_test.cpp:479
ripple::test::ReportingETL_test
Definition: ReportingETL_test.cpp:34
ripple::test::jtx::addGrpcConfigWithSecureGateway
std::unique_ptr< Config > addGrpcConfigWithSecureGateway(std::unique_ptr< Config >, std::string const &secureGateway)
add a grpc address, port and secure_gateway to config
Definition: envconfig.cpp:149
ripple::test::ReportingETL_test::testNeedCurrentOrClosed
void testNeedCurrentOrClosed()
Definition: ReportingETL_test.cpp:773
ripple::test::jtx::envconfig
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition: envconfig.h:49
ripple::needCurrentOrClosed
bool needCurrentOrClosed(Request &request)
Definition: P2pProxy.h:47
ripple::test::ReportingETL_test::GrpcLedgerDiffClient::reply
org::xrpl::rpc::v1::GetLedgerDiffResponse reply
Definition: ReportingETL_test.cpp:604
ripple::test::ReportingETL_test::testGetLedger
void testGetLedger()
Definition: ReportingETL_test.cpp:55
ripple::test::GRPCTestClientBase::status
grpc::Status status
Definition: GRPCTestClientBase.h:42
ripple::test::BEAST_DEFINE_TESTSUITE_PRIO
BEAST_DEFINE_TESTSUITE_PRIO(AccountDelete, app, ripple, 2)
ripple::test::ReportingETL_test::GrpcLedgerDiffClient::GetLedgerDiff
void GetLedgerDiff()
Definition: ReportingETL_test.cpp:612
ripple::test::ReportingETL_test::GrpcLedgerClient::reply
org::xrpl::rpc::v1::GetLedgerResponse reply
Definition: ReportingETL_test.cpp:41
ripple::test::GRPCTestClientBase
Definition: GRPCTestClientBase.h:29
std::vector::push_back
T push_back(T... args)
ripple::test::getEnvLocalhostAddr
const char * getEnvLocalhostAddr()
Definition: envconfig.h:31
ripple::test::ReportingETL_test::GrpcLedgerEntryClient::GrpcLedgerEntryClient
GrpcLedgerEntryClient(std::string const &port)
Definition: ReportingETL_test.cpp:721
ripple::test::ReportingETL_test::testGetLedgerData
void testGetLedgerData()
Definition: ReportingETL_test.cpp:497
ripple::test::ReportingETL_test::GrpcLedgerClient::GrpcLedgerClient
GrpcLedgerClient(std::string const &port)
Definition: ReportingETL_test.cpp:43
std::to_string
T to_string(T... args)
ripple::Serializer::slice
Slice slice() const noexcept
Definition: Serializer.h:63
ripple::ReportingETL
This class is responsible for continuously extracting data from a p2p node, and writing that data to ...
Definition: ReportingETL.h:70
ripple::RPC::Tuning::pageLength
constexpr int pageLength(bool isBinary)
Maximum number of pages in a LedgerData response.
Definition: rpc/impl/Tuning.h:75
std::uint32_t
ripple::test::ReportingETL_test::GrpcLedgerEntryClient
Definition: ReportingETL_test.cpp:715
std::map
STL class.
ripple::test::GRPCTestClientBase::context
grpc::ClientContext context
Definition: GRPCTestClientBase.h:43
ripple::test::ReportingETL_test::GrpcLedgerDataClient::request
org::xrpl::rpc::v1::GetLedgerDataRequest request
Definition: ReportingETL_test.cpp:482
ripple::test::jtx::fee
Set the fee on a JTx.
Definition: fee.h:35
ripple::Serializer
Definition: Serializer.h:39
ripple
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition: RCLCensorshipDetector.h:29
ripple::test::ReportingETL_test::GrpcLedgerDataClient::GetLedgerData
void GetLedgerData()
Definition: ReportingETL_test.cpp:491
ripple::base_uint< 256 >::fromVoid
static base_uint fromVoid(void const *data)
Definition: base_uint.h:312
ripple::test::ReportingETL_test::GrpcLedgerClient::GetLedger
void GetLedger()
Definition: ReportingETL_test.cpp:49
ripple::test::ReportingETL_test::GrpcLedgerDiffClient::GrpcLedgerDiffClient
GrpcLedgerDiffClient(std::string const &port)
Definition: ReportingETL_test.cpp:606
ripple::test::ReportingETL_test::testSecureGateway
void testSecureGateway()
Definition: ReportingETL_test.cpp:891
ripple::test::ReportingETL_test::GrpcLedgerDataClient::GrpcLedgerDataClient
GrpcLedgerDataClient(std::string const &port)
Definition: ReportingETL_test.cpp:485
ripple::test::jtx::acctdelete
Json::Value acctdelete(Account const &account, Account const &dest)
Delete account.
Definition: acctdelete.cpp:29
ripple::test::ReportingETL_test::testGetLedgerEntry
void testGetLedgerEntry()
Definition: ReportingETL_test.cpp:734
ripple::test::ReportingETL_test::GrpcLedgerDataClient::reply
org::xrpl::rpc::v1::GetLedgerDataResponse reply
Definition: ReportingETL_test.cpp:483
ripple::test::jtx::Account
Immutable cryptographic account descriptor.
Definition: Account.h:37
ripple::test::ReportingETL_test::GrpcLedgerClient::request
org::xrpl::rpc::v1::GetLedgerRequest request
Definition: ReportingETL_test.cpp:40
std::make_pair
T make_pair(T... args)
ripple::test::ReportingETL_test::GrpcLedgerEntryClient::reply
org::xrpl::rpc::v1::GetLedgerEntryResponse reply
Definition: ReportingETL_test.cpp:719
ripple::test::ReportingETL_test::GrpcLedgerEntryClient::request
org::xrpl::rpc::v1::GetLedgerEntryRequest request
Definition: ReportingETL_test.cpp:718
std::numeric_limits::max
T max(T... args)
std::unique_ptr
STL class.
ripple::test::ReportingETL_test::GrpcLedgerDiffClient
Definition: ReportingETL_test.cpp:600
ripple::test::jtx::Env
A transaction testing environment.
Definition: Env.h:116
ripple::test::ReportingETL_test::GrpcLedgerEntryClient::GetLedgerEntry
void GetLedgerEntry()
Definition: ReportingETL_test.cpp:727