[bisq-network/bisq] Add transient tx map to DaoState to speed up getTx queries (#3773)
chimp1984
notifications at github.com
Wed Dec 11 20:36:50 UTC 2019
I created a patch with my recommended changes. Feel free to ignore the patch ;-), most of my points have been in the comments above as well, but might be easier to have all in code....
```
Index: core/src/main/java/bisq/core/dao/state/model/DaoState.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/src/main/java/bisq/core/dao/state/model/DaoState.java (date 1576079165000)
+++ core/src/main/java/bisq/core/dao/state/model/DaoState.java (date 1576096243000)
@@ -36,6 +36,8 @@
import javax.inject.Inject;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -47,6 +49,8 @@
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
+import javax.annotation.Nullable;
+
/**
* Root class for mutable state of the DAO.
@@ -103,9 +107,8 @@
private final List<DecryptedBallotsWithMerits> decryptedBallotsWithMeritsList;
// Transient data used only as an index - must be kept in sync with the block list
- @Getter
@JsonExclude
- private transient final Map<String, Tx> txMap; // key is txId
+ private transient final Map<String, Tx> txCache; // key is txId
///////////////////////////////////////////////////////////////////////////////////////////
@@ -155,7 +158,7 @@
this.evaluatedProposalList = evaluatedProposalList;
this.decryptedBallotsWithMeritsList = decryptedBallotsWithMeritsList;
- txMap = blocks.stream()
+ txCache = blocks.stream()
.flatMap(block -> block.getTxs().stream())
.collect(Collectors.toMap(Tx::getId, Function.identity(), (x, y) -> y, HashMap::new));
}
@@ -237,6 +240,28 @@
return getBsqStateBuilderExcludingBlocks().addBlocks(getBlocks().getLast().toProtoMessage()).build().toByteArray();
}
+ public void addToTxCache(Tx tx) {
+ txCache.put(tx.getId(), tx);
+ }
+
+ public void setTxCache(Map<String, Tx> txCache) {
+ this.txCache.clear();
+ this.txCache.putAll(txCache);
+ }
+
+ public Map<String, Tx> getTxCache() {
+ return Collections.unmodifiableMap(txCache);
+ }
+
+ public Collection<Tx> getTxsFromTxCache() {
+ return Collections.unmodifiableCollection(txCache.values());
+ }
+
+ @Nullable
+ public Tx getTxFromTxCache(String txId) {
+ return txCache.get(txId);
+ }
+
@Override
public String toString() {
return "DaoState{" +
@@ -250,7 +275,7 @@
",\n paramChangeList=" + paramChangeList +
",\n evaluatedProposalList=" + evaluatedProposalList +
",\n decryptedBallotsWithMeritsList=" + decryptedBallotsWithMeritsList +
- ",\n txMap=" + txMap +
+ ",\n txMap=" + txCache +
"\n}";
}
}
Index: desktop/src/main/java/bisq/desktop/main/dao/economy/transactions/BSQTransactionsView.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- desktop/src/main/java/bisq/desktop/main/dao/economy/transactions/BSQTransactionsView.java (date 1576079165000)
+++ desktop/src/main/java/bisq/desktop/main/dao/economy/transactions/BSQTransactionsView.java (date 1576090509000)
@@ -143,7 +143,7 @@
///////////////////////////////////////////////////////////////////////////////////////////
private void updateWithBsqBlockChainData() {
- allTxTextField.setText(String.valueOf(daoFacade.getTxs().size()));
+ allTxTextField.setText(String.valueOf(daoFacade.getNumTxs()));
utxoTextField.setText(String.valueOf(daoFacade.getUnspentTxOutputs().size()));
compensationIssuanceTxTextField.setText(String.valueOf(daoFacade.getNumIssuanceTransactions(IssuanceType.COMPENSATION)));
reimbursementIssuanceTxTextField.setText(String.valueOf(daoFacade.getNumIssuanceTransactions(IssuanceType.REIMBURSEMENT)));
Index: core/src/main/java/bisq/core/dao/node/parser/BlockParser.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/src/main/java/bisq/core/dao/node/parser/BlockParser.java (date 1576079165000)
+++ core/src/main/java/bisq/core/dao/node/parser/BlockParser.java (date 1576094143000)
@@ -53,7 +53,6 @@
// Constructor
///////////////////////////////////////////////////////////////////////////////////////////
- @SuppressWarnings("WeakerAccess")
@Inject
public BlockParser(TxParser txParser,
DaoStateService daoStateService) {
Index: core/src/main/java/bisq/core/dao/state/model/blockchain/Block.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/src/main/java/bisq/core/dao/state/model/blockchain/Block.java (date 1576079165000)
+++ core/src/main/java/bisq/core/dao/state/model/blockchain/Block.java (date 1576096243000)
@@ -24,11 +24,11 @@
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import lombok.EqualsAndHashCode;
-import lombok.Value;
/**
* The Block which gets persisted in the DaoState. During parsing transactions can be
@@ -44,8 +44,8 @@
*
*/
@EqualsAndHashCode(callSuper = true)
- at Value
public final class Block extends BaseBlock implements PersistablePayload, ImmutableDaoStateModel {
+ // We do not expose txs with a Lombok getter. We cannot make it immutable as we add transactions during parsing.
private final List<Tx> txs;
public Block(int height, long time, String hash, String previousBlockHash) {
@@ -93,6 +93,21 @@
txs);
}
+ public void addTx(Tx tx) {
+ txs.add(tx);
+ }
+
+ // We want to guarantee that no client can modify the list. We use unmodifiableList and not ImmutableList as
+ // we want that clients reflect any change to the source list. Also ImmutableList is more expensive as it
+ // creates a copy.
+ public List<Tx> getTxs() {
+ return Collections.unmodifiableList(txs);
+ }
+
+ public int getNumTxs() {
+ return txs.size();
+ }
+
@Override
public String toString() {
return "Block{" +
Index: core/src/main/java/bisq/core/dao/state/DaoStateService.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/src/main/java/bisq/core/dao/state/DaoStateService.java (date 1576079165000)
+++ core/src/main/java/bisq/core/dao/state/DaoStateService.java (date 1576096243000)
@@ -42,11 +42,7 @@
import javax.inject.Inject;
-import com.google.common.base.Preconditions;
-
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
@@ -119,8 +115,7 @@
daoState.setChainHeight(snapshot.getChainHeight());
- daoState.getTxMap().clear();
- daoState.getTxMap().putAll(snapshot.getTxMap());
+ daoState.setTxCache(snapshot.getTxCache());
daoState.getBlocks().clear();
daoState.getBlocks().addAll(snapshot.getBlocks());
@@ -235,17 +230,26 @@
// Third we add each successfully parsed BSQ tx to the last block
public void onNewTxForLastBlock(Block block, Tx tx) {
- // At least one block must be present else no rawTx would have been recognised as a BSQ tx.
- Preconditions.checkArgument(block == getLastBlock().orElseThrow());
+ assertDaoStateChange();
- block.getTxs().add(tx);
- daoState.getTxMap().put(tx.getId(), tx);
+ getLastBlock().ifPresent(lastBlock -> {
+ if (block == lastBlock) {
+ // We need to ensure that the txs in all blocks are in sync with the txs in our txMap (cache).
+ block.addTx(tx);
+ daoState.addToTxCache(tx);
+ } else {
+ // Not clear if this case can happen but at onNewBlockWithEmptyTxs we handle such a potential edge
+ // case as well, so we need to reflect that here as well.
+ log.warn("Block for parsing does not match last block. That might happen in edge cases at reorgs. " +
+ "Received block={}", block);
+ }
+ });
}
// Fourth we get the onParseBlockComplete called after all rawTxs of blocks have been parsed
public void onParseBlockComplete(Block block) {
if (parseBlockChainComplete)
- log.info("Parse block completed: Block height {}, {} BSQ transactions.", block.getHeight(), block.getTxs().size());
+ log.info("Parse block completed: Block height {}, {} BSQ transactions.", block.getHeight(), block.getNumTxs());
// Need to be called before onParseTxsCompleteAfterBatchProcessing as we use it in
// VoteResult and other listeners like balances usually listen on onParseTxsCompleteAfterBatchProcessing
@@ -359,17 +363,12 @@
// Tx
///////////////////////////////////////////////////////////////////////////////////////////
- public Stream<Tx> getTxStream() {
- return getBlocks().stream()
- .flatMap(block -> block.getTxs().stream());
- }
-
- private Stream<Tx> getUnorderedTxStream() {
- return getTxs().stream();
+ public Stream<Tx> getUnorderedTxStream() {
+ return daoState.getTxsFromTxCache().stream();
}
- public Collection<Tx> getTxs() {
- return Collections.unmodifiableCollection(daoState.getTxMap().values());
+ public int getNumTxs() {
+ return daoState.getTxsFromTxCache().size();
}
public List<Tx> getInvalidTxs() {
@@ -381,7 +380,7 @@
}
public Optional<Tx> getTx(String txId) {
- return Optional.ofNullable(daoState.getTxMap().get(txId));
+ return Optional.ofNullable(daoState.getTxFromTxCache(txId));
}
public boolean containsTx(String txId) {
Index: core/src/main/java/bisq/core/dao/DaoFacade.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/src/main/java/bisq/core/dao/DaoFacade.java (date 1576079165000)
+++ core/src/main/java/bisq/core/dao/DaoFacade.java (date 1576094143000)
@@ -94,7 +94,6 @@
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -625,9 +624,8 @@
return daoStateService.getUnspentTxOutputs();
}
- // Returns a view rather than a copy of all the txs.
- public Collection<Tx> getTxs() {
- return daoStateService.getTxs();
+ public int getNumTxs() {
+ return daoStateService.getNumTxs();
}
public Optional<TxOutput> getLockupTxOutput(String txId) {
Index: core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java (date 1576079165000)
+++ core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java (date 1576094143000)
@@ -140,7 +140,7 @@
// Access to daoStateService is single threaded, we must not access daoStateService from the thread.
List<JsonTxOutput> allJsonTxOutputs = new ArrayList<>();
- List<JsonTx> jsonTxs = daoStateService.getTxStream()
+ List<JsonTx> jsonTxs = daoStateService.getUnorderedTxStream()
.map(tx -> {
JsonTx jsonTx = getJsonTx(tx);
allJsonTxOutputs.addAll(jsonTx.getOutputs());
```
--
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/bisq-network/bisq/pull/3773#issuecomment-564721881
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.bisq.network/pipermail/bisq-github/attachments/20191211/d1e47b28/attachment-0001.html>
More information about the bisq-github
mailing list