From b4f7986d4ce2c20c11904dd5ec69f381df3e0682 Mon Sep 17 00:00:00 2001 From: Oleg Zhymolokhov Date: Fri, 5 Oct 2018 13:19:02 +0300 Subject: [PATCH] wallet-discovery-refactoring: Extended functionality of the discovery algorithm. --- .../controller/WalletController.java | 21 +++- .../walletservice/dto/AccountScanDto.java | 19 +++ .../exception/GlobalExceptionHandler.java | 3 + .../{CKDService.java => WalletService.java} | 118 ++++++++++++------ src/main/resources/application.yml | 4 +- 5 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/nynja/walletservice/dto/AccountScanDto.java rename src/main/java/com/nynja/walletservice/service/operation/impl/{CKDService.java => WalletService.java} (56%) diff --git a/src/main/java/com/nynja/walletservice/controller/WalletController.java b/src/main/java/com/nynja/walletservice/controller/WalletController.java index fca19df..a102f24 100644 --- a/src/main/java/com/nynja/walletservice/controller/WalletController.java +++ b/src/main/java/com/nynja/walletservice/controller/WalletController.java @@ -1,14 +1,16 @@ package com.nynja.walletservice.controller; +import com.nynja.walletservice.dto.AccountScanDto; import com.nynja.walletservice.dto.AddressesWithTxDto; import com.nynja.walletservice.dto.WalletImportDto; -import com.nynja.walletservice.service.operation.impl.CKDService; +import com.nynja.walletservice.service.operation.impl.WalletService; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import javax.validation.Valid; +import java.util.List; import java.util.concurrent.CompletableFuture; import static com.nynja.walletservice.constant.Constants.RestApi.API_VERSION; @@ -19,16 +21,23 @@ import static com.nynja.walletservice.controller.ControllerHelper.handleAndRespo @RequestMapping("/wallet-operations") public class WalletController { - private CKDService CKDService; + private WalletService walletService; - public WalletController(CKDService CKDService) { - this.CKDService = CKDService; + public WalletController(WalletService walletService) { + this.walletService = walletService; } @ApiOperation(value = "Import mnemonic and retrieve HD wallet addresses that participated in transactions.", httpMethod = "POST") @ApiResponse(code = 200, message = RETRIEVED) @PostMapping(API_VERSION + "/hd-wallet-scan") - public CompletableFuture> getHDWalletBalance(@Valid @RequestBody WalletImportDto dto) { - return handleAndRespondAsync(() -> CKDService.discoverHDWallet(dto)); + public CompletableFuture> discoverWallet(@Valid @RequestBody WalletImportDto dto) { + return handleAndRespondAsync(() -> walletService.discoverHDWallet(dto)); + } + + @ApiOperation(value = "Retrieve HD wallet addresses that participated in transactions for a public chain of a single account.", httpMethod = "POST") + @ApiResponse(code = 200, message = RETRIEVED) + @PostMapping(API_VERSION + "/scan-single-account") + public CompletableFuture>> scanSingleAccount(@Valid @RequestBody AccountScanDto dto) { + return handleAndRespondAsync(() -> walletService.scanSingleAccount(dto.getPublicKey(), dto.getChainCode())); } } diff --git a/src/main/java/com/nynja/walletservice/dto/AccountScanDto.java b/src/main/java/com/nynja/walletservice/dto/AccountScanDto.java new file mode 100644 index 0000000..ddf9d39 --- /dev/null +++ b/src/main/java/com/nynja/walletservice/dto/AccountScanDto.java @@ -0,0 +1,19 @@ +package com.nynja.walletservice.dto; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.validator.constraints.NotBlank; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class AccountScanDto { + + @NotBlank(message = "Public key should be present.") + private String publicKey; + @NotBlank(message = "ChainCode should be present.") + private String chainCode; +} diff --git a/src/main/java/com/nynja/walletservice/exception/GlobalExceptionHandler.java b/src/main/java/com/nynja/walletservice/exception/GlobalExceptionHandler.java index e965a67..1e81b3c 100644 --- a/src/main/java/com/nynja/walletservice/exception/GlobalExceptionHandler.java +++ b/src/main/java/com/nynja/walletservice/exception/GlobalExceptionHandler.java @@ -6,6 +6,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.MethodArgumentNotValidException; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.async.AsyncRequestTimeoutException; import org.web3j.exceptions.MessageDecodingException; import javax.validation.ConstraintViolation; @@ -30,6 +31,8 @@ public class GlobalExceptionHandler { return compose(HttpStatus.BAD_REQUEST, reason.getMessage()); if (reason instanceof MessageDecodingException) return compose(HttpStatus.BAD_REQUEST, reason.getMessage()); + if (reason instanceof AsyncRequestTimeoutException) + return compose(HttpStatus.GATEWAY_TIMEOUT, reason.getMessage()); return compose(HttpStatus.INTERNAL_SERVER_ERROR, reason.getMessage()); } diff --git a/src/main/java/com/nynja/walletservice/service/operation/impl/CKDService.java b/src/main/java/com/nynja/walletservice/service/operation/impl/WalletService.java similarity index 56% rename from src/main/java/com/nynja/walletservice/service/operation/impl/CKDService.java rename to src/main/java/com/nynja/walletservice/service/operation/impl/WalletService.java index a571a85..81a2c79 100644 --- a/src/main/java/com/nynja/walletservice/service/operation/impl/CKDService.java +++ b/src/main/java/com/nynja/walletservice/service/operation/impl/WalletService.java @@ -14,8 +14,10 @@ import org.bitcoinj.crypto.HDKeyDerivation; import org.bitcoinj.wallet.DeterministicSeed; import org.bitcoinj.wallet.UnreadableWalletException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.web3j.crypto.Credentials; +import org.web3j.crypto.Hash; +import org.web3j.utils.Numeric; import java.util.*; import java.util.concurrent.*; @@ -25,16 +27,18 @@ import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; @Service @Slf4j -public class CKDService { +public class WalletService { private static final int BIP44_GAP = 20; private static final int EXTERNAL_CHAIN = 0; - private static final int N_THREADS = 2; + + @Value("${wallet-discovery.search-job-pool-size}") + private int poolSize; private EtherScanConsumer etherScanConsumer; @Autowired - public CKDService(EtherScanConsumer etherScanConsumer) { + public WalletService(EtherScanConsumer etherScanConsumer) { this.etherScanConsumer = etherScanConsumer; } @@ -52,7 +56,7 @@ public class CKDService { } return Optional.ofNullable(seed.getSeedBytes()) - .map(bytes -> scanAddresses(deriveCoinPath(HDKeyDerivation.createMasterPrivateKey(bytes), dto.getCurrency()))) + .map(bytes -> scanAccounts(deriveCoinPath(HDKeyDerivation.createMasterPrivateKey(bytes), dto.getCurrency()))) .map(AddressesWithTxDto::new) .orElseThrow(() -> new RuntimeException("Error reading mnemonic bytes. Reason: the number is null.")); } @@ -61,45 +65,72 @@ public class CKDService { * Concurrent implementation of the BIP-44 Account discovery algorithm: * https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#account-discovery */ - private Map> scanAddresses(DeterministicKey coinKey) { + private Map> scanAccounts(DeterministicKey coinKey) { int accountNumber = 0; Map> accountToAddresses = new HashMap<>(); - ExecutorService ec = Executors.newFixedThreadPool(N_THREADS); - CompletionService cs = new ExecutorCompletionService<>(ec); + ExecutorService ec = Executors.newFixedThreadPool(poolSize); - do { - int firstIdx = 0, lastIdx = BIP44_GAP; - int[] counter = {0}; - DeterministicKey accountKey = coinKey.derive(accountNumber); - DeterministicKey externalChainKey = deriveNonHardened(accountKey, EXTERNAL_CHAIN); + try { + CompletionService cs = new ExecutorCompletionService<>(ec); do { - List> futureTasks = new LinkedList<>(); - List results = new ArrayList<>(BIP44_GAP - counter[0]); + DeterministicKey externalChainKey = deriveNonHardened(coinKey.derive(accountNumber), EXTERNAL_CHAIN); + List results = performSearch(cs, externalChainKey); + mergeResults(accountToAddresses, accountNumber, results); + } while (isNotEmpty(accountToAddresses.get(accountNumber++))); - for (int childIndex = firstIdx; childIndex < lastIdx; childIndex++) { - futureTasks.add(cs.submit(searchJob(externalChainKey, childIndex))); - } + } finally { + ec.shutdown(); + } - handleSearchResults(futureTasks, results, cs); - if (isNotEmpty(results)) { - mergeResults(accountToAddresses, accountNumber, results, counter); - } + return accountToAddresses; + } - firstIdx = lastIdx; - lastIdx = counter[0] == 0 ? lastIdx + BIP44_GAP : (BIP44_GAP - counter[0]) + BIP44_GAP; - } while (counter[0] != BIP44_GAP); - } while (isNotEmpty(accountToAddresses.get(accountNumber++))); + public List scanSingleAccount(String pub, String chainCode) { + ExecutorService ec = Executors.newFixedThreadPool(poolSize); + try { + CompletionService cs = new ExecutorCompletionService<>(ec); - ec.shutdown(); + DeterministicKey externalChainKey = HDKeyDerivation.createMasterPubKeyFromBytes( + Base64.getDecoder().decode(pub), + Base64.getDecoder().decode(chainCode) + ); - return accountToAddresses; + return performSearch(cs, externalChainKey); + } finally { + ec.shutdown(); + } + } + + private List performSearch(CompletionService cs, DeterministicKey externalChainKey) { + List addresses = new LinkedList<>(); + int firstIdx = 0, lastIdx = BIP44_GAP; + int[] counter = {0}; + + do { + List> futureTasks = new LinkedList<>(); + List results = new ArrayList<>(BIP44_GAP - counter[0]); + + for (int childIndex = firstIdx; childIndex < lastIdx; childIndex++) { + futureTasks.add(cs.submit(searchJob(externalChainKey, childIndex))); + } + + handleSearchResults(futureTasks, results, cs); + + if (isNotEmpty(results)) { + addresses.addAll(filterHistory(results, counter)); + } + + firstIdx = lastIdx; + lastIdx = counter[0] == 0 ? lastIdx + BIP44_GAP : (BIP44_GAP - counter[0]) + BIP44_GAP; + } while (counter[0] != BIP44_GAP); + + return addresses; } private Callable searchJob(DeterministicKey externalChainKey, int childIndex) { return () -> { - DeterministicKey childAddress = deriveNonHardened(externalChainKey, childIndex); - String address = Credentials.create(childAddress.getPrivateKeyAsHex()).getAddress(); + String address = prepareETHAddress(deriveNonHardened(externalChainKey, childIndex)); return new ScanJobResult( childIndex, @@ -109,6 +140,14 @@ public class CKDService { }; } + private String prepareETHAddress(DeterministicKey childAddressKey) { + byte[] decompressedWithPrefix = childAddressKey.decompress().getPubKey(); + byte[] decompressedWithoutPrefix = Arrays.copyOfRange(decompressedWithPrefix, 1, decompressedWithPrefix.length); + String kessak256Hex = Numeric.toHexString(Hash.sha3(decompressedWithoutPrefix)); + + return "0x".concat(kessak256Hex.substring(26, 66)); + } + private void handleSearchResults(List> futureTasks, List results, CompletionService cs) { for (Future future : futureTasks) { @@ -118,6 +157,7 @@ public class CKDService { future.cancel(true); } catch (ExecutionException e) { log.error("Error receiving scan job results.", e); + throw new RuntimeException(e); } } } @@ -126,8 +166,14 @@ public class CKDService { return new AddressData(scanJobResult.index, scanJobResult.address, isNotEmpty(scanJobResult.history)); } - private void mergeResults(Map> accountToAddresses, int accountNumber, List results, int[] counter) { - List data = results.stream() + private void mergeResults(Map> accountToAddresses, int accountNumber, List data) { + if (isNotEmpty(data)) { + accountToAddresses.computeIfAbsent(accountNumber, k -> new ArrayList<>()).addAll(data); + } + } + + private List filterHistory(List results, int[] counter) { + return results.stream() .filter(ar -> { if (ar.hasHistory) { counter[0] = 0; @@ -136,14 +182,10 @@ public class CKDService { } return ar.hasHistory; }).map(AddressData::getAddress).collect(Collectors.toList()); - - if (isNotEmpty(data)) { - accountToAddresses.computeIfAbsent(accountNumber, k -> new ArrayList<>()).addAll(data); - } } - private DeterministicKey deriveCoinPath(DeterministicKey masterPrivKey, Currency currency) { - return masterPrivKey.derive(44).derive(currency.getNumber()); + private DeterministicKey deriveCoinPath(DeterministicKey masterPrivateKey, Currency currency) { + return masterPrivateKey.derive(44).derive(currency.getNumber()); } private DeterministicKey deriveNonHardened(DeterministicKey parent, int childNumber) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c9bd497..3d76581 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,7 +3,7 @@ server: spring: mvc: async: - request-timeout: 600000 + request-timeout: 60000 datasource: url: jdbc:h2:file:./data/persistent_storage username: sa @@ -28,6 +28,8 @@ ethereum: nynja-coin-address: "0x5c015d5b7490cc329b96aebfb9d8e5ebd78e2bf6" gas-price: 2000000000000 gas-limit: 4700000 +wallet-discovery: + search-job-pool-size: 2 etherscan: url: http://api.etherscan.io/api #url: http://api-ropsten.etherscan.io/api -- GitLab