Upstream merge

contrib/build-linux/appimage/Dockerfile_ub1804

@@ -21,7 +21,7 @@ RUN echo deb ${UBUNTU_MIRROR} bionic main restricted universe multiverse > /etc/
         libncurses5-dev=6.1-1ubuntu1.18.04 \
         libsqlite3-dev=3.22.0-1ubuntu0.4 \
         libusb-1.0-0-dev=2:1.0.21-2 \
-        libudev-dev=237-3ubuntu10.45 \
+        libudev-dev=237-3ubuntu10.47 \
         gettext=0.19.8.1-6ubuntu0.3 \
         pkg-config=0.29.1-0ubuntu2 \
         libdbus-1-3=1.12.2-1ubuntu1.2 \
@@ -34,7 +34,7 @@ RUN echo deb ${UBUNTU_MIRROR} bionic main restricted universe multiverse > /etc/
         zlib1g-dev=1:1.2.11.dfsg-0ubuntu2 \
         libfreetype6=2.8.1-2ubuntu2.1 \
         libfontconfig1=2.12.6-0ubuntu2 \
-        libssl-dev=1.1.1-1ubuntu2.1~18.04.8 \
+        libssl-dev=1.1.1-1ubuntu2.1~18.04.9 \
         rustc=1.47.0+dfsg1+llvm-1ubuntu1~18.04.1 \
         && \
     rm -rf /var/lib/apt/lists/* && \

electroncash/commands.py

@@ -162,6 +162,16 @@ def EncodeNamedTupleObject(nt):
             d[k] = DoChk(d[k])
         return d
 
+    @staticmethod
+    def address_from_string_check_slp(address, wallet):
+        addr_str = address
+        address = Address.from_string(address)
+        assert not isinstance(address, str)
+        slp_addr_str = address.to_full_string(Address.FMT_SLPADDR)
+        if addr_str in slp_addr_str and not wallet.is_slp:
+            raise BaseException('Cannot check SLP addresses with a non-SLP type wallet.')
+        return address
+
     @command('')
     def addressconvert(self, address):
         """Convert to/from Legacy <-> Cash Address.  Address can be either
@@ -176,6 +186,28 @@ def addressconvert(self, address):
             'legacy'   : addr.to_full_string(Address.FMT_LEGACY),
         }
 
+    @command('')
+    def addressconvert_slp(self, address):
+        """Convert to/from Legacy <-> Cash Address or slp Address.  Address can be either
+        a legacy or a Cash Address and both forms will be returned as a JSON
+        dict."""
+        try:
+            addr = Address.from_string(address)
+        except Exception as e:
+            raise AddressError(f'Invalid address: {address}') from e
+
+        if self.config.get("allow_cli_slp_address_conversion") != True:
+            print("WARNING: If you are converting from legacy or cash address to slp format you need \n" + 
+                "to make sure the receiving wallet is compatible with slp tokens protocol. If the wallet \n" +
+                "is not compatible with slp, the receiver's wallet will easily burn tokens. To enable slp \n" +
+                "address conversion you must set the config key 'allow_cli_slp_address_conversion' to 'true'.")
+        else:
+            return {
+                'cashaddr' : addr.to_full_string(Address.FMT_CASHADDR),
+                'legacy'   : addr.to_full_string(Address.FMT_LEGACY),
+                'slpaddr'  : addr.to_full_string(Address.FMT_SLPADDR),
+            }
+
     @command('')
     def commands(self):
         """List of commands"""
@@ -376,31 +408,23 @@ def createmultisig(self, num, pubkeys):
         address = bitcoin.hash160_to_p2sh(hash_160(bfh(redeem_script)))
         return {'address':address, 'redeemScript':redeem_script}
 
-    def address_from_string_check_slp(address, wallet):
-        address = Address.from_string(address)
-        assert not isinstance(address, str)
-        slp_addr_str = address.to_full_string(Address.FMT_SLPADDR)
-        if address in slp_addr_str and not wallet.is_slp:
-            raise BaseException('Cannot check SLP addresses with a non-SLP type wallet.')
-        return address
-
     @command('w')
     def freeze(self, address):
         """Freeze address. Freeze the funds at one of your wallet\'s addresses"""
-        address = address_from_string_check_slp(address, self.wallet)
+        address = self.address_from_string_check_slp(address, self.wallet)
         return self.wallet.set_frozen_state([address], True)
 
     @command('w')
     def unfreeze(self, address):
         """Unfreeze address. Unfreeze the funds at one of your wallet\'s address"""
-        address = address_from_string_check_slp(address, self.wallet)
+        address = self.address_from_string_check_slp(address, self.wallet)
         return self.wallet.set_frozen_state([address], False)
 
     @command('wp')
     def getprivatekeys(self, address, password=None):
         """Get private keys of addresses. You may pass a single wallet address, or a list of wallet addresses."""
         def get_pk(address):
-            address = address_from_string_check_slp(address, self.wallet)
+            address = self.address_from_string_check_slp(address, self.wallet)
             return self.wallet.export_private_key(address, password)
 
         if isinstance(address, str):
@@ -411,7 +435,7 @@ def get_pk(address):
     @command('w')
     def ismine(self, address):
         """Check if address is in wallet. Return true if and only address is in wallet"""
-        address = address_from_string_check_slp(address, self.wallet)
+        address = self.address_from_string_check_slp(address, self.wallet)
         return self.wallet.is_mine(address)
 
     @command('')
@@ -427,7 +451,7 @@ def validateaddress(self, address):
     @command('w')
     def getpubkeys(self, address):
         """Return the public keys for a wallet address. """
-        address = address_from_string_check_slp(address, self.wallet)
+        address = self.address_from_string_check_slp(address, self.wallet)
         return self.wallet.get_public_keys(address)
 
     @command('w')
@@ -549,7 +573,7 @@ def sweep(self, privkey, destination, fee=None, nocheck=False, imax=100):
     def signmessage(self, address, message, password=None):
         """Sign a message with a key. Use quotes if your message contains
         whitespaces"""
-        address = address_from_string_check_slp(address, self.wallet)
+        address = self.address_from_string_check_slp(address, self.wallet)
         sig = self.wallet.sign_message(address, message, password)
         return base64.b64encode(sig).decode('ascii')
 
@@ -980,8 +1004,16 @@ def getunusedaddress(self):
         """Returns the first unused address of the wallet, or None if all addresses are used.
         An address is considered as used if it has received a transaction, or if it is used in a payment request."""
         fmt = Address.FMT_CASHADDR
-        if self.wallet.is_slp:
-            fmt = Address.FMT_SLPADDR
+        addr = self.wallet.get_unused_address()
+        if addr:
+            return addr.to_full_string(fmt)
+        return None
+
+    @command('w')
+    def getunusedaddress_slp(self):
+        """Returns the first unused address of the wallet using slp format, or None if all addresses are used.
+        An address is considered as used if it has received a transaction, or if it is used in a payment request."""
+        fmt = Address.FMT_SLPADDR
         addr = self.wallet.get_unused_address()
         if addr:
             return addr.to_full_string(fmt)
@@ -1019,13 +1051,13 @@ def signrequest(self, address, password=None):
         alias_addr = (data and data.get('address')) or None
         if not alias_addr:
             raise RuntimeError('Alias could not be resolved')
-        address_from_string_check_slp(address, self.wallet)  # throws with slp address format in non-slp wallets
+        self.address_from_string_check_slp(address, self.wallet)  # throws with slp address format in non-slp wallets
         self.wallet.sign_payment_request(address, alias, alias_addr, password)
 
     @command('w')
     def rmrequest(self, address):
         """Remove a payment request"""
-        address_from_string_check_slp(address, self.wallet)  # throws with slp address format in non-slp wallets
+        self.address_from_string_check_slp(address, self.wallet)  # throws with slp address format in non-slp wallets
         return self.wallet.remove_payment_request(address, self.config)
 
     @command('w')

electroncash/networks.py

@@ -43,6 +43,7 @@ class AbstractNet:
     LEGACY_POW_RETARGET_BLOCKS = LEGACY_POW_TARGET_TIMESPAN // LEGACY_POW_TARGET_INTERVAL  # 2016 blocks
     BASE_UNITS = {'BCH': 8, 'mBCH': 5, 'bits': 2}
     DEFAULT_UNIT = "BCH"
+    SLP_PREFLIGHT_CHECK = True  # pings other node implementations in the slp network before signing
 
 
 class MainNet(AbstractNet):
@@ -161,6 +162,8 @@ class TestNet4(TestNet):
     asert_daa = ASERTDaa(is_testnet=True)  # Redeclare to get instance for this subclass
     asert_daa.anchor = Anchor(height=16844, bits=486604799, prev_time=1605451779)
 
+    # disable slp pre-flight since this is a testnet with no known slp network servers
+    SLP_PREFLIGHT_CHECK = False
 
 
 class ScaleNet(TestNet):
@@ -188,6 +191,8 @@ class ScaleNet(TestNet):
     asert_daa = ASERTDaa(is_testnet=False)  # Despite being a "testnet", ScaleNet uses 2d half-life
     asert_daa.anchor = None  # Intentionally not specified because it's after checkpoint; blockchain.py will calculate
 
+    # disable slp pre-flight since this is a testnet with no known slp network servers
+    SLP_PREFLIGHT_CHECK = False
 
 # All new code should access this to get the current network config.
 net = MainNet

electroncash/slp_checker.py

@@ -1,6 +1,6 @@
 import json
 from .util import NotEnoughFundsSlp, NotEnoughUnfrozenFundsSlp, print_error
-from . import slp
+from . import slp, networks
 from .slp import SlpParsingError, SlpInvalidOutputMessage, SlpUnsupportedSlpTokenType
 from .slp_preflight_check import SlpPreflightCheck
 from .transaction import Transaction
@@ -250,10 +250,11 @@ def check_tx_slp(wallet, tx, *, coins_to_burn=None, amt_to_burn=None, require_tx
                             raise BadSlpOutpointType('Transaction token receiver vout is not P2PKH' \
                                                         + ' or P2SH output type')
 
-        # perform slp pre-flight check before signing (this check run here and also at signing)
-        slp_preflight = SlpPreflightCheck.query(tx, selected_slp_coins=coins_to_burn, amt_to_burn=amt_to_burn)
-        if not slp_preflight['ok']:
-            raise Exception("slp pre-flight check failed: %s\n\n(node: %s)"%(slp_preflight.get('invalid_reason', json.dumps(slp_preflight)), slp_preflight['node']))
+        # perform slp pre-flight check
+        if networks.net.SLP_PREFLIGHT_CHECK:
+            slp_preflight = SlpPreflightCheck.query(tx, selected_slp_coins=coins_to_burn, amt_to_burn=amt_to_burn)
+            if not slp_preflight['ok']:
+                raise Exception("slp pre-flight check failed: %s\n\n(node: %s)"%(slp_preflight.get('invalid_reason', json.dumps(slp_preflight)), slp_preflight['node']))
 
         # return True if this check passes
         print_error("Final SLP check passed")

electroncash/slp_dagging.py

@@ -250,8 +250,7 @@ def has_txid(self, txid):
     def run(self,):
         """ Wrapper for mainloop() to manage run state. """
 
-        # Check if we need to reset the graph due to having
-        # nodes in an undetermined state.
+        # Check if we need to reset the graph
         self.graph.maybe_reset()
 
         with self._statelock:
@@ -394,6 +393,7 @@ def skip_callback(txid):
             if self.debug > 0:
                 print("DEBUG-DAG: SKIPPING: " + txid)
             node = self.graph.get_node(txid)
+            node.graph.gs_skipped = True
             node.set_validity(False, 2)
 
             # temp for debugging
@@ -409,10 +409,17 @@ def dl_callback(tx):
             # f.write(txid+","+str(self.currentdepth)+",true,\n")
 
             node = self.graph.get_node(txid)
+
+            # try to get validity from wallet cache
             try:
                 val = self.validitycache[txid]
             except KeyError:
                 val = None
+
+            # check the graph's internal validity cache
+            if val == None and txid in self.graph._valid_txids:
+                val = 1
+
             try:
                 node.load_tx(tx, cached_validity=val)
             except DoubleLoadException:
@@ -627,7 +634,7 @@ def add_job(self, job):
             if job in self.all_jobs:
                 raise ValueError
             self.all_jobs.add(job)
-            self.jobs_pending.put((job.height, next(unique), job))
+        self.jobs_pending.put((job.height, next(unique), job))
         self.wakeup.set()
 
     def _stop_all_common(self, job):
@@ -811,7 +818,7 @@ class TokenGraph:
     """
     debugging = False
 
-    def __init__(self, validator):
+    def __init__(self, validator, valid_txids=None):
         self.validator = validator
 
         self._nodes = dict() # txid -> Node
@@ -827,13 +834,19 @@ def __init__(self, validator):
         # create singletons for pruning
         self.prunednodes = {v:NodeInactive(v, None) for v in validator.validity_states.keys()}
 
+        # internal validity cache when the graph needs reset
+        self.gs_skipped = False
+        self._valid_txids = set()
+        if valid_txids != None:
+            self._valid_txids = valid_txids
+
         # Threading rule: we never call node functions while locked.
         # self._lock = ... # threading not enabled.
 
     def reset(self, ):
         # copy nodes and reset self
         prevnodes = self._nodes
-        TokenGraph.__init__(self, self.validator)
+        TokenGraph.__init__(self, self.validator, self._valid_txids)
 
         # nuke Connections to encourage prompt GC
         for n in prevnodes.values():
@@ -844,14 +857,11 @@ def reset(self, ):
                 pass
 
     # This is used by a ValidationJob to determine if the graph
-    # should be reset.  At the beginning of a validation job
-    # having active nodes with validity==0 causes the job to fail.
+    # should be reset.  If graph search skipped any nodes, then
+    # the graph needs to be reset.
     def maybe_reset(self):
-        nodes = self._nodes.copy()
-        for node in nodes.values():
-            if node.validity == 0 and isinstance(node, Node):
-                self.reset()
-                return
+        if self.gs_skipped:
+            self.reset()
 
     def debug(self, formatstr, *args):
         if self.debugging:
@@ -966,6 +976,11 @@ def get_valid_txids(self, max_size=-1, exclude=None):
                     txid not in exclude:
                 candidate_txids.append(txid)
 
+        # join the different validity caches
+        candidate_txids = set(candidate_txids)
+        candidate_txids |= self._valid_txids
+        candidate_txids = list(candidate_txids)
+
         # depending on the number of txids available
         # either return all txids, or return random txids
         if len(candidate_txids) <= max_size:
@@ -1152,6 +1167,10 @@ def _inactivate_self(self, keepinfo, validity):
         # Replace self with NodeInactive instance according to keepinfo and validity
         # no thread locking here, this only gets called internally.
 
+        # if valid then we save as valid internally for graph search resets
+        if validity == 1:
+            self.graph._valid_txids.add(self.txid)
+
         if keepinfo:
             replacement = NodeInactive(validity, self.outputs)
         else:

electroncash/slp_graph_search.py

@@ -121,33 +121,27 @@ def get_job_cache(self, *, max_size=-1, is_mint=False):
         token_id = self.valjob.graph.validator.token_id_hex
         gs_cache = []
 
+        # get valid txid cache from the graph
+        gs_cache = self.valjob.graph.get_valid_txids(max_size=max_size, exclude=gs_cache)
+
         # pull valid txids from wallet storage
+        wallet_cache = []
         wallet_val = self.valjob.validitycache.copy()
         wallet_tok_info = wallet.tx_tokinfo.copy()
         for txid, val in wallet_val.items():
             _token_id = wallet_tok_info.get(txid, {}).get("token_id", None)
             if _token_id == token_id and val == 1:
-                gs_cache.append(txid)
-
-        # pull valid txids from the shared in-memory token graph
-        # and prioritize items from the wallet validity cache
-        if not is_mint:
-            sample_size = -1
-            if max_size > 0 and len(gs_cache) < max_size:
-                sample_size = max_size - len(gs_cache)
-            if sample_size > 0:
-                for txid in self.valjob.graph.get_valid_txids(max_size=sample_size, exclude=gs_cache):
-                    gs_cache.append(txid)
-
-        # TODO: pull valid txids from a "checkpoints" file shipped with the wallet
-        #   these txids can be selected intelligently through graph analysis.  Tokens
-        #   supported in the type of arrangement would likely be done through the
-        #   support of the token issuer for the purpose of improving user experience.
+                wallet_cache.append(txid)
 
         # if required limit the size of the cache
         gs_cache = list(set(gs_cache))
-        if gs_cache and max_size > 0 and len(gs_cache) > max_size:
-            gs_cache = list(set(random.choices(gs_cache, k=max_size)))
+        if gs_cache and max_size > 0 and len(gs_cache) + len(wallet_cache) > max_size:
+            gs_cache = random.choices(gs_cache, k=max_size)
+
+        # guarantee inclusion of txids in cache
+        for txid in wallet_cache:
+            gs_cache.append(txid)
+        gs_cache = list(set(gs_cache))
 
         # update the cache size variable used in the UI
         self.validity_cache_size = len(gs_cache)
@@ -396,7 +390,7 @@ def search_query(self, job):
         elif kind == 'bchd':
             root_hash_b64 = base64.standard_b64encode(codecs.decode(job.root_txid,'hex')[::-1]).decode("ascii") 
             url = host + "/v1/GetSlpGraphSearch"
-            cache = job.get_job_cache(max_size=100)
+            cache = job.get_job_cache(max_size=1000)
 
             # bchd needs the reverse txid and then use base64 encoding
             tx_hashes = []