Counterparty Release 9.55.3 (#1004)

* Rename docker-engine to docker-ce. (#982) * upgrade pip before installing requirements * CIP-11: Shorten Transaction Type ID Namespace (#977) * Implements CIP 9 (WIP) (#985) * instead of auto-correcting the quantity to the amount the address holds return invalid: insufficient funds * CIP12 REQUIRE_MEMO: * Fix locked issuance (#999) * Remove rpcthreads=100 (#1002) * Use python-bitcoinlib snapshot for segwit support (#1001) Signed-off-by: Devon Weller <[email protected]>
CounterpartyXCP · Sep 28, 2017 · 544d822 · 544d822
1 parent f645384
commit 544d822
Show file tree

Hide file tree

Showing 52 changed files with 3,663 additions and 2,105 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -15,7 +15,7 @@ before_install:
 - sudo apt-get -y install python3 python3-pip
 - sudo pip3 install coveralls
 # upgrade docker, for build argument support
-- sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-engine
+- sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-ce
 - docker version
 - docker ps -a
 # get the current PR and branch name

diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,4 +1,12 @@
 ## Library Versions ##
+* v9.55.3 (2017-09-26)
+    * Implemented CIP-9 Enhanced Send (https://github.com/CounterpartyXCP/cips/blob/master/cip-0009.md) (protocol change: 489956)
+    * Implemented CIP-11 Shorten Transaction Type ID Namespace (https://github.com/CounterpartyXCP/cips/blob/master/cip-0011.md) (protocol change: 489956)
+    * Implemented CIP-12 Memo Requirement through Broadcasts (https://github.com/CounterpartyXCP/cips/blob/master/cip-0012.md) (protocol change: 489956)
+    * Fixes locked issuance workaround (https://github.com/CounterpartyXCP/counterparty-lib/pull/999)
+    * Updated python-bitcoinlib library for handling blocks that include transactions with segwit outputs
+    * Test suite and services updates
+    * Remove rpcthreads recommendation
 * v9.55.2 (2017-05-01)
     * Implemented CIP-4 subassets (https://github.com/CounterpartyXCP/cips/blob/master/cip-0004.md) (protocol change: 467417)
     * Moved to bitcoind 0.13.2-addrindex (please use at least 0.13.2 with this version of counterparty-lib)

diff --git a/README.md b/README.md
@@ -29,10 +29,11 @@ rpcpassword=rpc
 server=1
 txindex=1
 addrindex=1
-rpcthreads=100
 rpctimeout=300
 ```
 
+**Note:** you can and should replace the RPC credentials. Remember to use the changed RPC credentials throughout this document.
+
 Then, download and install `counterparty-lib`:
 
 ```
@@ -115,7 +116,7 @@ Counterparty database files are by default named `counterparty.[testnet.]db` and
 
 ## Configuration File Format
 
-Manual configuration is not necessary for most use cases. 
+Manual configuration is not necessary for most use cases. "back-end" and "wallet" are used to access Bitcoin server RPC.
 
 A `counterparty-server` configuration file looks like this:
 

diff --git a/circle.yml b/circle.yml
@@ -3,6 +3,7 @@ dependencies:
     override:
         - rm -rf /home/ubuntu/virtualenvs/venv-*/bin/serpent
         - rm -rf /home/ubuntu/virtualenvs/venv-*/lib/python3.4/site-packages/apsw*
+        - pip install --upgrade pip
         - pip install -r requirements.txt
         - python setup.py install --with-serpent
         - python -c "import apsw; print(apsw.apswversion())"

diff --git a/counterpartylib/lib/address.py b/counterpartylib/lib/address.py
@@ -0,0 +1,22 @@
+import logging
+logger = logging.getLogger(__name__)
+import struct
+
+import bitcoin
+
+def pack(address):
+    """
+    Converts a base58 bitcoin address into a 21 byte bytes object
+    """
+
+    short_address_bytes = bitcoin.base58.decode(address)[:-4]
+    return short_address_bytes
+
+# retuns both the message type id and the remainder of the message data
+def unpack(short_address_bytes):
+    """
+    Converts a 21 byte prefix and public key hash into a full base58 bitcoin address
+    """
+
+    check = bitcoin.core.Hash(short_address_bytes)[0:4]
+    return bitcoin.base58.encode(short_address_bytes + check)
diff --git a/counterpartylib/lib/api.py b/counterpartylib/lib/api.py
@@ -39,6 +39,7 @@
 from counterpartylib.lib import transaction
 from counterpartylib.lib import blocks
 from counterpartylib.lib import script
+from counterpartylib.lib import message_type
 from counterpartylib.lib.messages import send
 from counterpartylib.lib.messages import order
 from counterpartylib.lib.messages import btcpay
@@ -189,6 +190,10 @@ def value_to_marker(value):
         if 'case_sensitive' in filter_ and not isinstance(filter_['case_sensitive'], bool):
             raise APIError("case_sensitive must be a boolean")
 
+    # special case for memo and memo_hex field searches
+    if table == 'sends':
+        adjust_get_sends_memo_filters(filters)
+
     # SELECT
     statement = '''SELECT * FROM {}'''.format(table)
     # WHERE
@@ -257,7 +262,44 @@ def value_to_marker(value):
         if offset:
             statement += ''' OFFSET {}'''.format(offset)
 
-    return db_query(db, statement, tuple(bindings))
+
+    query_result = db_query(db, statement, tuple(bindings))
+
+    if table == 'sends':
+        # for sends, handle the memo field properly
+        return adjust_get_sends_results(query_result)
+
+    return query_result
+
+def adjust_get_sends_memo_filters(filters):
+    """Convert memo to a byte string.  If memo_hex is supplied, attempt to decode it and use that instead."""
+    for filter_ in filters:
+        if filter_['field'] == 'memo':
+            filter_['value'] = bytes(filter_['value'], 'utf-8')
+        if filter_['field'] == 'memo_hex':
+            # search the indexed memo field with a byte string
+            filter_['field'] = 'memo'
+            try:
+                filter_['value'] = bytes.fromhex(filter_['value'])
+            except ValueError as e:
+                raise APIError("Invalid memo_hex value")
+
+def adjust_get_sends_results(query_result):
+    """Format the memo_hex field.  Try and decode the memo from a utf-8 uncoded string. Invalid utf-8 strings return an empty memo."""
+    filtered_results = []
+    for send_row in list(query_result):
+        try:
+            if send_row['memo'] is None:
+                send_row['memo_hex'] = None
+                send_row['memo'] = None
+            else:
+                send_row['memo_hex'] = binascii.hexlify(send_row['memo']).decode('utf8')
+                send_row['memo'] = send_row['memo'].decode('utf-8')
+        except UnicodeDecodeError:
+            send_row['memo'] = ''
+        filtered_results.append(send_row)
+    return filtered_results
+
 
 def compose_transaction(db, name, params,
                         encoding='auto',
@@ -721,8 +763,7 @@ def get_tx_info(tx_hex, block_index=None):
         @dispatcher.add_method
         def unpack(data_hex):
             data = binascii.unhexlify(data_hex)
-            message_type_id = struct.unpack(config.TXTYPE_FORMAT, data[:4])[0]
-            message = data[4:]
+            message_type_id, message = message_type.unpack(data)
 
             # TODO: Enabled only for `send`.
             if message_type_id == send.ID:

diff --git a/counterpartylib/lib/blocks.py b/counterpartylib/lib/blocks.py
@@ -31,7 +31,9 @@
 from counterpartylib.lib import backend
 from counterpartylib.lib import log
 from counterpartylib.lib import database
+from counterpartylib.lib import message_type
 from .messages import (send, order, btcpay, issuance, broadcast, bet, dividend, burn, cancel, rps, rpsresolve, publish, execute, destroy)
+from .messages.versions import enhanced_send
 
 from .kickstart.blocks_parser import BlockchainParser, ChainstateParser
 from .kickstart.utils import ib2h
@@ -46,7 +48,7 @@
          'cancels', 'dividends', 'issuances', 'sends',
          'rps_match_expirations', 'rps_expirations', 'rpsresolves',
          'rps_matches', 'rps', 'executions', 'storage', 'suicides', 'nonces',
-         'postqueue', 'contracts', 'destructions', 'assets']
+         'postqueue', 'contracts', 'destructions', 'assets', 'addresses']
 # Compose list of tables tracked by undolog
 UNDOLOG_TABLES = copy.copy(TABLES)
 UNDOLOG_TABLES.remove('messages')
@@ -75,20 +77,23 @@ def parse_tx(db, tx):
         burn.parse(db, tx, MAINNET_BURNS)
         return
 
-    if len(tx['data']) > 4:
+    if len(tx['data']) > 1:
         try:
-            message_type_id = struct.unpack(config.TXTYPE_FORMAT, tx['data'][:4])[0]
+            message_type_id, message = message_type.unpack(tx['data'], tx['block_index'])
         except struct.error:    # Deterministically raised.
             message_type_id = None
+            message = None
     else:
         message_type_id = None
+        message = None
 
     # Protocol change.
     rps_enabled = tx['block_index'] >= 308500 or config.TESTNET
 
-    message = tx['data'][4:]
     if message_type_id == send.ID:
         send.parse(db, tx, message)
+    elif message_type_id == enhanced_send.ID and util.enabled('enhanced_sends', block_index=tx['block_index']):
+        enhanced_send.parse(db, tx, message)
     elif message_type_id == order.ID:
         order.parse(db, tx, message)
     elif message_type_id == btcpay.ID:
@@ -350,6 +355,17 @@ def initialise(db):
         cursor.execute('''INSERT INTO assets VALUES (?,?,?,?)''', ('0', 'BTC', None, None))
         cursor.execute('''INSERT INTO assets VALUES (?,?,?,?)''', ('1', 'XCP', None, None))
 
+    # Addresses
+    # Leaving this here because in the future this could work for other things besides broadcast
+    cursor.execute('''CREATE TABLE IF NOT EXISTS addresses(
+                      address TEXT UNIQUE,
+                      options INTEGER,
+                      block_index INTEGER)
+                   ''')
+    cursor.execute('''CREATE INDEX IF NOT EXISTS
+                      addresses_idx ON addresses (address)
+                   ''')
+
     # Consolidated
     send.initialise(db)
     destroy.initialise(db)

diff --git a/counterpartylib/lib/check.py b/counterpartylib/lib/check.py
@@ -68,11 +68,17 @@
     500000: {'ledger_hash': '703632461af220490f6f9cb006a4741ed07d54dd8d5f0da81297308934745819', 'txlist_hash': '5f32a0d9c49c7788ce0f154c72e9e227c42f7d1ab8a2ff5031701fd46c15eec5'},
     550000: {'ledger_hash': '042f52c7944512e4386dd4a3a5c4666ae1ba6234ef9d7d7c14bcba1b39bd75c7', 'txlist_hash': '362613cc234336cb30f645894f3587db636c8b9cba45a01e74934e349063714c'},
     600000: {'ledger_hash': '5dfb8ca53d6820f268378cc4313890c14e86ed12623616bfce3800b288749847', 'txlist_hash': 'f7c4f03135d68d139ad4152d46f0a9ffa44c4f2a562d2b7abe15055a43b91dc2'},
-    650000: {'ledger_hash': '451a04386454fe0f9f1ff81770f70c762be675b88d1cec4fc1ec3d1fbe74d08c', 'txlist_hash': '7ac0d7fa2ec4553ca83c3687b9e16367f468c315df838aef55ae4fd1135adae9'},
-    700000: {'ledger_hash': 'aecd0c7e882b1770402678a96c413a3c7eb0141c3832e1a2f0ec13a0fa2e7e15', 'txlist_hash': '14acc1dd95aff8348b47866487f85e689dd40920178cf56cdd455d768dbad5cd'},
-    750000: {'ledger_hash': '955fb9d714eb6c05e3d64f05ddb2f2ff18423e1e16c05dfc6aea8838ce71b807', 'txlist_hash': '54f06faab3e94d30fda98df2a95f3f0a7ad7527b7301bfc824ba847c1b06bd17'},
-    800000: {'ledger_hash': '0a29c190507ba4cfbeb028e79060352788317143bb907532b0c202c25bf9b2b8', 'txlist_hash': 'd251bb3f1b7be7d6e912bbcb83e3bd554ce82e2e5fd9e725114107a4e2864602'},
-    850000: {'ledger_hash': '681aa15524dc39d864787362d1eac60a48b10a1d8a689fed4ef33e6f2780ba4d', 'txlist_hash': '94a37d05251ab4749722976929b888853b1b7662595955f94dd8dc97c50cb032'},
+    650000: {'ledger_hash': '3716fd39dc5e30867b1e78233b9b920e0a106932b03675e8c6a86fce99c87c59', 'txlist_hash': 'be8173266f3720bdc38b609d5418a728d62c416e31d6295218bc9cf10c69d412'},
+    700000: {'ledger_hash': 'a9948e54915a1bcfb034e59ce88d613ee5f7d3b798b9800c9e81d7e430729924', 'txlist_hash': '2d6c8995bb256c0a68507f322f84d02c335cbdd6ba4e329809905abefede8a4d'},
+    750000: {'ledger_hash': 'a0ad188ad5f037b3bd26114ebd5ace57d05d767ce5641e318998c9c193ce396a', 'txlist_hash': '6fa10581c0717ce6a6c9938905d2ae78a1265a66ee1b5dd0532025c6f562d95f'},
+    800000: {'ledger_hash': '2186547319dc5ca9eee16eb5ea298e3d4c8eced0309651e4c5d625c135db44e2', 'txlist_hash': '0e72e166609229e067601eadb7d77b179e363bd75453dc7bad0a8c52e6a15a1b'},
+    850000: {'ledger_hash': '203430101150a7ed630fc3ce895eec9925571ef8a3e66d9e5b7f0b0e8249dbb9', 'txlist_hash': '485615f9a45e147c5b30452f7ca6cecb704cc88f6e35845e4a5c6499daa29b9a'},
+    900000: {'ledger_hash': 'c82f249b9a793b507be4b76d02e23219378b4ed2dd2db76362143f881cd3c161', 'txlist_hash': '4088837b4f94ffe0bc542895eee32ff1496f94b7d6e1ed1091303042c320f8b2'},
+    950000: {'ledger_hash': '1b9b69f19f313c99a26c835cf3d9701ea477c3edd5b31812300658d9cc1ac351', 'txlist_hash': 'ec7435440523babb05eee73e68b72ef0aa66f9b4a04e92b62ed775780227ec8a'},
+    1000000: {'ledger_hash': '5c899627965bf3f334f3eaa22a17c7b127705b255f6e933c7bdf8aa21218d1bc', 'txlist_hash': 'e54ccfdb1966d4c0cd7b4a87a4a6ac91e41a0b5973356091d0e17bd165fc51c8'},
+    1050000: {'ledger_hash': '53580232f2ad500829906c6af21f4834574e432702e8cb8d3b5cf5398c073563', 'txlist_hash': '95ae6ae5b618faeda6176da5a18db9277d6e62309ff0837b4c21686e4c889918'},
+    1100000: {'ledger_hash': 'ba6f165f6453e0d58e2bed6fe19f6b1d36e1878a9b248e266c035ce51e0f6270', 'txlist_hash': '2d3c3e184f58e991f9a8f5e3d11ea5a96240649ff57a3042e7688917486b0a06'},
+    1150000: {'ledger_hash': 'f263033868a450338e26a72a6323230740889b76f488df60a43b92232b710779', 'txlist_hash': '63d35f206c2eb3337f0e3f698f714be898ce217c79f87da76f43ebed67de2071'},
 }
 
 

diff --git a/counterpartylib/lib/config.py b/counterpartylib/lib/config.py
@@ -7,12 +7,13 @@
 # Versions
 VERSION_MAJOR = 9
 VERSION_MINOR = 55
-VERSION_REVISION = 2
+VERSION_REVISION = 3
 VERSION_STRING = str(VERSION_MAJOR) + '.' + str(VERSION_MINOR) + '.' + str(VERSION_REVISION)
 
 
 # Counterparty protocol
 TXTYPE_FORMAT = '>I'
+SHORT_TXTYPE_FORMAT = 'B'
 
 TWO_WEEKS = 2 * 7 * 24 * 3600
 MAX_EXPIRATION = 4 * 2016   # Two months
@@ -109,4 +110,7 @@
 DEFAULT_UTXO_LOCKS_MAX_ADDRESSES = 1000
 DEFAULT_UTXO_LOCKS_MAX_AGE = 3.0 #in seconds
 
+ADDRESS_OPTION_REQUIRE_MEMO = 1
+ADDRESS_OPTION_MAX_VALUE = ADDRESS_OPTION_REQUIRE_MEMO # Or list of all the address options
+
 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
diff --git a/counterpartylib/lib/database.py b/counterpartylib/lib/database.py
@@ -63,6 +63,11 @@ def exectracer(cursor, sql, bindings):
         if category == 'issuances' and not util.enabled('subassets'):
             if isinstance(bindings, dict) and 'asset_longname' in bindings: del bindings['asset_longname']
 
+        # don't include memo as part of the messages hash 
+        #   until enhanced_sends are enabled
+        if category == 'sends' and not util.enabled('enhanced_sends'):
+            if isinstance(bindings, dict) and 'memo' in bindings: del bindings['memo']
+
         sorted_bindings = sorted(bindings.items()) if isinstance(bindings, dict) else [bindings,] 
         BLOCK_MESSAGES.append('{}{}{}'.format(command, category, sorted_bindings))
 

diff --git a/counterpartylib/lib/message_type.py b/counterpartylib/lib/message_type.py
@@ -0,0 +1,34 @@
+import logging
+logger = logging.getLogger(__name__)
+import struct
+
+from counterpartylib.lib import config
+from counterpartylib.lib import util
+
+def pack(message_type_id, block_index=None):
+    # pack message ID into 1 byte if not zero
+    if util.enabled('short_tx_type_id', block_index) and message_type_id > 0 and message_type_id < 256:
+        return struct.pack(config.SHORT_TXTYPE_FORMAT, message_type_id)
+
+    # pack into 4 bytes
+    return struct.pack(config.TXTYPE_FORMAT, message_type_id)
+
+# retuns both the message type id and the remainder of the message data
+def unpack(packed_data, block_index=None):
+    message_type_id = None
+    message_remainder = None
+
+    if len(packed_data) > 1:
+        # try to read 1 byte first
+        if util.enabled('short_tx_type_id', block_index):
+            message_type_id = struct.unpack(config.SHORT_TXTYPE_FORMAT, packed_data[:1])[0]
+            if message_type_id > 0:
+                message_remainder = packed_data[1:]
+                return (message_type_id, message_remainder)
+
+    # First message byte was 0.  We will read 4 bytes
+    if len(packed_data) > 4:
+        message_type_id = struct.unpack(config.TXTYPE_FORMAT, packed_data[:4])[0]
+        message_remainder = packed_data[4:]
+
+    return (message_type_id, message_remainder)
diff --git a/counterpartylib/lib/messages/bet.py b/counterpartylib/lib/messages/bet.py
@@ -24,6 +24,7 @@
 from counterpartylib.lib import exceptions
 from counterpartylib.lib import util
 from counterpartylib.lib import log
+from counterpartylib.lib import message_type
 
 FORMAT = '>HIQQdII'
 LENGTH = 2 + 4 + 8 + 8 + 8 + 4 + 4
@@ -301,7 +302,7 @@ def compose (db, source, feed_address, bet_type, deadline, wager_quantity,
         problems.append('deadline passed')
     if problems: raise exceptions.ComposeError(problems)
 
-    data = struct.pack(config.TXTYPE_FORMAT, ID)
+    data = message_type.pack(ID)
     data += struct.pack(FORMAT, bet_type, deadline,
                         wager_quantity, counterwager_quantity, target_value,
                         leverage, expiration)

diff --git a/counterpartylib/lib/messages/broadcast.py b/counterpartylib/lib/messages/broadcast.py
@@ -35,6 +35,7 @@
 from counterpartylib.lib import config
 from counterpartylib.lib import util
 from counterpartylib.lib import log
+from counterpartylib.lib import message_type
 from . import (bet)
 
 FORMAT = '>IdI'
@@ -103,6 +104,19 @@ def validate (db, source, timestamp, value, fee_fraction_int, text, block_index)
         if len(text) > 52:
             problems.append('text too long')
 
+    if util.enabled('options_require_memo') and text and text.lower().startswith('options'):
+        ops_spl = text.split(" ")
+        if len(ops_spl) == 2:
+            try:
+                options_int = int(ops_spl.pop())
+
+                if (options_int > config.MAX_INT) or (options_int < 0):
+                    problems.append('integer overflow')
+                elif options_int > config.ADDRESS_OPTION_MAX_VALUE:
+                    problems.append('options out of range')
+            except:
+                problems.append('options not an integer')
+
     return problems
 
 def compose (db, source, timestamp, value, fee_fraction, text):
@@ -113,7 +127,7 @@ def compose (db, source, timestamp, value, fee_fraction, text):
     problems = validate(db, source, timestamp, value, fee_fraction_int, text, util.CURRENT_BLOCK_INDEX)
     if problems: raise exceptions.ComposeError(problems)
 
-    data = struct.pack(config.TXTYPE_FORMAT, ID)
+    data = message_type.pack(ID)
 
     # always use custom length byte instead of problematic usage of 52p format and make sure to encode('utf-8') for length
     if util.enabled('broadcast_pack_text'):
@@ -197,6 +211,29 @@ def parse (db, tx, message):
     if util.enabled('broadcast_invalid_check') and status != 'valid':
         return
 
+    # Options? if the status is invalid the previous if should have catched it
+    if util.enabled('options_require_memo'):
+        if text and text.lower().startswith('options'):
+            ops_spl = text.split(" ")
+            if len(ops_spl) == 2:
+                change_ops = False
+                options_int = 0
+                try:
+                    options_int = int(ops_spl.pop())
+                    change_ops = True
+                except:
+                    pass
+
+                if change_ops:
+                    op_bindings = {
+                                'block_index': tx['block_index'],
+                                'address': tx['source'],
+                                'options': options_int
+                               }
+                    sql = 'insert or replace into addresses(address, options, block_index) values(:address, :options, :block_index)'
+                    cursor = db.cursor()
+                    cursor.execute(sql, op_bindings)
+
     # Negative values (default to ignore).
     if value is None or value < 0:
         # Cancel Open Bets?