Merge pull request #921 from CounterpartyXCP/develop

9.55.0

.travis.yml

@@ -1,15 +1,56 @@
-language: python
-python:
-- "3.4"
-install:
-- pip install -r requirements.txt
-- pip install python-coveralls
-- python setup.py install --with-serpent
-script: cd counterpartylib; py.test --skiptestbook=all --cov-config=../.coveragerc --cov-report=term-missing --cov=./
-notifications:
-  slack:
-    secure: cl0G5fWZNnIK5VQ6BPZ4RtwVO2/nfvX/zjhljfF6cQ20OKyvyiJHq+e67fl2pkCKDlqlEqSWaQJ6G52YzUpDQmf+o7qpH6YnkFxSqYp5h4YF1hJw4sCHJ7bVSLrjogWxU8QaNvH7YpL4YkGe+WOau2FgoIZOcMt6hhKrvWclfuc=
+sudo: required
+
+services:
+- docker
+
+env:
+  global:
+    # define DOCKER_USERNAME and DOCKER_PASSWORD
+    - secure: "NZZCFpX/ubtrMHxU3C6+HkV+mrSUIAqRLXKySBiUJeiKaOsiVUl1AMd1cKQOrDp59dIFmjMvH+HGFbAKt3QvTcTImx0yye09bsXNKUKIQbVElnoZvkP6pZSxXAFcmjZdUGTfMSMkLukrXHgLb5ob2ziBCinJBoG57tgcUJnoZTc="
+    - secure: "d4TXRnQ4QffUFILVN6Hn7DA55unGDk0H6vZsmDRxP3ucLrUC/j8ko3Tm9jDeHZvVxGUe/e0dNXQ1WjaialCpB/h+rrnCEFExk9+oeUhhzn9GVOp488jqoiIwQxYM8cEh5I8Fh8lztBwaLnaI/MgRougFuOs3Ocf3v2IhBgvcwGs="
+
+before_install:
+- sudo apt-get -qq update
+# add coveralls
+- sudo apt-get -y install python3 python3-pip
+- sudo pip3 install coveralls
+# upgrade docker, for build argument support
+- sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-engine
+- docker version
+- docker ps -a
+# get the current PR and branch name
+- sudo apt-get -y install jq
+- export PR=https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/$TRAVIS_PULL_REQUEST
+- export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo `curl -s $PR | jq -r .head.ref`; fi)
+- echo $PR
+- echo $BRANCH
+# determine the docker image tag name ("nopush" being a placeholder to signify the image will not be pushed to dockerhub)
+- export DOCKER_TAG="nopush"
+- |
+  if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+    if [ "$BRANCH" == "master" ]; then export DOCKER_TAG="latest"; fi
+    if [ "$BRANCH" == "develop" ]; then export DOCKER_TAG="develop"; fi
+  fi
+- echo $DOCKER_TAG
+# try to check out the same branch for counterparty-lib (if it exists), and fallback to the develop branch if not
+- if curl --output /dev/null --silent --head --fail "https://github.com/CounterpartyXCP/counterparty-cli/tree/$BRANCH"; then export DOCKERFILE_CLI_BRANCH="$BRANCH"; else export DOCKERFILE_CLI_BRANCH="develop"; fi
+- echo $DOCKERFILE_CLI_BRANCH
+# build docker image
+- docker build -t counterparty/counterparty-server:$DOCKER_TAG --build-arg CLI_BRANCH="$DOCKERFILE_CLI_BRANCH" .
+
+script:
+# run the test suite in the docker container we just made
+- docker run --entrypoint=/bin/bash counterparty/counterparty-server:$DOCKER_TAG -c "cd /counterparty-lib/counterpartylib; py.test --verbose --skiptestbook=all --cov-config=../.coveragerc --cov-report=term-missing --cov=./"
+
 after_success:
 - coveralls
+# if it's master or develop, then push the appropirately tagged docker image (otherwise don't push anything)
+- |
+  if [ "$DOCKER_TAG" != "nopush" ]; then
+    docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
+    docker push counterparty/counterparty-server:$DOCKER_TAG;
+  fi
 
-sudo: required
+notifications:
+  slack:
+    secure: cl0G5fWZNnIK5VQ6BPZ4RtwVO2/nfvX/zjhljfF6cQ20OKyvyiJHq+e67fl2pkCKDlqlEqSWaQJ6G52YzUpDQmf+o7qpH6YnkFxSqYp5h4YF1hJw4sCHJ7bVSLrjogWxU8QaNvH7YpL4YkGe+WOau2FgoIZOcMt6hhKrvWclfuc=

Dockerfile

@@ -0,0 +1,38 @@
+FROM counterparty/base
+
+MAINTAINER Counterparty Developers <[email protected]>
+
+# Install counterparty-lib
+COPY . /counterparty-lib
+WORKDIR /counterparty-lib
+RUN pip3 install -r requirements.txt
+RUN python3 setup.py develop
+RUN python3 setup.py install_apsw
+RUN python3 setup.py install_serpent
+
+# Install counterparty-cli
+# NOTE: By default, check out the counterparty-cli master branch. You can override the BRANCH build arg for a different
+# branch (as you should check out the same branch as what you have with counterparty-lib, or a compatible one)
+# NOTE2: In the future, counterparty-lib and counterparty-cli will go back to being one repo...
+ARG CLI_BRANCH=master
+ENV CLI_BRANCH ${CLI_BRANCH}
+RUN git clone -b ${CLI_BRANCH} https://github.com/CounterpartyXCP/counterparty-cli.git /counterparty-cli
+WORKDIR /counterparty-cli
+RUN pip3 install -r requirements.txt
+RUN python3 setup.py develop
+
+# Additional setup
+COPY docker/server.conf /root/.config/counterparty/server.conf
+COPY docker/start.sh /usr/local/bin/start.sh
+RUN chmod a+x /usr/local/bin/start.sh
+WORKDIR /
+
+# Pull the mainnet and testnet DB boostraps
+RUN counterparty-server bootstrap --quiet
+RUN counterparty-server --testnet bootstrap --quiet
+
+EXPOSE 4000 14000
+
+# NOTE: Defaults to running on mainnet, specify -e TESTNET=1 to start up on testnet
+ENTRYPOINT ["start.sh"]
+

README.md

@@ -4,40 +4,82 @@
 [![Latest Version](https://pypip.in/version/counterparty-lib/badge.svg)](https://pypi.python.org/pypi/counterparty-lib/)
 [![License](https://pypip.in/license/counterparty-lib/badge.svg)](https://pypi.python.org/pypi/counterparty-lib/)
 [![Slack Status](http://slack.counterparty.io/badge.svg)](http://slack.counterparty.io)
+[![Docker Pulls](https://img.shields.io/docker/pulls/counterparty/counterparty-server.svg?maxAge=2592000)](https://hub.docker.com/r/counterparty/counterparty-server/)
 
 
 # Description
 `counterparty-lib` is the reference implementation of the [Counterparty Protocol](https://counterparty.io).
 
-**Note:** for the command-line interface that used to be called `counterpartyd`, see [`counterparty-cli`](https://github.com/CounterpartyXCP/counterparty-cli).
+**Note:** for the command-line interface to `counterparty-lib`, see [`counterparty-cli`](https://github.com/CounterpartyXCP/counterparty-cli).
 
 
-# Requirements
-* [Patched Bitcoin Core](https://github.com/btcdrak/bitcoin/releases) with the following options set:
+# Installation
 
-	```
-	rpcuser=bitcoinrpc
-	rpcpassword=<password>
-	server=1
-	txindex=1
-	addrindex=1
-	rpcthreads=1000
-	rpctimeout=300
-	minrelaytxfee=0.00005
-	limitfreerelay=0
-	```
+For a simple Docker-based install of the Counterparty software stack, see [this guide](http://counterparty.io/docs/federated_node/).
 
 
-# Installation
+# Manual installation
+
+Download the newest [patched Bitcoin Core](https://github.com/btcdrak/bitcoin/releases) and create
+a `bitcoin.conf` file with the following options:
+
+```
+rpcuser=bitcoinrpc
+rpcpassword=rpc
+server=1
+txindex=1
+addrindex=1
+rpcthreads=100
+rpctimeout=300
+```
+
+Then, download and install `counterparty-lib`:
 
 ```
 $ git clone https://github.com/CounterpartyXCP/counterparty-lib.git
 $ cd counterparty-lib
-$ python3 setup.py install
+$ sudo pip3 install -r requirements.txt
+$ sudo python3 setup.py install
+```
+
+Followed by `counterparty-cli`:
+
+```
+$ git clone https://github.com/CounterpartyXCP/counterparty-cli.git
+$ cd counterparty-cli
+$ sudo pip3 install -r requirements.txt
+$ sudo python3 setup.py install
 ```
 
+Then, launch the daemon via:
+
+```
+$ counterparty-server bootstrap
+$ counterparty-server --backend-password=rpc start
+```
+
+# Basic Usage
+
+## Via command-line 
+
+(Requires `counterparty-cli` to be installed.)
+
+* The first time you run the server, you may bootstrap the local database with:
+	`$ counterparty-server bootstrap`
+
+* Start the server with:
+	`$ counterparty-server start`
+
+* Check the status of the server with:
+	`$ counterparty-client getinfo`
+
+* For additional command-line arguments and options:
+	`$ counterparty-server --help`
+	`$ counterparty-client --help`
 
-# Usage
+## Via Python
+
+Bare usage from Python is also possible, without installing `counterparty-cli`:
 
 ```
 $ python3
@@ -46,6 +88,68 @@ $ python3
 >>> server.start_all(db)
 ```
 
+# Configuration and Operation
+
+The paths to the **configuration** files, **log** files and **database** files are printed to the screen when starting the server in ‘verbose’ mode:
+	`$ counterparty-server --verbose start`
+
+By default, the **configuration files** are named `server.conf` and `client.conf` and located in the following directories:
+
+* Linux: `~/.config/counterparty/`
+* Windows: `%APPDATA%\Counterparty\`
+
+Client and Server log files are named `counterparty.client.[testnet.]log` and `counterparty.server.[testnet.]log`, and located in the following directories:
+
+* Linux: `~/.cache/counterparty/log/`
+* Windows: `%APPDATA%\Local\Counterparty\counterparty\Logs`
+
+Counterparty API activity is logged in `server.[testnet.]api.log` and `client.[testnet.]api.log`.
+
+Counterparty database files are by default named `counterparty.[testnet.]db` and located in the following directories:
+
+* Linux: `~/.local/share/counterparty`
+* Windows: `%APPDATA%\Roaming\Counterparty\counterparty`
+
+## Configuration File Format
+
+Manual configuration is not necessary for most use cases.
+
+A `counterparty-server` configuration file looks like this:
+
+	[Default]
+	backend-name = addrindex
+	backend-user = <user>
+	backend-password = <password>
+	rpc-host = 0.0.0.0
+	rpc-user = <rpcuser>
+	rpc-password = <rpcpassword>
+
+A `counterparty-client` configuration file looks like this:
+
+	[Default]
+	wallet-name = bitcoincore
+	wallet-connect = localhost
+	wallet-user = <user>
+	wallet-password = <password>
+	counterparty-rpc-connect = localhost
+	counterparty-rpc-user = <rpcuser>
+	counterparty-rpc-password = <password>
+
+
+# Developer notes
+
+## Versioning
+
+* Major version changes require a full (automatic) rebuild of the database.
+* Minor version changes require a(n automatic) database reparse.
+* All protocol changes are retroactive on testnet.
+
+## Continuous integration
+ - TravisCI is setup to run all tests with 1 command and generate a coverage report and let `python-coveralls` parse and upload it.
+   It does runs with `--skiptestbook=all` so it will not do the reparsing of the bootstrap files.
+ - CircleCI is setup to split the tests as much as possible to make it easier to read the error reports.
+   It also runs the `integration_test.test_book` tests, which reparse the bootstrap files.
+
 
 # Further Reading
 

circle.yml

@@ -1,14 +1,26 @@
+# Keep this as non-docker for now, to give some variety to our test configurations, as travis builds with docker
 dependencies:
     override:
+        - rm -rf /home/ubuntu/virtualenvs/venv-*/bin/serpent
+        - rm -rf /home/ubuntu/virtualenvs/venv-*/lib/python3.4/site-packages/apsw*
         - pip install -r requirements.txt
         - python setup.py install --with-serpent
+        - python -c "import apsw; print(apsw.apswversion())"
 test:
     override:
-        - cd counterpartylib; py.test --verbose --capture=no --cov-config=../.coveragerc --cov-report=term-missing --cov=./
+        - py.test --verbose --capture=no counterpartylib/test/unit_test.py
+        - py.test --verbose --capture=no counterpartylib/test/parse_block_test.py
+        - py.test --verbose --capture=no --skiptestbook=all counterpartylib/test/integration_test.py
+        - py.test --verbose --capture=no counterpartylib/test/contracts_test.py
+        - py.test --verbose --capture=no --skiptestbook=mainnet -k test_book counterpartylib/test/reparse_test.py
+        - py.test --verbose --capture=no --skiptestbook=testnet -k test_book counterpartylib/test/reparse_test.py
+        - py.test --verbose --capture=no counterpartylib/test/database_version_test.py
 machine:
     pre:
-        - mkdir -p ~/.local/share/counterparty
-        - cd ~/.local/share/counterparty; wget https://s3.amazonaws.com/counterparty-bootstrap/counterparty-db.latest.tar.gz; tar xvzf counterparty-db.latest.tar.gz;
-        - cd ~/.local/share/counterparty; wget https://s3.amazonaws.com/counterparty-bootstrap/counterparty-db-testnet.latest.tar.gz; tar xvzf counterparty-db-testnet.latest.tar.gz;
+        - mkdir -p ~/.local/share/counterparty;
+        - wget https://s3.amazonaws.com/counterparty-bootstrap/counterparty-db-testnet.latest.tar.gz -O ~/.local/share/counterparty/counterparty-db-testnet.latest.tar.gz;
+        - tar -C ~/.local/share/counterparty -xvzf ~/.local/share/counterparty/counterparty-db-testnet.latest.tar.gz;
+        - wget https://s3.amazonaws.com/counterparty-bootstrap/counterparty-db.latest.tar.gz -O ~/.local/share/counterparty/counterparty-db.latest.tar.gz;
+        - tar -C ~/.local/share/counterparty -xvzf ~/.local/share/counterparty/counterparty-db.latest.tar.gz;
     python:
         version: 3.4.1

counterpartylib/lib/api.py

@@ -23,7 +23,7 @@
 import struct
 import apsw
 import flask
-from flask.ext.httpauth import HTTPBasicAuth
+from flask_httpauth import HTTPBasicAuth
 import jsonrpc
 from jsonrpc import dispatcher
 from jsonrpc.exceptions import JSONRPCDispatchException
@@ -68,7 +68,7 @@
 COMMONS_ARGS = ['encoding', 'fee_per_kb', 'regular_dust_size',
                 'multisig_dust_size', 'op_return_value', 'pubkey',
                 'allow_unconfirmed_inputs', 'fee', 'fee_provided',
-                'unspent_tx_hash','custom_inputs']
+                'unspent_tx_hash', 'custom_inputs', 'dust_return_pubkey', 'disable_utxo_locks']
 
 API_MAX_LOG_SIZE = 10 * 1024 * 1024 #max log size of 20 MB before rotation (make configurable later)
 API_MAX_LOG_COUNT = 10
@@ -261,14 +261,15 @@ def value_to_marker(value):
 def compose_transaction(db, name, params,
                         encoding='auto',
                         fee_per_kb=config.DEFAULT_FEE_PER_KB,
+                        estimate_fee_per_kb=None, estimate_fee_per_kb_nblocks=config.ESTIMATE_FEE_NBLOCKS,
                         regular_dust_size=config.DEFAULT_REGULAR_DUST_SIZE,
                         multisig_dust_size=config.DEFAULT_MULTISIG_DUST_SIZE,
                         op_return_value=config.DEFAULT_OP_RETURN_VALUE,
                         pubkey=None,
                         allow_unconfirmed_inputs=False,
                         fee=None,
                         fee_provided=0,
-                        unspent_tx_hash=None, custom_inputs=None):
+                        unspent_tx_hash=None, custom_inputs=None, dust_return_pubkey=None, disable_utxo_locks=False):
     """Create and return a transaction."""
 
     # Get provided pubkeys.
@@ -303,14 +304,17 @@ def compose_transaction(db, name, params,
     tx_info = compose_method(db, **params)
     return transaction.construct(db, tx_info, encoding=encoding,
                                         fee_per_kb=fee_per_kb,
+                                        estimate_fee_per_kb=estimate_fee_per_kb, estimate_fee_per_kb_nblocks=estimate_fee_per_kb_nblocks,
                                         regular_dust_size=regular_dust_size,
                                         multisig_dust_size=multisig_dust_size,
                                         op_return_value=op_return_value,
                                         provided_pubkeys=provided_pubkeys,
                                         allow_unconfirmed_inputs=allow_unconfirmed_inputs,
                                         exact_fee=fee,
                                         fee_provided=fee_provided,
-                                        unspent_tx_hash=unspent_tx_hash, custom_inputs=custom_inputs)
+                                        unspent_tx_hash=unspent_tx_hash, custom_inputs=custom_inputs,
+                                        dust_return_pubkey=dust_return_pubkey,
+                                        disable_utxo_locks=disable_utxo_locks)
 
 def conditional_decorator(decorator, condition):
     """Checks the condition and if True applies specified decorator."""
@@ -565,6 +569,10 @@ def get_block_info(block_index):
             cursor.close()
             return block
 
+        @dispatcher.add_method
+        def fee_per_kb(nblocks=config.ESTIMATE_FEE_NBLOCKS):
+            return backend.fee_per_kb(nblocks)
+
         @dispatcher.add_method
         def get_blocks(block_indexes, min_message_index=None):
             """fetches block info and messages for the specified block indexes
@@ -721,7 +729,7 @@ def _set_cors_headers(response):
             if not config.RPC_NO_ALLOW_CORS:
                 response.headers['Access-Control-Allow-Origin'] = '*'
                 response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
-                response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'
+                response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization'
 
         @app.route('/', defaults={'args_path': ''}, methods=['GET', 'POST', 'OPTIONS'])
         @app.route('/<path:args_path>',  methods=['GET', 'POST', 'OPTIONS'])