app doc changes, benchmark scripts changes

parse/block_report.py

@@ -1,13 +1,28 @@
 import sys, urllib, json, datetime, time
-from parse_common import uvarint, parseutc, formatbytes, readjson, getmaxheight
+import matplotlib.pyplot as plt
+from common_parse_utils import uvarint, parseutc, formatbytes, readjson, getmaxheight
+
+def get_num_txs(json):
+	return json["result"]["block"]["header"]["num_txs"]
 
 if len(sys.argv) < 2:
-	print "usage: python parse_block.py host:port minheight [maxheight]"
+	print "usage: python parse_block.py host:port [report_name [minheight [maxheight]]]"
 	sys.exit()
 
 tmaddress = sys.argv[1]
-minheight = int(sys.argv[2])
+report_name = sys.argv[2] if len(sys.argv) > 2 else ""
-maxheight = int(sys.argv[3]) if len(sys.argv) > 3 else getmaxheight(tmaddress)
+if len(sys.argv) > 4:
+	maxheight = int(sys.argv[4])
+else:
+	maxheight = getmaxheight(tmaddress)
+	while maxheight >= 3 and get_num_txs(readjson(tmaddress + "/block?height=%d" % maxheight)) == 0:
+		maxheight -= 1
+if len(sys.argv) > 3:
+	minheight = int(sys.argv[3])
+else:
+	minheight = maxheight
+	while minheight >= 3 and get_num_txs(readjson(tmaddress + "/block?height=%d" % (minheight - 1))) > 0:
+		minheight -= 1
 	
 accsize = 0
 acclatency = 0
@@ -24,7 +39,7 @@ maxblocksize = 0
 txstat = []
 for height in range(minheight, maxheight + 1):
 	data = readjson(tmaddress + "/block?height=%d" % height)
-	numtxs = data["result"]["block"]["header"]["num_txs"]
+	numtxs = get_num_txs(data)
 	
 	blocktimetxt = data["result"]["block"]["header"]["time"]
 	blocktime = parseutc(blocktimetxt)
@@ -39,7 +54,7 @@ for height in range(minheight, maxheight + 1):
 	txs = data["result"]["block"]["data"]["txs"]
 	if txs:
 		for index, txhex in enumerate(txs):
-			txbytes = bytearray.fromhex(txhex)
+			txbytes = bytearray.fromhex(txhex)# if re.fullmatch(r"^[0-9a-fA-F]$", txhex) is not None 
 			key = chr(txbytes[0]) if chr(txbytes[1]) == '=' else "*" 
 			connindex = uvarint(txbytes[2:8])
 			txnumber = uvarint(txbytes[8:16])
@@ -63,11 +78,12 @@ for height in range(minheight, maxheight + 1):
 			txstat.append((txtime, 1))
 			txstat.append((blocktime, -1))
 			if index < 5:
-				print key, connindex, txnumber, hostnamehash, txtimetxt, latency
+				print txtimetxt, latency
+				#print key, connindex, txnumber, hostnamehash, txtimetxt, latency
 
 print "Transactions:    ", txcount, "=", formatbytes(accsize)
 print "                 ", "%.3f s" % (lasttx - firsttx), "from", datetime.datetime.fromtimestamp(firsttx), "to", datetime.datetime.fromtimestamp(lasttx)
-print "Blocks:          ", blockcount
+print "Blocks:          ", "%d: from %d to %d" % (blockcount, minheight, maxheight)
 print "                 ", "%.3f s" % (lastblock - firstblock), "from", datetime.datetime.fromtimestamp(firstblock), "to", datetime.datetime.fromtimestamp(lastblock)
 print "Tx send rate:    ", "%.3f tx/s" % (txcount / (lasttx - firsttx)), "=", formatbytes(accsize / (lasttx - firsttx)) + "/s"
 print "Tx throughput:   ", "%.3f tx/s" % (txcount / (lastblock - firsttx)), "=", formatbytes(accsize / (lastblock - firsttx)) + "/s"
@@ -87,15 +103,20 @@ for i in range(steps + 1):
 		cursum += txstat[curindex][1] 
 		curindex += 1
 	stepstat.append(cursum)
-import matplotlib.pyplot as plt
 f = plt.figure(figsize=(15, 5))
 plt.plot([i * (lastblock - firsttx) / steps for i in range(steps + 1)], stepstat)
-plt.title("Duration: %.1f s, Tx size: %s, Tx send rate: %.3f tx/s = %s/s, Tx throughput: %.3f tx/s = %s/s" %
+long_title = "Duration: %.1f s, Tx size: %s, Tx send rate: %.3f tx/s = %s/s, Tx throughput: %.3f tx/s = %s/s" % \
-	(lasttx - firsttx, formatbytes(accsize / txcount), 
+	(lasttx - firsttx, formatbytes(accsize / txcount), \
-	txcount / (lasttx - firsttx), formatbytes(accsize / (lasttx - firsttx)),
+	txcount / (lasttx - firsttx), formatbytes(accsize / (lasttx - firsttx)), \
-	txcount / (lastblock - firsttx), formatbytes(accsize / (lastblock - firsttx))))
+	txcount / (lastblock - firsttx), formatbytes(accsize / (lastblock - firsttx)))
+#plt.title(long_title)
+plt.title(report_name)
 plt.xlabel("seconds from first tx")
 plt.ylabel("txs in backlog")
-f.savefig("tdmnt-stat-%d-%d-%d-%.1f-%.0f-%.0f.pdf" % 
+
-	(minheight, maxheight, maxblocksize, lasttx - firsttx, accsize / txcount, txcount / (lasttx - firsttx)), bbox_inches='tight')
+if report_name != "":
+	long_filename = "tdmnt-stat-%d-%d-%d-%.1f-%.0f-%.0f.png" % \
+		(minheight, maxheight, maxblocksize, lasttx - firsttx, accsize / txcount, txcount / (lasttx - firsttx))
+	#f.savefig(long_filename, bbox_inches='tight')
+	f.savefig(report_name + ".png", bbox_inches='tight')
 plt.show(block=True)

parse/parse_chain.py

@@ -1,5 +1,5 @@
 import sys, urllib, json, datetime, time
-from parse_common import parseutc, readjson, getmaxheight
+from common_parse_utils import parseutc, readjson, getmaxheight
 
 if len(sys.argv) < 2:
 	print "usage: python parse_chain.py host:port [minheight]"
@@ -10,8 +10,9 @@ tmaddress = sys.argv[1]
 maxheight = getmaxheight(tmaddress)
 minheight = int(sys.argv[2]) if len(sys.argv) > 2 else max(1, maxheight - 49)
 
-lastempty = -1
+lastnonempty = -1
 last_fetched_height = minheight - 1
+print "%6s %26s %7s %7s %8s %30s %30s %30s %30s %30s" % ("height", "block time", "txs", "acc.txs", "app_hash", "tx1", "tx2", "tx3", "tx4", "tx5")
 for height in range(minheight, maxheight + 1):
 	if height > last_fetched_height:
 		last_fetched_height = min(height + blocks_fetch - 1, maxheight)
@@ -26,11 +27,12 @@ for height in range(minheight, maxheight + 1):
 	blocktimetxt = data["time"]
 	blocktime = parseutc(blocktimetxt)
 
-	if numtxs > 0 or height == maxheight:
+	if numtxs > 0 or height == maxheight or height == lastnonempty + 1:
 		blockdata = readjson(tmaddress + "/block?height=%d" % height)
 		txs = blockdata["result"]["block"]["data"]["txs"]
 		txsummary = ""
 		if txs:
+			lastnonempty = height
 			for tx in txs[0:5]:
 				txstr = tx.decode('base64')
 				if len(txstr) > 30:
@@ -39,8 +41,7 @@ for height in range(minheight, maxheight + 1):
 					txsummary += "%30s " % txstr
 			if len(txs) > 5:
 				txsummary += "..."
-		print "%5s: %s %7d %7d %s... %s" % (height, datetime.datetime.fromtimestamp(blocktime), numtxs, totaltxs, app_hash[0:6], txsummary)
+		print "%5s: %s %7d %7d" % (height, datetime.datetime.fromtimestamp(blocktime), numtxs, totaltxs), "0x" + app_hash[0:6], txsummary
 	else:
-		if lastempty < height - 1:
+		if height == lastnonempty + 2:
 			print "..."
-		lastempty = height

parse/report_to_file.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+python block_report.py $1 "$2" $3 $4 | tee "$2.txt"

tmdemoapp/docs/README.md

@@ -9,51 +9,58 @@ Because every computation is verified by the cluster nodes and computation outco
 ![Nodes in cluster](cluster_nodes.png)
 
 ## Motivation
-The application is a proof-of-concept of a system with the following properties:
+The application is a proof-of-concept of a distributed system with the following properties:
-* Support of arbitrary deterministic operations: simple reads/writes as well as complex and time-consuming calculations
+* Support of arbitrary deterministic operations: simple reads/writes as well as complex and time-consuming calculations.
-* Having high throughput (1000 transaction per second) and low latency (1-2 seconds) of operations
+* High availability: tolerance to simultaneous failures or Byzantine actions of some subset of nodes.
-* Having every operation response verifiable (and thus trusted by the client)
+* High throughput (1000 transactions per second) and low latency (1-2 seconds) of operations.
-* Ability to restore liveness and even safety after violating typical Byzantine quorum requirements (1/3 of failed nodes and more) – every node could rapidly detect problems in the blockchain or disagreement with the rest of nodes
+* Small blockchain finality time (several seconds).
+* Extremely low probability of consistency violation.
 
 ## Architecture overview
-The application use [Tendermint](https://github.com/tendermint/tendermint) platform which provides:
+The entire application is distributed over set of machines having the following roles:
-* Distributed transaction cache
+* client-side **Proxy**, which originates the requests to the cluster
-* Blockchain (to store transactions persistently)
+* cluster **Node**, which serves the requests
-* Consensus logic (to reach agreement about the order of transactions)
+* the **Judge**, which is in charge of resolving complicated disagreements between nodes
-* Peer-to-peer communication layer (between nodes)
 
-The application implements Tendermint's [ABCI interface](http://tendermint.readthedocs.io/projects/tools/en/master/abci-spec.html) to follow Tendermint's architecture which decomposes the application logic into 2 main parts:
+The application uses blockchain approach to decompose the application logic into 2 main parts:
-* Distributed replicated transaction log (managed by Tendermint)
+* replicated transaction log
-* And state machine with business logic (manages by the application itself).
+* state machine with the domain-specific logic
+This decomposition allows to simplify development process. This modularization is not only logical but also phisical: transaction log and state machine run in separate processes, developed in different languages.
 
-The application is written in Scala 2.12. It is compatible with `Tendermint v0.19.x` and uses `com.github.jtendermint.jabci` for Java ABCI definitions.
+The application uses [Tendermint](https://github.com/tendermint/tendermint) platform which provides replicated transaction log components (**TM Core**), in particular:
+* distributed transaction cache (*Mempool*)
+* blockchain (to store transactions persistently)
+* Byzantine-resistant **consensus** logic (to reach agreement about the order of transactions)
+* peer-to-peer layer to communicate with another nodes
+* entry point for client requests
 
-It models in-memory key-value string storage. Keys here are hierarchical, `/`-separated. This key hierarchy is *merkelized*, so every node stores Merkle hash of its associated value (if present) and its children.
+To perform domain-specific logic the application uses its own **State machine** implementing Tendermint's [ABCI interface](http://tendermint.readthedocs.io/projects/tools/en/master/abci-spec.html) to follow Tendermint's architecture. It is written in Scala 2.12, compatible with `Tendermint v0.19.x` and uses `com.github.jtendermint.jabci` for Java ABCI definitions.
-
-![Key-values in cluster](cluster_key_value.png)
-
-The entire application consists of the following components:
-* **Client** proxy (**Proxy**)
-* Node Tendermint (**TM** or **TM Core**) with notable modules: Mempool, Consensus and Query
-* Node ABCI Application itself (**App** or **ABCI App**)
 
 ![Architecture](architecture.png)
 
-### Operations
+As the application is intended to run normally in presence of some failures, including Byzantine failures, the following principles used:
-Clients typically interact with Fluence via some local **Proxy**. This Proxy might be implemented in any language (because it communicates with TM Core by queries RPC endpoints), for example, Scala implementation of *some operation* may look like `def doSomeOperation(req: SomeRequest): SomeResponse`. This application uses simple (but powerful) Python `query.sh` script as Proxy to perform arbitrary operations, including:
+* Every operation result is verifiable (and thus trusted by the client).
-* Write transactions
+* The application uses Tendermint's implementation of Byzantine fault-tolerant consensus algorithms to provide **safety** and **liveness** without external interference to the cluster – while more than 2/3 of cluster nodes are correct (*quorum* exists).
-`tx a/b=10`
+* It can restore liveness and even safety after violating quorum requirements – every node could rapidly detect problems with the blockchain or disagreement with the rest of nodes and raise a dispute to the **Judge**.
-* Key read queries
-`get a/b`
-* Arbitrary operations
-`op factorial:a/b`
-* Writing results of arbitrary operations
-`tx a/c=factorial:a/b`
 
-In terms of Tendermint architecture, these operations implemented in the following way:
+The **State machine** maintains its state using in-memory key-value string storage. Keys here are hierarchical, `/`-separated. This key tree is *merkelized*, so every key stores Merkle hash of its associated value (if present) and its children keys.
-* All writes (simple and containing operations) are Tendermint *transactions*: a transaction changes the application state and stored to the blockchain (the correctness ensured by the consensus).
+
-* Reads are Tendermint *ABCI queries*: they do not change the application state, App just return requested value together with Merkle proof (the correctness ensured by Merkle proof).
+![Key-values in cluster](cluster_key_value.png)
-* Operations are combinations of writes and reads: to perform operation trustfully, Proxy first requests writing the result of operation to some key and then queries its value (the correctness ensured by both the consensus and Merkle proof).
+
+### Operations
+Tendermint architecture suppose that the client typically interacts with the Application via the local **Proxy**. This application uses Python `query.py` script as client-side Proxy to request arbitrary operations to the cluster, including:
+* Simple `put` requests which specify a target key and a constant as its new value: `put a/b=10`.
+* Computational `put` requests which specify that a target key should be assigned to the result of some operation (with arguments) invocation: `put a/c=factorial:a/b`.
+* Requests to obtain a result of running an arbitrary operation: `run factorial:a/b`.
+* Requests to read the value of specified key: `get a/b`.
+
+`get` operations do not change the state of the application. They are implemented via Tendermint ABCI queries. As the result of a such query the **State machine** returns the value of the requested key together with the Merkle proof.
+
+`put` operations are *effectful* and change the application state explicitly. They are implemented via Tendermint **transactions** that combined into **blocks**. **TM Core** sends a transaction to the **State machine** and **State machine** applies this transaction to its state, typically changing the target key.
+
+`get` and `put` operations use different techniques to prove to the client that the operation is actually invoked and its result is correct. `get`s take advantage of Merkelized structure of the application state and provide Merkle proof of the result correctness. Any `put` invocation leads to adding the corresponding transaction to the blockchain, so observing this transaction in a correctly signed block means that there is a quorum in the cluster regarding this transaction's invocation.
+
+`run` operations are also *effectul*. They are implemented as combinations of `put`s and `get`s: to perform operation trustfully, **Proxy** first requests `put`-ting the result of operation to some key and then queries its value. Thus the correctness is ensured by both the consensus and Merkle proof.
 
 ## Installation and run
 For single-node run just launch the application:
@@ -75,7 +82,7 @@ In case Tendermint launched first, it would periodically try to connect the app
 
 After successful launch the client can communicate with the application via sending RPC calls to TM Core on local `46678` port.
 
-### Cluster
+### Local cluster
 There are scripts that automate deployment and running 4 Application nodes on the local machine.
 
 ```bash

tmdemoapp/src/main/scala/kvstore/ServerMonitor.scala

@@ -28,7 +28,7 @@ class ServerMonitor(handler: ABCIHandler) extends Runnable {
       val localHash = state.lastVerifiableAppHash.map(MerkleUtil.merkleHashToHex).getOrElse("")
 
       if (clusterHash != localHash) {
-        throw new IllegalStateException("Cluster quorum has unexpected app hash for previous block")
+        throw new IllegalStateException(s"Cluster quorum has unexpected app hash for previous block '$clusterHash' '$localHash'")
       }
 
       val timeWaiting = timeWaitingForEmptyBlock(state)