From 28adca9fe529d20420bd457d438e8b511fe69b1e Mon Sep 17 00:00:00 2001
From: Dmitry Sergeev <dmi.sergeev@gmail.com>
Date: Fri, 22 Jun 2018 00:08:53 +0500
Subject: [PATCH] /parse -> /cli

---
 cli/block_report.py       | 122 ++++++++++++++++++++++++++++++++++++++
 cli/common_parse_utils.py |  46 ++++++++++++++
 cli/demo_queries.sh       |  10 ++++
 cli/parse_chain.py        |  48 +++++++++++++++
 cli/query.py              |  96 ++++++++++++++++++++++++++++++
 cli/report_to_file.sh     |   2 +
 6 files changed, 324 insertions(+)
 create mode 100644 cli/block_report.py
 create mode 100644 cli/common_parse_utils.py
 create mode 100755 cli/demo_queries.sh
 create mode 100644 cli/parse_chain.py
 create mode 100644 cli/query.py
 create mode 100755 cli/report_to_file.sh

diff --git a/cli/block_report.py b/cli/block_report.py
new file mode 100644
index 0000000..826bc82
--- /dev/null
+++ b/cli/block_report.py
@@ -0,0 +1,122 @@
+import sys, urllib, json, datetime, time
+import matplotlib.pyplot as plt
+from common_parse_utils import uvarint, parseutc, formatbytes, readjson, getmaxheight
+
+def get_num_txs(json):
+	return json["result"]["block"]["header"]["num_txs"]
+
+if len(sys.argv) < 2:
+	print "usage: python parse_block.py host:port [report_name [minheight [maxheight]]]"
+	sys.exit()
+
+tmaddress = sys.argv[1]
+report_name = sys.argv[2] if len(sys.argv) > 2 else ""
+if len(sys.argv) > 4:
+	maxheight = int(sys.argv[4])
+else:
+	maxheight = getmaxheight(tmaddress)
+	while maxheight >= 3 and get_num_txs(readjson(tmaddress + "/block?height=%d" % maxheight)) == 0:
+		maxheight -= 1
+if len(sys.argv) > 3:
+	minheight = int(sys.argv[3])
+else:
+	minheight = maxheight
+	while minheight >= 3 and get_num_txs(readjson(tmaddress + "/block?height=%d" % (minheight - 1))) > 0:
+		minheight -= 1
+	
+accsize = 0
+acclatency = 0
+minlatency = 1e20
+maxlatency = 0
+txcount = 0
+blockcount = 0
+firsttx = 1e20
+lasttx = 0
+firstblock = 1e20
+lastblock = 0
+maxblocksize = 0
+
+txstat = []
+for height in range(minheight, maxheight + 1):
+	data = readjson(tmaddress + "/block?height=%d" % height)
+	numtxs = get_num_txs(data)
+	
+	blocktimetxt = data["result"]["block"]["header"]["time"]
+	blocktime = parseutc(blocktimetxt)
+
+	if numtxs > 0:
+		firstblock = min(firstblock, blocktime)
+		lastblock = max(lastblock, blocktime)
+		blockcount += 1
+		maxblocksize = max(maxblocksize, numtxs)
+
+	print height, numtxs, blocktimetxt
+	txs = data["result"]["block"]["data"]["txs"]
+	if txs:
+		for index, txhex in enumerate(txs):
+			txbytes = bytearray.fromhex(txhex)# if re.fullmatch(r"^[0-9a-fA-F]$", txhex) is not None 
+			key = chr(txbytes[0]) if chr(txbytes[1]) == '=' else "*" 
+			connindex = uvarint(txbytes[2:8])
+			txnumber = uvarint(txbytes[8:16])
+			hostnamehash = txhex[32:64]
+			
+			txtime = uvarint(txbytes[32:40]) / 1e6
+			if txtime < 1e9:
+				txtime *= 1e6 # legacy support
+			latency = blocktime - txtime
+
+			accsize += len(txbytes)
+			acclatency += latency
+			minlatency = min(minlatency, latency)
+			maxlatency = max(maxlatency, latency)
+			txcount += 1
+			firsttx = min(firsttx, txtime)
+			lasttx = max(lasttx, txtime)
+
+			txtimetxt = datetime.datetime.fromtimestamp(txtime)
+
+			txstat.append((txtime, 1))
+			txstat.append((blocktime, -1))
+			if index < 5:
+				print txtimetxt, latency
+				#print key, connindex, txnumber, hostnamehash, txtimetxt, latency
+
+print "Transactions:    ", txcount, "=", formatbytes(accsize)
+print "                 ", "%.3f s" % (lasttx - firsttx), "from", datetime.datetime.fromtimestamp(firsttx), "to", datetime.datetime.fromtimestamp(lasttx)
+print "Blocks:          ", "%d: from %d to %d" % (blockcount, minheight, maxheight)
+print "                 ", "%.3f s" % (lastblock - firstblock), "from", datetime.datetime.fromtimestamp(firstblock), "to", datetime.datetime.fromtimestamp(lastblock)
+print "Tx send rate:    ", "%.3f tx/s" % (txcount / (lasttx - firsttx)), "=", formatbytes(accsize / (lasttx - firsttx)) + "/s"
+print "Tx throughput:   ", "%.3f tx/s" % (txcount / (lastblock - firsttx)), "=", formatbytes(accsize / (lastblock - firsttx)) + "/s"
+print "Block throughput:", "%.3f block/s" % (blockcount / (lastblock - firsttx))
+print "Avg tx latency:  ", "%.3f s" % (acclatency / txcount)
+print "Min tx latency:  ", "%.3f s" % minlatency
+print "Max tx latency:  ", "%.3f s" % maxlatency
+
+txstat = sorted(txstat)
+cursum = 0
+curindex = 0
+steps = 1000
+stepstat = []
+for i in range(steps + 1):
+	t = firsttx + (lastblock - firsttx) / steps * i
+	while curindex < len(txstat) and txstat[curindex][0] <= t:
+		cursum += txstat[curindex][1] 
+		curindex += 1
+	stepstat.append(cursum)
+f = plt.figure(figsize=(15, 5))
+plt.plot([i * (lastblock - firsttx) / steps for i in range(steps + 1)], stepstat)
+long_title = "Duration: %.1f s, Tx size: %s, Tx send rate: %.3f tx/s = %s/s, Tx throughput: %.3f tx/s = %s/s" % \
+	(lasttx - firsttx, formatbytes(accsize / txcount), \
+	txcount / (lasttx - firsttx), formatbytes(accsize / (lasttx - firsttx)), \
+	txcount / (lastblock - firsttx), formatbytes(accsize / (lastblock - firsttx)))
+#plt.title(long_title)
+plt.title(report_name)
+plt.xlabel("seconds from first tx")
+plt.ylabel("txs in backlog")
+
+if report_name != "":
+	long_filename = "tdmnt-stat-%d-%d-%d-%.1f-%.0f-%.0f.png" % \
+		(minheight, maxheight, maxblocksize, lasttx - firsttx, accsize / txcount, txcount / (lasttx - firsttx))
+	#f.savefig(long_filename, bbox_inches='tight')
+	f.savefig(report_name + ".png", bbox_inches='tight')
+plt.show(block=True)
\ No newline at end of file
diff --git a/cli/common_parse_utils.py b/cli/common_parse_utils.py
new file mode 100644
index 0000000..2ad7e0d
--- /dev/null
+++ b/cli/common_parse_utils.py
@@ -0,0 +1,46 @@
+import sys, urllib, json, datetime, time
+
+def uvarint(buf):
+	x = long(0)
+	s = 0
+	for b in buf:
+		if b < 0x80:
+			return x | long(b) << s
+		x |= long(b & 0x7f) << s
+		s += 7
+	return 0
+
+def parseutc(utctxt):
+	#tz conversion may be wrong
+	now_timestamp = time.time()
+	offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
+	dt, _, tail = utctxt.partition(".")
+	if tail == "":
+		dt, _, _ = utctxt.partition("Z")
+		tail = "0Z"
+	pure = int((datetime.datetime.strptime(dt, '%Y-%m-%dT%H:%M:%S') + offset).strftime("%s"))
+	ns = int(tail.rstrip("Z").ljust(9, "0"), 10)
+	return pure + ns / 1e9
+	
+def formatbytes(value):
+	if value < 1024:
+		return "%.0f B" % value
+	elif value < 1024 * 1024:
+		return "%.3f KiB" % (value / 1024.0)
+	else:
+		return "%.3f MiB" % (value / 1024.0 / 1024.0)
+
+def readjson(url):
+	response = urllib.urlopen("http://" + url)
+	return json.loads(response.read())
+
+def getsyncinfo(tmaddress):
+	status = readjson(tmaddress + "/status")["result"]
+	if "sync_info" in status: # compatibility
+		return status["sync_info"]
+	else:
+		return status
+
+def getmaxheight(tmaddress):
+	return getsyncinfo(tmaddress)["latest_block_height"]
+
diff --git a/cli/demo_queries.sh b/cli/demo_queries.sh
new file mode 100755
index 0000000..70e37ce
--- /dev/null
+++ b/cli/demo_queries.sh
@@ -0,0 +1,10 @@
+python query.py localhost:46257 put a/b=10
+python query.py localhost:46257 put "a/c=copy(a/b)"
+python query.py localhost:46257 put "a/d=increment(a/c)"
+python query.py localhost:46257 put "a/d=increment(a/c)###again"
+python query.py localhost:46257 put "a/e=sum(a/c,a/d)"
+python query.py localhost:46257 put "a/f=factorial(a/b)"
+python query.py localhost:46257 put "c/asum=hiersum(a)"
+python query.py localhost:46257 get a/e
+python query.py localhost:46257 put "0-200:b/@1/@0=1"
+python query.py localhost:46257 put "c/bsum=hiersum(b)"
diff --git a/cli/parse_chain.py b/cli/parse_chain.py
new file mode 100644
index 0000000..ddba470
--- /dev/null
+++ b/cli/parse_chain.py
@@ -0,0 +1,48 @@
+import sys, urllib, json, datetime, time
+from common_parse_utils import parseutc, readjson, getmaxheight
+
+if len(sys.argv) < 2:
+	print "usage: python parse_chain.py host:port [minheight]"
+	sys.exit()
+
+blocks_fetch = 20 # tendermint can't return more blocks
+tmaddress = sys.argv[1]
+maxheight = getmaxheight(tmaddress)
+minheight = int(sys.argv[2]) if len(sys.argv) > 2 else max(1, maxheight - 49)
+
+lastnonempty = -1
+last_fetched_height = minheight - 1
+print "%6s %26s %7s %7s %8s %30s %30s %30s %30s %30s" % ("height", "block time", "txs", "acc.txs", "app_hash", "tx1", "tx2", "tx3", "tx4", "tx5")
+for height in range(minheight, maxheight + 1):
+	if height > last_fetched_height:
+		last_fetched_height = min(height + blocks_fetch - 1, maxheight)
+		bulk_data = (readjson(tmaddress + "/blockchain?minHeight=%d&maxHeight=%d" % (height, last_fetched_height)))["result"]["block_metas"]
+
+	data = bulk_data[last_fetched_height - height]["header"]
+	
+	numtxs = data["num_txs"]
+	totaltxs = data["total_txs"]
+	app_hash = data["app_hash"]
+
+	blocktimetxt = data["time"]
+	blocktime = parseutc(blocktimetxt)
+
+	if numtxs > 0 or height == maxheight or height == lastnonempty + 1:
+		blockdata = readjson(tmaddress + "/block?height=%d" % height)
+		txs = blockdata["result"]["block"]["data"]["txs"]
+		txsummary = ""
+		if txs:
+			lastnonempty = height
+			for tx in txs[0:5]:
+				txstr = tx.decode('base64')
+				if len(txstr) > 30:
+					txsummary += "%27s... " % txstr[0:27]
+				else:
+					txsummary += "%30s " % txstr
+			if len(txs) > 5:
+				txsummary += "..."
+		app_hash_to_show = "0x" + app_hash[0:6] if app_hash != "" else "--------"
+		print "%5s: %s %7d %7d" % (height, datetime.datetime.fromtimestamp(blocktime), numtxs, totaltxs), app_hash_to_show, txsummary
+	else:
+		if height == lastnonempty + 2:
+			print "..."
diff --git a/cli/query.py b/cli/query.py
new file mode 100644
index 0000000..916a0d3
--- /dev/null
+++ b/cli/query.py
@@ -0,0 +1,96 @@
+import sys, urllib, json, datetime, time, hashlib, sha3
+from common_parse_utils import readjson, getsyncinfo, getmaxheight
+
+CMD_PUT = "fastput"
+CMD_CHECKED_PUT = "put"
+CMD_RUN = "run"
+CMD_GET_QUERY = "get"
+CMD_LS_QUERY = "ls"
+
+def verify_merkle_proof(result, proof, app_hash):
+	parts = proof.split(", ")
+	parts_len = len(parts)
+	for index in range(parts_len, -1, -1):
+		low_string = parts[index] if index < parts_len else result
+		low_hash = hashlib.sha3_256(low_string).hexdigest()
+		high_hashes = parts[index - 1].split(" ") if index > 0 else [app_hash.lower()]
+		if not any(low_hash in s for s in high_hashes):
+			return False
+	return True
+
+def checked_abci_query(tmaddress, height, command, query, tentative_info):
+	if getmaxheight(tmaddress) < height + 1:
+		return (height, None, None, None, False, "Cannot verify tentative '%s'! Height is not verifiable" % (info or ""))
+
+	apphash = readjson('%s/block?height=%d' % (tmaddress, height + 1))["result"]["block"]["header"]["app_hash"]
+	response = readjson('%s/abci_query?height=%d&data="%s:%s"' % (tmaddress, height, command, query))["result"]["response"]
+	(result, proof) = (
+		response["value"].decode('base64') if "value" in response else None,
+		response["proof"].decode('base64') if "proof" in response else None
+		)
+	if result is None:
+		return (height, result, proof, apphash, False, "Result is empty")
+	elif tentative_info is not None and result != tentative_info:
+		return (height, result, proof, apphash, False, "Verified result '%s' doesn't match tentative '%s'!" % (result, info))
+	elif proof is None:
+		return (height, result, proof, apphash, False, "No proof")
+	elif not verify_merkle_proof(result, proof, apphash) :
+		return (height, result, proof, apphash, False, "Proof is invalid")
+	else:
+		return (height, result, proof, apphash, True, "")
+
+def print_checked_abci_query(tmaddress, height, command, query, tentative_info):
+	(height, result, proof, apphash, success, message) = checked_abci_query(tmaddress, height, command, query, tentative_info)
+	print "HEIGHT:", height
+	print "HASH  :", apphash or "NOT_READY"
+	print "PROOF :", (proof or "NO_PROOF").upper()
+	print "RESULT:", result or "EMPTY"
+	if success:
+		print "OK"
+	else:
+		print "BAD   :", message
+
+def latest_provable_height(tmaddress):
+	return getsyncinfo(tmaddress)["latest_block_height"] - 1
+
+def wait_for_height(tmaddress, height):
+	for w in range(0, 5):
+		if getmaxheight(tmaddress) >= height:
+			break
+		time.sleep(1)
+
+
+if len(sys.argv) < 3:
+	print "usage: python query.py host:port command arg"
+	sys.exit()
+
+tmaddress = sys.argv[1]
+command = sys.argv[2]
+arg = sys.argv[3]
+if command in {CMD_PUT, CMD_CHECKED_PUT, CMD_RUN}:
+	if command == CMD_RUN:
+		query_key = "optarg"
+		tx = query_key + "=" + arg
+	else:
+		tx = arg
+		query_key = tx.split("=")[0]
+	response = readjson(tmaddress + '/broadcast_tx_commit?tx="' + tx + '"')
+	if "error" in response:
+		print "ERROR :", response["error"]["data"]
+	else:
+		height = response["result"]["height"]
+		if response["result"].get("deliver_tx", {}).get("code", "0") != "0":
+			print "HEIGHT:", height
+			print "BAD   :", log or "NO_LOG"
+		else:
+			info = response["result"].get("deliver_tx", {}).get("info")
+			if command in {CMD_CHECKED_PUT, CMD_RUN} and info is not None:
+				wait_for_height(tmaddress, height + 1)
+				print_checked_abci_query(tmaddress, height, "get", query_key, info)
+			else:
+				print "HEIGHT:", height
+				print "INFO:  ", info or "EMPTY"
+				print "OK"
+elif command in {CMD_GET_QUERY, CMD_LS_QUERY}:
+	height = latest_provable_height(tmaddress)
+	print_checked_abci_query(tmaddress, height, command, arg, None)
diff --git a/cli/report_to_file.sh b/cli/report_to_file.sh
new file mode 100755
index 0000000..1813b73
--- /dev/null
+++ b/cli/report_to_file.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python block_report.py $1 "$2" $3 $4 | tee "$2.txt"
\ No newline at end of file