Add sync tests

Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/index.js

@@ -0,0 +1,24 @@
+var System = java.lang.System;
+
+var iri = com.iota.iri;
+var Callable = iri.service.CallableRequest;
+var Response = iri.service.dto.IXIResponse;
+var ErrorResponse = iri.service.dto.ErrorResponse;
+
+var tangle = IOTA.tangle;
+
+
+
+function getTotalTransactions(){
+    var count = tangle.getCount(com.iota.iri.model.persistables.Transaction.class)
+
+    return Response.create({
+        total: count
+    });
+}
+
+
+
+
+API.put("getTotalTransactions", new Callable({ call: getTotalTransactions }))
+

Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "index.js"
+}

Nightly-Tests/Sync-Tests/config.yml

@@ -0,0 +1,61 @@
+default_args: &args
+  ['--testnet-coordinator',
+   'KSAFREMKHHYHSXNLGZPFVHANVHVMKWSGEAHGTXZCSQMXTCZXOGBLVPCWFKVAEQYDJMQALKZRKOTWLGBSC',
+   '--mwm',
+   '1',
+   '--milestone-start',
+   '0',
+   '--testnet-no-coo-validation',
+   'true',
+   '--testnet',
+   'true',
+   '--snapshot',
+   './snapshot.txt',
+   '--remote',
+   'true',
+   '--remote-limit-api',
+   '""',
+   '--zmq-enable-tcp',
+   'true',
+   '--zmq-port',
+   '5566',
+   '--local-snapshots-enabled',
+   'false'
+  ]
+
+default_ixi: &ixi
+  ['IXI/TotalTransactions.ixi']
+
+java_options: -agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n -javaagent:/opt/jacoco/lib/jacocoagent.jar=destfile=/iri/jacoco.exec,output=file,append=true,dumponexit=true
+
+defaults: &db_full
+  db: https://s3.eu-central-1.amazonaws.com/iotaledger-dbfiles/dev/SyncTestsDbComplete.tar
+  db_checksum: 12f8ae2dda157097e8929ee1ef26bf5b4f5b0f93d5b49f966bbb19afd6245e88
+  iri_args: *args
+  ixis: *ixi
+
+db_empty: &db_empty
+  db: https://s3.eu-central-1.amazonaws.com/iotaledger-dbfiles/dev/EmptyDB.tar
+  db_checksum: f453e80b82ad5abd25102833f03b39379667fada962ee376a7f629a027f83a88
+  iri_args: *args
+  ixis: *ixi
+
+nodes:
+  nodeA: #name
+    <<: *db_full
+    neighbors: 
+      - tcp://nodeB:15600
+      - tcp://nodeC:15600
+
+  nodeB:
+    <<: *db_full
+    neighbors: 
+      - tcp://nodeA:15600
+      - tcp://nodeC:15600
+
+  nodeC:
+    <<: *db_empty
+    neighbors: 
+      - tcp://nodeA:15600
+      - tcp://nodeB:15600
+

Nightly-Tests/Sync-Tests/createCluster.sh

@@ -0,0 +1,86 @@
+set -x
+
+trap ctrl_c INT
+
+function ctrl_c() {
+    echo -e "\nExit called by user"
+    for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}');do
+        echo "Pulling node logs..." 
+        kubectl logs $pod > ./SyncOutput/$DATE/$(kubectl get pod $pod -o jsonpath='{.metadata.labels.nodenum}').log
+    done
+
+    timeout 10 tiab/teardown_cluster.py -t $UUID -n $K8S_NAMESPACE
+    deactivate
+    exit
+}
+
+if [ "$#" -ne 1 ]; then
+    echo "Please specify an image to test"
+    exit 1
+fi
+
+set -x
+
+echo "Downloading apt requirments "
+sed 's/#.*//' requirements.txt | xargs apt-get install -y
+
+UUID="$(uuidgen)"
+K8S_NAMESPACE=$(kubectl config get-contexts $(kubectl config current-context) | tail -n+2 | awk '{print $5}')
+base_dir=$(pwd)
+IMAGE=$1
+DATE=$(date '+%Y-%m-%d')
+
+if [ ! -d tiab ]; then
+  git clone --depth 1 https://github.com/iotaledger/tiab tiab
+fi
+
+virtualenv -p python2 venv
+source ./venv/bin/activate 
+
+cd tiab
+git fetch
+git pull
+echo "tiab revision: "; git rev-parse HEAD
+pip install -r requirements.txt
+cd $base_dir/../
+
+echo "Installing python requirements"
+pip install --upgrade pip
+pip install -e .
+cd $base_dir
+
+ERROR=0
+
+python tiab/create_cluster.py -i $IMAGE -t $UUID -n $K8S_NAMESPACE -c config.yml -o output.yml -x $base_dir -e "apt update && apt install unzip && wget http://search.maven.org/remotecontent\?filepath\=org/jacoco/jacoco/0.8.4/jacoco-0.8.4.zip -O jacoco.zip && mkdir /opt/jacoco && unzip jacoco.zip -d /opt/jacoco" -d
+
+if [ $? -ne 0 ]; then
+    ERROR=1
+    python <<EOF
+import yaml
+for (key,value) in yaml.load(open('output.yml'))['nodes'].iteritems():
+  if value['status'] == 'Error':
+    print(value['log'])
+EOF
+fi
+
+if [ $ERROR -eq 0 ]; then
+  sleep 20
+  echo "Sending milestone"
+  python milestone.py -i 2006
+
+  echo "Running ZMQ Tests"
+  python runZMQScans.py -o ./SyncOutput
+
+  for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}');do
+    echo $pod
+    kubectl logs $pod > ./SyncOutput/$DATE/$(kubectl get pod $pod -o jsonpath='{.metadata.labels.nodenum}').log
+  done
+fi
+
+echo "Tearing down cluster" 
+timeout 10 tiab/teardown_cluster.py -t $UUID -n $K8S_NAMESPACE
+
+echo "Deactivating"
+deactivate
+
+exit $ERROR

Nightly-Tests/Sync-Tests/finalize.py

@@ -0,0 +1,26 @@
+from iota import BundleHash, Fragment
+from iota.crypto import HASH_LENGTH
+from iota.crypto.kerl import Kerl
+
+def finalize(bundle): 
+    sponge = Kerl()
+    last_index = len(bundle) - 1
+
+    for (i, txn) in enumerate(bundle):  # type: Tuple[int, ProposedTransaction]
+        txn.current_index = i
+        txn.last_index = last_index
+        sponge.absorb(txn.get_signature_validation_trytes().as_trits())
+
+    bundle_hash_trits = [0] * HASH_LENGTH  # type: MutableSequence[int]
+    sponge.squeeze(bundle_hash_trits)
+    bundle_hash = BundleHash.from_trits(bundle_hash_trits)
+
+    for txn in bundle:
+        txn.bundle_hash = bundle_hash
+        # Initialize signature/message fragment.
+        txn.signature_message_fragment = Fragment(txn.message or b'')
+
+
+
+
+

Nightly-Tests/Sync-Tests/milestone.py

@@ -0,0 +1,56 @@
+from iota import TryteString, trits_from_int, ProposedTransaction, ProposedBundle, Tag, Address, Iota
+import finalize
+from yaml import full_load
+import sys
+
+index = 1 
+args = sys.argv
+
+print("Starting")
+for arg in args: 
+    if arg == '-i':
+        index = args[int(args.index(arg)) + 1]
+
+def int_to_trytes(int_input, length):
+    trits = trits_from_int(int(int_input))
+    trytes = TryteString.from_trits(trits)
+    if len(trytes) < length:
+        trytes += '9' * (length - len(trytes))
+    return trytes
+
+yaml_file = full_load(open('./output.yml', 'r'))
+nodes = {}
+keys = yaml_file.keys()
+for key in keys:
+    if key != 'seeds' and key != 'defaults':
+        nodes[key] = yaml_file[key]
+
+host = yaml_file['nodes']['nodeA']['host']
+port = yaml_file['nodes']['nodeA']['ports']['api']
+
+api = Iota('http://{}:{}'.format(host, port))
+
+txn = ProposedTransaction(
+address = Address('KSAFREMKHHYHSXNLGZPFVHANVHVMKWSGEAHGTXZCSQMXTCZXOGBLVPCWFKVAEQYDJMQALKZRKOTWLGBSC'),
+value = 0
+)
+
+bundle = ProposedBundle()
+bundle.add_transaction(txn)
+bundle.add_transaction(txn)
+
+index_trytes = str(int_to_trytes(index, 9))
+
+bundle[0]._legacy_tag = Tag(index_trytes)
+
+finalize.finalize(bundle)
+bundle_trytes = bundle.as_tryte_strings()
+
+tips = api.get_transactions_to_approve(depth=3)
+branch = tips['branchTransaction']
+trunk = tips['trunkTransaction']
+
+
+milestone_bundle = api.attach_to_tangle(trunk,branch, bundle_trytes,3)
+api.broadcast_and_store(milestone_bundle.get('trytes'))
+print("Milestone {} attached and stored: {}".format(index, milestone_bundle))

Nightly-Tests/Sync-Tests/requirements.txt

@@ -0,0 +1,6 @@
+virtualenv
+python
+python-pip
+python-dev
+python-tk
+build-essential