Merge pull request #15 from raphaelvrosa/master

Adds scaling and fuzzing (and support for Iroha)

.gitignore

@@ -1,10 +1,4 @@
-build/containernet/
-build/git/
-build/oflops/
-build/oftest/
-build/openflow/
-build/pox/
-examples/logs/
+build/
 .vscode/
 *.pyc
 html/
@@ -13,9 +7,7 @@ latex/
 *.egg-info
 *~
 dist/
-examples/fabric/fabric_configs/
-examples/.run.sh.lock
-
+.vagrant/
 
 # =======================
 # github/gitignore/Python

Dockerfile

@@ -0,0 +1,31 @@
+FROM ubuntu:20.04
+
+RUN apt update && apt install -y python3.8 python3-setuptools python3-pip make
+
+RUN python3.8 -m pip install \
+        "asyncio>=3.4.3" \
+        "protobuf>=3.11.0" \
+        "grpclib>=0.3.1" \
+        "grpcio-tools>=1.26.0" \
+        "PyYAML>=5.1.2" \
+        "networkx>=2.4" \
+        "psutil>=5.6.7" \
+        "docker<=4.1.0" \
+        "prompt_toolkit<=3.0.6" \
+        "paramiko<=2.6.0" \
+        "scp<=0.13.2"
+
+RUN mkdir -p /umbra
+
+COPY ./deps /umbra/deps
+COPY ./examples /umbra/examples
+COPY ./umbra /umbra/umbra
+COPY ./setup.py /umbra/
+COPY ./Makefile /umbra/
+COPY ./README.md /umbra/
+
+WORKDIR /umbra
+
+RUN make install
+
+CMD [ "umbra-cli" ]

Makefile

@@ -0,0 +1,171 @@
+.PHONY: clean-pyc clean-build
+.DEFAULT_GOAL := help
+
+TEST_PATH=./umbra/tests
+
+DEPS_FOLDER=./deps
+DEPS=deps.sh
+DEPS_FABRIC=deps_fabric.sh
+DEPS_IROHA=deps_iroha.sh
+
+AUX_FOLDER=./aux
+AUX_MONITOR=monitor.sh
+
+
+install-fabric:
+	sh -c "cd $(DEPS_FOLDER) && ./$(DEPS_FABRIC) install && cd - "
+
+uninstall-fabric:
+	sh -c "cd $(DEPS_FOLDER) && ./$(DEPS_FABRIC) uninstall && cd - "
+
+install-iroha:
+	sh -c "cd $(DEPS_FOLDER) && ./$(DEPS_IROHA) install && cd - "
+
+uninstall-iroha:
+	sh -c "cd $(DEPS_FOLDER) && ./$(DEPS_IROHA) uninstall && cd - "
+
+vagrant-requirements-virtualbox:
+	sudo apt install -y vagrant
+
+	sudo apt install -y virtualbox
+	sudo apt-get install -y linux-headers-generic
+	sudo dpkg-reconfigure virtualbox-dkms
+	sudo dpkg-reconfigure virtualbox
+	sudo modprobe vboxdrv
+	sudo modprobe vboxnetflt
+
+vagrant-requirements-libvirt:
+	sudo apt install -y vagrant
+	sudo apt-get install -y qemu-kvm qemu-utils libvirt-daemon bridge-utils virt-manager libguestfs-tools virtinst rsync
+	sudo apt-get install -y ruby-libvirt libvirt-dev
+	vagrant plugin install vagrant-libvirt
+
+requirements:
+	sudo apt update && sudo apt install -y git python3.8 python3-setuptools python3-pip
+	mkdir -p /tmp/umbra
+	mkdir -p /tmp/umbra/logs
+	mkdir -p /tmp/umbra/source
+
+install-deps:
+	sh -c "cd $(DEPS_FOLDER) && ./$(DEPS) install && cd - "
+
+uninstall-deps:
+	sh -c "cd $(DEPS_FOLDER) && ./$(DEPS) uninstall && cd - "
+
+install: requirements
+	sudo /usr/bin/python3.8 setup.py develop
+	# sudo /usr/bin/python3.8 -m pip install .
+
+develop: requirements
+	sudo /usr/bin/python3.8 setup.py develop
+
+uninstall:
+	sudo /usr/bin/python3.8 -m pip uninstall -y umbra
+
+clean-pyc:
+	sudo sh -c "find . -name '*.pyc' -exec rm --force {} + "
+	sudo sh -c "find . -name '*.pyo' -exec rm --force {} + "
+	sudo sh -c "find . -name '*~' -exec rm --force  {} + "
+
+clean-build:
+	sudo sh -c "rm --force --recursive build/"
+	sudo sh -c "rm --force --recursive dist/"
+	sudo sh -c "rm --force --recursive *.egg-info"
+
+clean: clean-build clean-pyc
+	sudo rm -R /tmp/umbra
+
+isort:
+	sh -c "isort --skip-glob=.tox --recursive . "
+
+lint:
+	flake8 --exclude=.tox
+
+test: clean-pyc
+	py.test --verbose --color=yes $(TEST_PATH)
+
+run: install
+	umbra-cli
+
+docker-build:
+	docker build \
+	--tag=umbra:latest .
+
+docker-run: docker-build
+	docker run \
+	--detach=false \
+	--name=umbra \
+	umbra:latest umbra-cli
+
+vagrant-run-virtualbox: requirements-vagrant-virtualbox
+	vagrant up --provider virtualbox
+
+vagrant-run-libvirt: vagrant-requirements-libvirt
+	vagrant up --provider libvirt
+
+start-aux-monitor:
+	sh -c "cd $(AUX_FOLDER) && ./$(AUX_MONITOR) start && cd - "
+
+stop-aux-monitor:
+	sh -c "cd $(AUX_FOLDER) && ./$(AUX_MONITOR) stop && cd - "
+
+all: requirements install run
+
+
+
+help:
+	@echo ""
+	@echo "     				umbra makefile help"
+	@echo "           ---------------------------------------------------------"
+	@echo ""
+	@echo "     requirements"
+	@echo "         Install umbra requirements (i.e., python3.8 python3-setuptools python3-pip)."
+	@echo "     vagrant-requirements-virtualbox"
+	@echo "         Install the requirements to build a virtual machine with umbra installed using virtualbox."
+	@echo "     vagrant-requirements-libvirt"
+	@echo "         Install the requirements to build a virtual machine with umbra installed using qemu-kvm."
+	@echo "     install-fabric"
+	@echo "         Install fabric dependencies (images, python SDK, golang)."
+	@echo "     uninstall-fabric"
+	@echo "         Remove fabric dependencies (images, python SDK, ...)."
+	@echo "     install-iroha"
+	@echo "         Install iroha dependencies (images, python SDK)."
+	@echo "     uninstall-iroha"
+	@echo "         Remove iroha dependencies (images, python SDK, ...)."
+	@echo "     install-deps"
+	@echo "         Install umbra-scenario dependencies (i.e., containernet)."
+	@echo "     uninstall-deps"
+	@echo "         Uninstall umbra-scenario dependencies (i.e., containernet)."
+	@echo "     develop"
+	@echo "         Run python3.8 setup.py develop."
+	@echo "     install"
+	@echo "         Setup with pip install . ."
+	@echo "     uninstall"
+	@echo "         Remove umbra with pip uninstall umbra."
+	@echo "     clean-pyc"
+	@echo "         Remove python artifacts."
+	@echo "     clean-build"
+	@echo "         Remove build artifacts."
+	@echo "     clean"
+	@echo "         Remove build and python artifacts (clean-build clean-pyc)."
+	@echo "     isort"
+	@echo "         Sort import statements."
+	@echo "     lint"
+	@echo "         Check style with flake8."
+	@echo "     test"
+	@echo "         Run py.test in umbra/tests folder"
+	@echo "     run"
+	@echo "         Run the umbra-cli on your local machine."
+	@echo "     docker-run"
+	@echo "         Build and run the umbra-cli in a Docker container."
+	@echo "     vagrant-run-virtualbox"
+	@echo "         Build and run a virtual machine with umbra installed using virtualbox."
+	@echo "     vagrant-run-libvirt"
+	@echo "         Build and run a virtual machine with umbra installed using qemu-kvm."
+	@echo "     start-aux-monitor"
+	@echo "         Create and start the aux containers for monitoring (influxdb and graphana)."
+	@echo "     stop-aux-monitor"
+	@echo "         Stop and remove the aux containers for monitoring (influxdb and graphana)."
+	@echo ""
+	@echo "           ---------------------------------------------------------"
+	@echo ""

README.md

@@ -2,16 +2,58 @@
 
 An emulation platform for Hyperledger blockchains.
 
-To understand and run Umbra, please take a look at the [Umbra Docs](https://umbra-labs.readthedocs.io/en/latest/).
+This lab is intended to be an ongoing project to provide a research tool for understanding and improving the Hyperledger blockchain platforms as well as for conducting future research in consensus algorithms and scalability. This lab is not intended to be a shipping product but rather an ongoing collaboration between academic, corporate, and hobbyist researchers improving upon existing capabilities and also trying new things with Hyperledger blockchains.
 
-Also, if needed download an [Umbra virtual machine](https://intrig.dca.fee.unicamp.br:8840/owncloud/index.php/s/aKYbP8TxlxymRHY). 
-Use VirtualBox to import the source file umbra.ova. The recommended settings are 4 cores and 4GB of memory. To use the Umbra virtual machine, login: umbra / password: lives. The umbra source folder is located inside the home directory (/home/umbra/umbra). All the umbra components are in place and installed. Check the [documentation example](https://umbra-labs.readthedocs.io/en/latest/example.html) to give it a try.
+This project is the outcome of the 2019 and 2020 Hyperledger internship projects (e.g., [Hyperledger-Labs Umbra](https://wiki.hyperledger.org/display/INTERN/Hyperledger+Umbra%3A+Simulating+Hyperledger+Blockchains+using+Mininet)).
 
-This project is the outcome of the 2019 [internship project named Hyperledger Umbra](https://wiki.hyperledger.org/display/INTERN/Hyperledger+Umbra%3A+Simulating+Hyperledger+Blockchains+using+Mininet).
 
-This lab is intended to be an ongoing project to provide a research tool for understanding and improving the Hyperledger blockchain platforms as well as for conducting future research in consensus algorithms and scalability.
+# Requirements
+
+Umbra is developed and tested in Ubuntu 20.04.
+
+The hardware requirements needed for umbra will depend on the scale of the experiments to be played with it (e.g., number of nodes in a blockchain topology, amount of events triggered into the topology, the topology resource settings, etc).
+For a simple setup its recommended to have available at least: 4 logical cpu cores, 8 GB of RAM, and 10GB of storage.
+
+To begin with umbra there is the need to only install the make package. All the python packages needed by umbra are listed in the file requirements.txt. 
+
+```bash
+sudo apt install make
+```
+
+
+# Installing
+
+First of all, obtain the umbra source code.
+
+```bash
+git clone https://github.com/raphaelvrosa/umbra
+cd umbra
+```
+
+Given the installation of the umbra requirements, the commands below install umbra.
+
+```bash
+sudo make install
+```
+
+Umbra can also be installed using a Vagrant virtual machine, either using qemu-kvm/libvirt or virtualbox as providers, as stated below:
+
+```bash
+sudo make vagrant-run-libvirt # Installs umbra in a virtual machine using qemu-kvm/libvirt, and turn it up
+sudo make vagrant-run-virtualbox # Installs umbra in a virtual machine using virtualbox, and turn it up
+```
+
+As any other Vagrant virtual machine (box), it is possible to interact with it via vagrant commands, such as using ssh to login (i.e., vagrant ssh).
+
+
+# Running
+
+Having umbra installed, it is possible to experiment with it using the provided examples, inside examples/ folder.
+
+The README.md file in the examples folder provide the instructions to experiment with umbra.
+
+Inside the configuration files of the examples, the comments provide guidelines on how to compose umbra experiments.
 
-This lab is not intended to be a shipping product but rather an ongoing collaboration between academic, corporate, and hobbyist researchers improving upon existing capabilities and also trying new things with Hyperledger blockchains.
 
 # License
 

Vagrantfile

@@ -0,0 +1,48 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+
+  config.ssh.insert_key = false
+
+  config.vm.define "umbra_virtualbox" do |umbra_virtualbox|
+
+    umbra_virtualbox.vm.box = "ubuntu/focal64"    
+    umbra_virtualbox.vm.network :private_network, ip: "192.168.56.101"
+    umbra_virtualbox.vm.hostname = "umbra"
+
+    umbra_virtualbox.vm.provider "virtualbox" do |vb|
+      vb.name = "umbra"
+      opts = ["modifyvm", :id, "--natdnshostresolver1", "on"]
+      vb.customize opts
+      vb.cpus = 2
+      vb.memory = "2048"
+    end
+
+    umbra_virtualbox.vm.synced_folder ".", "/home/umbra", mount_options: ["dmode=775"]
+    umbra_virtualbox.vm.provision :shell, inline: "apt update && apt install -y make"
+    umbra_virtualbox.vm.provision :shell, inline: "cd /home/umbra && make install"
+
+  end
+
+
+  config.vm.define "umbra_libvirt" do |umbra_libvirt|
+
+    umbra_libvirt.vm.box = "generic/ubuntu2004"
+    umbra_libvirt.vm.hostname = "umbra"
+
+    umbra_libvirt.vm.provider "libvirt" do |libvirt|
+      libvirt.cpus = 2
+      libvirt.memory = 2048
+    end
+
+    umbra_libvirt.vm.network :private_network, ip: "192.168.121.101"
+
+
+    umbra_libvirt.vm.synced_folder ".", "/home/umbra", mount_options: ["dmode=775"]
+    umbra_libvirt.vm.provision :shell, inline: "apt update && apt install -y make"
+    umbra_libvirt.vm.provision :shell, inline: "cd /home/umbra && make install"
+
+  end
+
+end