Merge "[CE-62] Add network and network_config module"

src/agent/docker/cluster.py

@@ -8,7 +8,7 @@
 from agent import compose_up, compose_clean, compose_start, compose_stop, \
     compose_restart
 
-from common import FABRIC_VERSION, CONSENSUS_PLUGINS, \
+from common import NETWORK_TYPES, CONSENSUS_PLUGINS, \
     CONSENSUS_MODES, CLUSTER_SIZES
 
 from ..cluster_base import ClusterBase
@@ -27,7 +27,7 @@ def __init__(self):
         pass
 
     def create(self, cid, mapped_ports, host, user_id="",
-               fabric_version=FABRIC_VERSION[0],
+               fabric_version=NETWORK_TYPES[0],
                consensus_plugin=CONSENSUS_PLUGINS[0],
                consensus_mode=CONSENSUS_MODES[0], size=CLUSTER_SIZES[0]):
         """ Create a cluster based on given data
@@ -51,7 +51,7 @@ def create(self, cid, mapped_ports, host, user_id="",
         logger.debug("Start compose project with name={}".format(cid))
         containers = compose_up(
             name=cid, mapped_ports=mapped_ports, host=host,
-            fabric_version=fabric_version,
+            network_type=fabric_version,
             consensus_plugin=consensus_plugin, consensus_mode=consensus_mode,
             cluster_size=size)
         if not containers or len(containers) != size:

src/agent/docker/docker_swarm.py

@@ -12,7 +12,7 @@
 from common import log_handler, LOG_LEVEL
 from common import \
     HOST_TYPES, \
-    CLUSTER_NETWORK, FABRIC_VERSION, \
+    CLUSTER_NETWORK, NETWORK_TYPES, \
     CONSENSUS_PLUGINS, CONSENSUS_MODES, \
     CLUSTER_LOG_TYPES, CLUSTER_LOG_LEVEL, \
     CLUSTER_SIZES, \
@@ -308,7 +308,7 @@ def get_project(template_path):
 
 
 def _compose_set_env(name, daemon_url, mapped_ports=SERVICE_PORTS,
-                     fabric_version=FABRIC_VERSION[0],
+                     fabric_version=NETWORK_TYPES[0],
                      consensus_plugin=[0],
                      consensus_mode=CONSENSUS_MODES[0],
                      cluster_size=CLUSTER_SIZES[0],
@@ -323,7 +323,7 @@ def _compose_set_env(name, daemon_url, mapped_ports=SERVICE_PORTS,
         'VM_DOCKER_HOSTCONFIG_NETWORKMODE':
             CLUSTER_NETWORK + "_{}".format(consensus_plugin),
         'PEER_VALIDATOR_CONSENSUS_PLUGIN': consensus_plugin,
-        'FABRIC_VERSION': fabric_version,
+        'NETWORK_TYPES': fabric_version,
         'PBFT_GENERAL_MODE': consensus_mode,
         'PBFT_GENERAL_N': str(cluster_size),
         'PEER_NETWORKID': name,
@@ -339,7 +339,7 @@ def _compose_set_env(name, daemon_url, mapped_ports=SERVICE_PORTS,
 
 
 def compose_up(name, host, mapped_ports,
-               fabric_version=FABRIC_VERSION[0],
+               network_type=NETWORK_TYPES[0],
                consensus_plugin=CONSENSUS_PLUGINS[0],
                consensus_mode=CONSENSUS_MODES[0],
                cluster_size=CLUSTER_SIZES[0],
@@ -349,7 +349,7 @@ def compose_up(name, host, mapped_ports,
     :param name: The name of the cluster
     :param mapped_ports: The mapped ports list of the cluster
     :param host: Docker host obj
-    :param fabric_version: Fabric version
+    :param network_type: Fabric version
     :param consensus_plugin: Cluster consensus plugin
     :param consensus_mode: Cluster consensus mode
     :param cluster_size: the size of the cluster
@@ -367,19 +367,13 @@ def compose_up(name, host, mapped_ports,
     if log_type != CLUSTER_LOG_TYPES[0]:  # not local
         os.environ['SYSLOG_SERVER'] = log_server
 
-    _compose_set_env(name, daemon_url, mapped_ports, fabric_version,
+    _compose_set_env(name, daemon_url, mapped_ports, network_type,
                      consensus_plugin, consensus_mode, cluster_size,
                      log_level, log_type, log_server)
 
-    if fabric_version == FABRIC_VERSION[1]:
-        cluster_version = 'fabric-0.6'
-    else:
-        logger.debug("log_type={}".format(log_type))
-        cluster_version = 'fabric-1.0'
-
     try:
         project = get_project(COMPOSE_FILE_PATH +
-                              "/{}/".format(cluster_version) + log_type)
+                              "/{}/".format(network_type) + log_type)
         containers = project.up(detached=True, timeout=timeout)
     except Exception as e:
         logger.warning("Exception when compose start={}".format(e))
@@ -432,7 +426,7 @@ def compose_clean(name, daemon_url, fabric_version,
 
 
 def compose_start(name, daemon_url, mapped_ports=SERVICE_PORTS,
-                  fabric_version=FABRIC_VERSION[0],
+                  fabric_version=NETWORK_TYPES[0],
                   consensus_plugin=[0],
                   consensus_mode=CONSENSUS_MODES[0],
                   log_type=CLUSTER_LOG_TYPES[0], log_server="",
@@ -462,7 +456,7 @@ def compose_start(name, daemon_url, mapped_ports=SERVICE_PORTS,
                      consensus_plugin, consensus_mode, cluster_size,
                      log_level, log_type, log_server)
 
-    if fabric_version == FABRIC_VERSION[1]:
+    if fabric_version == NETWORK_TYPES[1]:
         cluster_version = 'fabric-0.6'
     else:
         cluster_version = 'fabric-1.0'
@@ -480,7 +474,7 @@ def compose_start(name, daemon_url, mapped_ports=SERVICE_PORTS,
 
 
 def compose_restart(name, daemon_url, mapped_ports=SERVICE_PORTS,
-                    fabric_version=FABRIC_VERSION[0],
+                    fabric_version=NETWORK_TYPES[0],
                     consensus_plugin=[0],
                     consensus_mode=CONSENSUS_MODES[0],
                     log_type=CLUSTER_LOG_TYPES[0], log_server="",
@@ -511,7 +505,7 @@ def compose_restart(name, daemon_url, mapped_ports=SERVICE_PORTS,
                      consensus_mode, cluster_size, log_level, log_type,
                      log_server)
 
-    if fabric_version == FABRIC_VERSION[1]:
+    if fabric_version == NETWORK_TYPES[1]:
         cluster_version = 'fabric-0.6'
     else:
         cluster_version = 'fabric-1.0'
@@ -529,7 +523,7 @@ def compose_restart(name, daemon_url, mapped_ports=SERVICE_PORTS,
 
 
 def compose_stop(name, daemon_url, mapped_ports=SERVICE_PORTS,
-                 fabric_version=FABRIC_VERSION[0],
+                 fabric_version=NETWORK_TYPES[0],
                  consensus_plugin=[0],
                  consensus_mode=CONSENSUS_MODES[0],
                  log_type=CLUSTER_LOG_TYPES[0], log_server="",
@@ -561,7 +555,7 @@ def compose_stop(name, daemon_url, mapped_ports=SERVICE_PORTS,
                      consensus_mode, cluster_size, log_level, log_type,
                      log_server)
 
-    if fabric_version == FABRIC_VERSION[1]:
+    if fabric_version == NETWORK_TYPES[1]:
         cluster_version = 'fabric-0.6'
     else:
         cluster_version = 'fabric-1.0'
@@ -577,7 +571,7 @@ def compose_stop(name, daemon_url, mapped_ports=SERVICE_PORTS,
 
 
 def compose_down(name, daemon_url, mapped_ports=SERVICE_PORTS,
-                 fabric_version=FABRIC_VERSION[0],
+                 fabric_version=NETWORK_TYPES[0],
                  consensus_plugin=[0],
                  consensus_mode=CONSENSUS_MODES[0],
                  log_type=CLUSTER_LOG_TYPES[0], log_server="",
@@ -607,7 +601,7 @@ def compose_down(name, daemon_url, mapped_ports=SERVICE_PORTS,
                      consensus_mode, cluster_size, log_level, log_type,
                      log_server)
 
-    if fabric_version == FABRIC_VERSION[1]:
+    if fabric_version == NETWORK_TYPES[1]:
         cluster_version = 'fabric-0.6'
     else:
         cluster_version = 'fabric-1.0'

src/common/__init__.py

@@ -7,7 +7,7 @@
 from .log import log_handler, LOG_LEVEL
 from .utils import \
     PEER_SERVICE_PORTS, CA_SERVICE_PORTS, SERVICE_PORTS, \
-    FABRIC_VERSION, CONSENSUS_PLUGINS, CONSENSUS_MODES, CONSENSUS_TYPES, \
+    NETWORK_TYPES, CONSENSUS_PLUGINS, CONSENSUS_MODES, CONSENSUS_TYPES, \
     HOST_TYPES, \
     CLUSTER_PORT_START, CLUSTER_PORT_STEP, CLUSTER_SIZES, \
     CLUSTER_NETWORK, \

src/common/blockchain_network.py

@@ -0,0 +1,17 @@
+import abc
+
+
+class BlockchainNetwork(object):
+    """
+    BlockchainNetwork is an abstract class to represent a general model of a
+    blockchain, such as a Hyperledger Fabric network.
+    """
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def get_config(self):
+        """
+        Get the configuration data for the network
+        Returns: configuration dict struct
+        """
+        return

src/common/blockchain_network_config.py

@@ -0,0 +1,29 @@
+class BlockchainNetworkConfig(object):
+    """
+    BlockchainNetworkConfig includes those configuration data for a network.
+    """
+
+    def __init__(self, data=None, metadata=None):
+        """
+        Init.
+
+        Args:
+            data: config data related to network.
+            metadata: metadata is for cello usage.
+        """
+        self.data = data or {}  # include all config data we need for a network
+        self.metadata = metadata or {}  # metadata is for cello usage
+
+    def get_data(self):
+        """
+        Get the configuration data for the blockchain network
+        Returns: data dict
+        """
+        return self.data
+
+    def get_metadata(self):
+        """
+        Get the metadata
+        Returns: metadata dict
+        """
+        return self.metadata

src/common/fabric_network.py

@@ -0,0 +1,23 @@
+from common.blockchain_network import BlockchainNetwork
+from common.fabric_network_config import FabricNetworkConfig
+
+
+class FabricNetwork(BlockchainNetwork):
+    """
+    FabricNetwork represents a Hyperledger Fabric network.
+    """
+
+    def __init__(self, config=None):
+        """
+
+        Args:
+            config: configuration data of the fabric network
+        """
+        self.config = config or FabricNetworkConfig()
+
+    def get_config(self):
+        """
+        Get the configuration data for the network
+        Returns: configuration dict struct
+        """
+        return self.config

src/common/fabric_network_config.py

@@ -0,0 +1,45 @@
+from common.blockchain_network_config import BlockchainNetworkConfig
+from common.utils import NETWORK_TYPES
+
+
+class FabricNetworkConfig(BlockchainNetworkConfig):
+    """
+    FabricNetworkConfig includes those configuration data for a fabric network.
+    """
+
+    def __init__(self, data=None, metadata=None):
+        """
+        Init.
+
+        Args:
+            data: config data related to the fabric network.
+            metadata: metadata is for cello usage.
+        """
+        if not data:
+            data = {  # will be used in compose template as the network config
+                'for_test': NETWORK_TYPES[0]
+            }
+
+        if not metadata:
+            metadata = {  # for cello usage
+                'name': 'FABRIC_NETWORK_{}'.format('test'),  # TODO: randomlize
+                'type': NETWORK_TYPES[0],  # network of the blockchain
+                'worker_api': '',  # which worker will handle this network
+                'envs': {}  # env variables to setup, before sending request
+                # to worker. Can be used for compose template files
+            }
+        super(FabricNetworkConfig, self).__init__(data, metadata)
+
+    def get_data(self):
+        """
+        Get the configuration data
+        Returns: data dict
+        """
+        return self.data
+
+    def get_metadata(self):
+        """
+        Get the metadata
+        Returns: metadata dict
+        """
+        return self.metadata

src/common/utils.py

@@ -30,7 +30,7 @@
 SERVICE_PORTS = dict(list(PEER_SERVICE_PORTS.items()) +
                      list(CA_SERVICE_PORTS.items()))
 
-FABRIC_VERSION = ['1.0.0', '0.6.0']  # first one is the default one
+NETWORK_TYPES = ['fabric-0.6', 'fabric-1.0']  # first one is the default one
 
 CONSENSUS_PLUGINS = ['noops', 'pbft', 'solo']  # first one is the default one
 # CONSENSUS_MODES = ['classic', 'batch', 'sieve']  # pbft has various modes

src/modules/cluster.py

@@ -14,7 +14,7 @@
 
 from agent import get_swarm_node_ip
 
-from common import CLUSTER_PORT_START, CLUSTER_PORT_STEP, FABRIC_VERSION, \
+from common import CLUSTER_PORT_START, CLUSTER_PORT_STEP, NETWORK_TYPES, \
     CONSENSUS_PLUGINS, CONSENSUS_MODES, HOST_TYPES, SYS_CREATOR, SYS_DELETER, \
     SYS_USER, SYS_RESETTING, CLUSTER_SIZES, \
     PEER_SERVICE_PORTS, CA_SERVICE_PORTS
@@ -81,7 +81,7 @@ def get_by_id(self, id, col_name="active"):
         return self._serialize(cluster)
 
     def create(self, name, host_id, start_port=0, user_id="",
-               fabric_version=FABRIC_VERSION[0],
+               fabric_version=NETWORK_TYPES[0],
                consensus_plugin=CONSENSUS_PLUGINS[0],
                consensus_mode=CONSENSUS_MODES[0], size=CLUSTER_SIZES[0]):
         """ Create a cluster based on given data
@@ -249,7 +249,7 @@ def delete(self, id, record=False, forced=False):
                 {"$set": {"user_id": SYS_DELETER + user_id}})
         host_id, daemon_url, fabric_version, consensus_plugin, cluster_size = \
             c.get("host_id"), c.get("daemon_url"), \
-            c.get("fabric_version", FABRIC_VERSION[0]), \
+            c.get("fabric_version", NETWORK_TYPES[0]), \
             c.get("consensus_plugin", CONSENSUS_PLUGINS[0]), \
             c.get("size", CLUSTER_SIZES[0])
 

src/modules/stat.py

@@ -1,7 +1,7 @@
 import logging
 import time
 from threading import Thread
-from common import LOG_LEVEL, HOST_TYPES, FABRIC_VERSION, CONSENSUS_PLUGINS, \
+from common import LOG_LEVEL, HOST_TYPES, NETWORK_TYPES, CONSENSUS_PLUGINS, \
     log_handler, CONSENSUS_MODES
 
 from modules import host_handler, cluster_handler

src/resources/cluster_api.py

@@ -9,7 +9,7 @@
 from common import log_handler, LOG_LEVEL, \
     request_get, make_ok_response, make_fail_response, \
     request_debug, request_json_body, \
-    CODE_CREATED, CODE_NOT_FOUND, FABRIC_VERSION, \
+    CODE_CREATED, CODE_NOT_FOUND, NETWORK_TYPES, \
     CONSENSUS_PLUGINS, CONSENSUS_MODES, CLUSTER_SIZES
 from modules import cluster_handler, host_handler
 

src/resources/cluster_view.py

@@ -7,7 +7,7 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
 from common import log_handler, LOG_LEVEL, \
-    request_debug, FABRIC_VERSION, \
+    request_debug, NETWORK_TYPES, \
     CONSENSUS_PLUGINS, CONSENSUS_MODES, CLUSTER_SIZES
 from modules import cluster_handler, host_handler
 
@@ -68,7 +68,7 @@ def clusters_show():
     return render_template("clusters.html", type=show_type, col_name=col_name,
                            items_count=total_items, items=clusters,
                            hosts_available=hosts_avail,
-                           fabric_version=FABRIC_VERSION,
+                           fabric_version=NETWORK_TYPES,
                            consensus_plugins=CONSENSUS_PLUGINS,
                            consensus_modes=CONSENSUS_MODES,
                            cluster_sizes=CLUSTER_SIZES)

src/resources/index.py

@@ -5,7 +5,7 @@
 from flask import request as r
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
-from common import log_handler, LOG_LEVEL, FABRIC_VERSION, CONSENSUS_PLUGINS, \
+from common import log_handler, LOG_LEVEL, NETWORK_TYPES, CONSENSUS_PLUGINS, \
     CONSENSUS_MODES, HOST_TYPES, CLUSTER_SIZES, request_debug, \
     CLUSTER_LOG_TYPES, CLUSTER_LOG_LEVEL
 from version import version, homepage, author
@@ -50,7 +50,7 @@ def show():
                            clusters_inuse=clusters_inuse,
                            clusters_temp=clusters_temp,
                            cluster_sizes=CLUSTER_SIZES,
-                           fabric_version=FABRIC_VERSION,
+                           fabric_version=NETWORK_TYPES,
                            consensus_plugins=CONSENSUS_PLUGINS,
                            consensus_modes=CONSENSUS_MODES,
                            host_types=HOST_TYPES,