Switch peers from using variable IPs to hostnames. (#58)

* Switch peers from using variable IPs to hostnames. * Use k8s service hostname for quorum peers: permissioned-nodes.json and static-nodes.json. * Use k8s service hostname in tessera config for peer discovery. * The hostnames are obtained via the quorum services which are known at the time of generation and do not change. This removes the need to do sed and manipulation on the config files after they are deployed in order to set the IPs (which are not known until after k8s resources have been deployed). This also helps with automatically updating the config files when new nodes are deployed or removed, thus avoiding restarting the nodes. * Enabled by this PR: Consensys/quorum#885. * Remove support / testing for quorum <= 2.3.0 * hostname not supported pre 2.3.0, if there is demand to support them, we can include support, but for now removing support as these version should be phased out. * add sleep in quorum version tests after running initial public contract to allow block number to increase.
Consensys · Aug 20, 2020 · 15bf0ff · 15bf0ff
1 parent b6a6300
commit 15bf0ff
Show file tree

Hide file tree

Showing 12 changed files with 21 additions and 146 deletions.
diff --git a/qubernetes b/qubernetes
@@ -112,31 +112,6 @@ end
 # create the output directory if it doesn't exist
 `mkdir -p out`
 
-sed_string = ""
-sed_string_no_escape = ""
-
-# make all services
-# set the replacement string sed command which is run on the permissioned-nodes.json,
-# as the service host IPs of the nodes are not known until they are deployed, and need
-# to be properly set in the permissioned-nodes.json.
-#PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | sed \"s/%QUORUM_DEPLOYMENT_01_SERVICE_HOST%/$QUORUM_NODE01_SERVICE_HOST/g\" | sed \"s/\\$QUORUM_DEPLOYMENT_02_SERVICE_HOST/$QUORUM_NODE02_SERVICE_HOST/g\");
-@nodes.each do |node|
-  set_node_template_vars(node)
-#  puts ("#{@Node_UserIdent}")
-  k8_service_host_var = ("#{@Node_UserIdent}".upcase + "_SERVICE_HOST").gsub("-", "_")
-  permission_node_host="#{@Node_UserIdent}".upcase + "_SERVICE_HOST"
- # puts (k8_service_host_var)
-  sed_instruction = ' sed \"s/%' + permission_node_host + '%/$' + k8_service_host_var + '/g\"'
-  sed_instruction_no_escape = ' sed "s/%' + permission_node_host + '%/$' + k8_service_host_var + '/g"'
-  sed_string = sed_string + sed_instruction + " | "
-  sed_string_no_escape = sed_string_no_escape + sed_instruction_no_escape + " | "
-end
-
-# used/written inside each generated deployment file.
-@Sed_Set_Node_Service_Host = sed_string[0...-2]
-# used inside cofigMap script
-@Sed_Set_Node_Service_Host_No_Escape = sed_string_no_escape[0...-2]
-
 @base_template_path = "templates/k8s"
 #puts (sed_string)
 

diff --git a/templates/k8s/quorum-deployment.yaml.erb b/templates/k8s/quorum-deployment.yaml.erb
@@ -151,13 +151,10 @@ spec:
            CONFIG_TMPL=$(cat ${DDIR}/tessera-config${TESSERA_CONFIG_TYPE}.json.tmpl);
 
 
-           echo  \"<%= @Sed_Set_Node_Service_Host %>\";
-           CONFIG_WITH_OTHER_HOSTS=$(echo $CONFIG_TMPL | <%= @Sed_Set_Node_Service_Host %>) ;
-           CONFIG_WITH_ALL_HOSTS=$(echo $CONFIG_WITH_OTHER_HOSTS | sed \"s/%THIS_SERVICE_HOST%/$<%= @This_Host %>/g\");
            PRIV_KEY=$(cat $DDIR/tm.key)
            PUB_KEY=$(cat $DDIR/tm.pub)
-           CONFIG_FINAL=$(echo $CONFIG_WITH_ALL_HOSTS | sed \"s-%THIS_PRIV_KEY%-${PRIV_KEY}-g\" |  sed \"s-%THIS_PUB_KEY%-${PUB_KEY}-g\");
-           CONFIG_FINAL_9_0=$(echo $CONFIG_WITH_ALL_HOSTS | sed \"s-%THIS_PRIV_KEY%-${DDIR}/tm.key-g\" |  sed \"s-%THIS_PUB_KEY%-${DDIR}/tm.pub-g\");
+           CONFIG_FINAL=$(echo $CONFIG_TMPL | sed \"s-%THIS_PRIV_KEY%-${PRIV_KEY}-g\" |  sed \"s-%THIS_PUB_KEY%-${PUB_KEY}-g\");
+           CONFIG_FINAL_9_0=$(echo $CONFIG_TMPL | sed \"s-%THIS_PRIV_KEY%-${DDIR}/tm.key-g\" |  sed \"s-%THIS_PUB_KEY%-${DDIR}/tm.pub-g\");
            if [[ \"${Tess_Ver_Second}\" -ge \"9\" ]]; then CONFIG_FINAL=${CONFIG_FINAL_9_0}; fi;
            echo $CONFIG_FINAL >  ${DDIR}/tessera-config-with-hosts.json;
            cat  ${DDIR}/tessera-config-with-hosts.json;
@@ -345,7 +342,7 @@ spec:
           name: quorum-permissioned-config
           items:
           - key: permissioned-nodes.json
-            path: permissioned-nodes.json.tmpl
+            path: permissioned-nodes.json
           - key: permissioned-update.sh
             path: permissioned-update.sh
       - name: genesis-config-persistent-storage

diff --git a/templates/k8s/quorum-shared-config.yaml.erb b/templates/k8s/quorum-shared-config.yaml.erb
@@ -16,9 +16,7 @@ data:
 
   permissioned-update.sh: |
     #!/bin/bash
-    PERM_NODE_TMPL=$(cat $QHOME/permission-nodes/permissioned-nodes.json.tmpl);
-    PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | <%= @Sed_Set_Node_Service_Host_No_Escape %>);
-    echo $PERM_NODE_JSON >  $QUORUM_DATA_DIR/permissioned-nodes.json;
+    cp $QUORUM_HOME/permission-nodes/permissioned-nodes.json $QUORUM_DATA_DIR/permissioned-nodes.json;
     cp $QUORUM_DATA_DIR/permissioned-nodes.json $QUORUM_DATA_DIR/static-nodes.json;
 
 ---

diff --git a/templates/quorum/permissioned-nodes.json.erb b/templates/quorum/permissioned-nodes.json.erb
@@ -9,10 +9,11 @@ end
    # The enode value is obtained from that file and set in the permissioned-nodes.json
 -%>
 [
-<%- @nodes.each_with_index do |node, indexNode| -%>
-    <%- set_node_template_vars(node) -%>
-    <%- @Enode_File = @Key_Dir_Base + "/" + @Node_Key_Dir + "/enode" -%>
-    <%- File.readlines(@Enode_File).each do |line| @Enode = "#{line}".gsub(/\s+/, "") end -%>
-  "enode://<%= @Enode %>@%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @NodeP2P_ListenAddr%>?discport=0&raftport=<%= @Raft_Port%>"<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
-<% end -%>
+  <%- @nodes.each_with_index do |node, indexNode| -%>
+  <%- set_node_template_vars(node) -%>
+  <%- @Enode_File = @Key_Dir_Base + "/" + @Node_Key_Dir + "/enode" -%>
+  <%- File.readlines(@Enode_File).each do |line| @Enode = "#{line}".gsub(/\s+/, "") end -%>
+  <%- # use the service name / kube-dns to look up the nodes -%>
+  "enode://<%= @Enode %>@<%= "#{@Node_UserIdent}"%>:<%= @NodeP2P_ListenAddr%>?discport=0&raftport=<%= @Raft_Port%>"<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
+  <% end -%>
 ]
diff --git a/templates/quorum/tessera-config-9.0.json.erb b/templates/quorum/tessera-config-9.0.json.erb
@@ -24,7 +24,7 @@ end
   {
     "app":"ThirdParty",
     "enabled": true,
-    "serverAddress": "http://%THIS_SERVICE_HOST%:<%= @Tm_3Party_Port %>",
+    "serverAddress": "http://<%= @Node_UserIdent %>:<%= @Tm_3Party_Port %>",
     "cors": {
         "allowedMethods": [
             "GET",
@@ -45,7 +45,7 @@ end
   {
     "app":"P2P",
     "enabled": true,
-    "serverAddress":"http://%THIS_SERVICE_HOST%:<%= @Tm_Port %>",
+    "serverAddress":"http://<%= @Node_UserIdent %>:<%= @Tm_Port %>",
     "sslConfig": {
       "tls": "OFF",
       "generateKeyStoreIfNotExisted": true,
@@ -70,7 +70,7 @@ end
 <%- @nodes.each_with_index do |node, indexNode| %>
 <%= set_node_template_vars(node) -%>
      {
-           "url": "http://%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @Tm_Port %>"
+           "url": "http://<%= @Node_UserIdent %>:<%= @Tm_Port %>"
          }<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
 
 <% end -%>

diff --git a/templates/quorum/tessera-config-enhanced.json.erb b/templates/quorum/tessera-config-enhanced.json.erb
@@ -27,7 +27,7 @@ end
     "serverSocket":{
       "type":"INET",
       "port": <%= @Tm_3Party_Port %>,
-      "hostName": "http://%THIS_SERVICE_HOST%"
+      "hostName": "http://<%= @Node_UserIdent %>"
     },
     "communicationType" : "REST"
   },
@@ -46,7 +46,7 @@ end
     "serverSocket":{
       "type":"INET",
       "port": <%= @Tm_Port %>,
-      "hostName": "http://%THIS_SERVICE_HOST%",
+      "hostName": "http://<%= @Node_UserIdent %>",
       "bindingAddress": "http://0.0.0.0:<%= @Tm_Port %>"
     },
     "sslConfig": {
@@ -73,7 +73,7 @@ end
 <%- @nodes.each_with_index do |node, indexNode| %>
 <%= set_node_template_vars(node) -%>
      {
-           "url": "http://%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @Tm_Port %>"
+           "url": "http://<%= @Node_UserIdent %>:<%= @Tm_Port %>"
          }<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
 
 <% end -%>

diff --git a/templates/quorum/tessera-config.json.erb b/templates/quorum/tessera-config.json.erb
@@ -23,7 +23,7 @@ end
     },
     "server": {
         "port": <%= @Tm_Port %>,
-        "hostName": "http://%THIS_SERVICE_HOST%",
+        "hostName": "http://<%= @Node_UserIdent %>",
         "bindingAddress": "http://0.0.0.0:<%= @Tm_Port %>",
         "sslConfig": {
             "tls": "OFF",
@@ -46,7 +46,7 @@ end
 <%- @nodes.each_with_index do |node, indexNode| %>
     <%= set_node_template_vars(node) -%>
      {
-           "url": "http://%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @Tm_Port %>"
+           "url": "http://%<%= @Node_UserIdent%>%:<%= @Tm_Port %>"
          }<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
 
 <% end -%>

diff --git a/testing/qversions-config/qubes-ibft-2.2.0.yaml b/testing/qversions-config/qubes-ibft-2.2.0.yaml
diff --git a/testing/qversions-config/qubes-ibft-2.3.0.yaml b/testing/qversions-config/qubes-ibft-2.3.0.yaml
diff --git a/testing/qversions-config/qubes-raft-2.2.0.yaml b/testing/qversions-config/qubes-raft-2.2.0.yaml
diff --git a/testing/qversions-config/qubes-raft-2.3.0.yaml b/testing/qversions-config/qubes-raft-2.3.0.yaml
diff --git a/testing/test-qnet.sh b/testing/test-qnet.sh
@@ -160,6 +160,8 @@ then
 fi
 
 ## Test blockNumber for Public TX, should be > 1
+echo "sleeping for 5 seconds to give block time to increase."
+sleep 5
 get_block_number $NAMESPACE
 BLOCK_NUM=$?
 echo "BLOCK_NUM: [$BLOCK_NUM]"