various: major change to ndmtk-git (#6)

Makefile

@@ -3,7 +3,7 @@
 export USER
 PLUGIN_NAME="ndmtk"
 PLUGIN_NAME_EGG := $(subst -,_,$(PLUGIN_NAME))
-PLUGIN_VER=0.1.6
+PLUGIN_VER=0.1.7
 DOCKER_IMAGE_NAME="greenpau/ndmtk"
 DOCKER_CONTAINER_NAME="ndmtk"
 DOCKER_CONTAINER_SHELL="/bin/sh"

circle.yml

@@ -11,7 +11,7 @@ machine:
     version: 2.7.5
   environment:
     PACKAGE: ndmtk
-    VERSION: '0.1.6'
+    VERSION: '0.1.7'
     TAG1: ${VERSION}-$(date +%Y%m%dT%H%M)-git-${CIRCLE_SHA1:0:7}
     TAG2: ${CIRCLE_PR_USERNAME}_${CIRCLE_BRANCH/pull\//pr_}
   services:

docker/alpine/Dockerfile

@@ -23,7 +23,7 @@ COPY demo/firewall/files/ndmtk/spec/*.yml /etc/ansible/files/ndmtk/spec/
 COPY demo/firewall/files/ndmtk/os/*.yml /etc/ansible/files/ndmtk/os/
 COPY demo/firewall/files/ndmtk/host/*.yml /etc/ansible/files/ndmtk/host/
 COPY demo/firewall/files/ndmtk/exceptions.yml /etc/ansible/files/ndmtk/
-COPY dist/ndmtk-0.1.6.tar.gz /usr/local/src/
-RUN  pip install /usr/local/src/ndmtk-0.1.6.tar.gz
+COPY dist/ndmtk-0.1.7.tar.gz /usr/local/src/
+RUN  pip install /usr/local/src/ndmtk-0.1.7.tar.gz
 
 ENTRYPOINT ["/bin/sh"]

docker/centos/Dockerfile

@@ -19,7 +19,7 @@ COPY demo/firewall/files/ndmtk/spec/*.yml /etc/ansible/files/ndmtk/spec/
 COPY demo/firewall/files/ndmtk/os/*.yml /etc/ansible/files/ndmtk/os/
 COPY demo/firewall/files/ndmtk/host/*.yml /etc/ansible/files/ndmtk/host/
 COPY demo/firewall/files/ndmtk/exceptions.yml /etc/ansible/files/ndmtk/
-COPY dist/ndmtk-0.1.6.tar.gz /usr/local/src/
-RUN  pip install /usr/local/src/ndmtk-0.1.6.tar.gz
+COPY dist/ndmtk-0.1.7.tar.gz /usr/local/src/
+RUN  pip install /usr/local/src/ndmtk-0.1.7.tar.gz
 
 ENTRYPOINT ["/bin/bash"]

docs/conf.py

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'0.1.6'
+version = u'0.1.7'
 # The full version, including alpha/beta/rc tags.
-release = u'0.1.6'
+release = u'0.1.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

ndmtk/plugins/action/files/cli/os/arista_eos.yml

@@ -410,7 +410,23 @@ ndmtk:
   tags: ['management', 'snmp']
   conditions_match_any:
   - '^snmp-server host'
+- description: 'Collect SNMP ifIndex numbers of interface'  
+  cli: 'show snmp mib walk ifDescr'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
 
 - cli: 'show logging | head -250'
   tags: ['management', 'logging']
   saveas: '%h.show.logging.txt'
+
+- description: 'Collect sFlow configuration'
+  cli: 'show sflow detail'
+  tags: ['management', 'sflow']
+  conditions_match_any:
+  - '^sflow run'
+- description: 'Collect sFlow interfaces'
+  cli: 'show sflow interfaces'
+  tags: ['management', 'sflow']
+  conditions_match_any:
+  - '^sflow run'

ndmtk/plugins/action/files/cli/os/cisco_nxos.yml

@@ -214,8 +214,8 @@ ndmtk:
   tags: ['vxlan']
   conditions_match_any:
   - '^interface nve'
-- description: 'Displays VXLAN interface status.'
-  cli: 'show vxlan interface'
+- description: 'Displays VXLAN VNI to VLAN mapping'
+  cli: 'show vxlan'
   tags: ['vxlan']
   conditions_match_all:
   - '.*l2vpn evpn'
@@ -315,24 +315,24 @@ ndmtk:
 # TODO: detect when to perform checks for fex interfaces
 - description: 'Collects the status of consistency checks for L2'
   cli: 'show consistency-checker l2 module 1'
-  tags: ['fib']
+  tags: ['fib', 'tech-support']
 - description: 'Collects the status of consistency checks for L3'
   cli: 'show consistency-checker l3 module 1'
-  tags: ['fib']
+  tags: ['fib', 'tech-support']
 - description: 'Collects the status of consistency checks for link state'
   cli: 'show consistency-checker link-state module 1'
-  tags: ['fib']
+  tags: ['fib', 'tech-support']
 
 # TODO: dynamic checks for VLANs
 # and port channels: show consistency-checker membership port-channels interface port-channel
 # show consistency-checker stp-state vlan 1
 - description: 'Collects the status of consistency checks for VLAN membership'
   cli: 'show consistency-checker membership vlan 1'
-  tags: ['fib']
+  tags: ['fib', 'tech-support']
 
 - description: 'Collects the status of consistency checks for CoPP'
   cli: 'show consistency-checker copp'
-  tags: ['fib']
+  tags: ['fib', 'tech-support']
   conditions_precedent_all:
   - 'os_class eq cisco_nxos'
   - 'os_version_major ge 7'
@@ -342,3 +342,46 @@ ndmtk:
   tags: ['fib', 'vxlan']
   conditions_match_any:
   - '^interface nve'
+
+
+- description: 'Collect SNMP status'
+  cli: 'show snmp'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+- description: 'Collect SNMP community strings'
+  cli: 'show snmp community'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+- description: 'Collect SNMP engineID'
+  cli: 'show snmp engineID'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+- description: 'Collect SNMP roles'
+  cli: 'show snmp group'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+- description: 'Collect SNMP sessions'
+  cli: 'show snmp sessions'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+  allow_empty_response: yes
+- description: 'Collect SNMP notifications enabled or disabled'
+  cli: 'show snmp trap'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+- description: 'Collect SNMP users'
+  cli: 'show snmp user'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'
+- description: 'Collect SNMP ifIndex numbers of interface'  
+  cli: 'show interface snmp-ifindex'
+  tags: ['management', 'snmp']
+  conditions_match_any:
+  - '^snmp-server'

ndmtk/plugins/action/ndmtk.j2

@@ -137,6 +137,7 @@ proc get_credentials {tp} {
     global ERROR_INPUT_CREDENTIALS;
     global session_log;
     global transaction_id;
+    set response "UKNOWN";
 
     if {[catch {puts stdout "$tp:";} err]} {
         puts $session_log "timestamp='[exec date]' transaction_id='$transaction_id' msg='$err'";
@@ -148,13 +149,16 @@ proc get_credentials {tp} {
             switch -glob -- $tp {
                 "username" {
                     set username $expect_out(1,string);
+                    set response $username;
                     puts $session_log "timestamp='[exec date]' transaction_id='$transaction_id' msg='user sent username: $username'";
                 }
                 "password" {
                     set password $expect_out(1,string);
+                    set response $password;
                 }
                 "password_enable" {
                     set password_enable $expect_out(1,string);
+                    set response $password_enable;
                 }
                 default {
                     puts $session_log "timestamp='[exec date]' transaction_id='$transaction_id' msg='user failed to provide: $tp' rc=$ERROR_INPUT_CREDENTIALS";
@@ -175,6 +179,10 @@ proc get_credentials {tp} {
             ctrl_exit $ERROR_INPUT_CREDENTIALS;
         }
     }
+    if { $response == "INTERNAL_ERROR" } {
+        puts $session_log "timestamp='[exec date]' transaction_id='$transaction_id' msg='internal error when polling for user credentials: $tp' rc=$ERROR_INPUT_CREDENTIALS";
+        ctrl_exit $ERROR_INPUT_CREDENTIALS;
+    }
     set expect_out(buffer) {};
     expect *;
     return;

ndmtk/plugins/action/ndmtk.py

@@ -544,18 +544,15 @@ def run(self, tmp=None, task_vars=None):
             else:
                 raise AnsibleError(self.plugin_name + ' failed to locate access credentials for remote devices');
         else:
-            credentials = self._load_credentials(credentials);
+            self.keystore = self._load_credentials(credentials);
             if self.errors:
                 return dict(msg='\n'.join(self.errors), log_dir=self.conf['temp_dir'], failed=True);
             j = 0;
-            for c in credentials:
+            for c in self.keystore:
                 for k in c:
                     if k not in ['password', 'password_enable']:
                         display.vvv('credentials (' + str(j)  + '): ' + str(k) + ': ' + str(c[k]), host=self.info['host']);
                 j += 1;
-            primary_credentials = credentials.pop(0);
-            for c in ['username', 'password', 'password_enable']:
-                self.conf[c] = primary_credentials[c];
 
         '''
         Create network connection string via either ssh or telnet.
@@ -566,7 +563,21 @@ def run(self, tmp=None, task_vars=None):
         display.vvv('host information:\n' + json.dumps(self.info, indent=4, sort_keys=True), host=self.info['host']);
         display.vvv('plugin configuration:\n' + json.dumps(self.conf, indent=4, sort_keys=True), host=self.info['host']);
 
+        display.vv('keystore:\n' + pprint.pformat(self.keystore), host=self.info['host']);
+        display.vv('activekey:\n' + pprint.pformat(self.activekey), host=self.info['host']);
+
+        '''
+        Check whether there is a valid password in the active credentials set.
+        '''
+        _keystore_item, _keystore_error = self._get_item_from_key('password', check_mode=True);
+        if _keystore_error:
+            if _keystore_error in ['PIN_NOT_FOUND', 'TOKEN_NOT_FOUND', 'TOKEN_FILE_NOT_FOUND'] or re.match('NO_', _keystore_error):
+                raise AnsibleError(_keystore_error);
+
         if self.errors or self.conf['cliset_last_id'] == 0:
+            '''
+            Triggered when any of the previous steps produced an error ir cli set is empty.
+            '''
             self.status['return_code'] = 1;
             self.status['return_status'] = 'failed';
             self.status['skipped'] = 'yes';
@@ -646,12 +657,61 @@ def _cleanup(self):
         return;
 
 
-    def _get_ssh_connectivity_details(self, host):
+    def _get_item_from_key(self, item, check_mode=False):
+        attempts = 0;
+        token_wait_time = 6;
+        if item in self.activekey:
+            value = str(self.activekey[item]);
+            if value == 'pin,token':
+                if not check_mode:
+                    display.display('<' + self.info['host'] + '> requires One-Time-Password (OTP)', color='green');
+                while True:
+                    if 'pin' not in self.activekey:
+                        return None, "PIN_NOT_FOUND";
+                    if 'token' not in self.activekey:
+                        return None, "TOKEN_NOT_FOUND";
+                    token = None;
+                    try:
+                        with open(os.path.expanduser(self.activekey['token']),'r') as f:
+                            token = f.readlines();
+                    except:
+                        return None, "TOKEN_FILE_NOT_FOUND";
+                    if token is None:
+                        return None, "TOKEN_FILE_NOT_FOUND";
+                    if len(token) > 0:
+                        ts_now = int(time.time());
+                        m = re.match(r"(?P<ts>\d+);(?P<token>\d+);(?P<lifetime>\d+)", token[0].strip());
+                        if m:
+                            token = m.groupdict()['token'];
+                            ts_now = int(time.time());
+                            ts_token = int(m.groupdict()['ts']) + int(m.groupdict()['lifetime']);
+                            if ts_now > ts_token:
+                                if check_mode:
+                                    return self.activekey['pin'] + token, None;
+                                if attempts > 5:
+                                    return None, "TOKEN_EXPIRED";
+                                attempts += 1;
+                                display.display('<' + self.info['host'] + '> token expired in "' + self.activekey['token'] + '", sleep for ' + str(token_wait_time) + ' seconds', color='yellow');
+                                time.sleep(token_wait_time);
+                                continue;
+                            display.display('<' + self.info['host'] + '> using token: ' + token + ', lifetime: ' + str(ts_token - ts_now) + ' seconds', color='green');
+                            return self.activekey['pin'] + token, None;
+                        else:
+                            return None, "TOKEN_INVALID_FORMAT";
+                    else:
+                        return None, "TOKEN_NOT_FOUND";
+            return value, None;
+        else:
+            return None, "NO_" + item.upper();
+
+    def _get_key_from_keystore(self, host):
         '''
         TODO: request fingerprints for jumphosts.
         '''
-        return (self.conf['username'], None); 
-
+        if len(self.keystore) > 0:
+            self.activekey = self.keystore.pop(0);
+            return None;
+        return 'no more access credentials left to try';
 
     def _get_network_connectivity_details(self):
         '''
@@ -672,6 +732,11 @@ def _get_network_connectivity_details(self):
         protocol. Later, the plugin will add jumphost information, if necessary.
         '''
 
+        err = self._get_key_from_keystore(self.info['host']);
+        if err:
+            self.errors.append(err);
+            return;
+
         _proto = 'ssh';
         if self.info['host_protocol'] is not None:
             '''
@@ -692,7 +757,7 @@ def _get_network_connectivity_details(self):
             if self.info['host_port'] is not None:
                 self.conf['args'].extend(['-p', str(self.info['host_port'])]);
             self.conf['args'].append('-tt');
-            _ssh_user, _ssh_fingerprint = self._get_ssh_connectivity_details(self.info['host']);
+            _ssh_user = self.activekey['username'];
             _ssh_host = self.info['host'];
             if self.info['host_overwrite'] is not None:
                 _ssh_host = self.info['host_overwrite']
@@ -963,7 +1028,12 @@ def _remote_play(self):
                         if re.search(prompt + ':', remote_session_stdin):
                             _prompted = True;
                             display.vvv('detected "' + prompt + '" prompt, sending response', host=self.info['host']);
-                            os.write(remote_session_stdout_pw.fileno(), self.conf[prompt] + "\n");
+                            _prompt_response, response_error = self._get_item_from_key(prompt);
+                            if response_error:
+                                self.errors.append('received ' + response_error + ' when looking up ' + prompt);
+                                os.write(remote_session_stdout_pw.fileno(), 'INTERNAL_ERROR' + "\n");
+                            else:
+                                os.write(remote_session_stdout_pw.fileno(), _prompt_response + "\n");
                     if not _prompted:
                         if str(remote_session_stdin) == "":
                             pass;
@@ -1588,7 +1658,7 @@ def _lookup_host_facts(self, fn, cli_id):
         - `hardware_macaddr`
 
         '''
-        display.vv('checking host facts ... ');
+        display.vv('<' + self.info['host'] + '> checking host facts ... ');
         patterns = None;
         if 'fact_patterns' not in self.conf:
             return;
@@ -2096,6 +2166,7 @@ def _parse_cli_output(self, fn, cli_id):
         else:
             self.conf['cliset'][cli_id]['status'] = 'ok';
             return False;
+        self.conf['cliset'][cli_id]['system_err'] = lines;
         self.conf['cliset'][cli_id]['status'] = 'failed';
         return True;
 
@@ -2268,7 +2339,7 @@ def _load_credentials(self, db=dict()):
         dft_credentials = {};
         for c in db:
             for k in c:
-                if k not in ['regex', 'username', 'password', 'password_enable', 'priority', 'description', 'default']:
+                if k not in ['regex', 'username', 'password', 'password_enable', 'priority', 'description', 'default', 'token', 'pin']:
                     self.errors.append('access credentials dictionary contains invalid key: ' + k);
             required_keys = ['username', 'password', 'priority'];
             for k in required_keys:

setup.py

@@ -30,7 +30,7 @@
 import unittest;
 
 pkg_name = 'ndmtk';
-pkg_ver = '0.1.6';
+pkg_ver = '0.1.7';
 
 cmdclass = {};