Add netconf_get module (ansible#39869)

* Add netconf_get module

Implements part-1 of proposal ansible#104
ansible/proposals#104

*  Add netconf_get module
*  Refactor `get`, `get_config`, `lock`, `unlock`
   and `discard_changes` netconf plugin api's
*  Add netconf module_utils file which netconf module
   related common functions
*  Refactor junos and iosxr netconf plugins

* Fix source option handling

* Fix review comments

* Update botmeta file

* Update review comments and add support for lock

* Lock update fix

* Fix CI issue

* Add integration test and minor fixes

* Fix review comments

* Fix CI failure

* Fix CI issues

* Fix CI issues

* Fix review comments and update integration test

* Fix review comments

* Fix review comments

* Fix review comments

Fix reveiw comments

.github/BOTMETA.yml

@@ -476,7 +476,8 @@ files:
   $modules/network/layer2/: $team_networking
   $modules/network/layer3/: $team_networking
   $modules/network/meraki/: $team_meraki
-  $modules/network/netconf/netconf_config.py: ganeshrn lpenz userlerueda
+  $modules/network/netconf/netconf_config.py: lpenz userlerueda $team_networking
+  $modules/network/netconf/netconf_get.py: wisotzky $team_networking
   $modules/network/netscaler/: $team_netscaler
   $modules/network/netvisor/: $team_netvisor
   $modules/network/nuage/: pdellaert
@@ -864,6 +865,9 @@ files:
   $module_utils/network/meraki:
     maintainers: $team_meraki
     labels: networking
+  $module_utils/network/netconf:
+    maintainers: $team_networking
+    labels: networking
   $module_utils/network/netscaler:
     maintainers: $team_netscaler
     labels: networking

lib/ansible/module_utils/network/common/netconf.py

@@ -30,6 +30,12 @@
 from ansible.module_utils._text import to_text, to_bytes
 from ansible.module_utils.connection import Connection, ConnectionError
 
+try:
+    from ncclient.xml_ import NCElement
+    HAS_NCCLIENT = True
+except ImportError:
+    HAS_NCCLIENT = False
+
 try:
     from lxml.etree import Element, fromstring, XMLSyntaxError
 except ImportError:
@@ -100,3 +106,40 @@ def parse_rpc_error(self, rpc_error):
                 return warnings
             except XMLSyntaxError:
                 raise ConnectionError(rpc_error)
+
+
+def transform_reply():
+    reply = '''<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+    <xsl:output method="xml" indent="no"/>
+
+    <xsl:template match="/|comment()|processing-instruction()">
+        <xsl:copy>
+            <xsl:apply-templates/>
+        </xsl:copy>
+    </xsl:template>
+
+    <xsl:template match="*">
+        <xsl:element name="{local-name()}">
+            <xsl:apply-templates select="@*|node()"/>
+        </xsl:element>
+    </xsl:template>
+
+    <xsl:template match="@*">
+        <xsl:attribute name="{local-name()}">
+            <xsl:value-of select="."/>
+        </xsl:attribute>
+    </xsl:template>
+    </xsl:stylesheet>
+    '''
+    if sys.version < '3':
+        return reply
+    else:
+        return reply.encode('UTF-8')
+
+
+# Note: Workaround for ncclient 0.5.3
+def remove_namespaces(data):
+    if not HAS_NCCLIENT:
+        raise ImportError("ncclient is required but does not appear to be installed.  "
+                          "It can be installed using `pip install ncclient`")
+    return NCElement(data, transform_reply()).data_xml

lib/ansible/module_utils/network/netconf/netconf.py

@@ -0,0 +1,106 @@
+#
+# (c) 2018 Red Hat, Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+import json
+
+from contextlib import contextmanager
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.connection import Connection, ConnectionError
+from ansible.module_utils.network.common.netconf import NetconfConnection
+
+
+def get_connection(module):
+    if hasattr(module, '_netconf_connection'):
+        return module._netconf_connection
+
+    capabilities = get_capabilities(module)
+    network_api = capabilities.get('network_api')
+    if network_api == 'netconf':
+        module._netconf_connection = NetconfConnection(module._socket_path)
+    else:
+        module.fail_json(msg='Invalid connection type %s' % network_api)
+
+    return module._netconf_connection
+
+
+def get_capabilities(module):
+    if hasattr(module, '_netconf_capabilities'):
+        return module._netconf_capabilities
+
+    capabilities = Connection(module._socket_path).get_capabilities()
+    module._netconf_capabilities = json.loads(capabilities)
+    return module._netconf_capabilities
+
+
+def lock_configuration(x, target=None):
+    conn = get_connection(x)
+    return conn.lock(target=target)
+
+
+def unlock_configuration(x, target=None):
+    conn = get_connection(x)
+    return conn.unlock(target=target)
+
+
+@contextmanager
+def locked_config(module, target=None):
+    try:
+        lock_configuration(module, target=target)
+        yield
+    finally:
+        unlock_configuration(module, target=target)
+
+
+def get_config(module, source, filter, lock=False):
+    conn = get_connection(module)
+    try:
+        locked = False
+        if lock:
+            conn.lock(target='running')
+            locked = True
+        response = conn.get_config(source=source, filter=filter)
+
+    except ConnectionError as e:
+        module.fail_json(msg=to_text(e, errors='surrogate_then_replace').strip())
+
+    finally:
+        if locked:
+            conn.unlock(target='running')
+
+    return response
+
+
+def get(module, filter, lock=False):
+    conn = get_connection(module)
+    try:
+        locked = False
+        if lock:
+            conn.lock(target='running')
+            locked = True
+
+        response = conn.get(filter=filter)
+
+    except ConnectionError as e:
+        module.fail_json(msg=to_text(e, errors='surrogate_then_replace').strip())
+
+    finally:
+        if locked:
+            conn.unlock(target='running')
+
+    return response