CVE-2019-1010017

@@ -1,3 +1,4 @@
+v0.7.1, 22/11/2020 -- code clean-up + fix for CVE-2019-1010017
 v0.7.0, 28/02/2016 -- A few bugfixes
                     - fixe of endless loop in Nmap.Process. Fix provided by @rcarrillo, many thanks!
 v0.6.3, 18/08/2015 -- Merged pull requests for automatic pypi upload, thanks @bmx0r

@@ -5,27 +5,34 @@
 
 rep = NmapParser.parse_fromfile("libnmap/test/files/full_sudo6.xml")
 
-print("Nmap scan discovered {0}/{1} hosts up".format(rep.hosts_up,
-                                                     rep.hosts_total))
+print(
+    "Nmap scan discovered {0}/{1} hosts up".format(
+        rep.hosts_up, rep.hosts_total
+    )
+)
 for _host in rep.hosts:
     if _host.is_up():
-        print("+ Host: {0} {1}".format(_host.address,
-                                       " ".join(_host.hostnames)))
+        print(
+            "+ Host: {0} {1}".format(_host.address, " ".join(_host.hostnames))
+        )
 
         # get CPE from service if available
         for s in _host.services:
-            print("    Service: {0}/{1} ({2})".format(s.port,
-                                                      s.protocol,
-                                                      s.state))
+            print(
+                "    Service: {0}/{1} ({2})".format(
+                    s.port, s.protocol, s.state
+                )
+            )
             # NmapService.cpelist returns an array of CPE objects
             for _serv_cpe in s.cpelist:
                 print("        CPE: {0}".format(_serv_cpe.cpestring))
 
         if _host.os_fingerprinted:
             print("  OS Fingerprints")
             for osm in _host.os.osmatches:
-                print("    Found Match:{0} ({1}%)".format(osm.name,
-                                                          osm.accuracy))
+                print(
+                    "    Found Match:{0} ({1}%)".format(osm.name, osm.accuracy)
+                )
                 # NmapOSMatch.get_cpe() method return an array of string
                 # unlike NmapOSClass.cpelist which returns an array of CPE obj
                 for cpe in osm.get_cpe():

@@ -66,7 +66,9 @@ def print_diff(obj1, obj2):
 
 
 def main():
-    newrep = NmapParser.parse_fromfile("libnmap/test/files/2_hosts_achange.xml")
+    newrep = NmapParser.parse_fromfile(
+        "libnmap/test/files/2_hosts_achange.xml"
+    )
     oldrep = NmapParser.parse_fromfile("libnmap/test/files/1_hosts.xml")
 
     print_diff(newrep, oldrep)

@@ -27,10 +27,9 @@ def get_os(nmap_host):
         cpelist = nmap_host.os.os_cpelist()
         if len(cpelist):
             mcpe = cpelist.pop()
-            rval.update({
-                    "vendor": mcpe.get_vendor(),
-                    "product": mcpe.get_product()
-            })
+            rval.update(
+                {"vendor": mcpe.get_vendor(), "product": mcpe.get_product()}
+            )
     return rval
 
 

@@ -10,10 +10,7 @@
     if nmaptask:
         print(
             "Task {0} ({1}): ETC: {2} DONE: {3}%".format(
-                nmaptask.name,
-                nmaptask.status,
-                nmaptask.etc,
-                nmaptask.progress
+                nmaptask.name, nmaptask.status, nmaptask.etc, nmaptask.progress
             )
         )
 print("rc: {0} output: {1}".format(nmap_proc.rc, nmap_proc.summary))

@@ -9,10 +9,7 @@ def mycallback(nmaptask):
     if nmaptask:
         print(
             "Task {0} ({1}): ETC: {2} DONE: {3}%".format(
-                nmaptask.name,
-                nmaptask.status,
-                nmaptask.etc,
-                nmaptask.progress
+                nmaptask.name, nmaptask.status, nmaptask.etc, nmaptask.progress
             )
         )
 

@@ -2,9 +2,13 @@
 
 
 try:
-    import xml.etree.cElementTree as ET
+    import defusedxml.ElementTree as ET
 except ImportError:
-    import xml.etree.ElementTree as ET
+    try:
+        import xml.etree.cElementTree as ET
+    except ImportError:
+        import xml.etree.ElementTree as ET
+from xml.etree.ElementTree import iselement as et_iselement
 from libnmap.objects import NmapHost, NmapService, NmapReport
 
 
@@ -701,7 +705,7 @@ def __format_element(elt_data):
                     "to instanciate XML Element from "
                     "string {0} - {1}".format(elt_data, e)
                 )
-        elif ET.iselement(elt_data):
+        elif et_iselement(elt_data):
             xelement = elt_data
         else:
             raise NmapParserException(
@@ -724,7 +728,7 @@ def __format_attributes(elt_data):
         """
 
         rval = {}
-        if not ET.iselement(elt_data):
+        if not et_iselement(elt_data):
             raise NmapParserException(
                 "Error while trying to parse supplied "
                 "data attributes: format is not XML or "

@@ -0,0 +1,6 @@
+<!-- comment -->
+<root>
+    <element key='value'>text</element>
+    <element>text</element>tail
+    <empty-element/>
+ </root>
\ No newline at end of file

@@ -0,0 +1,5 @@
+<?xml version="1.0"?>
+<!DOCTYPE external [
+<!ENTITY ee SYSTEM "file://./simple.xml">
+]>
+<root>&ee;</root>
\ No newline at end of file

@@ -1,175 +1,175 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# import unittest
-# import os
-# from libnmap.parser import NmapParser
-# from libnmap.plugins.backendplugin import NmapBackendPlugin
-# from libnmap.plugins.backendpluginFactory import BackendPluginFactory
-# 
-# 
-# class TestNmapBackendPlugin(unittest.TestCase):
-#     """
-#     This testing class will tests each plugins
-#     The following test need to be done :
-#        - test the factory
-#        - test all the method of the class NmapBackendPlugin:
-#           - Verify implmented/notImplemented
-#           - Verify the behaviour (ie insert must insert)
-#     To support a new plugin or a new way to instanciate a plugin, add a dict
-#     with the necessary parameter in the urls table define in setUp
-#     All testcase must loop thru theses urls to validate a plugins
-#     """
-#     def setUp(self):
-#         fdir = os.path.dirname(os.path.realpath(__file__))
-#         self.flist_full = [
-#             {
-#                 'file': "{0}/{1}".format(fdir, "files/2_hosts.xml"),
-#                 'hosts': 2
-#             },
-#             {
-#                 'file': "{0}/{1}".format(fdir, "files/1_hosts.xml"),
-#                 'hosts': 1
-#             },
-#             {
-#                 'file': "{0}/{1}".format(
-#                                     fdir,
-#                                     "files/1_hosts_banner_ports_notsyn.xml"
-#                                 ),
-#                 'hosts': 1
-#             },
-#             {
-#                 'file': "{0}/{1}".format(
-#                                     fdir,
-#                                     'files/1_hosts_banner_ports.xml'
-#                                 ),
-#                 'hosts': 1
-#             },
-#             {
-#                 'file': "{0}/{1}".format(
-#                                     fdir,
-#                                     'files/1_hosts_banner.xml'
-#                                 ),
-#                 'hosts': 1
-#             },
-#             {
-#                 'file': "{0}/{1}".format(
-#                                     fdir,
-#                                     'files/2_hosts_version.xml'
-#                                 ),
-#                 'hosts': 2
-#             },
-#             {
-#                 'file': "{0}/{1}".format(
-#                                     fdir,
-#                                     'files/2_tcp_hosts.xml'
-#                                 ),
-#                 'hosts': 2
-#             },
-#             {
-#                 'file': "{0}/{1}".format(
-#                                     fdir,
-#                                     'files/1_hosts_nohostname.xml'
-#                                 ),
-#                 'hosts': 1
-#             }
-#         ]
-#         self.flist = self.flist_full
-#         # build a list of NmapReport
-#         self.reportList = []
-#         for testfile in self.flist:
-#             fd = open(testfile['file'], 'r')
-#             s = fd.read()
-#             fd.close()
-#             nrp = NmapParser.parse(s)
-#             self.reportList.append(nrp)
-# 
-#         self.urls = [
-#             {
-#                 'plugin_name': "mongodb"
-#             },
-#             {
-#                 'plugin_name': 'sql',
-#                 'url': 'sqlite:////tmp/reportdb.sql',
-#                 'echo': False
-#             },
-#             {
-#                 'plugin_name': 'sql',
-#                 'url': 'mysql+pymysql://root@localhost/poulet',
-#                 'echo': False
-#             }
-#         ]
-# 
-#     def test_backend_factory(self):
-#         """ test_factory BackendPluginFactory.create(**url)
-#             Invoke factory and test that the object is of the right classes
-#         """
-#         for url in self.urls:
-#             backend = BackendPluginFactory.create(**url)
-#             self.assertEqual(isinstance(backend, NmapBackendPlugin), True)
-#             className = "Nmap%sPlugin" % url['plugin_name'].title()
-#             self.assertEqual(backend.__class__.__name__, className, True)
-# 
-#     def test_backend_insert(self):
-#         """ test_insert
-#             best way to insert is to call save() of nmapreport :P
-#         """
-#         for nrp in self.reportList:
-#             for url in self.urls:
-#                 # create the backend factory object
-#                 backend = BackendPluginFactory.create(**url)
-#                 # save the report
-#                 returncode = nrp.save(backend)
-#                 # test return code
-#                 self.assertNotEqual(returncode, None)
-# 
-#     def test_backend_get(self):
-#         """ test_backend_get
-#             inset all report and save the returned id in a list
-#             then get each id and create a new list of report
-#             compare each report (assume eq)
-#         """
-#         id_list = []
-#         result_list = []
-#         for url in self.urls:
-#             backend = BackendPluginFactory.create(**url)
-#             for nrp in self.reportList:
-#                 id_list.append(nrp.save(backend))
-#             for rep_id in id_list:
-#                 result_list.append(backend.get(rep_id))
-#             self.assertEqual(len(result_list), len(self.reportList))
-#             self.assertEqual((result_list), (self.reportList))
-#             id_list = []
-#             result_list = []
-# 
-#     def test_backend_getall(self):
-#         pass
-# 
-#     def test_backend_delete(self):
-#         """ test_backend_delete
-#             inset all report and save the returned id in a list
-#             for each id remove the item and test if not present
-#         """
-#         id_list = []
-#         result_list = []
-#         for url in self.urls:
-#             backend = BackendPluginFactory.create(**url)
-#             for nrp in self.reportList:
-#                 id_list.append(nrp.save(backend))
-#             for rep_id in id_list:
-#                 result_list.append(backend.delete(rep_id))
-#                 self.assertEqual(backend.get(rep_id), None)
-#             id_list = []
-#             result_list = []
-# 
-# 
-# if __name__ == '__main__':
-#     test_suite = [
-#         'test_backend_factory',
-#         'test_backend_insert',
-#         'test_backend_get',
-#         'test_backend_getall',
-#         'test_backend_delete'
-#     ]
-#     suite = unittest.TestSuite(map(TestNmapBackendPlugin, test_suite))
-#     test_result = unittest.TextTestRunner(verbosity=5).run(suite)
+import unittest
+import os
+from libnmap.parser import NmapParser
+from libnmap.plugins.backendplugin import NmapBackendPlugin
+from libnmap.plugins.backendpluginFactory import BackendPluginFactory
+
+
+class TestNmapBackendPlugin(unittest.TestCase):
+    """
+    This testing class will tests each plugins
+    The following test need to be done :
+       - test the factory
+       - test all the method of the class NmapBackendPlugin:
+          - Verify implmented/notImplemented
+          - Verify the behaviour (ie insert must insert)
+    To support a new plugin or a new way to instanciate a plugin, add a dict
+    with the necessary parameter in the urls table define in setUp
+    All testcase must loop thru theses urls to validate a plugins
+    """
+    def setUp(self):
+        fdir = os.path.dirname(os.path.realpath(__file__))
+        self.flist_full = [
+            {
+                'file': "{0}/{1}".format(fdir, "files/2_hosts.xml"),
+                'hosts': 2
+            },
+            {
+                'file': "{0}/{1}".format(fdir, "files/1_hosts.xml"),
+                'hosts': 1
+            },
+            {
+                'file': "{0}/{1}".format(
+                                    fdir,
+                                    "files/1_hosts_banner_ports_notsyn.xml"
+                                ),
+                'hosts': 1
+            },
+            {
+                'file': "{0}/{1}".format(
+                                    fdir,
+                                    'files/1_hosts_banner_ports.xml'
+                                ),
+                'hosts': 1
+            },
+            {
+                'file': "{0}/{1}".format(
+                                    fdir,
+                                    'files/1_hosts_banner.xml'
+                                ),
+                'hosts': 1
+            },
+            {
+                'file': "{0}/{1}".format(
+                                    fdir,
+                                    'files/2_hosts_version.xml'
+                                ),
+                'hosts': 2
+            },
+            {
+                'file': "{0}/{1}".format(
+                                    fdir,
+                                    'files/2_tcp_hosts.xml'
+                                ),
+                'hosts': 2
+            },
+            {
+                'file': "{0}/{1}".format(
+                                    fdir,
+                                    'files/1_hosts_nohostname.xml'
+                                ),
+                'hosts': 1
+            }
+        ]
+        self.flist = self.flist_full
+        # build a list of NmapReport
+        self.reportList = []
+        for testfile in self.flist:
+            fd = open(testfile['file'], 'r')
+            s = fd.read()
+            fd.close()
+            nrp = NmapParser.parse(s)
+            self.reportList.append(nrp)
+
+        self.urls = [
+            {
+                'plugin_name': "mongodb"
+            },
+            {
+                'plugin_name': 'sql',
+                'url': 'sqlite:////tmp/reportdb.sql',
+                'echo': False
+            },
+            {
+                'plugin_name': 'sql',
+                'url': 'mysql+pymysql://root@localhost/poulet',
+                'echo': False
+            }
+        ]
+
+    def test_backend_factory(self):
+        """ test_factory BackendPluginFactory.create(**url)
+            Invoke factory and test that the object is of the right classes
+        """
+        for url in self.urls:
+            backend = BackendPluginFactory.create(**url)
+            self.assertEqual(isinstance(backend, NmapBackendPlugin), True)
+            className = "Nmap%sPlugin" % url['plugin_name'].title()
+            self.assertEqual(backend.__class__.__name__, className, True)
+
+    def test_backend_insert(self):
+        """ test_insert
+            best way to insert is to call save() of nmapreport :P
+        """
+        for nrp in self.reportList:
+            for url in self.urls:
+                # create the backend factory object
+                backend = BackendPluginFactory.create(**url)
+                # save the report
+                returncode = nrp.save(backend)
+                # test return code
+                self.assertNotEqual(returncode, None)
+
+    def test_backend_get(self):
+        """ test_backend_get
+            inset all report and save the returned id in a list
+            then get each id and create a new list of report
+            compare each report (assume eq)
+        """
+        id_list = []
+        result_list = []
+        for url in self.urls:
+            backend = BackendPluginFactory.create(**url)
+            for nrp in self.reportList:
+                id_list.append(nrp.save(backend))
+            for rep_id in id_list:
+                result_list.append(backend.get(rep_id))
+            self.assertEqual(len(result_list), len(self.reportList))
+            self.assertEqual((result_list), (self.reportList))
+            id_list = []
+            result_list = []
+
+    def test_backend_getall(self):
+        pass
+
+    def test_backend_delete(self):
+        """ test_backend_delete
+            inset all report and save the returned id in a list
+            for each id remove the item and test if not present
+        """
+        id_list = []
+        result_list = []
+        for url in self.urls:
+            backend = BackendPluginFactory.create(**url)
+            for nrp in self.reportList:
+                id_list.append(nrp.save(backend))
+            for rep_id in id_list:
+                result_list.append(backend.delete(rep_id))
+                self.assertEqual(backend.get(rep_id), None)
+            id_list = []
+            result_list = []
+
+
+if __name__ == '__main__':
+    test_suite = [
+        'test_backend_factory',
+        'test_backend_insert',
+        'test_backend_get',
+        'test_backend_getall',
+        'test_backend_delete'
+    ]
+    suite = unittest.TestSuite(map(TestNmapBackendPlugin, test_suite))
+    test_result = unittest.TextTestRunner(verbosity=5).run(suite)

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from libnmap.parser import NmapParser, NmapParserException
+import unittest
+import os
+
+
+class TestDefusedXML(unittest.TestCase):
+    def setUp(self):
+        self.billionlaugh = """<?xml version="1.0"?>
+<!DOCTYPE lolz [
+<!ENTITY lol "lol">
+<!ELEMENT lolz (#PCDATA)>
+<!ENTITY lol1 "&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;">
+<!ENTITY lol2 "&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;">
+<!ENTITY lol3 "&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;">
+<!ENTITY lol4 "&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;">
+<!ENTITY lol5 "&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;">
+<!ENTITY lol6 "&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;">
+<!ENTITY lol7 "&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;">
+<!ENTITY lol8 "&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;">
+<!ENTITY lol9 "&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;">
+]>
+<lolz>&lol9;</lolz>
+        """
+        self.fdir = os.path.dirname(os.path.realpath(__file__))
+        self.billionlaugh_file = "{0}/files/{1}".format(self.fdir, "billion_laugh.xml")
+        self.external_entities_file = "{0}/files/{1}".format(self.fdir, "defused_et_local_includer.xml")
+
+    def test_billion_laugh(self):
+        self.assertRaisesRegex(NmapParserException, ".*EntitiesForbidden", NmapParser.parse_fromstring, self.billionlaugh)
+
+    def test_external_entities(self):
+        self.assertRaisesRegex(NmapParserException, ".*EntitiesForbidden", NmapParser.parse_fromfile, self.external_entities_file)
+
+
+if __name__ == "__main__":
+    #test_suite = ["test_external_entities"]
+    test_suite = ["test_billion_laugh", "test_external_entities" ]
+    suite = unittest.TestSuite(map(TestDefusedXML, test_suite))
+    test_result = unittest.TextTestRunner(verbosity=2).run(suite)

@@ -1,3 +1,8 @@
+0.7.1: - clean-up blacked code and pylint it 
+0.7.1: - add unittest for defusedxml
+           - billionlaugh and external entities
+0.7.1: - add CSV backend support
+0.7.1: - Change License
 - improve API for NSE scripts
 - add support for post,pre and host scripts
 - complete unit tests with coverall support

@@ -1,12 +1,21 @@
 [tox]
-envlist = py27, py32, py38, flake8, pycodestyle, formatting
+envlist = py27, py32, py38, flake8, pycodestyle, formatting, defusedxml
 
 [testenv]
+deps=pytest
+commands=pytest --ignore=libnmap/test/test_defusedxml.py --ignore=libnmap/test/test_backend_plugin_factory.py
+
+[testenv:defusedxml]
+deps=pytest
+     defusedxml
+commands=pytest --ignore=libnmap/test/test_backend_plugin_factory.py
+
+[testenv:dbbackend]
 deps=pytest
      pymongo
      sqlalchemy
      pymysql
-commands=pytest
+commands=pytest --ignore=libnmap/test/test_defusedxml.py
 
 [testenv:flake8]
 deps =

@@ -16,6 +16,7 @@ before_install:
   - "sudo apt-get install nmap -qq"
 install:
   - "pip install flake8"
+  - "pip install defusedxml"
   # - "pip install boto" # disabled: since boto not supporting py3
   # - "pip install pymongo sqlalchemy MySQL-python" # disabled MySQL-python (not py3 compatible)
   # - "pip install pymongo sqlalchemy pymysql"