From 2b8bf9de72f59e98d6ac24a4fc0b8df676f1b4dd Mon Sep 17 00:00:00 2001 From: Rony Xavier Date: Mon, 15 Mar 2021 16:05:25 -0400 Subject: [PATCH 1/7] netsparker mapper initial push Signed-off-by: Rony Xavier --- lib/heimdall_tools/cli.rb | 12 ++ lib/heimdall_tools/netsparker_mapper.rb | 175 ++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 lib/heimdall_tools/netsparker_mapper.rb diff --git a/lib/heimdall_tools/cli.rb b/lib/heimdall_tools/cli.rb index 6ec08c7..2db7088 100644 --- a/lib/heimdall_tools/cli.rb +++ b/lib/heimdall_tools/cli.rb @@ -122,6 +122,18 @@ def aws_config_mapper puts "\r\HDF Generated:\n" puts "#{options[:output]}" end + + desc 'netsparker_mapper', 'netsparker_mapper translates netsparker enterprise results xml to HDF format Json be viewed on Heimdall' + long_desc Help.text(:netsparker_mapper) + option :xml, required: true, aliases: '-x' + option :output, required: true, aliases: '-o' + option :verbose, type: :boolean, aliases: '-V' + def netsparker_mapper + hdf = HeimdallTools::NetsparkerMapper.new(File.read(options[:xml])).to_hdf + File.write(options[:output], hdf) + puts "\r\HDF Generated:\n" + puts "#{options[:output]}" + end desc 'version', 'prints version' def version diff --git a/lib/heimdall_tools/netsparker_mapper.rb b/lib/heimdall_tools/netsparker_mapper.rb new file mode 100644 index 0000000..3fb28c2 --- /dev/null +++ b/lib/heimdall_tools/netsparker_mapper.rb @@ -0,0 +1,175 @@ +require 'json' +require 'csv' +require 'heimdall_tools/hdf' +require 'utilities/xml_to_hash' + +RESOURCE_DIR = Pathname.new(__FILE__).join('../../data') + +CWE_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'cwe-nist-mapping.csv') +OWASP_NIST_MAPPING_FILE = File.join(RESOURCE_DIR, 'owasp-nist-mapping.csv') + +IMPACT_MAPPING = { + Critical: 1.0, + High: 0.7, + Medium: 0.5, + Low: 0.3, + Best_Practice: 0.0, + Information: 0.0 +}.freeze + +CWE_REGEX = 'CWE-(\d*):'.freeze + +DEFAULT_NIST_TAG = ["SA-11", "RA-5", "Rev_4"].freeze + +# rubocop:disable Metrics/AbcSize + +module HeimdallTools + class NetsparkerMapper + def initialize(xml, name=nil, verbose = false) + @verbose = verbose + + begin + @cwe_nist_mapping = parse_mapper(CWE_NIST_MAPPING_FILE) + @owasp_nist_mapping = parse_mapper(OWASP_NIST_MAPPING_FILE) + data = xml_to_hash(xml) + + @vulnerabilities = data['netsparker-enterprise']['vulnerabilities']['vulnerability'] + @scan_info = data['netsparker-enterprise']['target'] + + rescue StandardError => e + raise "Invalid Netsparker XML file provided Exception: #{e}" + end + + end + + + def to_hdf + controls = [] + @vulnerabilities.each do |vulnerability| + @item = {} + @item['id'] = vulnerability['LookupId'].to_s + @item['title'] = vulnerability['name'].to_s + @item['desc'] = format_control_desc(vulnerability) + @item['impact'] = impact(vulnerability['severity']) + @item['tags'] = {} + @item['descriptions'] = [] + + @item['descriptions'] << desc_tags(format_check_text(vulnerability), 'check') + @item['descriptions'] << desc_tags(format_fix_text(vulnerability), 'fix') + @item['refs'] = NA_ARRAY + @item['source_location'] = NA_HASH + @item['tags']['nist'] = nist_tag(vulnerability['classification']) + @item['code'] = '' + @item['results'] = finding(vulnerability) + + controls << @item + end + controls = collapse_duplicates(controls) + results = HeimdallDataFormat.new(profile_name: 'Netsparker Enterprise Scan', + title: "Netsparker Enterprise Scan ID: #{@scan_info['scan-id']} URL: #{@scan_info['url']}", + summary: "Netsparker Enterprise Scan", + target_id: @scan_info['url'], + controls: controls) + results.to_hdf + end + + private + + def parse_html(block) + block['#cdata-section'].to_s.strip unless block.nil? + end + + def finding(vulnerability) + finding = {} + finding['status'] = 'failed' + finding['code_desc'] = [] + finding['code_desc'] << "http-request : #{parse_html(vulnerability['http-request']['content']) }" + finding['code_desc'] << "method : #{vulnerability['http-request']['method']}" + finding['code_desc'] = finding['code_desc'].join("\n") + + finding['message'] = [] + finding['message'] << "http-response : #{parse_html(vulnerability['http-response']['content']) }" + finding['message'] << "duration : #{vulnerability['http-response']['duration']}" + finding['message'] << "status-code : #{vulnerability['http-response']['status-code']}" + finding['message'] = finding['message'].join("\n") + finding['run_time'] = NA_FLOAT + + finding['start_time'] = @scan_info['initiated'] + [finding] + end + + def format_control_desc(vulnerability) + text = [] + text << "#{parse_html(vulnerability['description'])}" unless vulnerability['description'].nil? + text << "Exploitation-skills: #{parse_html(vulnerability['exploitation-skills'])}" unless vulnerability['exploitation-skills'].nil? + text << "Extra-information: #{vulnerability['extra-information']}" unless vulnerability['extra-information'].nil? + text << "Classification: #{vulnerability['classification']}" unless vulnerability['classification'].nil? + text << "Impact: #{parse_html(vulnerability['impact'])}" unless vulnerability['impact'].nil? + text << "FirstSeenDate: #{vulnerability['FirstSeenDate']}" unless vulnerability['FirstSeenDate'].nil? + text << "LastSeenDate: #{vulnerability['LastSeenDate']}" unless vulnerability['LastSeenDate'].nil? + text << "Certainty: #{vulnerability['certainty']}" unless vulnerability['certainty'].nil? + text << "Type: #{vulnerability['type']}" unless vulnerability['type'].nil? + text << "Confirmed: #{vulnerability['confirmed']}" unless vulnerability['confirmed'].nil? + text.join("
") + end + + def format_check_text(vulnerability) + text = [] + text << "Exploitation-skills: #{parse_html(vulnerability['exploitation-skills'])}" unless vulnerability['exploitation-skills'].nil? + text << "Proof-of-concept: #{parse_html(vulnerability['proof-of-concept'])}" unless vulnerability['proof-of-concept'].nil? + text.join("
") + end + + def format_fix_text(vulnerability) + text = [] + text << "Remedial-actions: #{parse_html(vulnerability['remedial-actions'])}" unless vulnerability['remedial-actions'].nil? + text << "Remedial-procedure: #{parse_html(vulnerability['remedial-procedure'])}" unless vulnerability['remedial-procedure'].nil? + text << "Remedy-references: #{parse_html(vulnerability['remedy-references'])}" unless vulnerability['remedy-references'].nil? + text.join("
") + end + + def nist_tag(classification) + puts classification + tags = [] + entries = @cwe_nist_mapping.select { |x| classification['cwe'].include?(x[:cweid].to_s) && !x[:nistid].nil? } + tags << entries.map { |x| x[:nistid] } + puts "cwe #{tags.to_s}" + entries = @owasp_nist_mapping.select { |x| classification['owasp'].include?(x[:owaspid].to_s) && !x[:nistid].nil? } + tags << entries.map { |x| x[:nistid] } + puts "owasp: #{tags.to_s}" + tags.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq + end + + def impact(severity) + IMPACT_MAPPING[severity.to_sym] + end + + def parse_mapper(mapping_file) + csv_data = CSV.read(mapping_file, { encoding: 'UTF-8', + headers: true, + header_converters: :symbol, + converters: :all }) + csv_data.map(&:to_hash) + end + + def desc_tags(data, label) + { "data": data || NA_STRING, "label": label || NA_STRING } + end + + # Netsparker report could have multiple issue entries for multiple findings of same issue type. + # The meta data is identical across entries + # method collapse_duplicates return unique controls with applicable findings collapsed into it. + def collapse_duplicates(controls) + unique_controls = [] + + controls.map { |x| x['id'] }.uniq.each do |id| + collapsed_results = controls.select { |x| x['id'].eql?(id) }.map {|x| x['results']} + unique_control = controls.find { |x| x['id'].eql?(id) } + unique_control['results'] = collapsed_results.flatten + unique_controls << unique_control + end + unique_controls + end + + end +end From b99829e14f4ba394db143ef48afa9da30ce3e717 Mon Sep 17 00:00:00 2001 From: Rony Xavier Date: Mon, 15 Mar 2021 16:06:20 -0400 Subject: [PATCH 2/7] netsparker mapper autoload Signed-off-by: Rony Xavier --- lib/heimdall_tools.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/heimdall_tools.rb b/lib/heimdall_tools.rb index 3932476..ac028c5 100644 --- a/lib/heimdall_tools.rb +++ b/lib/heimdall_tools.rb @@ -15,4 +15,5 @@ module HeimdallTools autoload :JfrogXrayMapper, 'heimdall_tools/jfrog_xray_mapper' autoload :DBProtectMapper, 'heimdall_tools/dbprotect_mapper' autoload :AwsConfigMapper, 'heimdall_tools/aws_config_mapper' + autoload :NetsparkerMapper, 'heimdall_tools/netsparker_mapper' end From 711d9136b2081b6300b903c52fef78327a8f8908 Mon Sep 17 00:00:00 2001 From: Rony Xavier Date: Mon, 15 Mar 2021 16:14:56 -0400 Subject: [PATCH 3/7] netsparker mapper documentation update Signed-off-by: Rony Xavier --- README.md | 18 ++++++++++++++++++ lib/heimdall_tools/help/netsparker_mapper.md | 7 +++++++ 2 files changed, 25 insertions(+) create mode 100644 lib/heimdall_tools/help/netsparker_mapper.md diff --git a/README.md b/README.md index d83a94a..42efe91 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ HeimdallTools supplies several methods to convert output from various tools to " - **jfrog_xray_mapper** - package vulnerability scanner - **dbprotect_mapper** - database vulnerability scanner - **aws_config_mapper** - assess, audit, and evaluate AWS resources +- **netsparker_mapper** - web application security scanner Ruby 2.4 or higher (check using "ruby -v") @@ -234,6 +235,23 @@ FLAGS: example: heimdall_tools aws_config_mapper -o aws_config_results_hdf.json ``` +## netsparker_mapper + +netsparker_mapper translates an Netsparker XML results file into HDF format JSON to be viewable in Heimdall. + + The current iteration only works with Netsparker Enterprise Scan. + +``` +USAGE: heimdall_tools netsparker_mapper [OPTIONS] -x -o + +FLAGS: + -x : path to netsparker results XML file. + -o --output : path to output scan-results json. + -V --verbose : verbose run [optional]. + +example: heimdall_tools netsparker_mapper -x netsparker_results.xml -o netsparker_hdf.json +``` + ## version Prints out the gem version diff --git a/lib/heimdall_tools/help/netsparker_mapper.md b/lib/heimdall_tools/help/netsparker_mapper.md new file mode 100644 index 0000000..99f0427 --- /dev/null +++ b/lib/heimdall_tools/help/netsparker_mapper.md @@ -0,0 +1,7 @@ + netsparker_mapper translates an Netsparker XML results file into HDF format json to be viewable in Heimdall + + The current iteration only works with Netsparker Enterprise Scan. + +Examples: + + heimdall_tools netsparker_mapper -x netsparker_results.xml -o netsparker_hdf.json \ No newline at end of file From 532ac61f7bf86a99b04844212426579b520012f4 Mon Sep 17 00:00:00 2001 From: Rony Xavier Date: Mon, 15 Mar 2021 16:50:59 -0400 Subject: [PATCH 4/7] code cleanup Signed-off-by: Rony Xavier --- lib/heimdall_tools/netsparker_mapper.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/heimdall_tools/netsparker_mapper.rb b/lib/heimdall_tools/netsparker_mapper.rb index 3fb28c2..d317bfa 100644 --- a/lib/heimdall_tools/netsparker_mapper.rb +++ b/lib/heimdall_tools/netsparker_mapper.rb @@ -17,8 +17,6 @@ Information: 0.0 }.freeze -CWE_REGEX = 'CWE-(\d*):'.freeze - DEFAULT_NIST_TAG = ["SA-11", "RA-5", "Rev_4"].freeze # rubocop:disable Metrics/AbcSize @@ -129,15 +127,12 @@ def format_fix_text(vulnerability) end def nist_tag(classification) - puts classification tags = [] entries = @cwe_nist_mapping.select { |x| classification['cwe'].include?(x[:cweid].to_s) && !x[:nistid].nil? } tags << entries.map { |x| x[:nistid] } - puts "cwe #{tags.to_s}" entries = @owasp_nist_mapping.select { |x| classification['owasp'].include?(x[:owaspid].to_s) && !x[:nistid].nil? } tags << entries.map { |x| x[:nistid] } - puts "owasp: #{tags.to_s}" - tags.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq + tags.flatten.empty? ? DEFAULT_NIST_TAG : tags.flatten.uniq end def impact(severity) From 26fc074072afa71753c2d5b89565ccdd1f515b67 Mon Sep 17 00:00:00 2001 From: Rony Xavier Date: Mon, 15 Mar 2021 16:56:50 -0400 Subject: [PATCH 5/7] docs fix Signed-off-by: Rony Xavier --- README.md | 2 +- lib/heimdall_tools/help/netsparker_mapper.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42efe91..0e294e6 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ example: heimdall_tools aws_config_mapper -o aws_config_results_hdf.json netsparker_mapper translates an Netsparker XML results file into HDF format JSON to be viewable in Heimdall. - The current iteration only works with Netsparker Enterprise Scan. + The current iteration only works with Netsparker Enterprise Vulnerabilities Scan. ``` USAGE: heimdall_tools netsparker_mapper [OPTIONS] -x -o diff --git a/lib/heimdall_tools/help/netsparker_mapper.md b/lib/heimdall_tools/help/netsparker_mapper.md index 99f0427..8813ac6 100644 --- a/lib/heimdall_tools/help/netsparker_mapper.md +++ b/lib/heimdall_tools/help/netsparker_mapper.md @@ -1,6 +1,6 @@ netsparker_mapper translates an Netsparker XML results file into HDF format json to be viewable in Heimdall - The current iteration only works with Netsparker Enterprise Scan. + The current iteration only works with Netsparker Enterprise Vulnerabilities Scan. Examples: From a05d4b4f68d78d67c0a179ee79c479e42f2e2d28 Mon Sep 17 00:00:00 2001 From: Rony Xavier Date: Mon, 15 Mar 2021 16:59:08 -0400 Subject: [PATCH 6/7] spacing fix Signed-off-by: Rony Xavier --- lib/heimdall_tools/netsparker_mapper.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/heimdall_tools/netsparker_mapper.rb b/lib/heimdall_tools/netsparker_mapper.rb index d317bfa..b8ac959 100644 --- a/lib/heimdall_tools/netsparker_mapper.rb +++ b/lib/heimdall_tools/netsparker_mapper.rb @@ -37,10 +37,8 @@ def initialize(xml, name=nil, verbose = false) rescue StandardError => e raise "Invalid Netsparker XML file provided Exception: #{e}" end - end - def to_hdf controls = [] @vulnerabilities.each do |vulnerability| @@ -165,6 +163,5 @@ def collapse_duplicates(controls) end unique_controls end - end end From 318b1bcdac63f52fd71e50fe8e65ad760046c77b Mon Sep 17 00:00:00 2001 From: Robert Clark Date: Tue, 16 Mar 2021 09:13:24 -0400 Subject: [PATCH 7/7] Remove Rev_4 from NIST tags Signed-off-by: Robert Clark --- lib/heimdall_tools/netsparker_mapper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/heimdall_tools/netsparker_mapper.rb b/lib/heimdall_tools/netsparker_mapper.rb index b8ac959..00aba60 100644 --- a/lib/heimdall_tools/netsparker_mapper.rb +++ b/lib/heimdall_tools/netsparker_mapper.rb @@ -17,7 +17,7 @@ Information: 0.0 }.freeze -DEFAULT_NIST_TAG = ["SA-11", "RA-5", "Rev_4"].freeze +DEFAULT_NIST_TAG = ["SA-11", "RA-5"].freeze # rubocop:disable Metrics/AbcSize @@ -150,7 +150,7 @@ def desc_tags(data, label) end # Netsparker report could have multiple issue entries for multiple findings of same issue type. - # The meta data is identical across entries + # The meta data is identical across entries # method collapse_duplicates return unique controls with applicable findings collapsed into it. def collapse_duplicates(controls) unique_controls = []