From ce38c1e795ffa6ad7cb3cd139ae97a81edd0c9b3 Mon Sep 17 00:00:00 2001 From: Raphael Salas Date: Thu, 30 May 2019 11:18:13 -0400 Subject: [PATCH 1/5] exposed patternmatch options for including extensions (whitelist) --- lib/salus/scanners/pattern_search.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/salus/scanners/pattern_search.rb b/lib/salus/scanners/pattern_search.rb index c5a47e04..5c85feeb 100644 --- a/lib/salus/scanners/pattern_search.rb +++ b/lib/salus/scanners/pattern_search.rb @@ -16,7 +16,8 @@ module Salus::Scanners class PatternSearch < Base def run global_exclude_directory_flags = flag_list('--exclude-dirs', @config['exclude_directory']) - global_exclude_extension_flags = extension_flag(@config['exclude_extension']) + global_exclude_extension_flags = extension_flag('--exclude-ext', @config['exclude_extension']) + global_include_extension_flags = extension_flag('--ext', @config['include_extension']) # For each pattern, keep a running history of failures, errors, and hits # These will be reported on at the end. @@ -35,12 +36,14 @@ def run match_exclude_directory_flags = flag_list( '--exclude-dirs', match['exclude_directory'] ) - match_exclude_extension_flags = extension_flag(match['exclude_extension']) + match_exclude_extension_flags = extension_flag('--exclude-ext', match['exclude_extension']) + match_include_extension_flags = extension_flag('--ext', match['include_extension']) command_string = [ "sift -n -e \"#{match['regex']}\" .", match_exclude_directory_flags || global_exclude_directory_flags, - match_exclude_extension_flags || global_exclude_extension_flags + match_exclude_extension_flags || global_exclude_extension_flags, + match_include_extension_flags || global_include_extension_flags ].compact.join(' ') shell_return = run_shell(command_string) @@ -108,13 +111,13 @@ def should_run? private - def extension_flag(file_extensions) - if file_extensions.nil? + def extension_flag(flag, file_extensions) + if file_extensions.nil? or flag.nil? nil - elsif file_extensions.empty? + elsif file_extensions.empty? or flag.empty? "" else - flag = '--exclude-ext=' + flag << '=' flag << file_extensions.join(',') end end From 37fa5ffdeafa90e8dac1fb173bcf46aeee482933 Mon Sep 17 00:00:00 2001 From: Raphael Salas Date: Fri, 31 May 2019 16:53:18 -0400 Subject: [PATCH 2/5] added tests and fixtures for include_extension option --- spec/fixtures/pattern_search/fancy.md | 3 + .../lib/salus/scanners/pattern_search_spec.rb | 110 ++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 spec/fixtures/pattern_search/fancy.md diff --git a/spec/fixtures/pattern_search/fancy.md b/spec/fixtures/pattern_search/fancy.md new file mode 100644 index 00000000..ab4c516a --- /dev/null +++ b/spec/fixtures/pattern_search/fancy.md @@ -0,0 +1,3 @@ +# Fancy + +This is _fancy_ **formatted**. diff --git a/spec/lib/salus/scanners/pattern_search_spec.rb b/spec/lib/salus/scanners/pattern_search_spec.rb index 6ac9a6a5..355d8258 100644 --- a/spec/lib/salus/scanners/pattern_search_spec.rb +++ b/spec/lib/salus/scanners/pattern_search_spec.rb @@ -194,6 +194,116 @@ end end + context 'global inclusions are given' do + it 'should search only through included material' do + repo = Salus::Repo.new('spec/fixtures/pattern_search') + + config = { + 'matches' => [ + { regex: 'UN' }, + { 'regex' => 'lance', 'forbidden' => true }, + { regex: 'fancy'} + ], + 'include_extension' => ['md'] + } + + scanner = Salus::Scanners::PatternSearch.new(repository: repo, config: config) + scanner.run + + expect(scanner.report.passed?).to eq(true) + end + end + + context 'local inclusions are given' do + it 'should only search through included material' do + repo = Salus::Repo.new('spec/fixtures/pattern_search') + + config = { + 'matches' => [ + { 'regex' => 'lance', 'forbidden' => true, 'include_extension' => ['md'] } + ] + } + + scanner = Salus::Scanners::PatternSearch.new(repository: repo, config: config) + scanner.run + + expect(scanner.report.passed?).to eq(true) + end + + it 'should not search through extensions not explicitly included' do + repo = Salus::Repo.new('spec/fixtures/pattern_search') + + config = { + 'matches' => [ + { 'regex' => 'UN', 'include_extension' => ['md'] }, + { 'regex' => 'fancy', 'include_extension' => ['md'] } + ] + } + + scanner = Salus::Scanners::PatternSearch.new(repository: repo, config: config) + scanner.run + + expect(scanner.report.passed?).to eq(true) + + info = scanner.report.to_h.fetch(:info) + expect(info[:hits].map { |hit| hit[:regex] }).to_not include('UN') + expect(info[:hits].map { |hit| hit[:regex] }).to include('fancy') + end + + it 'should coexist with exclusions' do + repo = Salus::Repo.new('spec/fixtures/pattern_search') + config = { + 'matches' => [ + { 'regex' => 'fancy', 'include_extension' => ['md'] }, + { 'regex' => 'lance', 'forbidden' => true, 'exclude_extension' => ['txt'], 'include_extension' => ['md'] } + ] + } + + scanner = Salus::Scanners::PatternSearch.new(repository: repo, config: config) + scanner.run + + expect(scanner.report.passed?).to eq(true) + + info = scanner.report.to_h.fetch(:info) + expect(info[:hits].map { |hit| hit[:regex] }).to include('fancy') + end + + it 'should handle conflicting local exclusions' do + repo = Salus::Repo.new('spec/fixtures/pattern_search') + config = { + 'matches' => [ + { 'regex' => 'fancy', 'include_extension' => ['md'], 'exclude_extension' => ['md'] }, + { 'regex' => 'lance', 'forbidden' => true, 'exclude_extension' => ['txt'], 'include_extension' => ['md'] } + ] + } + + scanner = Salus::Scanners::PatternSearch.new(repository: repo, config: config) + scanner.run + + expect(scanner.report.passed?).to eq(true) + info = scanner.report.to_h.fetch(:info) + expect(info[:hits].map { |hit| hit[:regex] }).to_not include('fancy') + end + + it 'should handle conflicting global exclusions' do + repo = Salus::Repo.new('spec/fixtures/pattern_search') + config = { + 'matches' => [ + { 'regex' => 'fancy', 'include_extension' => ['md'] }, + { 'regex' => 'lance', 'forbidden' => true, 'include_extension' => ['md'] } + ], + 'exclude_extension' => %w[txt md] + } + + scanner = Salus::Scanners::PatternSearch.new(repository: repo, config: config) + scanner.run + + expect(scanner.report.passed?).to eq(true) + info = scanner.report.to_h.fetch(:info) + expect(info[:hits].map { |hit| hit[:regex] }).to_not include('fancy') + end + end + context 'invalid regex or settings which causes error' do it 'should record the STDERR of bundle-audit' do repo = Salus::Repo.new('spec/fixtures/pattern_search') From 83de12a9d08b5e11e22ee8ba279d00c3aa19de40 Mon Sep 17 00:00:00 2001 From: Raphael Salas Date: Fri, 31 May 2019 17:05:19 -0400 Subject: [PATCH 3/5] updated pattern_search docs to show include_extension example --- docs/scanners/pattern_search.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/scanners/pattern_search.md b/docs/scanners/pattern_search.md index cb844c75..814b1120 100644 --- a/docs/scanners/pattern_search.md +++ b/docs/scanners/pattern_search.md @@ -15,8 +15,11 @@ scanner_configs: forbidden: true exclude_directory: - node_modules - exclude_extension: - - md + include_extension: + - js + - erb + - html + - htm - regex: "# Threat Model" message: All repos must contain a documented threat model. required: true From 14f59f593fba93e6aadae19c6b5130763dc066e0 Mon Sep 17 00:00:00 2001 From: Raphael Salas Date: Tue, 4 Jun 2019 17:01:39 -0400 Subject: [PATCH 4/5] added note for include and exclude rule behavior --- docs/scanners/pattern_search.md | 2 ++ lib/salus/scanners/pattern_search.rb | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/scanners/pattern_search.md b/docs/scanners/pattern_search.md index 814b1120..6f639edc 100644 --- a/docs/scanners/pattern_search.md +++ b/docs/scanners/pattern_search.md @@ -2,6 +2,8 @@ This scanner can flag anti-patterns found in a codebase or require that certain strings be present. This might be useful for preventing the use of dangerous methods like `eval()` in Ruby (which might allow for RCE) or `dangerouslySetInnerHTML` in React (which might allow for XSS). By default, all found patterns are added to the info section of the report. If a found pattern is forbidden, this scanner will fail and the `message` will be show to the developer in the report to give additional context on why this was an issue. A `required` pattern must be found in order for the scan to pass. +The scanner also allows options `exclude_extension` and `include_extension` for excluding and including file extensions, respectively. These options can be set globally and per-match. While these options can be combined, exclusions take precedence when extensions conflict (are both included and excluded) in declarations. + The tool [sift](https://sift-tool.org), written in Go, is used to perform the pattern matching. ## Configuration diff --git a/lib/salus/scanners/pattern_search.rb b/lib/salus/scanners/pattern_search.rb index 5c85feeb..340b5ae4 100644 --- a/lib/salus/scanners/pattern_search.rb +++ b/lib/salus/scanners/pattern_search.rb @@ -5,7 +5,9 @@ # Config file can provide: # - exclude_directories: Array of directories (GLOB) in the repo to exclude from the search. # - exclude_extensions: Array of file extensions to exclude from the search. -# The above can also be provided per-match, and will override the global values. +# - include_extensions: Array of file extensions to scan exclusively. +# The above can also be provided per-match, and will override the global values. Exclusions +# take precedence over inclusions if they conflict. # - matches: Array[Hash] # regex: (required) regex to match against. # forbidden: (default false) if true, a hit on this regex will fail the test. From c65c4d0e295d7e5b6726380b07e2653cf217cdd4 Mon Sep 17 00:00:00 2001 From: Nishil Shah Date: Wed, 5 Jun 2019 16:39:28 -0700 Subject: [PATCH 5/5] fix rubocop errors --- lib/salus/scanners/pattern_search.rb | 7 ++++--- spec/lib/salus/scanners/pattern_search_spec.rb | 12 +++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/salus/scanners/pattern_search.rb b/lib/salus/scanners/pattern_search.rb index 340b5ae4..86f0a691 100644 --- a/lib/salus/scanners/pattern_search.rb +++ b/lib/salus/scanners/pattern_search.rb @@ -38,7 +38,8 @@ def run match_exclude_directory_flags = flag_list( '--exclude-dirs', match['exclude_directory'] ) - match_exclude_extension_flags = extension_flag('--exclude-ext', match['exclude_extension']) + match_exclude_extension_flags = extension_flag('--exclude-ext', \ + match['exclude_extension']) match_include_extension_flags = extension_flag('--ext', match['include_extension']) command_string = [ @@ -114,9 +115,9 @@ def should_run? private def extension_flag(flag, file_extensions) - if file_extensions.nil? or flag.nil? + if file_extensions.nil? || flag.nil? nil - elsif file_extensions.empty? or flag.empty? + elsif file_extensions.empty? || flag.empty? "" else flag << '=' diff --git a/spec/lib/salus/scanners/pattern_search_spec.rb b/spec/lib/salus/scanners/pattern_search_spec.rb index 355d8258..dadcc033 100644 --- a/spec/lib/salus/scanners/pattern_search_spec.rb +++ b/spec/lib/salus/scanners/pattern_search_spec.rb @@ -202,7 +202,7 @@ 'matches' => [ { regex: 'UN' }, { 'regex' => 'lance', 'forbidden' => true }, - { regex: 'fancy'} + { regex: 'fancy' } ], 'include_extension' => ['md'] } @@ -255,7 +255,8 @@ config = { 'matches' => [ { 'regex' => 'fancy', 'include_extension' => ['md'] }, - { 'regex' => 'lance', 'forbidden' => true, 'exclude_extension' => ['txt'], 'include_extension' => ['md'] } + { 'regex' => 'lance', 'forbidden' => true, 'exclude_extension' => ['txt'], \ + 'include_extension' => ['md'] } ] } @@ -272,8 +273,9 @@ repo = Salus::Repo.new('spec/fixtures/pattern_search') config = { 'matches' => [ - { 'regex' => 'fancy', 'include_extension' => ['md'], 'exclude_extension' => ['md'] }, - { 'regex' => 'lance', 'forbidden' => true, 'exclude_extension' => ['txt'], 'include_extension' => ['md'] } + { 'regex' => 'fancy', 'include_extension' => ['md'], 'exclude_extension' => ['md'] }, + { 'regex' => 'lance', 'forbidden' => true, 'exclude_extension' => ['txt'], \ + 'include_extension' => ['md'] } ] } @@ -289,7 +291,7 @@ repo = Salus::Repo.new('spec/fixtures/pattern_search') config = { 'matches' => [ - { 'regex' => 'fancy', 'include_extension' => ['md'] }, + { 'regex' => 'fancy', 'include_extension' => ['md'] }, { 'regex' => 'lance', 'forbidden' => true, 'include_extension' => ['md'] } ], 'exclude_extension' => %w[txt md]