Dev additions

privacyscanner/scanmodules/chromedevtools/__init__.py

@@ -9,7 +9,7 @@
     TLSDetailsExtractor, CertificateExtractor, ThirdPartyExtractor, InsecureContentExtractor, \
     FailedRequestsExtractor, SecurityHeadersExtractor, TrackerDetectExtractor, \
     CookieStatsExtractor, JavaScriptLibsExtractor, ScreenshotExtractor, ImprintExtractor, \
-    HSTSPreloadExtractor, FingerprintingExtractor
+    GeneratorTagExtractor, HSTSPreloadExtractor, FingerprintingExtractor, SriExtractor
 from privacyscanner.scanmodules.chromedevtools.utils import TLDEXTRACT_CACHE_FILE, parse_domain
 from privacyscanner.utils import file_is_outdated, set_default_options, calculate_jaccard_index
 
@@ -19,7 +19,8 @@
                      CertificateExtractor, ThirdPartyExtractor, InsecureContentExtractor,
                      FailedRequestsExtractor, SecurityHeadersExtractor, TrackerDetectExtractor,
                      CookieStatsExtractor, JavaScriptLibsExtractor, ScreenshotExtractor,
-                     ImprintExtractor, HSTSPreloadExtractor, FingerprintingExtractor]
+                     ImprintExtractor, HSTSPreloadExtractor, FingerprintingExtractor,
+                     GeneratorTagExtractor, SriExtractor]
 
 EXTRACTOR_CLASSES_HTTPS_RUN = [FinalUrlExtractor, TLSDetailsExtractor, CertificateExtractor,
                                InsecureContentExtractor, SecurityHeadersExtractor,
@@ -79,3 +80,4 @@ def update_dependencies(self):
         for extractor_class in EXTRACTOR_CLASSES:
             if hasattr(extractor_class, 'update_dependencies'):
                 extractor_class.update_dependencies(self.options)
+

privacyscanner/scanmodules/chromedevtools/chromescan.py

@@ -305,6 +305,9 @@ def scan(self, browser, result, logger, options):
         self._tab.Security.enable()
         self._tab.Security.setIgnoreCertificateErrors(ignore=True)
 
+        self._register_log_callbacks()
+        self._tab.Log.enable()
+
         self._tab.Page.loadEventFired = self._cb_load_event_fired
         self._tab.Page.frameScheduledNavigation = self._cb_frame_scheduled_navigation
         self._tab.Page.frameClearedScheduledNavigation = self._cb_frame_cleared_scheduled_navigation
@@ -485,6 +488,9 @@ def _cb_security_state_changed(self, **state):
     def _cb_loading_failed(self, **failed_request):
         self._page.add_failed_request(failed_request)
 
+    def _cb_log_entryAdded(self, **log):
+        self._page.add_log_event(log)
+
     def _register_network_callbacks(self):
         self._tab.Network.requestWillBeSent = self._cb_request_will_be_sent
         self._tab.Network.responseReceived = self._cb_response_received
@@ -498,6 +504,9 @@ def _unregister_network_callbacks(self):
     def _register_security_callbacks(self):
         self._tab.Security.securityStateChanged = self._cb_security_state_changed
 
+    def _register_log_callbacks(self):
+        self._tab.Log.entryAdded = self._cb_log_entryAdded
+
     def _unregister_security_callbacks(self):
         self._tab.Security.securityStateChanged = None
 
@@ -573,6 +582,7 @@ def __init__(self, tab=None):
         self.failed_request_log = []
         self.response_log = []
         self.security_state_log = []
+        self.logging_log = []
         self.scan_start = None
         self.tab = tab
         self._response_lookup = defaultdict(list)
@@ -594,6 +604,10 @@ def add_request(self, request):
     def add_failed_request(self, failed_request):
         self.failed_request_log.append(failed_request)
 
+    def add_log_event(self, log_event):
+        self.logging_log.append(log_event)
+
+
     def add_response(self, response):
         self.response_log.append(response)
         self._response_lookup[response['requestId']].append(response)

privacyscanner/scanmodules/chromedevtools/extractors/__init__.py

@@ -3,6 +3,7 @@
 from .cookiestats import CookieStatsExtractor
 from .failedrequests import FailedRequestsExtractor
 from .finalurl import FinalUrlExtractor
+from .generatortag import GeneratorTagExtractor
 from .googleanalytics import GoogleAnalyticsExtractor
 from .insecurecontent import InsecureContentExtractor
 from .javascriptlibs import JavaScriptLibsExtractor
@@ -15,4 +16,5 @@
 from .screenshot import ScreenshotExtractor
 from .imprint import ImprintExtractor
 from .hstspreload import HSTSPreloadExtractor
-from .fingerprinting import FingerprintingExtractor
+from .fingerprinting import FingerprintingExtractor
+from .sricheck import SriExtractor

privacyscanner/scanmodules/chromedevtools/extractors/generatortag.py

@@ -0,0 +1,48 @@
+import pychrome
+
+from privacyscanner.scanmodules.chromedevtools.extractors.base import Extractor
+from privacyscanner.scanmodules.chromedevtools.utils import scripts_disabled
+
+
+ELEMENT_NODE = 1
+
+
+class GeneratorTagExtractor(Extractor):
+
+    GENERATOR_KEYWORDS = ['generator', 'Generator']
+
+    def extract_information(self):
+        # Disable scripts to avoid DOM changes while searching for generator tags, see imprint.py / pull request
+        with scripts_disabled(self.page.tab, self.options):
+            self._extract_information()
+
+    def extract_information(self):
+        tags = []
+
+        node_id = self.page.tab.DOM.getDocument()['root']['nodeId']
+        meta_node_ids = self.page.tab.DOM.querySelectorAll(nodeId=node_id, selector='meta')['nodeIds']
+
+        for node_id in meta_node_ids:
+            while node_id is not None:
+                try:
+                    node = self.page.tab.DOM.describeNode(nodeId=node_id)['node']
+                except pychrome.CallMethodException:
+                    # For some reason, nodes seem to disappear in-between,
+                    # so just ignore these cases.
+                    break
+                if node['nodeType'] == ELEMENT_NODE and node['nodeName'].lower() == 'meta':
+                    if node['attributes'][1] == 'generator':
+                        tags.append(node['attributes'][3])
+                        break
+                node_id = node.get('parentId')
+
+        tags = list(set(tags))
+        generator_tags = {}
+        if tags:
+            i = 0
+            for element in tags:
+                generator_tags[str(i + 1)] = tags[i]
+                i += 1
+            self.result['generator'] = generator_tags
+        else:
+            self.result['generator'] = None

privacyscanner/scanmodules/chromedevtools/extractors/securityheaders.py

@@ -24,6 +24,7 @@ def extract_information(self):
         csp_value = None
         if 'content-security-policy' in headers:
             csp_value = self._parse_csp(headers['content-security-policy'])
+
         security_headers['Content-Security-Policy'] = csp_value
 
         xss_protection = None

privacyscanner/scanmodules/chromedevtools/extractors/sricheck.py

@@ -0,0 +1,135 @@
+import pychrome
+
+from privacyscanner.scanmodules.chromedevtools.extractors.base import Extractor
+from privacyscanner.scanmodules.chromedevtools.utils import scripts_disabled
+
+ELEMENT_NODE = 3
+
+
+class SriExtractor(Extractor):
+
+    def extract_information(self):
+        # Disable scripts to avoid DOM changes while searching for generator tags, see imprint.py / pull request
+        with scripts_disabled(self.page.tab, self.options):
+            self.extract_sri()
+
+    def extract_sri(self):
+        sri_dict = {}
+        final_sri_list = []
+        failed_urls = []
+
+        sri_dict['require_sri_for'] = None
+        sri_dict['all_sri_active_and_valid'] = None
+        sri_dict['at_least_one_sri_active'] = None
+        sri_dict['all_sri_active'] = None
+
+        # Check already read CSP Values in _self
+        # Currently Chrome is configured to IGNORE require_sri_for. Only if the flag
+        # #enable-experimental-web-platform-features is enabled, it correctly throws an error if a script / style
+        # has no integrity-hash.
+
+        security_headers = self.result['security_headers']
+        if security_headers['Content-Security-Policy'] is not None:
+            if 'require-sri-for' in security_headers['Content-Security-Policy']:
+                sri_dict['require_sri_for'] = security_headers['Content-Security-Policy']['require-sri-for'][0]
+        # This results in privacyscanner reading the CSP header for SRI but chromedevtools is currently not enforcing it
+
+        node_id = self.page.tab.DOM.getDocument()['root']['nodeId']
+        links = self.page.tab.DOM.querySelectorAll(nodeId=node_id, selector='link')['nodeIds']
+
+        for node_id in links:
+            while node_id is not None:
+                try:
+                    node = self.page.tab.DOM.describeNode(nodeId=node_id)['node']
+                except pychrome.CallMethodException:
+                    # For some reason, nodes seem to disappear in-between,
+                    # so just ignore these cases.
+                    break
+
+                if node['nodeType'] == 1 and 'href' in node['attributes']:
+                    if "stylesheet" in node['attributes']:
+                        self._add_element_to_linklist(final_sri_list, None, node['attributes'])
+                        break
+                    if "script" in node['attributes']:
+                        self._add_element_to_linklist(final_sri_list, None, node['attributes'])
+                        break
+                node_id = node.get('parentId')
+
+        # Check if href is in entry list, if yes set attributes accordingly.
+        logging_log = self.page.logging_log
+        for element in logging_log:
+            if element['entry']['source'] == 'security' and element['entry']['level'] == 'error':
+                if 'Failed to find a valid digest' in element['entry']['text']:
+                    failed_urls.append(element['entry']['text'].split('\'')[3])
+
+        for element in final_sri_list:
+            if len(failed_urls) == 0:
+                if element['integrity_active']:
+                    element['integrity_valid'] = True
+            for final_url in failed_urls:
+                if '/' + element['href'].replace('/', '', 1) in final_url:
+                    element['integrity_valid'] = False
+                elif element['integrity_active']:
+                    element['integrity_valid'] = True
+                else:
+                    element['integrity_valid'] = None
+
+        # Check if all links have SRI enabled and have a valid hash
+
+        active_counter, valid_counter = 0, 0
+
+        for element in final_sri_list:
+            if element['integrity_active'] and not None:
+                active_counter += 1
+            if element['integrity_valid'] and not None:
+                valid_counter += 1
+
+        # Case 1: All CSS/JS have SRI active
+
+        if len(final_sri_list) == active_counter:
+            sri_dict['all_sri_active'] = True
+        else:
+            sri_dict['all_sri_active'] = False
+
+        # Case 2: At least one of CSS/JS has SRI active (but can be invalid)
+        # This is to not punish websites for using SRI and having a bad hash due to changed code.
+
+        if active_counter > 0:
+            sri_dict['at_least_one_sri_active'] = True
+        else:
+            sri_dict['at_least_one_sri_active'] = False
+
+        # Case 3: All of the used CSS and JS have SRI enabled and all hashes match.
+
+        if active_counter == valid_counter == len(final_sri_list):
+            sri_dict['all_sri_active_and_valid'] = True
+        else:
+            sri_dict['all_sri_active_and_valid'] = False
+
+        sri_dict['link-list'] = final_sri_list
+
+        self.result['sri-info'] = sri_dict
+
+    def _add_element_to_linklist(self, final_sri_list, node_value, node_attributes):
+        global new_entry
+        new_entry = dict(href=None, type=None, integrity_active=False, integrity_hash=None, integrity_valid=None)
+        if node_value is not None:
+            value_parts = node_value.split()
+            for element in value_parts:
+                if 'href=' in element:
+                    new_entry['href'] = element.split('"')[1]
+                if 'integrity' in element:
+                    new_entry['integrity_active'] = True
+                    new_entry['integrity_hash'] = element.split('"')[1]
+
+        if node_attributes is not None:
+            new_entry['href'] = node_attributes[node_attributes.index('href') + 1]
+            new_entry['type'] = node_attributes[node_attributes.index('rel') + 1]
+            if new_entry['type'] == 'preload':
+                new_entry['type'] = node_attributes[node_attributes.index('preload') + 2]
+            if 'integrity' in node_attributes:
+                new_entry['integrity_active'] = True
+                new_entry['integrity_hash'] = node_attributes[node_attributes.index('integrity') + 1]
+
+        if new_entry not in final_sri_list:
+            final_sri_list.append(new_entry)

privacyscanner/scanmodules/serverleaks.py

@@ -23,10 +23,18 @@ def scan_site(self, result, meta):
 
 def _match_db_dump(content):
     targets = ["SQLite", "CREATE TABLE", "INSERT INTO", "DROP TABLE"]
-    matched = False
-    for target in targets:
-        matched |= target in content
-    return matched
+    return any(target in content for target in targets)
+
+
+def _match_env_file(content):
+    targets = ["TERM", "PATH", "COMPOSER", "INSTALL"]
+    return any(target in content for target in targets)
+
+
+def _match_package_file(content):
+    targets = ["name", "author", "contributors", "bugs", "homepage", "version", "license", "keywords", "description",
+               "repository", "main", "private", "scripts", "dependencies", "devDependencies", "engines", "browserslist"]
+    return any(target in content for target in targets)
 
 
 def _concat_sub(url, suffix):
@@ -102,7 +110,11 @@ def _gen_db_full_domain_pem(url):
     ('.svn/wc.db', 'SQLite'),
     ('core', 'ELF'),
     ('.DS_Store', 'Bud1'),
-
+    ('.npmrc', '='),
+    ('package.json', _match_package_file),
+    # ('.htaccess', 'unknown'),
+    ('workspace.xml', 'FileEditorManager'),
+    ('.gitlab-ci.yml', 'job'),
     # Check for Database dumps
     # sqldump - MySQL/MariaDB
     ('dump.db', _match_db_dump),
@@ -140,8 +152,7 @@ def _gen_db_full_domain_pem(url):
     # https://infosec.rm-it.de/2018/08/19/scanning-the-alexa-top-1m-sites-for-dockerfiles/
     ('Dockerfile', 'FROM'),
     # https://twitter.com/svblxyz/status/1045013939904532482
-    ('docker.env', '='),
-    ('.env', '='),
+    ('docker.env', _match_env_file),
     # Docker Compose
     ('docker-compose.yml', 'version:'),
 ]