Configurable file exclusions

README.md

@@ -93,4 +93,8 @@ You can use the tool with the following options:
 --branch                    name of the branch the tool is to run on
 --PR                        only when the tool is running on a pull request
 --allow-folder-id-mismatch  allow the addon's folder name and id to mismatch
+--exclude-file              files/folders to exclude from permissions and file extension checks
+--exclude-file-ext          file extensions to exclude from permissions and file extension checks
+--whitelist-file            files/folders to exclude from file extension checks
+--whitelist-file-ext        file extensions to exclude from file extension checks
 ```

kodi_addon_checker/__main__.py

@@ -76,6 +76,14 @@ def main():
     parser.add_argument("--PR", help="Tell if tool is to run on a pull requests or not", action='store_true')
     parser.add_argument("--allow-folder-id-mismatch", help="Allow the addon's folder name and id to mismatch",
                         action="store_true")
+    parser.add_argument("--exclude-file", nargs="*", default=[],
+                        help="Files/folders to exclude from permissions and file extension checks")
+    parser.add_argument("--exclude-file-ext", nargs="*", default=[],
+                        help="File extensions to exclude from permissions and file extension checks")
+    parser.add_argument("--whitelist-file", nargs="*", default=[],
+                        help="Files/folders to exclude from file extension checks")
+    parser.add_argument("--whitelist-file-ext", nargs="*", default=[],
+                        help="File extensions to exclude from file extension checks")
     ConfigManager.fill_cmd_args(parser)
     args = parser.parse_args()
 

kodi_addon_checker/check_addon.py

@@ -58,7 +58,7 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None):
 
             check_dependencies.check_reverse_dependencies(addon_report, addon_id, branch_name, all_repo_addons)
 
-            check_files.check_file_permission(addon_report, file_index)
+            check_files.check_file_permission(addon_report, file_index, args.exclude_file, args.exclude_file_ext)
 
             check_files.check_for_invalid_xml_files(addon_report, file_index)
 
@@ -96,7 +96,9 @@ def start(addon_path, branch_name, all_repo_addons, args, config=None):
             # General blacklist
             check_string.find_blacklisted_strings(addon_report, addon_path, [], [], [])
 
-            check_files.check_file_whitelist(addon_report, file_index, addon_path)
+            whitelist_files = args.exclude_file + args.whitelist_file
+            whitelist_exts = args.exclude_file_ext + args.whitelist_file_ext
+            check_files.check_file_whitelist(addon_report, file_index, addon_path, whitelist_files, whitelist_exts)
         else:
             addon_report.add(
                 Record(INFORMATION, "Addon marked as broken - skipping"))

kodi_addon_checker/check_files.py

@@ -103,23 +103,37 @@ def check_for_legacy_language_path(report: Report, addon_path: str):
                 break
 
 
-def check_file_whitelist(report: Report, file_index: list, addon_path: str):
+def check_file_whitelist(report: Report, file_index: list, addon_path: str,
+                         whitelisted_files: list, whitelisted_exts: list):
     """check whether the files present in addon are in whitelist or not
         It ignores README.md and .gitignore file
         :file_index: list having names and path of all the files present in addon
         :addon_path: path to the addon folder
+        :whitelisted_files: list of files/folders to whitelist provided by --exclude-file + --whitelist-file
+        :whitelisted_exts: list of file extensions to whitelist provided by --exclude-file-ext + --whitelist-file-ext
     """
     if ".module." in addon_path:
         report.add(Record(INFORMATION, "Module skipping whitelist"))
         return
 
+    whitelisted_exts = "|".join(set([re.escape(ext.lstrip(".")) for ext in whitelisted_exts]))
+    if whitelisted_exts:
+        whitelisted_exts = "|" + whitelisted_exts
+
     whitelist = (
         r"\.?(py|xml|gif|png|jpg|jpeg|md|txt|po|json|gitignore|markdown|yml|"
         r"rst|ini|flv|wav|mp4|html|css|lst|pkla|g|template|in|cfg|xsd|directory|"
-        r"help|list|mpeg|pls|info|ttf|xsp|theme|yaml|dict|crt)?$"
+        r"help|list|mpeg|pls|info|ttf|xsp|theme|yaml|dict|crt{})?$".format(whitelisted_exts)
     )
 
+    whitelisted_files = set(whitelisted_files)
+
     for file in file_index:
+        full_path = os.path.join(file["path"], file["name"])
+        if any(f == file["name"] or f == full_path or (os.path.isdir(f) and file["path"].startswith(f))
+               for f in whitelisted_files):
+            continue
+
         file_parts = file["name"].rsplit(".")
         if len(file_parts) > 1:
             file_ending = "." + file_parts[len(file_parts) - 1]
@@ -129,13 +143,31 @@ def check_file_whitelist(report: Report, file_index: list, addon_path: str):
                                   relative_path(os.path.join(file["path"], file["name"]))))
 
 
-def check_file_permission(report: Report, file_index: list):
+def check_file_permission(report: Report, file_index: list, whitelisted_files: list, whitelisted_exts: list):
     """Check whether the files present in addon are marked executable
        or not
         :file_index: list having names and path of all the files present in addon
+        :whitelisted_files: list of files/folders to whitelist provided by argument --exclude-file
+        :whitelisted_exts: list of file extensions to whitelist provided by argument --exclude-file-ext
     """
 
+    whitelisted_exts = "|".join(set([re.escape(ext.lstrip(".")) for ext in whitelisted_exts]))
+    whitelist = r"\.?({})?$".format(whitelisted_exts)
+
+    whitelisted_files = set(whitelisted_files)
+
     for file in file_index:
-        file = os.path.join(file["path"], file["name"])
-        if os.path.isfile(file) and os.access(str(file), os.X_OK):
-            report.add(Record(PROBLEM, "%s is marked as stand-alone executable" % relative_path(str(file))))
+        full_path = os.path.join(file["path"], file["name"])
+        if any(f == file["name"] or f == full_path or (os.path.isdir(f) and file["path"].startswith(f))
+               for f in whitelisted_files):
+            continue
+
+        if whitelisted_exts:
+            file_parts = file["name"].rsplit(".")
+            if len(file_parts) > 1:
+                file_ending = "." + file_parts[len(file_parts) - 1]
+                if re.match(whitelist, file_ending, re.IGNORECASE):
+                    continue
+
+        if os.path.isfile(full_path) and os.access(str(full_path), os.X_OK):
+            report.add(Record(PROBLEM, "%s is marked as stand-alone executable" % relative_path(str(full_path))))

tests/test_check_addon.py

@@ -10,6 +10,10 @@
 class Args(object):
     PR = False
     allow_folder_id_mismatch = False
+    exclude_file = []
+    exclude_file_ext = []
+    whitelist_file = []
+    whitelist_file_ext = []
 
 
 class TestCheckAddon(unittest.TestCase):

tests/test_check_files.py

@@ -4,6 +4,7 @@
 from os.path import abspath, dirname, join
 
 from kodi_addon_checker.check_files import check_file_permission
+from kodi_addon_checker.check_files import check_file_whitelist
 from kodi_addon_checker.handle_files import create_file_index
 
 from kodi_addon_checker.common import load_plugins
@@ -25,14 +26,65 @@ def setUp(self):
 
     def test_check_file_permission_is_true(self):
         self.path = join(HERE, 'test_data', 'Executable file')
-        self.string = "ERROR: .{path}/file_permission.py is marked as stand-alone executable".format(path=self.path)
+        self.string = "ERROR: {path} is marked as stand-alone executable"\
+            .format(path=relative_path(join(self.path, "file_permission.py")))
         file_index = create_file_index(self.path)
-        check_file_permission(self.report, file_index)
+        check_file_permission(self.report, file_index, [], [])
         records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports]
         flag = any(s == self.string for s in records)
         self.assertTrue(flag)
 
     def test_check_file_permission_is_None(self):
         self.path = join(HERE, 'test_data', 'Non-Executable file')
         file_index = create_file_index(self.path)
-        self.assertIsNone(check_file_permission(self.report, file_index))
+        self.assertIsNone(check_file_permission(self.report, file_index, [], []))
+
+    def test_check_file_permission_is_excluded_by_name(self):
+        self.path = join(HERE, 'test_data', 'Executable file')
+        self.string = "ERROR: {path} is marked as stand-alone executable"\
+            .format(path=relative_path(join(self.path, "file_permission.py")))
+        file_index = create_file_index(self.path)
+        check_file_permission(self.report, file_index, ["file_permission.py"], [])
+        records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports]
+        flag = any(s == self.string for s in records)
+        self.assertFalse(flag)
+
+    def test_check_file_permission_is_excluded_by_ext(self):
+        self.path = join(HERE, 'test_data', 'Executable file')
+        self.string = "ERROR: {path} is marked as stand-alone executable"\
+            .format(path=relative_path(join(self.path, "file_permission.py")))
+        file_index = create_file_index(self.path)
+        check_file_permission(self.report, file_index, [], ["py"])
+        records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports]
+        flag = any(s == self.string for s in records)
+        self.assertFalse(flag)
+
+    def test_check_file_whitelist_is_true(self):
+        self.path = join(HERE, 'test_data', 'File whitelist')
+        self.string = "WARN: Found non whitelisted file ending in filename {path}"\
+            .format(path=relative_path(join(self.path, "file_whitelist.ext")))
+        file_index = create_file_index(self.path)
+        check_file_whitelist(self.report, file_index, self.path, [], [])
+        records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports]
+        flag = any(s == self.string for s in records)
+        self.assertTrue(flag)
+
+    def test_check_file_whitelist_is_excluded_by_name(self):
+        self.path = join(HERE, 'test_data', 'File whitelist')
+        self.string = "WARN: Found non whitelisted file ending in filename {path}"\
+            .format(path=relative_path(join(self.path, "file_whitelist.ext")))
+        file_index = create_file_index(self.path)
+        check_file_whitelist(self.report, file_index, self.path, ["file_whitelist.ext"], [])
+        records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports]
+        flag = any(s == self.string for s in records)
+        self.assertFalse(flag)
+
+    def test_check_file_whitelist_is_excluded_by_ext(self):
+        self.path = join(HERE, 'test_data', 'File whitelist')
+        self.string = "WARN: Found non whitelisted file ending in filename {path}"\
+            .format(path=relative_path(join(self.path, "file_whitelist.ext")))
+        file_index = create_file_index(self.path)
+        check_file_whitelist(self.report, file_index, self.path, [], ["ext"])
+        records = [Record.__str__(r) for r in ReportManager.getEnabledReporters()[0].reports]
+        flag = any(s == self.string for s in records)
+        self.assertFalse(flag)

tests/test_data/File whitelist/file_whitelist.ext

@@ -0,0 +1 @@
+Non whitelisted file ending