From 5c5eeeac4014000709db44da187b976f14ea90fa Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Wed, 13 Mar 2024 17:04:49 +0100
Subject: [PATCH 01/10] improving python-linter_harness exit codes and script
 refactoring

---
 travisci/python-linter_harness.sh | 76 ++++++++++++++++++++++++-------
 1 file changed, 59 insertions(+), 17 deletions(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index ad7ad6052d1..2096679472e 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -2,42 +2,84 @@
 
 # See the NOTICE file distributed with this work for additional information
 # regarding copyright ownership.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #      http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PYTHON_SOURCE_LOCATIONS=('scripts' 'src/python')
-
 # Setup the environment variables
 # shellcheck disable=SC2155
 export PYTHONPATH=$PYTHONPATH:$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')
 # more info: https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules
 export MYPYPATH=$MYPYPATH:src/python/lib
 
-PYLINT_OUTPUT_FILE=$(mktemp)
-PYLINT_ERRORS=$(mktemp)
-# CITest project is on hold and it needs to be updated before resuming its linter checker
-find "${PYTHON_SOURCE_LOCATIONS[@]}" -type f -name "*.py" \! -name "Ortheus.py" \! -name "*citest*.py" \! -path "*/citest/*" -print0 | xargs -0 pylint --rcfile pyproject.toml --verbose | tee "$PYLINT_OUTPUT_FILE"
-grep -v "\-\-\-\-\-\-\-\-\-" "$PYLINT_OUTPUT_FILE" | grep -v "Your code has been rated" | grep -v "\n\n" | sed '/^$/d' > "$PYLINT_ERRORS"
-! [ -s "$PYLINT_ERRORS" ]
+# Function to run pylint
+run_pylint() {
+  local pylint_output_file=$(mktemp)
+  local pylint_errors=$(mktemp)
+
+  # Run pylint, excluding specific files and directories
+  find "${PYTHON_SOURCE_LOCATIONS[@]}" -type f -name "*.py" \
+    ! -name "Ortheus.py" \
+    ! -name "*citest*.py" \
+    ! -path "*/citest/*" -print0 |
+    xargs -0 pylint --rcfile=pyproject.toml --verbose \
+      --msg-template='{path}:{line}:{column}: {msg_id}: {msg} ({symbol})' |
+    tee "$pylint_output_file"
+
+  # Keep only lines with pylint messages
+  grep -E '^.+:[0-9]+:[0-9]+: [A-Z]+[0-9]+: .+ (\(.*\))?$' "$pylint_output_file" >"$pylint_errors"
+
+  # Return 1 if errors were found, otherwise 0
+  ! [ -s "$pylint_errors" ]
+  local result=$(
+    ! [ -s "$pylint_errors" ]
+    echo $?
+  )
+
+  # Cleanup
+  rm "$pylint_output_file" "$pylint_errors"
+
+  return $result
+}
+
+# Function to run mypy, excluding certain files and paths, and capturing the outcome
+run_mypy() {
+  find "${PYTHON_SOURCE_LOCATIONS[@]}" -type f -name "*.py" \
+    \! -name "Ortheus.py" \
+    \! -name "*citest*.py" \
+    \! -path "*/citest/*" -print0 |
+    xargs -0 mypy --config-file pyproject.toml --namespace-packages --explicit-package-bases
+
+  # Return the exit status of mypy
+  return $?
+}
+
+# Define Python source locations
+PYTHON_SOURCE_LOCATIONS=('scripts' 'src/python')
+
+# Run pylint and mypy, capturing their return codes
+run_pylint
 rt1=$?
-rm "$PYLINT_OUTPUT_FILE" "$PYLINT_ERRORS"
 
-# CITest project is on hold and it needs to be updated before resuming its static type checker
-find "${PYTHON_SOURCE_LOCATIONS[@]}" -type f -name "*.py" \! -name "Ortheus.py" \! -name "*citest*.py" \! -path "*/citest/*" -print0 | xargs -0 mypy --config-file pyproject.toml --namespace-packages --explicit-package-bases
+run_mypy
 rt2=$?
 
-if [[ ($rt1 -eq 0) && ($rt2 -eq 0) ]]; then
-  exit 0
+# Determine exit code based on results
+if [[ $rt1 -eq 0 && $rt2 -eq 0 ]]; then
+  exit 0 # success
+elif [[ $rt1 -ne 0 ]]; then
+  exit 1 # pylint error
+elif [[ $rt2 -ne 0 ]]; then
+  exit 2 # mypy error
 else
-  exit 255
+  exit 3 # error on both
 fi

From 9b8c37df2078c1f9ecb6aa87db9375e58780d9f1 Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Wed, 13 Mar 2024 17:07:53 +0100
Subject: [PATCH 02/10] fixing Double quote to prevent globbing and word
 splitting.

---
 travisci/python-linter_harness.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index 2096679472e..1e3bc15b3ff 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -48,7 +48,7 @@ run_pylint() {
   # Cleanup
   rm "$pylint_output_file" "$pylint_errors"
 
-  return $result
+  return "$result"
 }
 
 # Function to run mypy, excluding certain files and paths, and capturing the outcome

From c055287b5277049dc968a10817c12b23fbc5772e Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Wed, 13 Mar 2024 17:23:52 +0100
Subject: [PATCH 03/10] returning the unused character spaces to avoid
 unnecessary changes

---
 travisci/python-linter_harness.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index 1e3bc15b3ff..10b9e765119 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -2,13 +2,13 @@
 
 # See the NOTICE file distributed with this work for additional information
 # regarding copyright ownership.
-#
+# 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#
+# 
 #      http://www.apache.org/licenses/LICENSE-2.0
-#
+# 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

From 12f56cedcb49c6e7ff0a1cac145bbb522c29b4b3 Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Wed, 13 Mar 2024 17:29:46 +0100
Subject: [PATCH 04/10] removing duplicate check of empty file

---
 travisci/python-linter_harness.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index 10b9e765119..33cc4b5e1a0 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -39,7 +39,6 @@ run_pylint() {
   grep -E '^.+:[0-9]+:[0-9]+: [A-Z]+[0-9]+: .+ (\(.*\))?$' "$pylint_output_file" >"$pylint_errors"
 
   # Return 1 if errors were found, otherwise 0
-  ! [ -s "$pylint_errors" ]
   local result=$(
     ! [ -s "$pylint_errors" ]
     echo $?

From b3f5f616d927ba7890db98012ff9352fe07e59c5 Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Fri, 15 Mar 2024 11:21:16 +0100
Subject: [PATCH 05/10] Update travisci/python-linter_harness.sh

Co-authored-by: twalsh-ebi <twalsh@ebi.ac.uk>
---
 travisci/python-linter_harness.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index 33cc4b5e1a0..fa6fdf99110 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -57,9 +57,6 @@ run_mypy() {
     \! -name "*citest*.py" \
     \! -path "*/citest/*" -print0 |
     xargs -0 mypy --config-file pyproject.toml --namespace-packages --explicit-package-bases
-
-  # Return the exit status of mypy
-  return $?
 }
 
 # Define Python source locations

From 579d5d5d8412f67d5adff6a74dc935177d0f2ced Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Fri, 15 Mar 2024 11:26:37 +0100
Subject: [PATCH 06/10] Update travisci/python-linter_harness.sh

Co-authored-by: twalsh-ebi <twalsh@ebi.ac.uk>
---
 travisci/python-linter_harness.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index fa6fdf99110..b45097c808f 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -32,7 +32,7 @@ run_pylint() {
     ! -name "*citest*.py" \
     ! -path "*/citest/*" -print0 |
     xargs -0 pylint --rcfile=pyproject.toml --verbose \
-      --msg-template='{path}:{line}:{column}: {msg_id}: {msg} ({symbol})' |
+      --msg-template='COMPARA_PYLINT_MSG:{path}:{line}:{column}: {msg_id}: {msg} ({symbol})' |
     tee "$pylint_output_file"
 
   # Keep only lines with pylint messages

From 4c5af753a95b7d82686e4e34e5b609dbebcea33d Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Fri, 15 Mar 2024 11:28:26 +0100
Subject: [PATCH 07/10] -c option counts the number of matches, -m 1 stops
 after the first match to optimize performance,

Co-authored-by: twalsh-ebi <twalsh@ebi.ac.uk>
---
 travisci/python-linter_harness.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index b45097c808f..6f42f891a0d 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -35,14 +35,8 @@ run_pylint() {
       --msg-template='COMPARA_PYLINT_MSG:{path}:{line}:{column}: {msg_id}: {msg} ({symbol})' |
     tee "$pylint_output_file"
 
-  # Keep only lines with pylint messages
-  grep -E '^.+:[0-9]+:[0-9]+: [A-Z]+[0-9]+: .+ (\(.*\))?$' "$pylint_output_file" >"$pylint_errors"
-
-  # Return 1 if errors were found, otherwise 0
-  local result=$(
-    ! [ -s "$pylint_errors" ]
-    echo $?
-  )
+  # Return 1 if pylint messages were found, otherwise 0
+  local result=$(grep -c -m 1 -E '^COMPARA_PYLINT_MSG:' "$pylint_output_file")
 
   # Cleanup
   rm "$pylint_output_file" "$pylint_errors"

From 1d925b2cf77767f2e2b28f6602fee3a0c44bde84 Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Fri, 15 Mar 2024 11:29:19 +0100
Subject: [PATCH 08/10] Update travisci/python-linter_harness.sh

---
 travisci/python-linter_harness.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index 6f42f891a0d..6e5c8102e6e 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -36,6 +36,7 @@ run_pylint() {
     tee "$pylint_output_file"
 
   # Return 1 if pylint messages were found, otherwise 0
+  #  -c option counts the number of matches, -m 1 stops after the first match to optimize performance,
   local result=$(grep -c -m 1 -E '^COMPARA_PYLINT_MSG:' "$pylint_output_file")
 
   # Cleanup

From 376a328cc1a2a41e1dfeb12b08ace0ec44221738 Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Fri, 15 Mar 2024 11:48:09 +0100
Subject: [PATCH 09/10] removing pylint_error_file as it isn't needed anymore

---
 travisci/python-linter_harness.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index 6e5c8102e6e..c38b79face4 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -24,7 +24,6 @@ export MYPYPATH=$MYPYPATH:src/python/lib
 # Function to run pylint
 run_pylint() {
   local pylint_output_file=$(mktemp)
-  local pylint_errors=$(mktemp)
 
   # Run pylint, excluding specific files and directories
   find "${PYTHON_SOURCE_LOCATIONS[@]}" -type f -name "*.py" \
@@ -40,7 +39,7 @@ run_pylint() {
   local result=$(grep -c -m 1 -E '^COMPARA_PYLINT_MSG:' "$pylint_output_file")
 
   # Cleanup
-  rm "$pylint_output_file" "$pylint_errors"
+  rm "$pylint_output_file"
 
   return "$result"
 }

From af29e9a60e72cb30dcb994c8a0132202aa1aa411 Mon Sep 17 00:00:00 2001
From: Thiago Genez <thiagogenez@ebi.ac.uk>
Date: Fri, 15 Mar 2024 11:55:51 +0100
Subject: [PATCH 10/10] using \! to make the script more robust and safer

Co-authored-by: twalsh-ebi <twalsh@ebi.ac.uk>
---
 travisci/python-linter_harness.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/travisci/python-linter_harness.sh b/travisci/python-linter_harness.sh
index c38b79face4..863a48a61d2 100755
--- a/travisci/python-linter_harness.sh
+++ b/travisci/python-linter_harness.sh
@@ -27,9 +27,9 @@ run_pylint() {
 
   # Run pylint, excluding specific files and directories
   find "${PYTHON_SOURCE_LOCATIONS[@]}" -type f -name "*.py" \
-    ! -name "Ortheus.py" \
-    ! -name "*citest*.py" \
-    ! -path "*/citest/*" -print0 |
+    \! -name "Ortheus.py" \
+    \! -name "*citest*.py" \
+    \! -path "*/citest/*" -print0 |
     xargs -0 pylint --rcfile=pyproject.toml --verbose \
       --msg-template='COMPARA_PYLINT_MSG:{path}:{line}:{column}: {msg_id}: {msg} ({symbol})' |
     tee "$pylint_output_file"