Scorecard Integration

scanpipe/migrations/0067_packagescore_scorecardcheck.py

@@ -0,0 +1,44 @@
+# Generated by Django 5.0.7 on 2024-07-27 10:20
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0066_alter_webhooksubscription_options_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PackageScore',
+            fields=[
+                ('scoring_tool', models.CharField(blank=True, choices=[('ossf-scorecard', 'Ossf'), ('others', 'Others')], help_text='Defines the source of a score or any other scoring metricsFor example: ossf-scorecard for scorecard data', max_length=100)),
+                ('scoring_tool_version', models.CharField(blank=True, help_text='Defines the version of the scoring tool used for scanning thepackageFor Eg : 4.6 current version of OSSF - scorecard', max_length=50)),
+                ('score', models.CharField(blank=True, help_text='Score of the package which is scanned', max_length=50)),
+                ('scoring_tool_documentation_url', models.CharField(blank=True, help_text='Documentation URL of the scoring tool used', max_length=100)),
+                ('score_date', models.DateTimeField(blank=True, editable=False, help_text='Date when the scoring was calculated on the package', null=True)),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+                ('discovered_package', models.ForeignKey(blank=True, editable=False, help_text='The package for which the score is given', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='discovered_packages_score', to='scanpipe.discoveredpackage')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ScorecardCheck',
+            fields=[
+                ('check_name', models.CharField(blank=True, help_text='Defines the name of check corresponding to the OSSF scoreFor example: Code-Review or CII-Best-PracticesThese are the some of the checks which are performed on a scanned package', max_length=100)),
+                ('check_score', models.CharField(blank=True, help_text='Defines the score of the check for the package scannedFor Eg : 9 is a score given for Code-Review', max_length=50)),
+                ('reason', models.CharField(blank=True, help_text='Gives a reason why a score was given for a specific checkFor eg, : Found 9/10 approved changesets -- score normalized to 9', max_length=300)),
+                ('details', models.JSONField(blank=True, default=list, help_text='A list of details/errors regarding the score')),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+                ('for_package_score', models.ForeignKey(blank=True, editable=False, help_text='The checks for which the score is given', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='discovered_packages_score_checks', to='scanpipe.packagescore')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

scanpipe/migrations/0068_merge_20240820_1656.py

@@ -0,0 +1,14 @@
+# Generated by Django 5.0.7 on 2024-08-20 16:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0067_discoveredpackage_notes'),
+        ('scanpipe', '0067_packagescore_scorecardcheck'),
+    ]
+
+    operations = [
+    ]

scanpipe/models.py

@@ -29,6 +29,7 @@
 from collections import Counter
 from collections import defaultdict
 from contextlib import suppress
+from datetime import datetime
 from itertools import groupby
 from operator import itemgetter
 from pathlib import Path
@@ -74,6 +75,8 @@
 from licensedcode.cache import build_spdx_license_expression
 from licensedcode.cache import get_licensing
 from matchcode_toolkit.fingerprinting import IGNORED_DIRECTORY_FINGERPRINTS
+from ossf_scorecard.contrib.django.models import PackageScoreMixin
+from ossf_scorecard.contrib.django.models import ScorecardChecksMixin
 from packagedcode.models import build_package_uid
 from packagedcode.utils import get_base_purl
 from packageurl import PackageURL
@@ -1833,10 +1836,10 @@ class Run(UUIDPKModel, ProjectRelatedModel, AbstractTaskFieldsModel):
     scancodeio_version = models.CharField(max_length=100, blank=True)
     description = models.TextField(blank=True)
     current_step = models.CharField(max_length=256, blank=True)
-    selected_groups = models.JSONField(
+    selected_steps = models.JSONField(
         null=True, blank=True, validators=[validate_none_or_list]
     )
-    selected_steps = models.JSONField(
+    selected_groups = models.JSONField(
         null=True, blank=True, validators=[validate_none_or_list]
     )
 
@@ -3898,6 +3901,106 @@ def as_spdx(self):
         )
 
 
+class PackageScore(UUIDPKModel, PackageScoreMixin):
+    def __str__(self):
+        return self.score or str(self.uuid)
+
+    discovered_package = models.ForeignKey(
+        DiscoveredPackage,
+        related_name="discovered_packages_score",
+        help_text=_("The package for which the score is given"),
+        on_delete=models.CASCADE,
+        editable=False,
+        blank=True,
+        null=True,
+    )
+
+    @classmethod
+    @transaction.atomic()
+    def create_from_data(cls, DiscoveredPackage, scorecard_data, scoring_tool=None):
+        """Create ScoreCard Object from ScoreCard Object"""
+        final_data = {
+            "score": scorecard_data.score,
+            "scoring_tool_version": scorecard_data.scoring_tool_version,
+            "scoring_tool_documentation_url": (
+                scorecard_data.scoring_tool_documentation_url
+            ),
+        }
+
+        date_str = scorecard_data.score_date
+
+        formats = ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%SZ"]
+
+        if date_str:
+            naive_datetime = None
+
+            for fmt in formats:
+                try:
+                    naive_datetime = datetime.strptime(date_str, fmt)
+                except ValueError:
+                    continue
+
+            score_date = timezone.make_aware(
+                naive_datetime, timezone.get_current_timezone()
+            )
+
+        else:
+            score_date = timezone.now()
+
+        final_data["score_date"] = score_date
+
+        scorecard_object = cls.objects.create(
+            **final_data,
+            discovered_package=DiscoveredPackage,
+            scoring_tool=scoring_tool,
+        )
+
+        # Create associated scorecard_checks
+        checks_data = scorecard_data.checks
+
+        ScorecardCheck.objects.bulk_create(
+            [
+                ScorecardCheck(
+                    check_name=check_data.check_name,
+                    check_score=check_data.check_score,
+                    reason=check_data.reason or "",
+                    details=check_data.details or [],
+                    for_package_score=scorecard_object,
+                )
+                for check_data in checks_data
+            ]
+        )
+
+        return scorecard_object
+
+
+class ScorecardCheck(UUIDPKModel, ScorecardChecksMixin):
+    def __str__(self):
+        return self.check_score or str(self.uuid)
+
+    for_package_score = models.ForeignKey(
+        PackageScore,
+        related_name="discovered_packages_score_checks",
+        help_text=_("The checks for which the score is given"),
+        on_delete=models.CASCADE,
+        editable=False,
+        blank=True,
+        null=True,
+    )
+
+    @classmethod
+    def create_from_data(cls, package_score, check_data):
+        """Create a ScorecardCheck instance from provided data."""
+        final_data = {
+            "check_name": check_data.get("name"),
+            "check_score": check_data.get("score"),
+            "reason": check_data.get("reason"),
+            "details": check_data.get("details", []),
+            "for_package_score": package_score,
+        }
+        return cls.objects.create(**final_data)
+
+
 def normalize_package_url_data(purl_mapping, ignore_nulls=False):
     """
     Normalize a mapping of purl data so database queries with

scanpipe/pipelines/get_scorecard_info_packages.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from ossf_scorecard import scorecard
+
+from scanpipe.models import PackageScore
+from scanpipe.pipelines import Pipeline
+
+
+class FetchScoreCodeInfo(Pipeline):
+    """
+    Pipeline to fetch ScoreCode information for packages and dependencies.
+
+    This pipeline retrieves ScoreCode data for each package in the project and
+    stores it in the corresponding package instances
+
+    Attributes
+    ----------
+        download_inputs (bool): Indicates whether inputs should be downloaded.
+        is_addon (bool): Indicates whether this pipeline is an add-on.
+
+    Methods
+    -------
+        steps(cls):
+            Defines the steps for the pipeline.
+
+        check_scorecode_service_availability(self):
+            Checks if the ScoreCode service is configured and available.
+
+        lookup_save_packages_scorecode_info(self):
+            Fetches ScoreCode information for each discovered package in the project
+            and saves the information to the respective package instances.
+
+    """
+
+    download_inputs = False
+    is_addon = True
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_scorecode_service_availability,
+            cls.lookup_save_packages_scorecode_info,
+        )
+
+    def check_scorecode_service_availability(self):
+        """Check if the scorecode service is configured and available."""
+        if not scorecard.is_configured():
+            raise Exception("scorecode service is not configured.")
+
+        if not scorecard.is_available():
+            raise Exception("scorecode service is not available.")
+
+    def lookup_save_packages_scorecode_info(self):
+        """Fetch scorecode information for each of the project's discovered packages."""
+        packages = self.project.discoveredpackages.all()
+        scorecard_packages_data = scorecard.fetch_scorecard_info(
+            packages=packages,
+            logger=self.log,
+        )
+
+        if scorecard_packages_data:
+            scorecard.save_scorecard_info(
+                package_scorecard_data=scorecard_packages_data,
+                cls=PackageScore,
+                logger=self.log,
+            )
+
+        else:
+            raise Exception("No Data Found for the packages")

scanpipe/tests/__init__.py

@@ -20,8 +20,10 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+import json
 import os
 from datetime import datetime
+from pathlib import Path
 from unittest import mock
 
 from django.apps import apps
@@ -264,3 +266,10 @@ def make_dependency(project, **extra):
         "license_key": "mpl-2.0",
     },
 }
+
+scorecard_data = None
+
+data = Path(__file__).parent / "data"
+
+with open(f"{data}/scorecode/scorecard_response.json") as file:
+    scorecard_data = json.load(file)