cnf ran: add reporting for CRs on each cluster (#90)

Currently, the cnf/ran test suites do not log the CRs used in each test case, which becomes particularly important for the TALM and ZTP test suites. This PR uses the existing infrastructure in the repo around reporting to generate dumps with these resources.

There is an additional function in the internal/reporter package that allows reports to dump the CRs from a specific kubeconfig, not just the one specified by the KUBECONFIG environment variable.

tests/cnf/ran/talm/internal/tsparams/consts.go

@@ -123,6 +123,8 @@ const (
 	PrometheusPodName = "prometheus-k8s-0"
 	// PrometheusNamespace is the namespace for prometheus.
 	PrometheusNamespace = "openshift-monitoring"
+	// TalmBackupNamespace is the namespace that the CGU operator uses when running backups.
+	TalmBackupNamespace = "openshift-talo-backup"
 
 	// MasterNodeSelector when used in a label selector finds all master nodes.
 	MasterNodeSelector = "node-role.kubernetes.io/master="

tests/cnf/ran/talm/internal/tsparams/talmvars.go

@@ -3,23 +3,51 @@ package tsparams
 import (
 	"fmt"
 
+	cguv1alpha1 "github.com/openshift-kni/cluster-group-upgrades-operator/pkg/api/clustergroupupgrades/v1alpha1"
 	"github.com/openshift-kni/eco-gotests/tests/cnf/ran/internal/ranparam"
 	"github.com/openshift-kni/k8sreporter"
-	v1 "k8s.io/api/core/v1"
+	olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/ptr"
+	policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
+	policiesv1beta1 "open-cluster-management.io/governance-policy-propagator/api/v1beta1"
+	placementrulev1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/placementrule/v1"
 )
 
 var (
 	// Labels represents the range of labels that can be used for test cases selection.
 	Labels = append(ranparam.Labels, LabelSuite)
 
-	// ReporterNamespacesToDump tells to the reporter from where to collect logs.
-	ReporterNamespacesToDump = map[string]string{
-		TestNamespace: "",
+	// ReporterHubNamespacesToDump tells to the reporter which namespaces on the hub to collect pod logs from.
+	ReporterHubNamespacesToDump = map[string]string{
+		TestNamespace:                       "",
+		ranparam.OpenshiftOperatorNamespace: "",
 	}
 
-	// ReporterCRDsToDump tells to the reporter what CRs to dump.
-	ReporterCRDsToDump = []k8sreporter.CRData{
-		{Cr: &v1.PodList{}},
+	// ReporterSpokeNamespacesToDump tells the reporter which namespaces on the spokes to collect pod logs from.
+	ReporterSpokeNamespacesToDump = map[string]string{
+		TemporaryNamespace:  "",
+		PreCacheSpokeNS:     "",
+		TalmBackupNamespace: "",
+	}
+
+	// ReporterHubCRsToDump is the CRs the reporter should dump on the hub.
+	ReporterHubCRsToDump = []k8sreporter.CRData{
+		{Cr: &corev1.NamespaceList{}},
+		{Cr: &corev1.PodList{}},
+		{Cr: &cguv1alpha1.ClusterGroupUpgradeList{}, Namespace: ptr.To(TestNamespace)},
+		{Cr: &cguv1alpha1.PreCachingConfigList{}, Namespace: ptr.To(TestNamespace)},
+		{Cr: &policiesv1.PolicyList{}, Namespace: ptr.To(TestNamespace)},
+		{Cr: &policiesv1.PlacementBindingList{}, Namespace: ptr.To(TestNamespace)},
+		{Cr: &placementrulev1.PlacementRuleList{}, Namespace: ptr.To(TestNamespace)},
+		{Cr: &policiesv1beta1.PolicySetList{}, Namespace: ptr.To(TestNamespace)},
+	}
+
+	// ReporterSpokeCRsToDump is the CRs the reporter should dump on the spokes.
+	ReporterSpokeCRsToDump = []k8sreporter.CRData{
+		{Cr: &corev1.NamespaceList{}},
+		{Cr: &corev1.PodList{}},
+		{Cr: &olmv1alpha1.CatalogSourceList{}, Namespace: ptr.To(TemporaryNamespace)},
 	}
 
 	// TalmNonExistentClusterMessage is the condition message for when a cluster is non-existent.

tests/cnf/ran/talm/talm_suite_test.go

@@ -1,6 +1,8 @@
 package talm
 
 import (
+	"fmt"
+	"path"
 	"runtime"
 	"testing"
 
@@ -42,9 +44,36 @@ var _ = AfterSuite(func() {
 	Expect(err).ToNot(HaveOccurred(), "Failed to delete TALM test namespace")
 })
 
-var _ = ReportAfterEach(func(report SpecReport) {
+var _ = JustAfterEach(func() {
+	var (
+		currentDir, currentFilename = path.Split(currentFile)
+		hubReportPath               = fmt.Sprintf("%shub_%s", currentDir, currentFilename)
+		spoke2ReportPath            = fmt.Sprintf("%sspoke2_%s", currentDir, currentFilename)
+		report                      = CurrentSpecReport()
+	)
+
 	reporter.ReportIfFailed(
-		report, currentFile, tsparams.ReporterNamespacesToDump, tsparams.ReporterCRDsToDump, clients.SetScheme)
+		report, currentFile, tsparams.ReporterSpokeNamespacesToDump, tsparams.ReporterSpokeCRsToDump, clients.SetScheme)
+
+	if HubAPIClient != nil {
+		reporter.ReportIfFailedOnCluster(
+			RANConfig.HubKubeconfig,
+			report,
+			hubReportPath,
+			tsparams.ReporterHubNamespacesToDump,
+			tsparams.ReporterHubCRsToDump,
+			clients.SetScheme)
+	}
+
+	if Spoke2APIClient != nil {
+		reporter.ReportIfFailedOnCluster(
+			RANConfig.Spoke2Kubeconfig,
+			report,
+			spoke2ReportPath,
+			tsparams.ReporterSpokeNamespacesToDump,
+			tsparams.ReporterSpokeCRsToDump,
+			clients.SetScheme)
+	}
 })
 
 var _ = ReportAfterSuite("", func(report Report) {

tests/internal/reporter/reporter.go

@@ -20,7 +20,8 @@ var (
 )
 
 func newReporter(
-	reportPath string,
+	reportPath,
+	kubeconfig string,
 	namespacesToDump map[string]string,
 	apiScheme func(scheme *runtime.Scheme) error,
 	cRDs []k8sreporter.CRData) (*k8sreporter.KubernetesReporter, error) {
@@ -37,7 +38,7 @@ func newReporter(
 		}
 	}
 
-	res, err := k8sreporter.New("", apiScheme, nsToDumpFilter, reportPath, cRDs...)
+	res, err := k8sreporter.New(kubeconfig, apiScheme, nsToDumpFilter, reportPath, cRDs...)
 
 	if err != nil {
 		return nil, err
@@ -48,6 +49,18 @@ func newReporter(
 
 // ReportIfFailed dumps requested cluster CRs if TC is failed to the given directory.
 func ReportIfFailed(
+	report types.SpecReport,
+	testSuite string,
+	nSpaces map[string]string,
+	cRDs []k8sreporter.CRData,
+	apiScheme func(scheme *runtime.Scheme) error) {
+	ReportIfFailedOnCluster("", report, testSuite, nSpaces, cRDs, apiScheme)
+}
+
+// ReportIfFailedOnCluster dumps the requested cluster CRs on the cluster specified by kubeconfig if TC is failed to the
+// given directory.
+func ReportIfFailedOnCluster(
+	kubeconfig string,
 	report types.SpecReport,
 	testSuite string,
 	nSpaces map[string]string,
@@ -60,7 +73,7 @@ func ReportIfFailed(
 	dumpDir := GeneralConfig.GetDumpFailedTestReportLocation(testSuite)
 
 	if dumpDir != "" {
-		reporter, err := newReporter(dumpDir, nSpaces, apiScheme, cRDs)
+		reporter, err := newReporter(dumpDir, kubeconfig, nSpaces, apiScheme, cRDs)
 
 		if err != nil {
 			glog.Fatalf("Failed to create log reporter due to %s", err)