From e28c6826ea847021c27950d37bd23523afa4a07b Mon Sep 17 00:00:00 2001 From: Rob Moffat Date: Wed, 31 Jan 2024 17:50:32 +0000 Subject: [PATCH] Updated to scan 2 orgs, config file --- .gitignore | 1 + src/main/java/org/finos/ls/LandscapeApp.java | 33 +++++-- src/main/java/org/finos/ls/QueryService.java | 2 +- .../java/org/finos/ls/ReadmeGenerator.java | 85 +++++++++++++------ src/main/resources/application.yml | 41 ++++++++- src/test/java/org/finos/ls/BasicTest.java | 4 +- 6 files changed, 129 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 408294ed8..ae037b11e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ target application-local.yml /.settings .DS_Store +/bin \ No newline at end of file diff --git a/src/main/java/org/finos/ls/LandscapeApp.java b/src/main/java/org/finos/ls/LandscapeApp.java index fc7aec512..e7ca5305a 100644 --- a/src/main/java/org/finos/ls/LandscapeApp.java +++ b/src/main/java/org/finos/ls/LandscapeApp.java @@ -3,6 +3,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; import org.finos.ls.queries.SecurityCSVSummarizer; import org.finos.ls.search.FinanceCSVSummarizer; @@ -13,11 +14,15 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ConfigurableApplicationContext; import com.graphql_java_generator.client.GraphQLConfiguration; @SpringBootApplication(scanBasePackageClasses = { LandscapeApp.class, GraphQLConfiguration.class, QueryExecutor.class, SpringConfiguration.class }) +@EnableConfigurationProperties +@ConfigurationProperties(prefix = "scanning") public class LandscapeApp implements CommandLineRunner { public static void main(String[] args) { @@ -38,10 +43,7 @@ public static void main(String[] args) { @Value("${spring.profiles.active:}") String activeProfiles; - - @Value("${scanning.org:finos}") - String org; - + @Value("${scanning.readme:README.md}") String readmeFile; @@ -68,6 +70,16 @@ public static void main(String[] args) { @Value("${scanning.csv.ignore:}") String[] ignore; + + List orgs; + + public List getOrgs() { + return orgs; + } + + public void setOrgs(List orgs) { + this.orgs = orgs; + } @Autowired ConfigurableApplicationContext ctx; @@ -80,13 +92,20 @@ public static void main(String[] args) { public void run(String... args) throws Exception { if (activeProfiles.contains("summarize")) { // first, write the readme - String readmeContent = readme.generate(25, org); + String readmeContent = readme.generate(25, orgs); commit.commitFile(readmeFile, readmeContent.getBytes(), head, repo, owner); // then write the csv SecurityCSVSummarizer summ = new SecurityCSVSummarizer(Arrays.asList(ignore), Arrays.asList(priority)); - String csvContent = csv.generateOrg(org, summ); - commit.commitFile(csvFile, csvContent.getBytes(), head, repo, owner); + StringBuilder sb = new StringBuilder(); + orgs.forEach(o -> { + try { + sb.append( csv.generateOrg(o, summ)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + commit.commitFile(csvFile, sb.toString().getBytes(), head, repo, owner); FinanceCSVSummarizer fcsv = new FinanceCSVSummarizer(); String fcsvContent = csv.generateTopic("finance", fcsv); diff --git a/src/main/java/org/finos/ls/QueryService.java b/src/main/java/org/finos/ls/QueryService.java index f867e51a7..e178efec7 100644 --- a/src/main/java/org/finos/ls/QueryService.java +++ b/src/main/java/org/finos/ls/QueryService.java @@ -96,7 +96,7 @@ public Map getAllRepositoriesInOrg(QueryType qt, String org) t cursor = conn.getPageInfo().getEndCursor(); } - return out.stream().collect(Collectors.toMap(r -> r.getName(), r-> qt.convert(r, qe))); + return out.stream().collect(Collectors.toMap(r -> r.getOwner().getLogin()+","+r.getName(), r-> qt.convert(r, qe))); } diff --git a/src/main/java/org/finos/ls/ReadmeGenerator.java b/src/main/java/org/finos/ls/ReadmeGenerator.java index 653573557..223795e9d 100644 --- a/src/main/java/org/finos/ls/ReadmeGenerator.java +++ b/src/main/java/org/finos/ls/ReadmeGenerator.java @@ -2,7 +2,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -18,12 +17,16 @@ import org.finos.scan.github.client.Repository; import org.finos.scan.github.client.util.QueryExecutor; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.stereotype.Service; import com.graphql_java_generator.exception.GraphQLRequestExecutionException; import com.graphql_java_generator.exception.GraphQLRequestPreparationException; @Service +@EnableConfigurationProperties +@ConfigurationProperties(prefix = "scanning.readme") public class ReadmeGenerator { @Autowired @@ -32,10 +35,38 @@ public class ReadmeGenerator { @Autowired QueryExecutor qe; + private List remove; + + public List getRemove() { + return remove; + } + + public void setRemove(List remove) { + this.remove = remove; + } + + private Map> buckets; + + public Map> getBuckets() { + return buckets; + } + + public void setBuckets(Map> buckets) { + this.buckets = buckets; + } + Map cache = new HashMap<>(); - public String generate(int cutoff, String org) throws GraphQLRequestExecutionException, GraphQLRequestPreparationException { - Map activeProjects = qs.getAllRepositoriesInOrg(BasicQueries.COMBINED_ACTIVITY, org); + public String generate(int cutoff, List orgs) throws GraphQLRequestExecutionException, GraphQLRequestPreparationException { + Map activeProjects = new HashMap<>(); + orgs.forEach(o -> { + try { + activeProjects.putAll(qs.getAllRepositoriesInOrg(BasicQueries.COMBINED_ACTIVITY, o)); + } catch (GraphQLRequestExecutionException | GraphQLRequestPreparationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }); List names = activeProjects.entrySet().stream() .filter(r -> r.getValue().getScore() > cutoff) @@ -84,7 +115,8 @@ private String tableOfContents(Map> bucketedProjects) throw private String getTitleForRepo(String name) { try { MarkdownSummarizer s = new MarkdownSummarizer(SummaryLevel.SUBITEM); - Repository repo = getRepoDetails(s, name); + String[] parts = name.split(","); + Repository repo = getRepoDetails(s, parts[1], parts[0]); String title = s.getTitleFromNameOrH1(name, repo); return title; } catch (Exception e) { @@ -112,19 +144,23 @@ private String report(Map> bucketedProjects) { return out.toString(); } - private void appendUsing(MarkdownSummarizer l, String name, StringBuilder out) { + private void appendUsing(MarkdownSummarizer l, String id, StringBuilder out) { try { - out.append(l.convert(getRepoDetails(l, name), qe)); + String[] parts = id.split(","); + String name = parts[1]; + String org = parts[0]; + + out.append(l.convert(getRepoDetails(l, name, org), qe)); } catch (Exception e) { throw new RuntimeException("Couldn't process: ", e); } } - private Repository getRepoDetails(MarkdownSummarizer l, String name) + private Repository getRepoDetails(MarkdownSummarizer l, String name, String org) throws GraphQLRequestExecutionException, GraphQLRequestPreparationException { if (!cache.containsKey(name)) { - cache.put(name, qs.getRawRepository(l, "finos", name)); + cache.put(name, qs.getRawRepository(l, org, name)); } return cache.get(name); @@ -132,20 +168,13 @@ private Repository getRepoDetails(MarkdownSummarizer l, String name) private Map> bucketNames(List names) { Map> out = new HashMap>(); - names = remove(names, n -> n.contains("juju")); - names = bucket(out, names, "Legend", n -> n.contains("legend")); - names = bucket(out, names, "Morphir", n -> n.contains("morphir")); - names = bucket(out, names, "Symphony", n -> n.contains("symphony")); - names = bucket(out, names, "FDC3", n -> n.contains("fdc3")); - names = bucket(out, names, "Waltz", n -> n.contains("waltz")); - names = remove(names, - "finos-landscape", - "software-project-blueprint", - "standards-project-blueprint", - "clabot-config", - "finos-parent-pom"); - names = bucketItems(out, names, "Symphony", "messageml-utils"); - names = bucketItems(out, names, "SIGs", "dei-sig", "innersource", "curref-data", "open-source-readiness", "compliant-financial-infrastructure"); + + names = remove(names, this.remove); + + for (Map.Entry> bucket : this.buckets.entrySet()) { + names = bucketItems(out, names, bucket.getKey(), bucket.getValue()); + } + singleBucketTheRest(out, names); return out; } @@ -155,8 +184,7 @@ private void singleBucketTheRest(Map> out, List nam .forEach(n -> out.put(n, Collections.singletonList(n))); } - private List remove(List items, String... toRemove) { - List toGo = Arrays.asList(toRemove); + private List remove(List items, List toGo) { return remove(items, n -> toGo.contains(n)); } @@ -166,9 +194,12 @@ private List remove(List items, Predicate toRemove) { .collect(Collectors.toList()); } - private List bucketItems(Map> out, List in, String key, String... toInclude) { - List toGo = Arrays.asList(toInclude); - return bucket(out, in, key, n -> toGo.indexOf(n) > -1); + private List bucketItems(Map> out, List in, String key, List toGo) { + List lcToGo = toGo.stream().map(e -> e.toLowerCase()).collect(Collectors.toList()); + Predicate theTest = j -> lcToGo.stream() + .map(e -> j.toLowerCase().indexOf(e) > -1) + .reduce(false, (a, b) -> a || b); + return bucket(out, in, key, theTest); } private List bucket(Map> out, List in, String key, Predicate test) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 056f95053..1157587dd 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,10 +9,49 @@ logging: debug: true scanning: - org: finos + orgs: + - finos + - finos-labs csv: +<<<<<<< HEAD priority: morphir, fdc3, cfi ignore: none +======= + priority: legend, morphir, fdc3, cfi + ignore: none + readme: + remove: + - juju + - finos-landscape + - software-project-blueprint + - standards-project-blueprint + - clabot-config + - finos-parent-pom + buckets: + FDC3: + - fdc3 + Jupyter: + - ipy + - regular-table + - jupyter + Symphony: + - Symphony + - messageml-utils + Legend: + - Legend + Morphir: + - Morphir + Accessibility: + - a11y + SIGs: + - dei-sig + - innersource + - curref-data + - open-source-readiness + - compliant-financial-infrastructure + - common-cloud-controls + - zenith +>>>>>>> a5fd8f3 (Updated to scan 2 orgs, config file) write-to: repo: landscape-scanning diff --git a/src/test/java/org/finos/ls/BasicTest.java b/src/test/java/org/finos/ls/BasicTest.java index 18f1aaad7..2dd4d1370 100644 --- a/src/test/java/org/finos/ls/BasicTest.java +++ b/src/test/java/org/finos/ls/BasicTest.java @@ -26,7 +26,9 @@ public class BasicTest { private static final String ORG = "finos"; + private static final String ORG2 = "finos-labs"; + @Autowired QueryService qs; @@ -180,7 +182,7 @@ public void testBadReadme() throws GraphQLRequestExecutionException, GraphQLRequ @Test public void testWholeReadme() throws GraphQLRequestExecutionException, GraphQLRequestPreparationException { - String out = readme.generate(25, ORG); + String out = readme.generate(25, Arrays.asList(ORG, ORG2)); System.out.println(out); }