Updated to scan 2 orgs, config file

.gitignore

@@ -2,3 +2,4 @@ target
 application-local.yml
 /.settings
 .DS_Store
+/bin

src/main/java/org/finos/ls/LandscapeApp.java

@@ -3,6 +3,7 @@
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 import org.finos.ls.queries.SecurityCSVSummarizer;
 import org.finos.ls.search.FinanceCSVSummarizer;
@@ -13,11 +14,15 @@
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.ConfigurableApplicationContext;
 
 import com.graphql_java_generator.client.GraphQLConfiguration;
 
 @SpringBootApplication(scanBasePackageClasses = { LandscapeApp.class, GraphQLConfiguration.class, QueryExecutor.class, SpringConfiguration.class })
+@EnableConfigurationProperties
+@ConfigurationProperties(prefix = "scanning")
 public class LandscapeApp implements CommandLineRunner {
 
 	public static void main(String[] args) {
@@ -38,10 +43,7 @@ public static void main(String[] args) {
 
 	@Value("${spring.profiles.active:}")
 	String activeProfiles;
-
-	@Value("${scanning.org:finos}")
-	String org;
-
+
 	@Value("${scanning.readme:README.md}")
 	String readmeFile;
 
@@ -68,6 +70,16 @@ public static void main(String[] args) {
 
 	@Value("${scanning.csv.ignore:}")
 	String[] ignore;
+
+	List<String> orgs;
+
+	public List<String> getOrgs() {
+		return orgs;
+	}
+
+	public void setOrgs(List<String> orgs) {
+		this.orgs = orgs;
+	}
 
 	@Autowired
 	ConfigurableApplicationContext ctx;
@@ -80,13 +92,20 @@ public static void main(String[] args) {
 	public void run(String... args) throws Exception {
 		if (activeProfiles.contains("summarize")) {
 			// first, write the readme
-			String readmeContent = readme.generate(25, org);
+			String readmeContent = readme.generate(25, orgs);
 			commit.commitFile(readmeFile, readmeContent.getBytes(), head, repo, owner);
 
 			// then write the csv
 			SecurityCSVSummarizer summ = new SecurityCSVSummarizer(Arrays.asList(ignore), Arrays.asList(priority));
-			String csvContent = csv.generateOrg(org, summ);
-			commit.commitFile(csvFile, csvContent.getBytes(), head, repo, owner);
+			StringBuilder sb = new StringBuilder();
+			orgs.forEach(o -> {
+				try {
+					sb.append( csv.generateOrg(o, summ));
+				} catch (Exception e) {
+					e.printStackTrace();
+				}
+			});
+			commit.commitFile(csvFile, sb.toString().getBytes(), head, repo, owner);
 
 			FinanceCSVSummarizer fcsv = new FinanceCSVSummarizer();
 			String fcsvContent = csv.generateTopic("finance", fcsv);

src/main/java/org/finos/ls/QueryService.java

@@ -96,7 +96,7 @@ public <X> Map<String, X> getAllRepositoriesInOrg(QueryType<X> qt, String org) t
 			cursor = conn.getPageInfo().getEndCursor();
 		}
 
-		return out.stream().collect(Collectors.toMap(r -> r.getName(), r-> qt.convert(r, qe)));
+		return out.stream().collect(Collectors.toMap(r -> r.getOwner().getLogin()+","+r.getName(), r-> qt.convert(r, qe)));
 
 	}
 

src/main/java/org/finos/ls/ReadmeGenerator.java

@@ -2,7 +2,6 @@
 
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -18,12 +17,16 @@
 import org.finos.scan.github.client.Repository;
 import org.finos.scan.github.client.util.QueryExecutor;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.stereotype.Service;
 
 import com.graphql_java_generator.exception.GraphQLRequestExecutionException;
 import com.graphql_java_generator.exception.GraphQLRequestPreparationException;
 
 @Service
+@EnableConfigurationProperties
+@ConfigurationProperties(prefix = "scanning.readme")
 public class ReadmeGenerator {
 
 	@Autowired
@@ -32,10 +35,38 @@ public class ReadmeGenerator {
 	@Autowired 
 	QueryExecutor qe;
 
+	private List<String> remove;
+
+	public List<String> getRemove() {
+		return remove;
+	}
+
+	public void setRemove(List<String> remove) {
+		this.remove = remove;
+	}
+
+	private Map<String, List<String>> buckets;
+
+	public Map<String, List<String>> getBuckets() {
+		return buckets;
+	}
+
+	public void setBuckets(Map<String, List<String>> buckets) {
+		this.buckets = buckets;
+	}
+
 	Map<String, Repository> cache = new HashMap<>();
 
-	public String generate(int cutoff, String org) throws GraphQLRequestExecutionException, GraphQLRequestPreparationException {
-		Map<String, Activity> activeProjects = qs.getAllRepositoriesInOrg(BasicQueries.COMBINED_ACTIVITY, org);
+	public String generate(int cutoff, List<String> orgs) throws GraphQLRequestExecutionException, GraphQLRequestPreparationException {
+		Map<String, Activity> activeProjects = new HashMap<>(); 
+		orgs.forEach(o -> {
+			try {
+				activeProjects.putAll(qs.getAllRepositoriesInOrg(BasicQueries.COMBINED_ACTIVITY, o));
+			} catch (GraphQLRequestExecutionException | GraphQLRequestPreparationException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+		});
 
 		List<String> names = activeProjects.entrySet().stream()
 			.filter(r -> r.getValue().getScore() > cutoff)
@@ -84,7 +115,8 @@ private String tableOfContents(Map<String, List<String>> bucketedProjects) throw
 	private String getTitleForRepo(String name) {
 		try {
 			MarkdownSummarizer s = new MarkdownSummarizer(SummaryLevel.SUBITEM);
-			Repository repo = getRepoDetails(s, name);
+			String[] parts = name.split(",");
+			Repository repo = getRepoDetails(s, parts[1], parts[0]);
 			String title = s.getTitleFromNameOrH1(name, repo);
 			return title;
 		} catch (Exception e) {
@@ -112,40 +144,37 @@ private String report(Map<String, List<String>> bucketedProjects) {
 		return out.toString();
 	}
 
-	private void appendUsing(MarkdownSummarizer l, String name, StringBuilder out) {
+	private void appendUsing(MarkdownSummarizer l, String id, StringBuilder out) {
 		try {
-			out.append(l.convert(getRepoDetails(l, name), qe));
+			String[] parts = id.split(",");
+			String name = parts[1];
+			String org = parts[0];
+
+			out.append(l.convert(getRepoDetails(l, name, org), qe));
 		} catch (Exception e) {
 			throw new RuntimeException("Couldn't process: ", e);
 		}
 	}
 
-	private Repository getRepoDetails(MarkdownSummarizer l, String name)
+	private Repository getRepoDetails(MarkdownSummarizer l, String name, String org)
 			throws GraphQLRequestExecutionException, GraphQLRequestPreparationException {
 
 		if (!cache.containsKey(name)) {
-			cache.put(name, qs.getRawRepository(l, "finos", name));
+			cache.put(name, qs.getRawRepository(l, org, name));
 		}
 
 		return cache.get(name);
 	}
 
 	private Map<String, List<String>> bucketNames(List<String> names) {
 		Map<String, List<String>> out = new HashMap<String, List<String>>();
-		names = remove(names, n -> n.contains("juju"));
-		names = bucket(out, names, "Legend", n -> n.contains("legend"));
-		names = bucket(out, names, "Morphir", n -> n.contains("morphir"));
-		names = bucket(out, names, "Symphony", n -> n.contains("symphony"));
-		names = bucket(out, names, "FDC3", n -> n.contains("fdc3"));
-		names = bucket(out, names, "Waltz", n -> n.contains("waltz"));
-		names = remove(names, 
-				"finos-landscape", 
-				"software-project-blueprint", 
-				"standards-project-blueprint", 
-				"clabot-config",
-				"finos-parent-pom");
-		names = bucketItems(out, names, "Symphony", "messageml-utils");
-		names = bucketItems(out, names, "SIGs", "dei-sig", "innersource", "curref-data", "open-source-readiness", "compliant-financial-infrastructure");
+
+		names = remove(names, this.remove);
+
+		for (Map.Entry<String, List<String>> bucket : this.buckets.entrySet()) {
+			names = bucketItems(out, names, bucket.getKey(), bucket.getValue());
+		}
+
 		singleBucketTheRest(out, names);
 		return out;
 	}
@@ -155,8 +184,7 @@ private void singleBucketTheRest(Map<String, List<String>> out, List<String> nam
 			.forEach(n -> out.put(n, Collections.singletonList(n)));
 	}
 
-	private List<String> remove(List<String> items, String... toRemove) {
-		List<String> toGo = Arrays.asList(toRemove);
+	private List<String> remove(List<String> items, List<String> toGo) {
 		return remove(items, n -> toGo.contains(n));
 	}
 
@@ -166,9 +194,12 @@ private List<String> remove(List<String> items, Predicate<String> toRemove) {
 				.collect(Collectors.toList());
 	}
 
-	private List<String> bucketItems(Map<String, List<String>> out, List<String> in, String key, String... toInclude) {
-		List<String> toGo = Arrays.asList(toInclude);
-		return bucket(out, in, key, n -> toGo.indexOf(n) > -1);
+	private List<String> bucketItems(Map<String, List<String>> out, List<String> in, String key, List<String> toGo) {
+		List<String> lcToGo = toGo.stream().map(e -> e.toLowerCase()).collect(Collectors.toList());
+		Predicate<String> theTest = j -> lcToGo.stream()
+				.map(e -> j.toLowerCase().indexOf(e) > -1)
+				.reduce(false, (a, b) -> a || b);
+		return bucket(out, in, key, theTest);
 	}
 
 	private List<String> bucket(Map<String, List<String>> out, List<String> in, String key, Predicate<String> test) {

src/main/resources/application.yml

@@ -9,10 +9,49 @@ logging:
 debug: true
 
 scanning:
-  org: finos
+  orgs: 
+   - finos
+   - finos-labs
   csv:
+<<<<<<< HEAD
     priority: morphir, fdc3, cfi
     ignore: none  
+=======
+    priority: legend, morphir, fdc3, cfi
+    ignore: none    
+  readme:
+    remove: 
+     - juju
+     - finos-landscape
+     - software-project-blueprint
+     - standards-project-blueprint
+     - clabot-config
+     - finos-parent-pom
+    buckets: 
+      FDC3: 
+        - fdc3
+      Jupyter:
+        - ipy
+        - regular-table
+        - jupyter
+      Symphony: 
+        - Symphony 
+        - messageml-utils 
+      Legend: 
+        - Legend
+      Morphir: 
+        - Morphir
+      Accessibility:
+        - a11y
+      SIGs: 
+        - dei-sig
+        - innersource
+        - curref-data
+        - open-source-readiness
+        - compliant-financial-infrastructure
+        - common-cloud-controls
+        - zenith
+>>>>>>> a5fd8f3 (Updated to scan 2 orgs, config file)
 
   write-to:
     repo: landscape-scanning

src/test/java/org/finos/ls/BasicTest.java

@@ -26,7 +26,9 @@
 public class BasicTest {
 
 	private static final String ORG = "finos";
+	private static final String ORG2 = "finos-labs";
 
+
 	@Autowired
 	QueryService qs;
 
@@ -180,7 +182,7 @@ public void testBadReadme() throws GraphQLRequestExecutionException, GraphQLRequ
 
 	@Test
 	public void testWholeReadme() throws GraphQLRequestExecutionException, GraphQLRequestPreparationException {
-		String out = readme.generate(25, ORG);
+		String out = readme.generate(25, Arrays.asList(ORG, ORG2));
 		System.out.println(out);
 	}