Update News.md

core/src/main/java/org/mskcc/oncotree/crosswalk/CrosswalkRepository.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 - 2019 Memorial Sloan-Kettering Cancer Center.
+ * Copyright (c) 2017 - 2020 Memorial Sloan-Kettering Cancer Center.
  *
  * This library is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF
@@ -18,20 +18,15 @@
 
 package org.mskcc.oncotree.crosswalk;
 
+import java.io.*;
+import java.nio.charset.Charset;
 import java.util.*;
-
-import org.mskcc.oncotree.error.*;
+import org.mskcc.oncotree.crosswalk.CrosswalkStaticResourceParsingException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Repository;
-import org.springframework.web.client.HttpServerErrorException;
-import org.springframework.web.client.HttpStatusCodeException;
-import org.springframework.web.client.RestClientException;
-import org.springframework.web.client.RestTemplate;
 
 /**
  *
@@ -42,48 +37,70 @@ public class CrosswalkRepository {
 
     private static final Logger logger = LoggerFactory.getLogger(CrosswalkRepository.class);
 
-    // URI variables should be vocabularyId={vocabularyId}&conceptId={conceptId}&histologyCode={histologyCode}&siteCode={siteCode}
-    @Value("${crosswalk.url}")
-    private String crosswalkURL;
+    private static final String STATIC_CROSSWALK_FILENAME = "staticCrosswalkOncotreeMappings.txt";
+    private Map<String, MSKConcept> parsedStaticResource = null;
 
-    @Value("${crosswalk.disable_cvs_querying:false}")
-    private Boolean DISABLE_CVS_QUERYING;
+    public MSKConcept getByOncotreeCode(String oncotreeCode) {
+        parseCrosswalkResourceFileIfNeeded();
+        MSKConcept concept = parsedStaticResource.get(oncotreeCode);
+        return concept;
+    }
 
-    public MSKConcept getByOncotreeCode(String oncotreeCode)
-            throws CrosswalkException {
-        return queryCVS("ONCOTREE", oncotreeCode, null, null);
+    private void parseCrosswalkResourceFileIfNeeded() {
+        if (parsedStaticResource == null) {
+            parsedStaticResource = new HashMap<String, MSKConcept>();
+            try {
+                parseCrosswalkResourceFile();
+            } catch (CrosswalkStaticResourceParsingException e) {
+                logger.error(e.toString());
+                parsedStaticResource.clear();
+                logger.error("external reference map empty following parsing error");
+            }
+        }
     }
 
-    public MSKConcept queryCVS(String vocabularyId, String conceptId, String histologyCode, String siteCode)
-            throws CrosswalkException {
-        if (DISABLE_CVS_QUERYING) {
-            throw new CrosswalkServiceUnavailableException("CVS server queries have been disabled (crosswalk.disable_cvs_querying set to true)", new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE));
+    private String[] parseValuesIfPresent(String valueString) {
+        if (valueString.trim().length() > 0) {
+            return valueString.split(",");
         }
-        RestTemplate restTemplate = new RestTemplate();
+        return new String[0];
+    }
+
+    private void parseCrosswalkResourceFile() throws CrosswalkStaticResourceParsingException {
         try {
-            ResponseEntity<MSKConcept> response = restTemplate.getForEntity(crosswalkURL, MSKConcept.class, vocabularyId, conceptId, histologyCode, siteCode);
-            return response.getBody();
-        } catch (HttpStatusCodeException e) {
-            logger.error("queryCVS() -- caught HttpStatusCodeException: " + e);
-            String errorString = "URI: " + crosswalkURL + "\n" +
-                "vocabularyId: " + vocabularyId + "\n" +
-                "conceptId: " + conceptId + "\n" +
-                "histologyCode: " + histologyCode + "\n" +
-                "siteCode: " + siteCode + "\n";
-            if (e.getStatusCode().is4xxClientError()) {
-                throw new CrosswalkConceptNotFoundException("4xx Error while getting data from CVS Service with: \n" + errorString, e);
-            } else if (e.getStatusCode().is5xxServerError()) {
-                throw new CrosswalkServiceUnavailableException("5xx Error while getting data from CVS", e);
+            Resource resource = new ClassPathResource(STATIC_CROSSWALK_FILENAME);
+            InputStream inputStream = resource.getInputStream();
+            InputStreamReader isreader = new InputStreamReader(inputStream, Charset.forName("UTF-8"));
+            BufferedReader reader = new BufferedReader(isreader);
+            while (reader.ready()) {
+                String line = reader.readLine();
+                String columns[] = line.split("\t");
+                if (columns.length != 3) {
+                    throw new CrosswalkStaticResourceParsingException("could not parse static file with crosswalk mappings - wrong column count");
+                }
+                String code = columns[0];
+                String nci[] = parseValuesIfPresent(columns[1]);
+                String umln[] = parseValuesIfPresent(columns[2]);
+                MSKConcept concept = new MSKConcept();
+                List<String> oncotreeCodes = new ArrayList<>();
+                oncotreeCodes.add(code);
+                concept.setOncotreeCodes(oncotreeCodes);
+                List<String> conceptIdList = Arrays.asList(umln);
+                if (conceptIdList.size() > 0) {
+                    concept.setConceptIds(conceptIdList);
+                }
+                HashMap<String, List<String>> crosswalks = new HashMap<String, List<String>>();
+                List<String> nciList = Arrays.asList(nci);
+                if (nciList.size() > 0) {
+                    crosswalks.put("NCI", nciList);
+                }
+                concept.setCrosswalks(crosswalks);
+                parsedStaticResource.put(code, concept);
             }
-            throw new UnexpectedCrosswalkResponseException("Exception while getting data from CVS Service with: \n" + errorString, e);
-        } catch (RestClientException e) {
-            logger.error("queryCVS() -- caught RestClientErrorException: " + e);
-            String errorString = "URI: " + crosswalkURL + "\n" +
-                "vocabularyId: " + vocabularyId + "\n" +
-                "conceptId: " + conceptId + "\n" +
-                "histologyCode: " + histologyCode + "\n" +
-                "siteCode: " + siteCode + "\n";
-            throw new UnexpectedCrosswalkResponseException("RestClientException while getting data from CVS Service" + errorString, e);
+        } catch (IOException e) {
+            throw new CrosswalkStaticResourceParsingException("error : could not parse static file with crosswalk mappings");
         }
     }
+
 }
+

core/src/main/java/org/mskcc/oncotree/error/CrosswalkException.java → core/src/main/java/org/mskcc/oncotree/crosswalk/CrosswalkStaticResourceParsingException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 Memorial Sloan-Kettering Cancer Center.
+ * Copyright (c) 2020 Memorial Sloan-Kettering Cancer Center.
  *
  * This library is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF
@@ -16,20 +16,24 @@
  * has been advised of the possibility of such damage.
 */
 
-package org.mskcc.oncotree.error;
+package org.mskcc.oncotree.crosswalk;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-/**
- *
- * @author Manda Wilson
- **/
-public class CrosswalkException extends RuntimeException {
 
-    private static final Logger logger = LoggerFactory.getLogger(CrosswalkException.class);
+public class CrosswalkStaticResourceParsingException extends Exception {
+
+    private static final Logger logger = LoggerFactory.getLogger(CrosswalkStaticResourceParsingException.class);
+
+    public CrosswalkStaticResourceParsingException() {
+        super();
+    }
+
+    public CrosswalkStaticResourceParsingException(String message) {
+        super(message);
+    }
 
-    public CrosswalkException(String message, Throwable cause) {
+    public CrosswalkStaticResourceParsingException(String message, Throwable cause) {
         super(message, cause);
-        logger.error(message);
     }
 }

core/src/main/java/org/mskcc/oncotree/crosswalk/MSKConceptCache.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 - 2019 Memorial Sloan-Kettering Cancer Center.
+ * Copyright (c) 2017 - 2020 Memorial Sloan-Kettering Cancer Center.
  *
  * This library is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF
@@ -83,18 +83,13 @@ private void resetCache() throws Exception {
             try {
                 oncoTreeNodes = oncoTreePersistentCache.getOncoTreeNodesFromPersistentCache(version);
             } catch (RuntimeException e) {
-                throw new FailedCacheRefreshException("Failed to refresh MSKConceptCache");
+                throw new FailedCacheRefreshException("Failed to refresh MSKConceptCache : " + e.toString());
             }
             for (OncoTreeNode node : oncoTreeNodes) {
                 // skip querying repeated nodes/MSKConcepts
                 if (!latestOncoTreeCodesToMSKConcepts.containsKey(node.getCode())) {
                     // pull from crosswalk first
-                    try {
-                        oncoTreePersistentCache.updateMSKConceptInPersistentCache(node.getCode());
-                    } catch (CrosswalkException e) {
-                        // only thrown if can't connect to crosswalk (5XX error)
-                        logger.error("Unable to update oncotree node with code " + node.getCode() + " from crosswalk...");
-                    }
+                    oncoTreePersistentCache.updateMSKConceptInPersistentCache(node.getCode());
                     MSKConcept concept = oncoTreePersistentCache.getMSKConceptFromPersistentCache(node.getCode());
                     latestOncoTreeCodesToMSKConcepts.put(node.getCode(), concept);
                 }

core/src/main/java/org/mskcc/oncotree/topbraid/OncoTreeRepository.java

@@ -26,6 +26,7 @@
 
 import org.mskcc.oncotree.model.Version;
 
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.stereotype.Repository;
 
@@ -38,24 +39,34 @@ public class OncoTreeRepository extends TopBraidRepository<OncoTreeNode> {
 
     private static final Logger logger = LoggerFactory.getLogger(OncoTreeRepository.class);
 
-    private String query = "PREFIX skos:<http://www.w3.org/2004/02/skos/core#> " +
-        "PREFIX onc:<http://data.mskcc.org/ontologies/oncotree#> " +
-        "SELECT DISTINCT (?s AS ?uri) ?code ?name ?mainType ?color ?parentCode ?revocations ?precursors " +
-        "WHERE { " +
-        "   GRAPH <%s> { " +
-        "       ?s skos:prefLabel ?name;" +
-        "       skos:notation ?code." +
-        "       OPTIONAL{?s skos:broader ?broader." +
-        "           ?broader skos:notation ?parentCode}." +
-        "       OPTIONAL{?s onc:mainType ?mainType}." +
-        "       OPTIONAL{?s onc:color ?color}." +
-        "       OPTIONAL{?s onc:revocations ?revocations}." +
-        "       OPTIONAL{?s onc:precursors ?precursors}." +
-        "   }" +
-        "}";
+    @Value("${topbraid.oncotree_namespace_prefix:http://data.mskcc.org/ontologies/oncotree#}")
+    private String topBraidOncotreeNamespacePrefix;
+
+    private String query = null;
+
+    private String getQuery() {
+        if (query == null) {
+            query = "PREFIX skos:<http://www.w3.org/2004/02/skos/core#> " +
+                    "PREFIX onc:<" + topBraidOncotreeNamespacePrefix + "> " +
+                    "SELECT DISTINCT (?s AS ?uri) ?code ?name ?mainType ?color ?parentCode ?revocations ?precursors " +
+                    "WHERE { " +
+                    "   GRAPH <%s> { " +
+                    "       ?s skos:prefLabel ?name;" +
+                    "       skos:notation ?code." +
+                    "       OPTIONAL{?s skos:broader ?broader." +
+                    "           ?broader skos:notation ?parentCode}." +
+                    "       OPTIONAL{?s onc:mainType ?mainType}." +
+                    "       OPTIONAL{?s onc:color ?color}." +
+                    "       OPTIONAL{?s onc:revocations ?revocations}." +
+                    "       OPTIONAL{?s onc:precursors ?precursors}." +
+                    "   }" +
+                    "}";
+        }
+        return query;
+    }
 
     public ArrayList<OncoTreeNode> getOncoTree(Version version) throws TopBraidException {
-        return new ArrayList<OncoTreeNode>(super.query(String.format(query, version.getGraphURI()), new ParameterizedTypeReference<List<OncoTreeNode>>(){}));
+        return new ArrayList<OncoTreeNode>(super.query(String.format(getQuery(), version.getGraphURI()), new ParameterizedTypeReference<List<OncoTreeNode>>(){}));
     }
 
 }