Merge pull request #244 from charvolant/master

Release 1.5.1
AtlasOfLivingAustralia · May 13, 2021 · e78f4f7 · e78f4f7
2 parents c601651 + 147a6ad
commit e78f4f7
Show file tree

Hide file tree

Showing 25 changed files with 723 additions and 340 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,4 @@ plugin.xml
 grails-bie-plugin-1.0-SNAPSHOT.zip
 
 grails-bie-plugin-1.0-SNAPSHOT.zip.sha1
+/out/
diff --git a/README.md b/README.md
@@ -20,6 +20,31 @@ Customisations of code and i18n files should be made in the copy of `ala-bie` (n
 
 The bie-plugin uses ISO-639 language codes, particularly ISO-639-3, drawn from http://www.sil.org/iso639-3/ and the AIATSIS codes, drawn from https://aiatsis.gov.au/
 
+### Blacklisted External Information
+
+It is possible to blacklist sources of external information
+that is either incorrect or not relevant.
+Blacklisting is performed by pattern matching and can be configured by URLs that give a specific blacklist.
+Blacklists are configured as a map of possible blacklists against the information in a document.
+For example:
+
+```yaml
+external:
+  blacklist: file:///data/ala-bie/config/blacklist.json
+```
+
+An example blacklist file can be found [here](src/test/resources/test-blacklist.json).
+It contains metadata descibing the intent of the blacklist and
+a list of entries that will cause the blacklist to trigger.
+
+Each blacklist entry can trigger on some combination of:
+
+* **source** The URL of the original source of the data.
+* **name** The supplied name of taxon.
+* **title** The title of the article
+
+Currently, the blacklist is only used with the Encyclopedia of Life external source.
+
 
 ### Common Names Pull
 

diff --git a/build.gradle b/build.gradle
@@ -10,7 +10,7 @@ buildscript {
     }
 }
 
-version "1.5.0"
+version "1.5.1"
 
 group "au.org.ala.plugins.grails"
 
@@ -47,6 +47,8 @@ dependencies {
     provided "org.grails:grails-plugin-services"
     provided "org.grails:grails-plugin-domain-class"
     compile "com.bertramlabs.plugins:asset-pipeline-grails:2.14.2"
+    runtime "org.hibernate:hibernate-validator:5.4.2.Final"
+    runtime "dom4j:dom4j:1.6.1"
     testCompile "org.grails:grails-plugin-testing"
     testCompile "org.grails.plugins:geb"
     testRuntime "org.seleniumhq.selenium:selenium-htmlunit-driver:2.47.1"
@@ -70,15 +72,11 @@ dependencies {
     compile "org.codehaus.jackson:jackson-mapper-asl:1.8.6"
     compile "org.jsoup:jsoup:1.8.3"
 
-    compile group: 'org.grails.plugins', name: 'ala-bootstrap3', version: '3.0.6'
-    compile group: 'org.grails.plugins', name: 'ala-auth', version:'3.1.2', changing: true
+    compile group: 'org.grails.plugins', name: 'ala-bootstrap3', version: '3.2.3'
+    compile group: 'org.grails.plugins', name: 'ala-auth', version:'3.1.3'
     compile group: 'au.org.ala.plugins.grails', name: 'ala-citation-plugin', version: '1.0'
     runtime group: 'au.org.ala.plugins.grails', name: 'ala-charts-plugin', version: '2.0.1'
     compile group: 'au.org.ala.plugins.grails', name: 'images-client-plugin', version: '1.2'
-    //runtime group: 'org.grails.plugins', name: 'ala-charts-plugin', version: '1.3', changing: true
-    //compile (group: 'org.grails.plugins', name: 'images-client-plugin', version: '0.8', changing: true) {
-    //    exclude group: 'org.grails.plugins', module: 'ala-auth'
-    //}
 }
 
 bootRun {

diff --git a/grails-app/assets/javascripts/species.show.js b/grails-app/assets/javascripts/species.show.js
@@ -356,47 +356,46 @@ function loadExternalSources(){
             var eolIdentifier = data.taxonConcept.identifier;
             $.each(data.taxonConcept.dataObjects, function(idx, dataObject){
                 //console.log('Loading EOL content -> ' + dataObject.description);
-                if(dataObject.language == SHOW_CONF.eolLanguage || !dataObject.language){
-                    var $description = $('#descriptionTemplate').clone();
-                    $description.css({'display':'block'});
-                    $description.attr('id', dataObject.id);
-                    $description.find(".title").html(dataObject.title ?  dataObject.title : 'Description');
-
-                    var descriptionDom = $.parseHTML( dataObject.description );
-                    var body = $(descriptionDom).find('#bodyContent > p:lt(2)').html(); // for really long EOL blocks
+                // Data objects are now language filtered on the server
+                var $description = $('#descriptionTemplate').clone();
+                $description.css({'display':'block'});
+                $description.attr('id', dataObject.id);
+                $description.find(".title").html(dataObject.title ?  dataObject.title : 'Description');
 
-                    if (body) {
-                        $description.find(".content").html(body);
-                    } else {
-                        $description.find(".content").html(dataObject.description);
-                    }
+                var descriptionDom = $.parseHTML( dataObject.description );
+                var body = $(descriptionDom).find('#bodyContent > p:lt(2)').html(); // for really long EOL blocks
 
+                if (body) {
+                    $description.find(".content").html(body);
+                } else {
+                    $description.find(".content").html(dataObject.description);
+                }
 
-                    if(dataObject.source && dataObject.source.trim().length != 0){
-                        var sourceText = dataObject.source;
-                        var sourceHtml = "";
 
-                        if (sourceText.match("^http")) {
-                            sourceHtml = "<a href='" + sourceText + "' target='eol'>"  + sourceText + "</a>"
-                        } else {
-                            sourceHtml = sourceText;
-                        }
+                if(dataObject.source && dataObject.source.trim().length != 0){
+                    var sourceText = dataObject.source;
+                    var sourceHtml = "";
 
-                        $description.find(".sourceText").html(sourceHtml);
+                    if (sourceText.match("^http")) {
+                        sourceHtml = "<a href='" + sourceText + "' target='eol'>"  + sourceText + "</a>"
                     } else {
-                        $description.find(".source").css({'display':'none'});
-                    }
-                    if(dataObject.rightsHolder && dataObject.rightsHolder.trim().length != 0){
-                        $description.find(".rightsText").html(dataObject.rightsHolder);
-                    } else {
-                        $description.find(".rights").css({'display':'none'});
+                        sourceHtml = sourceText;
                     }
 
-                    $description.find(".providedBy").attr('href', 'https://eol.org/pages/' + eolIdentifier);
-                    $description.find(".providedBy").attr('target', '_blank');
-                    $description.find(".providedBy").html("Encyclopedia of Life");
-                    $description.appendTo('#descriptiveContent');
+                    $description.find(".sourceText").html(sourceHtml);
+                } else {
+                    $description.find(".source").css({'display':'none'});
                 }
+                if(dataObject.rightsHolder && dataObject.rightsHolder.trim().length != 0){
+                    $description.find(".rightsText").html(dataObject.rightsHolder);
+                } else {
+                    $description.find(".rights").css({'display':'none'});
+                }
+
+                $description.find(".providedBy").attr('href', 'https://eol.org/pages/' + eolIdentifier);
+                $description.find(".providedBy").attr('target', '_blank');
+                $description.find(".providedBy").html("Encyclopedia of Life");
+                $description.appendTo('#descriptiveContent');
             });
         }
     });

diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
@@ -94,6 +94,10 @@ endpoints:
 app:
   name: bie-plugin
   dataDir: /tmp/gbif-ecat
+security:
+  cas:
+    casServerName: https://auth.ala.org.au
+    appServerName: http://dev.ala.org.au:8080
 skin:
   layout: generic
   orgNameLong: BIE Plugin
@@ -111,7 +115,7 @@ projectNameShort: ALA
 projectName: Atlas of Living Australia
 languageCodesUrl: /languages.json
 biocache:
-  baseURL: "http://biocache.ala.org.au"
+  baseURL: http://biocache.ala.org.au
 biocacheService:
   baseURL: http://biocache.ala.org.au/ws
   queryContext:
@@ -178,7 +182,8 @@ external:
       service: https://eol.org/api/search/1.0.json?q={0}&page=1&exact=true&cache_ttl=
     page:
       # {0} = page id
-      service: https://eol.org/api/pages/1.0/{0,number,#0}.json?images_per_page=0&videos_per_page=0&sounds_per_page=0&maps_per_page=0&texts_per_page=30&subjects=overview&licenses=all&details=true&references=true&vetted=0&cache_ttl=
+      service: https://eol.org/api/pages/1.0/{0,number,#0}.json?language={1}&images_per_page=0&videos_per_page=0&sounds_per_page=0&maps_per_page=0&texts_per_page=30&subjects=overview&licenses=all&details=true&references=true&vetted=0&cache_ttl=
+  blacklist: file:./src/test/resources/test-blacklist.json
 literature:
   bhl:
     url: https://biodiversitylibrary.org

diff --git a/grails-app/controllers/au/org/ala/bie/ExternalSiteController.groovy b/grails-app/controllers/au/org/ala/bie/ExternalSiteController.groovy
@@ -36,39 +36,14 @@ class ExternalSiteController {
     RateLimiter eolRateLimiter = RateLimiter.create(1.0) // rate max requests per second (Double)
     RateLimiter genbankRateLimiter = RateLimiter.create(3.0) // rate max requests per second (Double)
 
-    // by default do not sanitise EOL response
-    boolean sanitiseEol = grailsApplication.config.getProperty("eol.sanitise", Boolean, false)
-
     def index() {}
 
     def eol = {
         eolRateLimiter.acquire()
-        String jsonOutput = "{}" // default is empty JSON object
-        def nameEncoded = URLEncoder.encode(params.s, 'UTF-8')
-        def filterString  = URLEncoder.encode(params.f ?: '', 'UTF-8')
-        String search = grailsApplication.config.external.eol.search.service
-        search =  MessageFormat.format(search, nameEncoded, filterString)
-        log.debug "Initial EOL url = ${search}"
-        def js = new JsonSlurper()
-        def jsonText = new URL(search).text
-        def json = js.parseText(jsonText ?: '{}')
-
-        //get first pageId
-        if (json.results) {
-            def match = json.results.find { it.title.equalsIgnoreCase(params.s) }
-            if (match) {
-                def pageId = match.id
-                String page = grailsApplication.config.external.eol.page.service
-                page = MessageFormat.format(page, pageId)
-                log.debug("EOL page url = ${page}")
-                def pageText = new URL(page).text ?: '{}'
-                def updatedPageText = updateEolOutput(pageText)
-                jsonOutput = sanitiseEol ? sanitiseEolOutput(updatedPageText) : updatedPageText
-            }
-        }
-
-        response.setContentType("application/json")
-        render jsonOutput
+        def name = params.s
+        def filter = params.f
+        def results = externalSiteService.searchEol(name, filter)
+        render results as JSON
     }
 
     def genbank = {
@@ -185,83 +160,4 @@ class ExternalSiteController {
             }
         }
     }
-
-    /**
-     * Update EOL content before rendering, rules specified in an external file.
-     */
-    String updateEolOutput(String text){
-        String updateFile = grailsApplication.config.update.file.location
-        if (updateFile != null && new File(updateFile).exists()){
-            new File(updateFile).eachLine { line ->
-                if (!line.startsWith("#")) {
-                    String[] valuePairs = line.split('--')
-                    String replacement = valuePairs.length==1 ? "''" :valuePairs[1]
-                    text = text.replace(valuePairs[0], replacement)
-                }
-            }
-        }
-        text
-    }
-
-    /**
-     * Sanitise EOL response with defined policy.
-     * @param text EOL response
-     * @return processed EOL response
-     */
-    String sanitiseEolOutput(String text) {
-        def json = new JsonSlurper().parseText(text)
-
-        if(json.taxonConcept?.dataObjects){
-            PolicyFactory policy = getPolicyFactory()
-            json.taxonConcept.dataObjects.each { dataObject ->
-                String desc = dataObject.description
-                String processedDesc = sanitiseBodyText(policy, desc)
-                dataObject.description = processedDesc
-            }
-        }
-        JsonOutput.toJson(json)
-    }
-
-    /**
-     * Utility to sanitise HTML text and only allow links to be kept, removing any
-     * other HTML markup.
-     * @param policy PolicyFactory
-     * @param input HTML String
-     * @return output sanitized HTML String
-     */
-    String sanitiseBodyText(PolicyFactory policy, String input) {
-        // Sanitize the HTML based on given policy
-        String sanitisedHtml = policy.sanitize(input)
-        sanitisedHtml
-    }
-
-    private PolicyFactory getPolicyFactory(){
-        HtmlPolicyBuilder builder = new HtmlPolicyBuilder()
-                .allowStandardUrlProtocols()
-                .requireRelNofollowOnLinks()
-
-        String allowedElements = grailsApplication.config.eol.html.allowedElements
-        if (allowedElements){
-            String[] elements = allowedElements.split(",")
-            elements.each {
-                builder.allowElements(it)
-            }
-        }
-
-        String allowedAttributes = grailsApplication.config.eol.html.allowAttributes
-        if (allowedAttributes){
-            String[] attributes = allowedAttributes.split(",")
-            attributes.each { attribute ->
-                String[] values = attribute.split (";")
-                if (values.length == 2){
-                    builder.allowAttributes(values[0]).onElements(values[1])
-                } else {
-                    builder.allowAttributes(values[0]).matching(Pattern.compile(values[2], Pattern.CASE_INSENSITIVE)).onElements(values[1])
-                }
-
-            }
-        }
-
-        builder.toFactory()
-    }
 }