Merge branch 'release/1.5.0'

.gitignore

@@ -3,6 +3,9 @@
 ##
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
 
+# XXX abrooks added
+*.dll
+
 # User-specific files
 *.suo
 *.user

.travis.yml

@@ -13,7 +13,7 @@ jobs:
       - $HOME/.nuget
       - $HOME/.m2
     addons:
-      postgresql: 10
+      postgresql: "10"
       apt:
         sources:
         - sourceline: 'deb http://archive.ubuntu.com/ubuntu bionic main universe multiverse restricted'
@@ -28,6 +28,7 @@ jobs:
           key_url: 'https://packages.microsoft.com/keys/microsoft.asc'
         - sourceline: 'deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/ubuntu/16.04/mssql-server-2017 xenial main'
         packages:
+        - rabbitmq-server
         - postgresql-10
         - postgresql-client-10
         - rabbitmq-server
@@ -67,6 +68,8 @@ jobs:
     - dotnet test ./tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj
     - dotnet test ./tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj
     - dotnet test ./tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj
+
+    - ( cd data/tessdata && ./download.sh && cd - )
     - dotnet test ./tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj
     - mvn -f src/common/com.smi.microservices.parent/pom.xml clean test
     - if [ -d src/microservices/uk.ac.dundee.hic.nerd/ ]; then mvn -f src/microservices/uk.ac.dundee.hic.nerd/ clean test; fi

BRANCH_TODO.md

@@ -0,0 +1,8 @@
+
+# TODO for feature/extraction-refactoring
+
+- Ensure that any changes to the RabbitMQ messages are reflected in both the C# and Java codebases
+  - Yes, we could definitely use Protobuf here but that would take more effort than we have at the moment
+
+- Java services need to support the SMI_LOGS_ROOT env var
+- Java services need to support the control messages

CHANGELOG.md

@@ -1,4 +1,3 @@
-
 # Changelog
 
 All notable changes to this project will be documented in this file.
@@ -7,7 +6,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
-...
+## [1.5.0] - 2020-03-05
+
+- \[Breaking\] Updated RabbitMQ extraction config to match extraction plan v2
+- Refactor Java exception handling and use of threads
+- `TessDirectory` option in [IsIdentifiable] now expects tesseract models file to exist (no longer downloads it on demand)
+- Addeed support for outsourcing classification (e.g. NLP) to other processes via TCP (entered in [SocketRules] in `Rules.yaml`)
+- IsIdentifiable NLP text classification now outsourced via TCP to any services configured in 
+  - [StanfordNER implementation written in java](./src/microservices/uk.ac.dundee.hic.nerd/README.md)
+- New CohortExtractor yaml config option `ProjectPathResolverType` which determines the folder structure for extracted images
+- Added [script](./utils/rabbitmq-config-tester/rabbitmq-config-tester.py) to verify RabbitMQ config files
+- Added `DynamicRejector` which takes its cohort extraction rules from a script file (of CSharp code)
+- Added new application for reviewing IsIdentifiable output files
+
+### Fixed
+
+- Corrected the GetHashCode implementation in the MessageHeader class
 
 ## [1.4.5] - 2020-02-26
 
@@ -78,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added support for custom rules in IsIdentifiable (entered in `Rules.yaml`)
   - Rules are applied in the order they appear in this file
   - Rules are applied before any other classifiers (i.e. to allow whitelisting rules)
+- Added `RedisSwapper` which caches answers from any other swapper.  Set `RedisHost` option in yaml to use.
 
 ### Changed
 
@@ -93,7 +108,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 - Improved logging in IdentifierSwappers
-- Added `RedisSwapper` which caches answers from any other swapper.  Set `RedisHost` option in yaml to use.
 
 ### Changed
 
@@ -154,16 +168,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Changed
 
-#### C#
-
+### C\# dependencies
 - Bumped HIC.DicomTypeTranslation from 1.0.0.3 to 2.1.2
 - Bumped HIC.RDMP.Plugin from 3.1.1 to 4.0.1-rc2
 - Bumped Newtonsoft.Json from 12.0.2 to 12.0.3
 - Bumped RabbitMQ.Client from 5.1.0 to 5.1.2
 - Bumped System.IO.Abstractions from 4.2.17 to 7.0.7
 - Bumped MongoDB.Driver from 2.8.0 to 2.9.3
 
-#### Java
+### Java dependencies
 
 - Bumped jackson-databind from 2.9.6 to 2.9.10.0
 
@@ -182,10 +195,11 @@ First stable release after importing the repository from the private [SMIPlugin]
 
 ### Removed
 
-- Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`).  Previously skipping database portion was supported.
+- Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported.
 
 
-[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.4.5...develop
+[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.5.0...develop
+[1.5.0]:  https://github.com/SMI/SmiServices/compare/v1.4.5...v1.5.0
 [1.4.5]:  https://github.com/SMI/SmiServices/compare/v1.4.4...v1.4.5
 [1.4.4]:  https://github.com/SMI/SmiServices/compare/v1.4.3...v1.4.4
 [1.4.3]:  https://github.com/SMI/SmiServices/compare/v1.4.2...v1.4.3
@@ -197,7 +211,10 @@ First stable release after importing the repository from the private [SMIPlugin]
 [1.2.3]:  https://github.com/SMI/SmiServices/compare/v1.2.2...v1.2.3
 [1.2.2]:  https://github.com/SMI/SmiServices/compare/v1.2.1...v1.2.2
 [1.2.1]:  https://github.com/SMI/SmiServices/compare/1.2.0...v1.2.1
-[1.2.0]:  https://github.com/SMI/SmiServices/compare/1.1.0-rc1...1.2.0
+[1.2.0]:  https://github.com/SMI/SmiServices/compare/1.1.0...1.2.0
 [1.1.0]: https://github.com/SMI/SmiServices/compare/1.0.0...1.1.0
 [1.0.0]: https://github.com/SMI/SmiServices/releases/tag/1.0.0
 
+
+[IsIdentifiable]: ./src/microservices/Microservices.IsIdentifiable/README.md
+[SocketRules]: ./src/microservices/Microservices.IsIdentifiable/README.md#socket-rules

PACKAGES.md

@@ -15,6 +15,8 @@
 | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | |
 | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
 | JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | |
+| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.15.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.15.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | |
+| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.5.0-beta2-final](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.5.0-beta2-final) | [MIT](https://opensource.org/licenses/MIT)  | Supports dynamic rules for cohort extraction logic | |
 | Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.2](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.2) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.2/License) | Caching ID mappings retrieved from Redis/MySQL |
 | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)|  [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases|
 | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
@@ -30,4 +32,4 @@
 | YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet)  | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files|
 | fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | |
 | fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | |
-| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | |
+| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | |

README.md

@@ -3,7 +3,7 @@
 ![GitHub](https://img.shields.io/github/license/SMI/SmiServices)
 [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/)
 
-Version: `1.4.5`
+Version: `1.5.0`
 
 # SMI Services
 
@@ -112,25 +112,31 @@ The CTP dependency first needs to be manually installed:
 
 - Linux
 
-```shell
-> cd lib/java/
-> ./installDat.sh
+```bash
+$ cd lib/java/
+$ ./installDat.sh
 ```
 
 - Windows
 
-```shell
-> cd lib\java\
-> .\installDat.bat
+```bash
+$ cd lib\java\
+$ .\installDat.bat
 ```
 
-The projects can then be built by returning to the top level directory and running:
+The projects can then be built and tested by returning to the top level directory and running:
 
-```shell
-> mvn -f src/common/com.smi.microservices.parent/pom.xml clean install
+```bash
+$ mvn -f src/common/com.smi.microservices.parent/pom.xml clean test
 ```
 
-This will compile and run the tests for the projects. The full test suite requires a local RabbitMQ server, however these can be skipped by passing `-PunitTests`. The entire test sutie can be skipped by passing `-DskipTests`.
+This will compile and run the tests for the projects. The full test suite requires a local RabbitMQ server, however these can be skipped by passing `-PunitTests`. The entire test suite can be skipped by instead running `compile`, or by passing `-DskipTests`.
+
+To build a single project and its dependencies, you can do:
+
+```bash
+$ mvn -f src/common/com.smi.microservices.parent/pom.xml test -pl com.smi.microservices:extractorcli -am
+```
 
 Note: If you have Maven `>=3.6.1` then you can pass `-ntp` to each of the above commands in order to hide the large volume of messages related to the downloading of dependencies.
 

SmiServices.sln

@@ -73,6 +73,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiabl
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiable.Tests", "tests\microservices\Microservices.IsIdentifiable.Tests\Microservices.IsIdentifiable.Tests.csproj", "{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IsIdentifiableReviewer", "src\applications\IsIdentifiableReviewer\IsIdentifiableReviewer.csproj", "{C2031E86-81B4-405A-A923-9B82E0CE196F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -175,6 +177,10 @@ Global
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}.Debug|x64.Build.0 = Debug|x64
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}.Release|x64.ActiveCfg = Release|x64
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2}.Release|x64.Build.0 = Release|x64
+		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Debug|x64.ActiveCfg = Debug|x64
+		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Debug|x64.Build.0 = Debug|x64
+		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.ActiveCfg = Release|x64
+		{C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -204,6 +210,7 @@ Global
 		{9CC69AAC-2F7C-4F36-B35E-5007FC075155} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{1A27E9E8-F16E-43F9-927E-5FE92E2F97D8} = {421CCD37-3817-4748-B184-A134E19DD75C}
 		{E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2} = {421CCD37-3817-4748-B184-A134E19DD75C}
+		{C2031E86-81B4-405A-A923-9B82E0CE196F} = {8B943F2C-835B-484A-86D2-3F1462970605}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {11CDEA53-71E8-4A9B-BC0D-74F4EB54F740}

data/microserviceConfigs/default.yaml

@@ -71,6 +71,7 @@ CohortExtractorOptions:
     # also doable on a single line with [2,4,5] :)
     AuditorType: 'Microservices.CohortExtractor.Audit.NullAuditExtractions'
     RequestFulfillerType: 'Microservices.CohortExtractor.Execution.RequestFulfillers.FromCataloguesExtractionRequestFulfiller'
+    ProjectPathResolverType: 'Microservices.CohortExtractor.Execution.ProjectPathResolvers.DefaultProjectPathResolver'
     # Writes (Producer) to this exchange
     ExtractFilesProducerOptions: 
         ExchangeName: 'TEST.ExtractFileExchange'
@@ -165,4 +166,12 @@ ExtractorClOptions:
         MaxConfirmAttempts: 1
     ExtractionRequestInfoProducerOptions:
         ExchangeName: 'TEST.RequestInfoExchange'
+        MaxConfirmAttempts: 1
+
+IsIdentifiableOptions:
+    QueueName: 'TEST.IsIdentifiableQueue'
+    QoSPrefetchCount: 1
+    AutoAck: false
+    IsIdentifiableProducerOptions:
+        ExchangeName: 'TEST.IsIdentifiableExchange'
         MaxConfirmAttempts: 1