Merge pull request #199 from SMI/release/v1.8.0

Release v1.8.0
SMI · Apr 16, 2020 · c1b8512 · c1b8512
2 parents 6f98d46 + 7ad0065
commit c1b8512
Show file tree

Hide file tree

Showing 77 changed files with 2,904 additions and 368 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,17 +1,17 @@
+merge_mode: replace
 dist: bionic
 os: linux
-language: csharp
+language: generic
 
 jobs:
   include:
-  - language: csharp
-    mono: none
-    dotnet: 2.2.100
     cache:
       directories:
       - $HOME/.local/share/NuGet/
       - $HOME/.nuget
       - $HOME/.m2
+      - $HOME/rdmp-cli
+      - $HOME/data
     addons:
       postgresql: "10"
       apt:
@@ -31,7 +31,6 @@ jobs:
         - rabbitmq-server
         - postgresql-10
         - postgresql-client-10
-        - rabbitmq-server
         - libgdiplus
         - tesseract-ocr
         - libtesseract-dev
@@ -45,66 +44,58 @@ jobs:
     - MSSQL_SA_PASSWORD="YourStrong!Passw0rd"
     - ACCEPT_EULA=Y
     - MSSQL_PID='evaluation'
+
     before_install:
-    - sudo apt-get install -y mssql-tools mssql-server
+    - sudo apt-get install -y --no-install-recommends mssql-tools mssql-server dotnet-runtime-2.2 dotnet-sdk-3.1
     - sudo /opt/mssql/bin/mssql-conf -n setup accept-eula
+
     install:
     - ( cd ./lib/java && ./installDat.sh && cd - )
-    - wget https://github.com/HicServices/RDMP/releases/download/v3.2.1/rdmp-cli-linux-x64.zip
-    # RDMP 3.2.1 was packaged using a ZIP tool with a bug Microsoft fixed in .Net 4.6.1; excluding Chinese language support file and keyword help avoids unzip being affected by the bug: https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/mitigation-ziparchiveentry-fullname-path-separator?redirectedfrom=MSDN
-    - unzip -d rdmp-cli rdmp-cli-linux-x64.zip -x "Curation*" "zh-*"
-    - chmod +x rdmp-cli/rdmp
-    - rdmp-cli/rdmp install localhost TEST_ -u sa -p 'YourStrong!Passw0rd'
+    - |
+      if [ ! -x $HOME/rdmp-cli/rdmp ]
+      then
+        wget https://github.com/HicServices/RDMP/releases/download/v4.0.3/rdmp-cli-linux-x64.zip
+        # RDMP 3.2.1 was packaged using a ZIP tool with a bug Microsoft fixed in .Net 4.6.1; excluding Chinese language support file and keyword help avoids unzip being affected by the bug: https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/mitigation-ziparchiveentry-fullname-path-separator?redirectedfrom=MSDN
+        unzip -d $HOME/rdmp-cli rdmp-cli-linux-x64.zip -x "Curation*" "zh-*"
+        chmod +x $HOME/rdmp-cli/rdmp
+      fi
+    - $HOME/rdmp-cli/rdmp install localhost TEST_ -u sa -p 'YourStrong!Passw0rd'
+    - if [ -e $HOME/data/eng.traineddata ]; then ln $HOME/data/eng.traineddata data/tessdata/; fi
+    - ( cd data/tessdata && ./download.sh && ln -f eng.traineddata $HOME/data/ && cd - )
+
     script:
     - dotnet build --verbosity quiet
-    - dotnet test ./tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj
-    - dotnet test ./tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj
-    - dotnet test ./tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj
-    - dotnet test ./tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj
-
-    - ( cd data/tessdata && ./download.sh && cd - )
-    - dotnet test ./tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj
+    - find ./tests -iname "*.csproj" -print0 | xargs -0 -L 1 dotnet test -p:Platform=x64 --verbosity=normal --no-build
+    - 'if [ -f lib/java/Util/source/java/org/rsna/util/ChunkedInputStream.java ] ; then iconv -f iso-8859-1 -t utf-8 lib/java/Util/source/java/org/rsna/util/ChunkedInputStream.java > tmpiconv && mv tmpiconv lib/java/Util/source/java/org/rsna/util/ChunkedInputStream.java; fi'
     - mvn -f src/common/com.smi.microservices.parent/pom.xml clean test
-    - if [ -d src/microservices/uk.ac.dundee.hic.nerd/ ]; then mvn -f src/microservices/uk.ac.dundee.hic.nerd/ clean test; fi
+    - mvn -f src/microservices/uk.ac.dundee.hic.nerd/ clean test
 
-    after_success:
-    - if [ -z "$TRAVIS_TAG" ]; then travis_terminate 0; fi
-    - mkdir -p ./dist
-    - |
-      for platform in linux win
-      do
-        dotnet publish -c Release -r $platform-x64 -o $(pwd)/dist/$platform-x64 --self-contained false -nologo -v q
-      done
-    - ( cd dist && zip -r smi-services-win-x64-$TRAVIS_TAG.zip ./win-x64 && tar czf smi-services-linux-x64-$TRAVIS_TAG.tar.gz ./linux-x64 && cd - )
-    - mvn -f src/common/com.smi.microservices.parent/pom.xml install assembly:single@create-deployable
-    - |
-      for filename in $(find ./src -name "*deploy-distribution.zip")
-      do
-        base=$(ls $filename | awk -F'-' '{print $1}' | awk -F'/' '{print $6}')
-        mv $filename dist/$base-${TRAVIS_TAG}.zip
-      done
     - |
-      if [ -d src/microservices/uk.ac.dundee.hic.nerd/ ]
+      if [ ! -z "$TRAVIS_TAG" ]
       then
-      mvn -f src/microservices/uk.ac.dundee.hic.nerd/ package
-      mv src/microservices/uk.ac.dundee.hic.nerd/target/nerd-*.jar dist/nerd-${TRAVIS_TAG}.jar
+        mkdir -p ./dist
+        for platform in linux win
+        do
+          dotnet publish -c Release -r $platform-x64 -o $(pwd)/dist/$platform-x64 --self-contained false -nologo -v q
+        done
+        ( cd dist && zip -r smi-services-win-x64-$TRAVIS_TAG.zip ./win-x64 && tar czf smi-services-linux-x64-$TRAVIS_TAG.tar.gz ./linux-x64 && cd - )
+        mvn -f src/common/com.smi.microservices.parent/pom.xml install assembly:single@create-deployable
+        mvn -f src/microservices/uk.ac.dundee.hic.nerd/ package
+        for filename in $(find ./src -name "*deploy-distribution.zip")
+        do
+          base=$(ls $filename | awk -F'-' '{print $1}' | awk -F'/' '{print $6}')
+          mv $filename dist/$base-${TRAVIS_TAG}.zip
+        done
+        mv src/microservices/uk.ac.dundee.hic.nerd/target/nerd-*.jar dist/nerd-${TRAVIS_TAG}.jar
       fi
 
-# NOTE: This is inherited by both jobs, so actually runs twice
 deploy:
       provider: releases
       token:
         secure: DdirbOkYUePnKisYPlqYOpyti2HQFWmSIwY5spWflTUglzEZbnp+Q3sEltTebdlWo18i9skz6P5seAAXbX2ye1c3EZhqEyirWMxz1lvGdlNoQGHhWK6x+suM1DrG6Uun5dmcG/CEpWhDpcEbF8xQ/revtyCHEJj4xK9TXUHyFFG/KrzKSLGtsiBG1BOPkHJr4CoHjcpLzpmDB2hn+pYENtl3Nk0EeKmJgWdY/BWctEmUmRAXgfwsO/ejm9IbC5mIy5KBxbIUco/mf2JEKQB1drwdbINMxJXkbx+mWZYiDJBo5CWuWIzNOxv0Z3+Kg+N2cSwPX833v+W5YRTHyef/7Dum1yUrD3T7LMLY+3EIeCoGvARfRnQbi+BM9n1rFobPy/PWHWYxYRDnhDEfXrMcAyH/66InlQIhUh7XD01oZLEbsdZAbGnEsv5Ttz3F9LaEpk3EfoGqLxOWkL50OCxw6Jk6TfxsNtMoLpK1g7Jp8sI/EkjxYISer7S5myzLhKtgbtBwI0RcKh/sUo2m69QX++1JYy6CeN7/EwRmV4zJLk0WgalJuxgNFYrdzPLvg09okgxr1stHZQ7a5RkLH+r8KzA/Lmj6vTQG9HB44lHvX1hMiD0v8aCii+/wfr9AZZlNHjYq8RuHsP5uQ5U5F8G9uMqfiVb75MOPKY7qdYCZZZg=
       file_glob: true
       file: dist/*.*
-      skip-cleanup: true
+      skip_cleanup: true
       on:
         all_branches: true
         tags: true
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,7 +6,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
--
+## [1.8.0] - 2020-04-16
+
+### Added
+
+- Added Terminal.Gui at version 0.81.0
+- Added data/IsIdentifiableRules
+
+### Changed
+
+- \[Breaking\] Promote the PT modality to its own collection in MongoDB
+- \[Breaking\] Renamed `RedisHost` to `RedisConnectionString` in the config options for clarity
+- Update to .Net Core 3.1 (supported until Dec 2022) since 2.2 support ended last year
+- Switch CohortExtractor to use batched message producers
+- Simplify the Travis build script
+- Fail any integration tests in CI if a required service is not available (instead of skipping)
+- Specified LangVersion 8.0 in all project files
+- Upgraded CommandLineParser from 2.5.0 to 2.7.82
+- Upgraded CsvHelper from 12.1.2 to 15.0.4
+- Upgraded HIC.Rdmp.Dicom from 2.0.8 to 2.0.9
+- Upgraded JetBrains.Annotations from 2019.1.3 to 2020.1.0
+- Upgraded Magick.NET-Q16-AnyCPU from 7.15.1 to 7.16.0
+- Upgraded Microsoft.CodeAnalysis.CSharp.Scripting from 3.5.0-beta2-final to 3.5.0
+- Upgraded MongoDB.Driver from 2.9.3 to 2.10.3
+- Upgraded StackExchange.Redis from 2.0.601 to 2.1.30
+- Upgraded System.Drawing.Common from 4.6.0 to 4.7.0
+- Upgraded System.IO.Abstractions from 7.0.7 to 10.0.8
+- Upgraded YamlDotNet from 6.0.0 to 8.1.0
+
+### Fixed
+
+- Fixed logging to directories in the Java services
 
 ## [1.7.0] - 2020-03-30
 
@@ -243,7 +273,8 @@ First stable release after importing the repository from the private [SMIPlugin]
 - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported.
 
 
-[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.7.0...develop
+[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.8.0...develop
+[1.8.0]:  https://github.com/SMI/SmiServices/compare/v1.7.0...v1.8.0
 [1.7.0]:  https://github.com/SMI/SmiServices/compare/v1.6.0...v1.7.0
 [1.6.0]:  https://github.com/SMI/SmiServices/compare/v1.5.2...v1.6.0
 [1.5.2]:  https://github.com/SMI/SmiServices/compare/v1.5.1...v1.5.2

diff --git a/PACKAGES.md b/PACKAGES.md
@@ -9,25 +9,26 @@
 
 | Package | Source Code | Version | License | Purpose | Additional Risk Assessment |
 | ------- | ------------| --------| ------- | ------- | -------------------------- |
-| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing |  |
-| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | |
+| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.7.82](https://www.nuget.org/packages/CommandLineParser/2.7.82) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing |  |
+| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [15.0.4](https://www.nuget.org/packages/CsvHelper/15.0.4) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | |
 | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.2.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.2.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | |
-| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.8](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | |
+| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.9](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.9) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | |
 | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
-| JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | |
-| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.15.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.15.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | |
-| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.5.0-beta2-final](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.5.0-beta2-final) | [MIT](https://opensource.org/licenses/MIT)  | Supports dynamic rules for cohort extraction logic | |
+| JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | |
+| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.16.0](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.16.0) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | |
+| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.5.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.5.0) | [MIT](https://opensource.org/licenses/MIT)  | Supports dynamic rules for cohort extraction logic | |
 | Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.3](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.3) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.3/License) | Caching ID mappings retrieved from Redis/MySQL |
-| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)|  [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases|
+| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.10.3](https://www.nuget.org/packages/MongoDB.Driver/2.10.3)|  [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.10.3/License) | For writting/reading dicom tags into MongoDb databases|
 | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
 | Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission |
 | RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | |
-| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.0.601](https://www.nuget.org/packages/StackExchange.Redis/2.0.601) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | |
+| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.1.30](https://www.nuget.org/packages/StackExchange.Redis/2.1.30) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | |
 | Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [3.9.2](https://www.nuget.org/packages/Stanford.NLP.CoreNLP/3.9.2) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | |
-| System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.6.0](https://www.nuget.org/packages/System.Drawing.Common/4.6.0) | [MIT](https://opensource.org/licenses/MIT)  | Supports reading pixel data | |
-| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [7.0.7](https://www.nuget.org/packages/System.IO.Abstractions/7.0.7) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | |
+| System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Drawing.Common/4.7.0) | [MIT](https://opensource.org/licenses/MIT)  | Supports reading pixel data | |
+| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [10.0.8](https://www.nuget.org/packages/System.IO.Abstractions/10.0.8) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | |
 | System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.IO.FileSystem/4.3.0) |[MIT](https://opensource.org/licenses/MIT)  | File I/O | |
 | System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT)  | File access perimssions| |
+| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) | [0.81.0](https://www.nuget.org/packages/Terminal.Gui/0.81.0) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | |
 | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt)  | Optical Character Recognition in Dicom Pixel data| |
-| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet)  | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files|
+| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet)  | [8.1.0](https://www.nuget.org/packages/YamlDotNet/8.1.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files|
 | fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.4](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | |
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 ![GitHub](https://img.shields.io/github/license/SMI/SmiServices)
 [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/)
 
-Version: `1.7.0`
+Version: `1.8.0`
 
 # SMI Services
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -0,0 +1,19 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- master
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+steps:
+- script: echo Hello, world!
+  displayName: 'Run a one-line script'
+
+- script: |
+    echo Add other tasks to build, test, and deploy your project.
+    echo See https://aka.ms/yaml
+  displayName: 'Run a multi-line script'