From 66ef00a44097c4826cbf9f34cefe0e0cdec0104b Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Wed, 26 Feb 2020 22:28:56 +0000 Subject: [PATCH 01/24] Upgrade fo-dicom packages to v4.0.4 --- PACKAGES.md | 6 +++--- src/common/Smi.Common/Smi.Common.csproj | 2 +- .../Microservices.IsIdentifiable.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 98357c34e..2e2e63bf6 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -28,6 +28,6 @@ | System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| | | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | | YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| -| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | | -| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | -| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | | +| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.4](https://www.nuget.org/packages/fo-dicom/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | | +| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.4](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | +| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.4](https://www.nuget.org/packages/fo-dicom.Json/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | | diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index f529ff329..74832e241 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index ce3aa58ed..b0f6b4108 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -20,7 +20,7 @@ - + From f46ddcf9cc4cd87f7d25b47460f0499f4d4ca9a0 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Wed, 26 Feb 2020 22:41:19 +0000 Subject: [PATCH 02/24] Update message for fo-dicom FileReadOption.SkipLargeTags setting Although the bug surrounding this issue has now been fixed in fo-dicom, we still want to disallow it here since its usage may result in missing data. --- src/common/Smi.Common/Options/GlobalOptions.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index dcaefb498..8b92859f9 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -249,9 +249,8 @@ public FileReadOption GetReadOption() { var opt = (FileReadOption)Enum.Parse(typeof(FileReadOption), FileReadOption); - //TODO(Ruairidh 2019-08-28) Monitor the status of this if (opt == Dicom.FileReadOption.SkipLargeTags) - throw new ApplicationException("SkipLargeTags option is currently disabled due to issues in fo-dicom. See: https://github.com/fo-dicom/fo-dicom/issues/893"); + throw new ApplicationException("SkipLargeTags is disallowed here to ensure data consistency"); return opt; } From 09a8c75415e024ba4ea6cec0357aa580ab14d049 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Wed, 26 Feb 2020 23:13:15 +0000 Subject: [PATCH 03/24] Pre-update DicomTypeTranslation to 2.2.0, awaiting release --- src/common/Smi.Common/Smi.Common.csproj | 2 +- .../Microservices.IsIdentifiable.csproj | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 74832e241..4b0723230 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -29,7 +29,7 @@ - + diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index b0f6b4108..2d09b54a3 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -21,7 +21,6 @@ - From 2a903e0ff7966c378e8146e02bd444352e4e80a8 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 09:31:04 +0000 Subject: [PATCH 04/24] Fixed cancelling entering a Regex ignore/update pattern exiting the application --- .../IsIdentifiableReviewer/MainWindow.cs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 5d3de8e9b..a2c711194 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -253,7 +253,15 @@ private void Ignore() if(_valuePane.CurrentFailure == null) return; - Ignorer.Add(_valuePane.CurrentFailure); + try + { + Ignorer.Add(_valuePane.CurrentFailure); + } + catch (OperationCanceledException) + { + //if user cancels adding the ignore then stay on the same record + return; + } Next(); } private void Update() @@ -264,7 +272,12 @@ private void Update() try { Updater.Update(_cbRulesOnly.Checked ? null : CurrentTarget?.Discover() - ,_valuePane.CurrentFailure,null /*create one yourself*/); + , _valuePane.CurrentFailure, null /*create one yourself*/); + } + catch (OperationCanceledException) + { + //if user cancels updating then stay on the same record + return; } catch (Exception e) { @@ -521,7 +534,7 @@ public string GetPattern(object sender,Failure failure) } - throw new Exception("User chose not to enter a pattern"); + throw new OperationCanceledException("User chose not to enter a pattern"); } } From 15351e5d12f4ef376d6c042d94c0f6392607f9e0 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 10:34:21 +0000 Subject: [PATCH 05/24] Added more precanned patterns you can pick from in custom pattern mode --- .../IsIdentifiableReviewer/MainWindow.cs | 32 +++++++++++++------ .../Out/SymbolsRulesFactory.cs | 31 ++++++++++++++++++ .../ReviewerTests/SymbolsRulesFactoryTests.cs | 27 ++++++++++++++++ 3 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs create mode 100644 tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index a2c711194..6ee7d9490 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -424,7 +424,8 @@ bool RunDialog(string title, string message,out T chosen, params T[] options) chosen = result; return optionChosen; } - private bool GetText(string title, string message, string initialValue, out string chosen) + private bool GetText(string title, string message, string initialValue, out string chosen, + Dictionary buttons) { bool optionChosen = false; @@ -475,13 +476,17 @@ private bool GetText(string title, string message, string initialValue, out stri }; dlg.Add(btn); - var btnClear = new Button(15, line, "Clear") - { - Clicked = () => { txt.Text = ""; } - }; - dlg.Add(btnClear); - - + int x = 10; + if(buttons != null) + foreach (var kvp in buttons) + { + var button = new Button(x, line,kvp.Key) + { + Clicked = () => { txt.Text = kvp.Value; } + }; + dlg.Add(button); + x += 13; + } dlg.FocusFirst(); @@ -503,8 +508,15 @@ public string GetPattern(object sender,Failure failure) var defaultFactory = sender == Updater ? _origUpdaterRulesFactory : _origIgnorerRulesFactory; var recommendedPattern = defaultFactory.GetPattern(sender,failure); - - if (GetText("Pattern", "Enter pattern to match failure", recommendedPattern, out string chosen)) + + Dictionary buttons = new Dictionary(); + buttons.Add("Clear",""); + buttons.Add("Full",_origIgnorerRulesFactory.GetPattern(sender,failure)); + buttons.Add("Captures",_origUpdaterRulesFactory.GetPattern(sender,failure)); + + buttons.Add("Symbols",new SymbolsRulesFactory().GetPattern(sender,failure)); + + if (GetText("Pattern", "Enter pattern to match failure", recommendedPattern, out string chosen,buttons)) { Regex regex; diff --git a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs new file mode 100644 index 000000000..440e9ad9c --- /dev/null +++ b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs @@ -0,0 +1,31 @@ +using System.Text; +using System.Text.RegularExpressions; +using Microservices.IsIdentifiable.Reporting; + +namespace IsIdentifiableReviewer.Out +{ + public class SymbolsRulesFactory : IRulePatternFactory + { + public string GetPattern(object sender, Failure failure) + { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < failure.ProblemValue.Length; i++) + { + char cur = failure.ProblemValue[i]; + + if (char.IsDigit(cur)) + sb.Append("\\d"); + else + if (char.IsLetter(cur)) + sb.Append(char.IsUpper(cur) ? "[A-Z]" : "[a-z]"); + else + { + sb.Append(Regex.Escape(cur.ToString())); + } + } + + return "^" + sb + "$"; + } + } +} \ No newline at end of file diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs new file mode 100644 index 000000000..c129402dc --- /dev/null +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs @@ -0,0 +1,27 @@ +using IsIdentifiableReviewer.Out; +using Microservices.IsIdentifiable.Failures; +using Microservices.IsIdentifiable.Reporting; +using NUnit.Framework; + +namespace Microservices.IsIdentifiable.Tests.ReviewerTests +{ + public class SymbolsRulesFactoryTests + { + + [TestCase("12/34/56",@"^\d\d/\d\d/\d\d$")] + + [TestCase("1 6",@"^\d\ \d$")] + + [TestCase(@"abc +123",@"^[a-z][a-z][a-z]\r\n\d\d\d$")] + public void TestSymbols(string input, string expectedOutput) + { + var f = new SymbolsRulesFactory(); + + Assert.AreEqual( + expectedOutput, + f.GetPattern(this, new Failure(new FailurePart[0]) {ProblemValue = input}) + ); + } + } +} \ No newline at end of file From fa29caca1781dff465cf1f67938ffa583c920d17 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 11:05:53 +0000 Subject: [PATCH 06/24] Added undo feature --- CHANGELOG.md | 4 ++ .../IsIdentifiableReviewer/MainWindow.cs | 29 ++++++++ .../MainWindowHistory.cs | 16 +++++ .../IsIdentifiableReviewer/Out/OutBase.cs | 34 +++++++-- .../Out/OutBaseHistory.cs | 16 +++++ .../ReviewerTests/TestIgnoreRuleGenerator.cs | 72 +++++++++++++++++-- 6 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 src/applications/IsIdentifiableReviewer/MainWindowHistory.cs create mode 100644 src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index a03cae1f0..3b279e250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ... +### Added + +- Added undo feature to IsIdentifiableReviewer + ## [1.6.0] - 2020-03-17 ### Changed diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 6ee7d9490..577332e40 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -34,6 +34,8 @@ class MainWindow : View,IRulePatternFactory private Label _updateRuleLabel; private CheckBox _cbRulesOnly; + Stack History = new Stack(); + public MainWindow(List targets, IsIdentifiableReviewerOptions opts, IgnoreRuleGenerator ignorer, RowUpdater updater) { _targets = targets; @@ -116,6 +118,13 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno Clicked = ()=>GoToRelative(1) }); + frame.Add(new Button("unDo") + { + X = 11, + Y = 2, + Clicked = ()=>Undo() + }); + frame.Add(new Label(0,4,"Default Patterns")); _ignoreRuleLabel = new Label(0,5,"Ignore:"); @@ -146,6 +155,23 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno OpenReport(opts.FailuresCsv,(e)=>throw e, (t)=>throw new Exception("Mode only supported when a single Target is configured")); } + private void Undo() + { + if (History.Count == 0) + { + ShowMessage("History Empty","Cannot undo, history is empty"); + return; + } + + var popped = History.Pop(); + + //undo file history + popped.OutputBase.Undo(); + + //wind back UI + GoTo(popped.Index); + } + private void GoToRelative(int offset) { if(CurrentReport == null) @@ -256,6 +282,7 @@ private void Ignore() try { Ignorer.Add(_valuePane.CurrentFailure); + History.Push(new MainWindowHistory(CurrentReport.CurrentIndex,Ignorer)); } catch (OperationCanceledException) { @@ -273,6 +300,8 @@ private void Update() { Updater.Update(_cbRulesOnly.Checked ? null : CurrentTarget?.Discover() , _valuePane.CurrentFailure, null /*create one yourself*/); + + History.Push(new MainWindowHistory(CurrentReport.CurrentIndex,Updater)); } catch (OperationCanceledException) { diff --git a/src/applications/IsIdentifiableReviewer/MainWindowHistory.cs b/src/applications/IsIdentifiableReviewer/MainWindowHistory.cs new file mode 100644 index 000000000..d72c7a204 --- /dev/null +++ b/src/applications/IsIdentifiableReviewer/MainWindowHistory.cs @@ -0,0 +1,16 @@ +using IsIdentifiableReviewer.Out; + +namespace IsIdentifiableReviewer +{ + internal class MainWindowHistory + { + public int Index { get;} + public OutBase OutputBase { get; } + + public MainWindowHistory(int index, OutBase outputBase) + { + Index = index; + OutputBase = outputBase; + } + } +} \ No newline at end of file diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs index 204f276c0..43f3844e2 100644 --- a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs +++ b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs @@ -13,11 +13,13 @@ namespace IsIdentifiableReviewer.Out { public abstract class OutBase { - protected List Rules { get;} + public List Rules { get;} public FileInfo RulesFile { get; } public IRulePatternFactory RulesFactory { get; set; } = new MatchWholeStringRulePatternFactory(); + public Stack History = new Stack(); + protected OutBase(FileInfo rulesFile) { RulesFile = rulesFile; @@ -72,13 +74,35 @@ protected IsIdentifiableRule Add(Failure f, RuleAction action) var serializer = new Serializer(); var yaml = serializer.Serialize(new List {rule}); - File.AppendAllText(RulesFile.FullName, - $"#{Environment.UserName} - {DateTime.Now}" + Environment.NewLine + - yaml); + + var contents = $"#{Environment.UserName} - {DateTime.Now}" + Environment.NewLine + + yaml; + + File.AppendAllText(RulesFile.FullName,contents); + History.Push(new OutBaseHistory(rule,contents)); return rule; } - + + public void Undo() + { + if(History.Count == 0) + return; + + var popped = History.Pop(); + + if (popped != null) + { + //clear the rule from the serialized text file + var oldText = File.ReadAllText(RulesFile.FullName); + var newText = oldText.Replace(popped.Yaml, ""); + File.WriteAllText(RulesFile.FullName,newText); + + //clear the rule from memory + Rules.Remove(popped.Rule); + } + } + /// /// Returns true if there are any rules that already exactly cover the given /// diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs b/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs new file mode 100644 index 000000000..bdad48d04 --- /dev/null +++ b/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs @@ -0,0 +1,16 @@ +using Microservices.IsIdentifiable.Rules; + +namespace IsIdentifiableReviewer.Out +{ + public class OutBaseHistory + { + public IsIdentifiableRule Rule { get; } + public string Yaml { get; } + + public OutBaseHistory(IsIdentifiableRule rule, string yaml) + { + Rule = rule; + Yaml = yaml; + } + } +} \ No newline at end of file diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestIgnoreRuleGenerator.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestIgnoreRuleGenerator.cs index ced4cf9f2..b5999fa4f 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestIgnoreRuleGenerator.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestIgnoreRuleGenerator.cs @@ -28,13 +28,13 @@ public void TestRepeatedIgnoring() if(File.Exists(newRules.FullName)) File.Delete(newRules.FullName); - IgnoreRuleGenerator updater = new IgnoreRuleGenerator(newRules); + IgnoreRuleGenerator ignorer = new IgnoreRuleGenerator(newRules); //it should be novel i.e. require user decision - Assert.IsTrue(updater.OnLoad(failure,out _)); + Assert.IsTrue(ignorer.OnLoad(failure,out _)); //we tell it to ignore this value - updater.Add(failure); + ignorer.Add(failure); StringAssert.Contains( @"- Action: Ignore @@ -43,8 +43,72 @@ public void TestRepeatedIgnoring() ",File.ReadAllText(newRules.FullName)); //btw slash space is a 'literal space' so legit //it should be no longer be novel - Assert.IsFalse(updater.OnLoad(failure, out _)); + Assert.IsFalse(ignorer.OnLoad(failure, out _)); } + + [Test] + public void TestUndo() + { + var failure = new Reporting.Failure( + new FailurePart[] + { + new FailurePart("Kansas", FailureClassification.Location, 13), + new FailurePart("Toto", FailureClassification.Location, 28) + }) + { + ProblemValue = "We aren't in Kansas anymore Toto", + ProblemField = "Narrative", + ResourcePrimaryKey = "1.2.3.4" + }; + + var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "IgnoreList.yaml")); + + //make sure repeat test runs work properly + if(File.Exists(newRules.FullName)) + File.Delete(newRules.FullName); + + //create an existing rule to check that Undo doesn't just nuke the entire file + File.WriteAllText(newRules.FullName,@"- Action: Ignore + IfColumn: Narrative + IfPattern: ^Joker Wuz Ere$ +"); + + IgnoreRuleGenerator ignorer = new IgnoreRuleGenerator(newRules); + + //it should be novel i.e. require user decision + Assert.IsTrue(ignorer.OnLoad(failure,out _)); + + //we tell it to ignore this value + ignorer.Add(failure); + + StringAssert.Contains( + @"- Action: Ignore + IfColumn: Narrative + IfPattern: ^We\ aren't\ in\ Kansas\ anymore\ Toto$ +",File.ReadAllText(newRules.FullName)); //btw slash space is a 'literal space' so legit + + //it should be no longer be novel + Assert.IsFalse(ignorer.OnLoad(failure, out _)); + + //Undo + Assert.AreEqual(1,ignorer.History.Count); + Assert.AreEqual(2,ignorer.Rules.Count); + ignorer.Undo(); + + Assert.AreEqual(0,ignorer.History.Count); + Assert.AreEqual(1,ignorer.Rules.Count); + + //only the original one should be there + Assert.AreEqual(@"- Action: Ignore + IfColumn: Narrative + IfPattern: ^Joker Wuz Ere$ +",File.ReadAllText(newRules.FullName)); + + //repeated undo calls do nothing + ignorer.Undo(); + ignorer.Undo(); + ignorer.Undo(); + } } } \ No newline at end of file From 29a6189861d1b1dcd25379996ac7f1b8356a6f0d Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 11:27:10 +0000 Subject: [PATCH 07/24] Fixed horizontal button layout --- src/applications/IsIdentifiableReviewer/MainWindow.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 577332e40..276a2c33a 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -514,7 +514,7 @@ private bool GetText(string title, string message, string initialValue, out stri Clicked = () => { txt.Text = kvp.Value; } }; dlg.Add(button); - x += 13; + x += kvp.Key.Length + 5; } dlg.FocusFirst(); From 764539276dbedeb15ecc6d6e5816944482728b97 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 11:29:30 +0000 Subject: [PATCH 08/24] Fixed line ending in test (test was failing in linux) --- .../ReviewerTests/SymbolsRulesFactoryTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs index c129402dc..af3e287c2 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs @@ -12,8 +12,7 @@ public class SymbolsRulesFactoryTests [TestCase("1 6",@"^\d\ \d$")] - [TestCase(@"abc -123",@"^[a-z][a-z][a-z]\r\n\d\d\d$")] + [TestCase("abc\n123",@"^[a-z][a-z][a-z]\n\d\d\d$")] public void TestSymbols(string input, string expectedOutput) { var f = new SymbolsRulesFactory(); From b234730a8b230128d4172f3e4aa6ee3e1410cf87 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 11:57:36 +0000 Subject: [PATCH 09/24] Changed undo to use a temp file during the replace operation. --- src/applications/IsIdentifiableReviewer/Out/OutBase.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs index 43f3844e2..8e3ce281d 100644 --- a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs +++ b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs @@ -96,8 +96,14 @@ public void Undo() //clear the rule from the serialized text file var oldText = File.ReadAllText(RulesFile.FullName); var newText = oldText.Replace(popped.Yaml, ""); - File.WriteAllText(RulesFile.FullName,newText); + //write to a new temp file + File.WriteAllText(RulesFile.FullName + ".tmp",newText); + + //then hot swap them + File.Delete(RulesFile.FullName); + File.Move(RulesFile.FullName + ".tmp",RulesFile.FullName); + //clear the rule from memory Rules.Remove(popped.Rule); } From 178ad8b367786142468eca4c6fe55a13358c966f Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 13:19:46 +0000 Subject: [PATCH 10/24] no message --- .../IsIdentifiableReviewer/MainWindow.cs | 2 +- .../IsIdentifiableReviewer/Out/RowUpdater.cs | 7 ++- .../Out/UpdateStrategies/UpdateStrategy.cs | 6 ++- .../UnattendedReviewer.cs | 48 +++++++++++++++++-- .../ReviewerTests/TestUpdater.cs | 8 ++-- 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 276a2c33a..c30c6a8b0 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -242,7 +242,7 @@ private void Next() var next = CurrentReport.Current; //prefer rules that say we should update the database with redacted over rules that say we should ignore the problem - if (!Updater.OnLoad(CurrentTarget?.Discover(),next)) + if (!Updater.OnLoad(CurrentTarget?.Discover(),next, out _)) updated++; else if (!Ignorer.OnLoad(next,out _)) skipped++; diff --git a/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs b/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs index 5c95fbf36..c47b29b79 100644 --- a/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs +++ b/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs @@ -95,15 +95,18 @@ public void Update(DiscoveredServer server, Failure failure, IsIdentifiableRule /// /// /// + /// The first rule that covered the /// True if is novel and not seen before - public bool OnLoad(DiscoveredServer server,Failure failure) + public bool OnLoad(DiscoveredServer server,Failure failure, out IsIdentifiableRule rule) { + rule = null; + //we have bigger problems than if this is novel! if (server == null) return true; //if we have seen this before - if (IsCoveredByExistingRule(failure, out IsIdentifiableRule rule)) + if (IsCoveredByExistingRule(failure, out rule)) { //since user has issued an update for this exact problem before we can update this one too Update(server,failure,rule); diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs index 157de031f..878933e68 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using Microservices.IsIdentifiable.Reporting; @@ -23,6 +24,9 @@ public abstract class UpdateStrategy : IUpdateStrategy protected string GetUpdateWordSql(DiscoveredTable table, Dictionary primaryKeys, IQuerySyntaxHelper syntax, Failure failure,string word) { + if(string.IsNullOrEmpty(failure.ResourcePrimaryKey)) + throw new ArgumentException("Failure record's primary key is blank, cannot update database"); + return $@"update {table.GetFullyQualifiedName()} SET {syntax.EnsureWrapped(failure.ProblemField)} = REPLACE({syntax.EnsureWrapped(failure.ProblemField)},'{syntax.Escape(word)}', 'SMI_REDACTED') diff --git a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs index bee102994..2d5def3c8 100644 --- a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs +++ b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs @@ -1,10 +1,15 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using IsIdentifiableReviewer.Out; using Microservices.IsIdentifiable.Options; using Microservices.IsIdentifiable.Reporting.Destinations; using Microservices.IsIdentifiable.Reporting.Reports; +using Microservices.IsIdentifiable.Rules; +using NLog; +using NLog.Fluent; namespace IsIdentifiableReviewer { @@ -20,9 +25,15 @@ public class UnattendedReviewer public int Ignores = 0; public int Unresolved = 0; public int Total = 0; + private Logger _log; + + Dictionary _updateRulesUsed = new Dictionary(); + Dictionary _ignoreRulesUsed = new Dictionary(); public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, IgnoreRuleGenerator ignorer, RowUpdater updater) { + _log = LogManager.GetCurrentClassLogger(); + if (string.IsNullOrWhiteSpace(opts.FailuresCsv)) throw new Exception("Unattended requires a file of errors to process"); @@ -62,33 +73,60 @@ public int Run() while(_reportReader.Next()) { //is it novel for updater - if(_updater.OnLoad(server,_reportReader.Current)) + if(_updater.OnLoad(server,_reportReader.Current, out IsIdentifiableRule updateRule)) //is it novel for ignorer - if (_ignorer.OnLoad(_reportReader.Current,out _)) + if (_ignorer.OnLoad(_reportReader.Current,out IsIdentifiableRule ignoreRule)) { //we can't process it unattended storeReport.Add(_reportReader.Current); Unresolved++; } else + { + + if (!_updateRulesUsed.ContainsKey(ignoreRule)) + _updateRulesUsed.Add(ignoreRule, 1); + else + _updateRulesUsed[ignoreRule]++; + Ignores++; + } else + { + if (!_updateRulesUsed.ContainsKey(updateRule)) + _updateRulesUsed.Add(updateRule, 1); + else + _updateRulesUsed[updateRule]++; + Updates++; + } Total++; if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000) { - Console.WriteLine($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0}"); + Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0}"); sw.Restart(); } } storeReport.CloseReport(); } - - Console.WriteLine($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0}"); + + Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, + _ignoreRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}"))); + + Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, + _updateRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}"))); + + Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0}"); return 0; } + + private void Log(string msg) + { + _log.Info(msg); + Console.WriteLine(msg); + } } } \ No newline at end of file diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestUpdater.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestUpdater.cs index 7f9590025..50f4a572e 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestUpdater.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/TestUpdater.cs @@ -60,7 +60,7 @@ public void Test(DatabaseType dbType) updater.UpdateStrategy = new ProblemValuesUpdateStrategy(); //it should be novel i.e. require user decision - Assert.IsTrue(updater.OnLoad(db.Server,failure)); + Assert.IsTrue(updater.OnLoad(db.Server,failure, out _)); updater.Update(db.Server,failure,null); @@ -75,7 +75,7 @@ public void Test(DatabaseType dbType) ",File.ReadAllText(newRules.FullName)); //btw slash space is a 'literal space' so legit //it should be updated automatically and not require user decision - Assert.IsFalse(updater.OnLoad(db.Server,failure)); + Assert.IsFalse(updater.OnLoad(db.Server,failure,out _)); } @@ -137,7 +137,7 @@ public void Test_RegexUpdateStrategy(DatabaseType dbType, bool provideCaptureGro updater.UpdateStrategy = new RegexUpdateStrategy(); //it should be novel i.e. require user decision - Assert.IsTrue(updater.OnLoad(db.Server,failure)); + Assert.IsTrue(updater.OnLoad(db.Server,failure,out _)); updater.Update(db.Server,failure,null);//<- null here will trigger the rule pattern factory to prompt 'user' for pattern which is "(Toto)$" @@ -149,7 +149,7 @@ public void Test_RegexUpdateStrategy(DatabaseType dbType, bool provideCaptureGro Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"],"Because regex had no capture group we expected the update strategy to fallback on Failure Part matching"); //it should be updated automatically and not require user decision - Assert.IsFalse(updater.OnLoad(db.Server,failure)); + Assert.IsFalse(updater.OnLoad(db.Server,failure,out _)); } } From b87cd5ea1f75147d89184b37ea19db00a11ba86a Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 13:29:12 +0000 Subject: [PATCH 11/24] Fixed for last (accidental) commit --- .../IsIdentifiableReviewer/UnattendedReviewer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs index 2d5def3c8..90f954114 100644 --- a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs +++ b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs @@ -84,10 +84,10 @@ public int Run() else { - if (!_updateRulesUsed.ContainsKey(ignoreRule)) - _updateRulesUsed.Add(ignoreRule, 1); + if (!_ignoreRulesUsed.ContainsKey(ignoreRule)) + _ignoreRulesUsed.Add(ignoreRule, 1); else - _updateRulesUsed[ignoreRule]++; + _ignoreRulesUsed[ignoreRule]++; Ignores++; } From 60fa15e8f6a90d7cd2ba4b99dafa6274a42e2204 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 24 Mar 2020 13:34:13 +0000 Subject: [PATCH 12/24] Reduced console spam when there are many rules --- .../IsIdentifiableReviewer/UnattendedReviewer.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs index 90f954114..024aa0f00 100644 --- a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs +++ b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs @@ -105,7 +105,7 @@ public int Run() if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000) { - Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0}"); + Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0}",true); sw.Restart(); } } @@ -114,19 +114,20 @@ public int Run() } Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, - _ignoreRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}"))); + _ignoreRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}")),false); Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, - _updateRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}"))); + _updateRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}")),false); - Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0}"); + Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0}",true); return 0; } - private void Log(string msg) + private void Log(string msg, bool toConsole) { _log.Info(msg); - Console.WriteLine(msg); + if(toConsole) + Console.WriteLine(msg); } } } \ No newline at end of file From 41b561aefa253c1419a03cab2a0a2323e07e8f56 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Wed, 25 Mar 2020 22:08:50 +0000 Subject: [PATCH 13/24] Fix casing of some dependencies in csprojs causing issues with other tools --- .../Microservices.IsIdentifiable.csproj | 2 +- .../Applications.DicomDirectoryProcessor.Tests.csproj | 2 +- .../Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj | 2 +- .../Microservices.CohortExtractor.Tests.csproj | 2 +- .../Microservices.CohortPackager.Tests.csproj | 2 +- .../Microservices.DeadLetterReprocessor.Tests.csproj | 2 +- .../Microservices.DicomRelationalMapper.Tests.csproj | 2 +- .../Microservices.DicomReprocessor.Tests.csproj | 2 +- .../Microservices.DicomTagReader.Tests.csproj | 2 +- .../Microservices.IdentifierMapper.Tests.csproj | 2 +- .../Microservices.IsIdentifiable.Tests.csproj | 2 +- .../Microservices.MongoDBPopulator.Tests.csproj | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index 0b791c4dc..20ddeb51b 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -37,7 +37,7 @@ - + diff --git a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj index ee4adca56..8dff2e9c6 100644 --- a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj +++ b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj index 1cafcb9a2..139c77d76 100644 --- a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj +++ b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj @@ -10,7 +10,7 @@ true - + diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj index 8fac01e1e..6d0273f2d 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj index 37f4c9573..bc8d6fa12 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj +++ b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj index 3d44b0af9..2459e7874 100644 --- a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj @@ -10,7 +10,7 @@ true - + diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj index 1440487f6..6aaae56f2 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj @@ -20,7 +20,7 @@ - + diff --git a/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj b/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj index 3f739a8fc..ada9cb1bb 100644 --- a/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj @@ -10,7 +10,7 @@ true - + diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj index 046971185..5ecef0cd8 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj +++ b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj index 881f206b8..c2d5b328e 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj index a310a7e1c..43a31d9e9 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj index 56ae1815c..69443cda4 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj @@ -10,7 +10,7 @@ true - + From 08e5a081a5327792bb3d0169a8184f1ae08dfc28 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Wed, 25 Mar 2020 22:50:46 +0000 Subject: [PATCH 14/24] Remove explicit dependence on fo-dicom This should *only* be provided by the DicomTypeTranslation package, in order to avoid dependency issues. --- src/common/Smi.Common/Smi.Common.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 4b0723230..76e6e72ac 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -28,7 +28,6 @@ - From 8cc747ffac6d8db07697138810f7a85cfb0b394d Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Wed, 25 Mar 2020 22:56:01 +0000 Subject: [PATCH 15/24] Update Packages.md --- PACKAGES.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 2e2e63bf6..7b9d2aec2 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -11,7 +11,7 @@ | ------- | ------------| --------| ------- | ------- | -------------------------- | | CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | | CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | -| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.1.2](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.1.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | +| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.2.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.2.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | @@ -28,6 +28,4 @@ | System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| | | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | | YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| -| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.4](https://www.nuget.org/packages/fo-dicom/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | | | fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.4](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | -| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.4](https://www.nuget.org/packages/fo-dicom.Json/4.0.4) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | | From 162571efa5ead9b8d69b3a98cf2d909f2c6a1830 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Thu, 26 Mar 2020 00:08:12 +0000 Subject: [PATCH 16/24] Remove reference to Rdmp.Dicom in DicomReprocessor - should be unnecessary --- .../Microservices.DicomReprocessor.csproj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/microservices/Microservices.DicomReprocessor/Microservices.DicomReprocessor.csproj b/src/microservices/Microservices.DicomReprocessor/Microservices.DicomReprocessor.csproj index f2998790e..d057b330f 100644 --- a/src/microservices/Microservices.DicomReprocessor/Microservices.DicomReprocessor.csproj +++ b/src/microservices/Microservices.DicomReprocessor/Microservices.DicomReprocessor.csproj @@ -16,11 +16,10 @@ - - \ No newline at end of file + From 90db3f92ac4b9bae46cff55326e464a65c22dde5 Mon Sep 17 00:00:00 2001 From: tznind Date: Thu, 26 Mar 2020 09:07:10 +0000 Subject: [PATCH 17/24] Fix for when redaction process fails (e.g. where redacted text is too long for the column in which it is being redacted) --- .../UnattendedReviewer.cs | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs index 024aa0f00..b7d2a81d9 100644 --- a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs +++ b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs @@ -60,6 +60,7 @@ public int Run() { //In RulesOnly mode this will be null var server = _target?.Discover(); + List errors = new List(); var storeReport = new FailureStoreReport(_outputFile.Name,100); @@ -68,12 +69,26 @@ public int Run() using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile)) { + IsIdentifiableRule updateRule; + storeReport.AddDestination(storeReportDestination); while(_reportReader.Next()) { + bool noUpdate; + + try + { + noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule); + } + catch (Exception e) + { + errors.Add(e); + continue; + } + //is it novel for updater - if(_updater.OnLoad(server,_reportReader.Current, out IsIdentifiableRule updateRule)) + if(noUpdate) //is it novel for ignorer if (_ignorer.OnLoad(_reportReader.Current,out IsIdentifiableRule ignoreRule)) { @@ -105,7 +120,7 @@ public int Run() if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000) { - Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0}",true); + Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}",true); sw.Restart(); } } @@ -119,7 +134,9 @@ public int Run() Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, _updateRulesUsed.OrderBy(k=>k.Value).Select(k=>$"{k.Key.IfPattern} - {k.Value:N0}")),false); - Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0}",true); + Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine,errors.Select(e=>e.ToString())),false); + + Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}",true); return 0; } From 63eb0363aa5a60f9bf5fd406a5d35e6d2f12b729 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Thu, 26 Mar 2020 15:52:06 +0000 Subject: [PATCH 18/24] Update CHANGELOG --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b279e250..0b9770e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -... - ### Added - Added undo feature to IsIdentifiableReviewer +### Changed + +- Upgraded HIC.DicomTypeTranslation from `2.1.2` to `2.2.0` + - This includes an upgrade to fo-dicom from `4.0.1` to `4.0.4` +- Upgraded fo-dicom.Drawing from `4.0.1` to `4.0.4` + ## [1.6.0] - 2020-03-17 ### Changed From c3a1374d51fa7604e01b9a6a3b673ea8fd4f1a9f Mon Sep 17 00:00:00 2001 From: abrooks Date: Thu, 26 Mar 2020 17:04:32 +0000 Subject: [PATCH 19/24] IsIdentifiable Nacks message on failure --- .../Runners/DicomFileRunner.cs | 5 ++- .../Service/IsIdentifiableQueueConsumer.cs | 42 +++++++++++-------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/DicomFileRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/DicomFileRunner.cs index 08af7ecc0..37bc016e4 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/DicomFileRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/DicomFileRunner.cs @@ -120,7 +120,10 @@ public void ValidateDicomFile(FileInfo fi) ValidateDicomItem(fi, dicomFile, dataSet, dicomItem); } else + { _logger.Info("File does not contain valid preamble and header: " + fi.FullName); + throw new ApplicationException("File does not contain valid preamble and header: " + fi.FullName); + } DoneRows(1); } @@ -244,7 +247,7 @@ void ValidateDicomPixelData(FileInfo fi, DicomFile dicomFile, DicomDataset ds) { // An internal error should cause IsIdentifiable to exit _logger.Info(e, "Could not run Tesseract on '" + fi.FullName + "'"); - throw new Exception ("Could not run Tesseract on '" + fi.FullName + "'", e); + throw new ApplicationException ("Could not run Tesseract on '" + fi.FullName + "'", e); // OR add a message to the report saying we failed to run OCR //string problemField = "PixelData"; diff --git a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs index 411e3bdc8..aaf5bc37f 100644 --- a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs +++ b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs @@ -32,34 +32,40 @@ protected override void ProcessMessageImpl(IMessageHeader header, BasicDeliverEv if (!SafeDeserializeToMessage(header, basicDeliverEventArgs, out ExtractFileStatusMessage message)) return; - // We should only ever receive messages regarding anonymised images - if (message.Status != ExtractFileStatus.Anonymised) - throw new ApplicationException($"Received a message with anonymised status of {message.Status}"); + bool isClean = true; + object resultObject; - // TODO(rkm 2020-02-04) Check that CTP doesn't output rooted paths. Also need to use the extract root - // var toProcess = new FileInfo(Path.Combine(message.ExtractionDirectory, message.AnonymisedFileName)); + try + { + // We should only ever receive messages regarding anonymised images + if (message.Status != ExtractFileStatus.Anonymised) + throw new ApplicationException($"Received a message with anonymised status of {message.Status}"); - // The path is taken from the message, however maybe it should be FileSystemOptions|ExtractRoot in default.yaml - // If the filename has a rooted path then the ExtractionDirectory is ignored by Path.Combine - var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.AnonymisedFileName) ); + var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.AnonymisedFileName) ); - if(!toProcess.Exists) - // XXX this causes a fatal error and the whole service terminates - throw new FileNotFoundException("IsIdentifiable service cannot find file "+toProcess.FullName); + if(!toProcess.Exists) + throw new ApplicationException("IsIdentifiable service cannot find file "+toProcess.FullName); - var result = _classifier.Classify(toProcess); + var result = _classifier.Classify(toProcess); - bool isClean = true; - foreach (Failure f in result) + foreach (Failure f in result) + { + Logger.Log(LogLevel.Info,$"Validation failed for {f.Resource} Problem Value:{f.ProblemValue}"); + isClean = false; + } + resultObject = result; + } + catch (ApplicationException e) { - Logger.Log(LogLevel.Info,$"Validation failed for {f.Resource} Problem Value:{f.ProblemValue}"); - isClean = false; + // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper + ErrorAndNack(header, basicDeliverEventArgs, "Error while processing AnonSuccessMessage", e); + return; } - + _producer.SendMessage(new IsIdentifiableMessage(message) { IsIdentifiable = ! isClean, - Report = JsonConvert.SerializeObject(result) + Report = JsonConvert.SerializeObject(resultObject) }, header); Ack(header, basicDeliverEventArgs); From 2b0c731c54ea0a73b365cdcbb2cf112bcab6a12d Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 27 Mar 2020 12:49:06 +0000 Subject: [PATCH 20/24] Bumped RDMP.Dicom to 2.0.8 and fixed IsIdentifiable not targetting latest runtime patch --- PACKAGES.md | 2 +- .../Microservices.DicomRelationalMapper.csproj | 2 +- .../Microservices.IsIdentifiable.csproj | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index eaea0a147..c91ec14b3 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -12,7 +12,7 @@ | CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | | CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.2.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.2.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.8](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.15.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.15.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | diff --git a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj index 36646bf01..5f28dd2c9 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj +++ b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index 0c29892b0..9cd4f259f 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -6,6 +6,7 @@ false bin\$(Platform)\$(Configuration)\ x64 + true From 937a07fec27e6d1c46b907729d0102eef33d126b Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Fri, 27 Mar 2020 15:04:35 +0000 Subject: [PATCH 21/24] Feature/javalogging (#175) * Set log4j to point to SMI_LOGS_ROOT when applicable * Default to pwd if SMI_LOGS_ROOT not set, print warning when doing so * Add fallback getPid handling for JRE 8 - could/should be reused in other places PID is needed --- CHANGELOG.md | 2 + .../java/org/smi/extractorcl/Program.java | 2 +- .../org/smi/common/logging/SmiLogging.java | 131 ++++++-- .../res/SmiLogbackConfig.xml | 58 ---- .../java/org/smi/ctpanonymiser/Program.java | 2 +- .../test/execution/CTPAnonymiserHostTest.java | 309 +++++++++--------- .../test/messages/ExtractFileMessageTest.java | 2 +- 7 files changed, 266 insertions(+), 240 deletions(-) delete mode 100644 src/common/com.smi.microservices.parent/res/SmiLogbackConfig.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b9770e38..e36ee78b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +- Get Java microservices logging to SMI\_LOGS\_ROOT + ### Added - Added undo feature to IsIdentifiableReviewer diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java index 1c47172ea..e708f11b5 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java @@ -22,7 +22,7 @@ public static void main(String[] args) throws FileNotFoundException, FileProcessingException, LineProcessingException, IOException, ParseException, YAMLException, URISyntaxException, IllegalArgumentException, TimeoutException { - SmiLogging.Setup(); + SmiLogging.Setup(false); final Logger logger = LoggerFactory.getLogger(Program.class); CommandLine parsedArgs = ParseOptions(args); diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/logging/SmiLogging.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/logging/SmiLogging.java index 740fadc94..26373c527 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/logging/SmiLogging.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/logging/SmiLogging.java @@ -1,45 +1,122 @@ package org.smi.common.logging; +import java.io.File; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Iterator; +import java.util.Random; + +import org.apache.log4j.Appender; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.FileAppender; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.spi.Filter; /** * Static helper class to setup the SMI logging */ public final class SmiLogging { - private static final String ConfigFileName = "SmiLogbackConfig.xml"; - - private SmiLogging() { - } - - public static void Setup() { - Setup(-1); - } - - public static void Setup(int env) throws SmiLoggingException { - - // Turn off log4j warnings from library code - org.apache.log4j.Logger.getRootLogger().setLevel(org.apache.log4j.Level.OFF); + private SmiLogging() { + } - Path logConfigPath; + public static String getCaller() { + String prev=null; + StackTraceElement[] stElements = Thread.currentThread().getStackTrace(); + for (int i=1; i phclass = Class.forName("java.lang.ProcessHandle"); + Method phcurrent = phclass.getMethod("current", new Class[] {}); + Method getpid=phclass.getMethod("pid", new Class[] {}); + Object self=phcurrent.invoke(null, new Object[] {}); + Object pid=getpid.invoke(self, new Object[] {}); + if (pid instanceof Long) { + Long lpid=(Long) pid; + return lpid.longValue(); + } + } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + // Fallback for elderly JVMs: + try { + RuntimeMXBean rt=ManagementFactory.getRuntimeMXBean(); + Field jvm=rt.getClass().getField("jvm"); + jvm.setAccessible(true); + Object mgmt = jvm.get(rt); + Method getpid = mgmt.getClass().getDeclaredMethod("getProcessId"); + getpid.setAccessible(true); + int pid=(Integer)getpid.invoke(mgmt); + return pid; + } catch (NoSuchFieldException | IllegalArgumentException | IllegalAccessException | NoSuchMethodException | SecurityException | InvocationTargetException e) { + e.printStackTrace(); + } + + return new Random().nextLong(); + } - if (env == -1) { - logConfigPath = Paths.get(ConfigFileName); - } else { - try { - logConfigPath = Paths.get("./target", ConfigFileName); - } catch (InvalidPathException e) { - throw new SmiLoggingException("", e); - } - } + public static void Setup(boolean testing) throws SmiLoggingException { + DateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); + String logroot = System.getenv("SMI_LOGS_ROOT"); + if (logroot==null) { + System.err.println("WARNING: SMI_LOGS_ROOT not set, logging to pwd instead"); + logroot="."; + } + File logdir=new File(logroot+File.pathSeparator+getCaller()); + File logfile=new File(logroot+File.pathSeparator+getCaller()+File.pathSeparator+df.format(new Date())+"-"+getPid()); + if (!logdir.isDirectory()) { + logdir.mkdirs(); + } - if (Files.notExists(logConfigPath) || Files.isDirectory(logConfigPath)) - throw new SmiLoggingException("Could not find logback config file " + ConfigFileName); + // Turn off log4j warnings from library code + Logger l = Logger.getRootLogger(); + l.setLevel(Level.OFF); + + PatternLayout pl = new PatternLayout("%d{HH:mm:ss.SSS}|%thread|%-5level|%-15(%logger{0})| %msg%n"); - System.setProperty("logback.configurationFile", ConfigFileName); - } + ConsoleAppender ca = new ConsoleAppender(); + ca.setTarget("System.err"); + ca.setThreshold(testing?Level.ALL:Level.ERROR); + l.addAppender(ca); + + FileAppender fa = new FileAppender(); + fa.setFile(logfile.getAbsolutePath()); + fa.setThreshold(Level.ALL); + fa.setAppend(true); + fa.setBufferedIO(true); + fa.setImmediateFlush(false); + fa.setLayout(pl); + fa.activateOptions(); + l.addAppender(fa); + } } diff --git a/src/common/com.smi.microservices.parent/res/SmiLogbackConfig.xml b/src/common/com.smi.microservices.parent/res/SmiLogbackConfig.xml deleted file mode 100644 index 082ab98fb..000000000 --- a/src/common/com.smi.microservices.parent/res/SmiLogbackConfig.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - System.out - - INFO - - - ERROR - DENY - ACCEPT - - true - - %d{HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%class{0})| %msg%n - - - - - - - System.err - - ERROR - - true - - %d{HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%class{0})| %msg%n - - - - - - - logs/${fileTimestamp}.log - true - true - - %d{HH:mm:ss.SSS}|%thread|%-5level|%-15(%logger{0})| %msg%n - - - - - - - - - - - - - diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/Program.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/Program.java index 19c90d89c..c38e54bc5 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/Program.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/Program.java @@ -28,7 +28,7 @@ public class Program { public static void main(String[] args) throws ParseException, YAMLException, URISyntaxException, IOException, TimeoutException { - SmiLogging.Setup(); + SmiLogging.Setup(false); Logger logger = LoggerFactory.getLogger(Program.class); //TODO Make this into a helper class diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index a6d1e3f4f..906d48bec 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -26,220 +26,225 @@ public class CTPAnonymiserHostTest extends TestCase { - private static Logger _logger; + private static Logger _logger; - private static final String _fsRoot = System.getProperty("user.dir") + "/src/test/resources"; - private static final String _extractRoot = System.getProperty("user.dir") + "/src/test/resources"; + private static final String _fsRoot = System.getProperty("user.dir") + "/src/test/resources"; + private static final String _extractRoot = System.getProperty("user.dir") + "/src/test/resources"; - private static final String _testFile = "image-000001.dcm"; + private static final String _testFile = "image-000001.dcm"; - private static final String _inputExchName = "TEST.ExtractFileExchange"; - private static final String _outputQueueName = "TEST.FileStatusQueue"; + private static final String _inputExchName = "TEST.ExtractFileExchange"; + private static final String _outputQueueName = "TEST.FileStatusQueue"; - private static String _producerExchangeName; + private static String _producerExchangeName; - private ProducerOptions _extractFileProducerOptions; - private ConsumerOptions _extractFileStatusConsumerOptions; - private IProducerModel _extractFileMessageProducer; - private AnyConsumer _anonFileStatusMessageConsumer; + private ProducerOptions _extractFileProducerOptions; + private ConsumerOptions _extractFileStatusConsumerOptions; + private IProducerModel _extractFileMessageProducer; + private AnyConsumer _anonFileStatusMessageConsumer; - private ConnectionFactory _factory; - private Connection _conn; - private Channel _channel; + private ConnectionFactory _factory; + private Connection _conn; + private Channel _channel; - private GlobalOptions _options; - private RabbitMqAdapter _testAdapter; - private CTPAnonymiserHost _ctpHost; + private GlobalOptions _options; + private RabbitMqAdapter _testAdapter; + private CTPAnonymiserHost _ctpHost; - protected void setUp() throws Exception { + protected void setUp() throws Exception { - super.setUp(); + super.setUp(); - SmiLogging.Setup(0); - _logger = LoggerFactory.getLogger(Program.class); + try { + SmiLogging.Setup(true); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + _logger = LoggerFactory.getLogger(Program.class); - _options = GlobalOptions.Load(true); + _options = GlobalOptions.Load(true); - _options.FileSystemOptions.setFileSystemRoot(_fsRoot); - _options.FileSystemOptions.setExtractRoot(_extractRoot); + _options.FileSystemOptions.setFileSystemRoot(_fsRoot); + _options.FileSystemOptions.setExtractRoot(_extractRoot); - if (!_options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName.startsWith("TEST.")) - _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName = "TEST." - + _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; + if (!_options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName.startsWith("TEST.")) + _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName = "TEST." + + _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; - if (!_options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName.startsWith("TEST.")) - _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName = "TEST." - + _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; + if (!_options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName.startsWith("TEST.")) + _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName = "TEST." + + _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; - String _consumerQueueName = _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; - _producerExchangeName = _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; + String _consumerQueueName = _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; + _producerExchangeName = _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; - // Set up RMQ - _testAdapter = new RabbitMqAdapter(_options.RabbitOptions, "CTPAnonymiserHostTest"); + // Set up RMQ + _testAdapter = new RabbitMqAdapter(_options.RabbitOptions, "CTPAnonymiserHostTest"); - // Set up test producers, consumers + // Set up test producers, consumers - _extractFileStatusConsumerOptions = new ConsumerOptions(); - _extractFileStatusConsumerOptions.QueueName = _outputQueueName; - _extractFileStatusConsumerOptions.AutoAck = false; - _extractFileStatusConsumerOptions.QoSPrefetchCount = 1; + _extractFileStatusConsumerOptions = new ConsumerOptions(); + _extractFileStatusConsumerOptions.QueueName = _outputQueueName; + _extractFileStatusConsumerOptions.AutoAck = false; + _extractFileStatusConsumerOptions.QoSPrefetchCount = 1; - _anonFileStatusMessageConsumer = new AnyConsumer<>(ExtractFileStatusMessage.class); + _anonFileStatusMessageConsumer = new AnyConsumer<>(ExtractFileStatusMessage.class); - _extractFileProducerOptions = new ProducerOptions(); - _extractFileProducerOptions.ExchangeName = _inputExchName; + _extractFileProducerOptions = new ProducerOptions(); + _extractFileProducerOptions.ExchangeName = _inputExchName; - _extractFileMessageProducer = _testAdapter.SetupProducer(_extractFileProducerOptions); + _extractFileMessageProducer = _testAdapter.SetupProducer(_extractFileProducerOptions); - _factory = new ConnectionFactory(); - _conn = _factory.newConnection(); - _channel = _conn.createChannel(); + _factory = new ConnectionFactory(); + _conn = _factory.newConnection(); + _channel = _conn.createChannel(); - // Setup the input exch. / queue pair + // Setup the input exch. / queue pair - _channel.exchangeDeclare(_inputExchName, "direct", true); - _channel.queueDeclare(_consumerQueueName, true, false, false, null); - _channel.queueBind(_consumerQueueName, _inputExchName, ""); - System.out.println(String.format("Bound %s -> %s", _inputExchName, _consumerQueueName)); + _channel.exchangeDeclare(_inputExchName, "direct", true); + _channel.queueDeclare(_consumerQueueName, true, false, false, null); + _channel.queueBind(_consumerQueueName, _inputExchName, ""); + System.out.println(String.format("Bound %s -> %s", _inputExchName, _consumerQueueName)); - // Setup the output exch. / queue pair + // Setup the output exch. / queue pair - _channel.exchangeDeclare(_producerExchangeName, "direct", true); - _channel.queueDeclare(_outputQueueName, true, false, false, null); - _channel.queueBind(_outputQueueName, _producerExchangeName, ""); - _channel.queueBind(_outputQueueName, _producerExchangeName, "success"); - _channel.queueBind(_outputQueueName, _producerExchangeName, "failure"); - System.out.println(String.format("Bound %s -> %s", _producerExchangeName, _outputQueueName)); + _channel.exchangeDeclare(_producerExchangeName, "direct", true); + _channel.queueDeclare(_outputQueueName, true, false, false, null); + _channel.queueBind(_outputQueueName, _producerExchangeName, ""); + _channel.queueBind(_outputQueueName, _producerExchangeName, "success"); + _channel.queueBind(_outputQueueName, _producerExchangeName, "failure"); + System.out.println(String.format("Bound %s -> %s", _producerExchangeName, _outputQueueName)); - _channel.queuePurge(_consumerQueueName); - _channel.queuePurge(_outputQueueName); + _channel.queuePurge(_consumerQueueName); + _channel.queuePurge(_outputQueueName); - // Start our test consumer for receiving the anonymised message - _testAdapter.StartConsumer(_extractFileStatusConsumerOptions, _anonFileStatusMessageConsumer); + // Start our test consumer for receiving the anonymised message + _testAdapter.StartConsumer(_extractFileStatusConsumerOptions, _anonFileStatusMessageConsumer); - // Create the host for testing - String[] args = new String[]{"-a", _fsRoot + "/dicom-anonymizer.script"}; - _ctpHost = new CTPAnonymiserHost(_options, Program.ParseOptions(args)); + // Create the host for testing + String[] args = new String[]{"-a", _fsRoot + "/dicom-anonymizer.script"}; + _ctpHost = new CTPAnonymiserHost(_options, Program.ParseOptions(args)); - File inFile = new File(Paths.get(_fsRoot, _testFile).toString()); - assertTrue(inFile.exists()); + File inFile = new File(Paths.get(_fsRoot, _testFile).toString()); + assertTrue(inFile.exists()); - boolean ok = inFile.setWritable(false); - assertTrue(ok); - } + boolean ok = inFile.setWritable(false); + assertTrue(ok); + } - protected void tearDown() throws Exception { + protected void tearDown() throws Exception { - super.tearDown(); + super.tearDown(); - _ctpHost.Shutdown(); + _ctpHost.Shutdown(); - _testAdapter.Shutdown(); + _testAdapter.Shutdown(); - _channel.exchangeDelete(_inputExchName); - _channel.exchangeDelete(_producerExchangeName); + _channel.exchangeDelete(_inputExchName); + _channel.exchangeDelete(_producerExchangeName); - _channel.close(); - _conn.close(); - } + _channel.close(); + _conn.close(); + } - public void testBasicAnonymise_Success() throws InterruptedException { + public void testBasicAnonymise_Success() throws InterruptedException { - _logger.info("Starting basic anonymise test - should succeed"); + _logger.info("Starting basic anonymise test - should succeed"); - // Send a test message - ExtractFileMessage exMessage = new ExtractFileMessage(); + // Send a test message + ExtractFileMessage exMessage = new ExtractFileMessage(); - exMessage.ExtractionJobIdentifier = UUID.randomUUID(); - exMessage.JobSubmittedAt = ""; - exMessage.ExtractionDirectory = ""; - exMessage.DicomFilePath = _testFile; - exMessage.OutputPath = "AnonymisedFiles/" + exMessage.DicomFilePath; - exMessage.ProjectNumber = "123-456"; + exMessage.ExtractionJobIdentifier = UUID.randomUUID(); + exMessage.JobSubmittedAt = ""; + exMessage.ExtractionDirectory = ""; + exMessage.DicomFilePath = _testFile; + exMessage.OutputPath = "AnonymisedFiles/" + exMessage.DicomFilePath; + exMessage.ProjectNumber = "123-456"; - TimeUnit.MILLISECONDS.sleep(1000); + TimeUnit.MILLISECONDS.sleep(1000); - _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); - _extractFileMessageProducer.SendMessage(exMessage, "", null); + _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); + _extractFileMessageProducer.SendMessage(exMessage, "", null); - _logger.info("Waiting..."); + _logger.info("Waiting..."); - int timeout = 10000; - final int deltaMs = 1000; + int timeout = 10000; + final int deltaMs = 1000; - while (!_anonFileStatusMessageConsumer.isMessageValid() && timeout > 0) { + while (!_anonFileStatusMessageConsumer.isMessageValid() && timeout > 0) { - TimeUnit.MILLISECONDS.sleep(deltaMs); - timeout -= deltaMs; - } + TimeUnit.MILLISECONDS.sleep(deltaMs); + timeout -= deltaMs; + } - if (timeout > 0) { - _logger.info("... message received, took " + (10000-timeout) + " milliseconds"); - } else { - fail("Message not received in 10000 milliseconds"); - } + if (timeout > 0) { + _logger.info("... message received, took " + (10000-timeout) + " milliseconds"); + } else { + fail("Message not received in 10000 milliseconds"); + } - if (_anonFileStatusMessageConsumer.isMessageValid()) { + if (_anonFileStatusMessageConsumer.isMessageValid()) { - ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); + ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); - _logger.info("Message received"); - _logger.info("\n" + recvd.toString()); + _logger.info("Message received"); + _logger.info("\n" + recvd.toString()); - assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.AnonymisedFileName); - assertEquals("Project numbers do not match", exMessage.ProjectNumber, recvd.ProjectNumber); - assertEquals(ExtractFileStatus.Anonymised, recvd.Status); - } else { - fail("Did not receive message"); - } - } + assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.AnonymisedFileName); + assertEquals("Project numbers do not match", exMessage.ProjectNumber, recvd.ProjectNumber); + assertEquals(ExtractFileStatus.Anonymised, recvd.Status); + } else { + fail("Did not receive message"); + } + } - public void testBasicAnonymise_Failure() throws InterruptedException { - // TODO: Nasty hack, run the success test case first to avoid the "failed first message" path - testBasicAnonymise_Success(); - - _logger.info("Starting basic anonymise test - failure handling"); + public void testBasicAnonymise_Failure() throws InterruptedException { + // TODO: Nasty hack, run the success test case first to avoid the "failed first message" path + testBasicAnonymise_Success(); - // Send an invalid message - should fail - ExtractFileMessage exMessage = new ExtractFileMessage(); - exMessage.ExtractionJobIdentifier = UUID.randomUUID(); - exMessage.JobSubmittedAt = ""; - exMessage.ExtractionDirectory = ""; - exMessage.DicomFilePath = "missing.dcm"; - exMessage.OutputPath = "AnonymisedFiles/" + exMessage.DicomFilePath; - exMessage.ProjectNumber = "123-456"; + _logger.info("Starting basic anonymise test - failure handling"); - _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); - _extractFileMessageProducer.SendMessage(exMessage, "", null); + // Send an invalid message - should fail + ExtractFileMessage exMessage = new ExtractFileMessage(); + exMessage.ExtractionJobIdentifier = UUID.randomUUID(); + exMessage.JobSubmittedAt = ""; + exMessage.ExtractionDirectory = ""; + exMessage.DicomFilePath = "missing.dcm"; + exMessage.OutputPath = "AnonymisedFiles/" + exMessage.DicomFilePath; + exMessage.ProjectNumber = "123-456"; - _logger.info("Waiting..."); + _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); + _extractFileMessageProducer.SendMessage(exMessage, "", null); - int timeout = 10000; - final int deltaMs = 1000; + _logger.info("Waiting..."); - while (!_anonFileStatusMessageConsumer.isMessageValid() && timeout > 0) { + int timeout = 10000; + final int deltaMs = 1000; - TimeUnit.MILLISECONDS.sleep(deltaMs); - timeout -= deltaMs; - } + while (!_anonFileStatusMessageConsumer.isMessageValid() && timeout > 0) { - if (timeout > 0) { - _logger.info("... message received, took " + timeout + " milliseconds"); - } else { - fail("Message not received in " + timeout + " milliseconds"); - } + TimeUnit.MILLISECONDS.sleep(deltaMs); + timeout -= deltaMs; + } - if (_anonFileStatusMessageConsumer.isMessageValid()) { + if (timeout > 0) { + _logger.info("... message received, took " + timeout + " milliseconds"); + } else { + fail("Message not received in " + timeout + " milliseconds"); + } - ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); + if (_anonFileStatusMessageConsumer.isMessageValid()) { - _logger.info("Message received"); - _logger.info("\n" + recvd.toString()); + ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); - assertEquals("FilePaths do not match", null, recvd.AnonymisedFileName); - assertEquals(ExtractFileStatus.ErrorWontRetry, recvd.Status); - } else { - fail("Did not receive message"); - } - } + _logger.info("Message received"); + _logger.info("\n" + recvd.toString()); + + assertEquals("FilePaths do not match", null, recvd.AnonymisedFileName); + assertEquals(ExtractFileStatus.ErrorWontRetry, recvd.Status); + } else { + fail("Did not receive message"); + } + } } diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java index 492e9ffdb..ef47b81bc 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java @@ -26,7 +26,7 @@ protected void setUp() throws Exception { super.setUp(); - SmiLogging.Setup(0); + SmiLogging.Setup(true); _exMessage = new ExtractFileMessage(); From 29eec257726b8072d6fb429c068c04df332d7a39 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2020 10:25:43 +0100 Subject: [PATCH 22/24] Bump Microsoft.Extensions.Caching.Memory from 3.1.2 to 3.1.3 (#170) * Bump Microsoft.Extensions.Caching.Memory from 3.1.2 to 3.1.3 * Update PACKAGES.md --- PACKAGES.md | 2 +- .../Microservices.IdentifierMapper.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index c91ec14b3..2875c9469 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -17,7 +17,7 @@ | JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.15.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.15.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | | Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.5.0-beta2-final](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.5.0-beta2-final) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | -| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.2](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.2) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.2/License) | Caching ID mappings retrieved from Redis/MySQL | +| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.3](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.3) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.3/License) | Caching ID mappings retrieved from Redis/MySQL | | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases| | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | | Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | diff --git a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj index 43a3ca705..3dd11b260 100644 --- a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj +++ b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj @@ -16,7 +16,7 @@ - + From ada2123fcc7df65cc56f5d4227fe27fe9decf5c0 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Mon, 30 Mar 2020 13:13:10 +0100 Subject: [PATCH 23/24] Start release branch for 1.7.0 --- CHANGELOG.md | 8 ++++++-- README.md | 2 +- src/SharedAssemblyInfo.cs | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e36ee78b3..26fd7b2e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -- Get Java microservices logging to SMI\_LOGS\_ROOT +- + +## [1.7.0] - 2020-03-30 ### Added - Added undo feature to IsIdentifiableReviewer +- Java microservices now log to SMI_LOGS_ROOT ### Changed @@ -239,7 +242,8 @@ First stable release after importing the repository from the private [SMIPlugin] - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported. -[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.6.0...develop +[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.7.0...develop +[1.7.0]: https://github.com/SMI/SmiServices/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/SMI/SmiServices/compare/v1.5.2...v1.6.0 [1.5.2]: https://github.com/SMI/SmiServices/compare/v1.5.1...v1.5.2 [1.5.1]: https://github.com/SMI/SmiServices/compare/v1.5.0...v1.5.1 diff --git a/README.md b/README.md index b762c1831..c0129d71b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![GitHub](https://img.shields.io/github/license/SMI/SmiServices) [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/) -Version: `1.6.0` +Version: `1.7.0` # SMI Services diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index e808d0281..5ad82bec5 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -7,6 +7,6 @@ [assembly: AssemblyCulture("")] // These should be overwritten by release builds -[assembly: AssemblyVersion("1.6.0")] -[assembly: AssemblyFileVersion("1.6.0")] -[assembly: AssemblyInformationalVersion("1.6.0")] // This one can have the extra build info after it +[assembly: AssemblyVersion("1.7.0")] +[assembly: AssemblyFileVersion("1.7.0")] +[assembly: AssemblyInformationalVersion("1.7.0")] // This one can have the extra build info after it From 41fbbc6ec58dce663be1c6dc1337f6c4442b0f4d Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Mon, 30 Mar 2020 13:17:10 +0100 Subject: [PATCH 24/24] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26fd7b2e7..498d585ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Upgraded HIC.DicomTypeTranslation from `2.1.2` to `2.2.0` - This includes an upgrade to fo-dicom from `4.0.1` to `4.0.4` - Upgraded fo-dicom.Drawing from `4.0.1` to `4.0.4` +- Upgraded HIC.RdmpDicom from `2.0.7` to `2.0.8` ## [1.6.0] - 2020-03-17