From befd53ef0ecd116528a99ffbeff242b756509ecf Mon Sep 17 00:00:00 2001 From: jjYBdx4IL Date: Sun, 6 Jun 2021 07:42:40 +0200 Subject: [PATCH] add jmon, add to parsers (zfs and smartctl), add svn-client-wrapper, plus some minor stuff --- .travis.sh | 12 - README.md | 2 +- cms-it/pom.xml | 6 +- env-utils/pom.xml | 4 +- .../jjYBdx4IL/utils/gfx/ImageUtils.java | 11 + .../utils/concurrent/CountDownLatchEx.java | 79 +++ .../github/jjYBdx4IL/utils/io/IoUtils.java | 9 +- .../github/jjYBdx4IL/utils/io/ZipUtils.java | 259 +++++++-- .../jjYBdx4IL/utils/lang/ArrayUtils.java | 56 ++ .../jjYBdx4IL/utils/io/ZipUtilsTest.java | 37 +- .../jjYBdx4IL/utils/lang/ArrayUtilsTest.java | 44 ++ .../test/resources/simplelogger.properties | 31 -- jmon/LICENSE.other | 34 ++ jmon/README.md | 62 +++ jmon/bin.xml | 41 ++ jmon/pom.xml | 310 +++++++++++ .../github/jjYBdx4IL/jmon/CheckThread.java | 131 +++++ .../com/github/jjYBdx4IL/jmon/Config.java | 236 ++++++++ .../github/jjYBdx4IL/jmon/IExecModule.java | 22 + .../jjYBdx4IL/jmon/IProblemReporter.java | 21 + .../jjYBdx4IL/jmon/IServiceStateReceiver.java | 23 + .../jjYBdx4IL/jmon/IShutdownHandler.java | 21 + .../github/jjYBdx4IL/jmon/InstanceLock.java | 76 +++ .../jjYBdx4IL/jmon/MailProblemReporter.java | 24 + .../java/com/github/jjYBdx4IL/jmon/Main.java | 72 +++ .../java/com/github/jjYBdx4IL/jmon/Model.java | 269 +++++++++ .../github/jjYBdx4IL/jmon/ReporterThread.java | 211 +++++++ .../github/jjYBdx4IL/jmon/RequestHandler.java | 113 ++++ .../com/github/jjYBdx4IL/jmon/RunServer.java | 89 +++ .../github/jjYBdx4IL/jmon/StoreInSpool.java | 52 ++ .../github/jjYBdx4IL/jmon/TimedExecution.java | 42 ++ .../java/com/github/jjYBdx4IL/jmon/Utils.java | 179 ++++++ .../com/github/jjYBdx4IL/jmon/XmitSpool.java | 64 +++ .../jmon/checks/CertExpiryCheck.java | 106 ++++ .../jjYBdx4IL/jmon/checks/CheckBase.java | 30 + .../jjYBdx4IL/jmon/checks/CheckResult.java | 34 ++ .../github/jjYBdx4IL/jmon/checks/ICheck.java | 21 + .../jjYBdx4IL/jmon/checks/UrlCheck.java | 114 ++++ .../jmon/cmdqueue/AddReportCommand.java | 28 + .../jmon/cmdqueue/IReportCommand.java | 20 + .../jmon/cmdqueue/RemoveReportCommand.java | 25 + .../jmon/cmdqueue/ShutdownCommand.java | 20 + .../github/jjYBdx4IL/jmon/dto/Defaults.java | 35 ++ .../github/jjYBdx4IL/jmon/dto/HostDef.java | 73 +++ .../github/jjYBdx4IL/jmon/dto/HostState.java | 82 +++ .../jjYBdx4IL/jmon/dto/ReportingStatus.java | 31 ++ .../github/jjYBdx4IL/jmon/dto/ServiceDef.java | 80 +++ .../jjYBdx4IL/jmon/dto/ServiceState.java | 88 +++ .../jjYBdx4IL/jmon/dto/ServiceStateXfer.java | 38 ++ .../main/resources/simplelogger.properties | 7 + .../github/jjYBdx4IL/jmon/RunServerTest.java | 515 ++++++++++++++++++ .../github/jjYBdx4IL/jmon/SslTestServer.java | 64 +++ .../jjYBdx4IL/jmon/TestRequestHandler.java | 61 +++ .../junit4/IgnoreTestExceptionsRule1Test.java | 2 - .../junit4/RetryRuleFailExpectedTest.java | 2 - logging-utils/pom.xml | 15 +- .../logging/JavaUtilLoggingUtilsTest.java | 1 + .../jjYBdx4IL/utils/math/ShuffleTest.java | 14 +- .../jjYBdx4IL/utils/math/ShuffleTestBase.java | 6 +- net-utils/pom.xml | 2 +- .../jjYBdx4IL/utils/net/NetIoUtils.java | 1 - .../jjYBdx4IL/utils/net/AddressUtilsTest.java | 15 +- .../jjYBdx4IL/utils/osdapp/ZFSPoller.java | 10 +- .../jjYBdx4IL/utils/osdapp/OSDAppIT.java | 4 +- .../parser/linux/SmartCtlParser.java | 124 +++++ .../parser/linux/ZFSStatusParser.java | 191 ++++--- .../parser/linux/ZfsPoolListParser.java | 75 +++ .../parser/linux/ProcNetDevParserTest.java | 4 - .../parser/linux/ProcPidStatDataTest.java | 11 +- .../parser/linux/ProcPidStatusParserTest.java | 11 +- .../parser/linux/SmartCtlParserTest.java | 47 ++ .../parser/linux/ZFSStatusParserTest.java | 102 +++- .../parser/linux/ZfsPoolListParserTest.java | 41 ++ .../parser/linux/smart_a_output_longok | 99 ++++ .../parser/linux/smart_a_output_nolong | 95 ++++ .../github/jjYBdx4IL/parser/linux/zpoolListHp | 2 + .../parser/linux/zpool_output_resilver | 39 ++ .../parser/linux/zpool_output_resilver_long | 39 ++ .../jjYBdx4IL/parser/linux/zpool_output_scrub | 32 ++ .../parser/linux/zpool_output_scrub_bug1 | 30 + .../parser/linux/zpool_output_scrub_long | 32 ++ .../parser/linux/zpool_output_scrub_old | 32 ++ pom.xml | 2 + .../jjYBdx4IL/utils/proc/ProcRunner.java | 24 +- release-parent/pom.xml | 14 +- remote-robot/README.md | 5 + svn-client-wrapper/LICENSE | 201 +++++++ svn-client-wrapper/README.md | 4 + svn-client-wrapper/pom.xml | 52 ++ .../utils/svncw/SvnClientWrapper.java | 257 +++++++++ .../utils/svncw/SvnClientWrapperTest.java | 47 ++ .../test/resources/simplelogger.properties | 8 + .../test/resources/simplelogger.properties | 33 +- text-utils/pom.xml | 6 +- .../github/jjYBdx4IL/utils/text/EtaTest.java | 4 +- .../jjYBdx4IL/utils/time/TimeUsageTest.java | 2 +- .../jjYBdx4IL/wildfly/WildFlyClient.java | 9 +- 97 files changed, 5575 insertions(+), 285 deletions(-) delete mode 100755 .travis.sh create mode 100755 io-utils/src/main/java/com/github/jjYBdx4IL/utils/concurrent/CountDownLatchEx.java create mode 100755 io-utils/src/main/java/com/github/jjYBdx4IL/utils/lang/ArrayUtils.java create mode 100755 io-utils/src/test/java/com/github/jjYBdx4IL/utils/lang/ArrayUtilsTest.java create mode 100755 jmon/LICENSE.other create mode 100644 jmon/README.md create mode 100644 jmon/bin.xml create mode 100644 jmon/pom.xml create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/CheckThread.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/Config.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/IExecModule.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/IProblemReporter.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/IServiceStateReceiver.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/IShutdownHandler.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/InstanceLock.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/MailProblemReporter.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/Main.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/Model.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/ReporterThread.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/RequestHandler.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/RunServer.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/StoreInSpool.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/TimedExecution.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/Utils.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/XmitSpool.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CertExpiryCheck.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckBase.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckResult.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/ICheck.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/UrlCheck.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/AddReportCommand.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/IReportCommand.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/RemoveReportCommand.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/ShutdownCommand.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/Defaults.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostDef.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostState.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ReportingStatus.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceDef.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceState.java create mode 100755 jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceStateXfer.java create mode 100755 jmon/src/main/resources/simplelogger.properties create mode 100755 jmon/src/test/java/com/github/jjYBdx4IL/jmon/RunServerTest.java create mode 100755 jmon/src/test/java/com/github/jjYBdx4IL/jmon/SslTestServer.java create mode 100755 jmon/src/test/java/com/github/jjYBdx4IL/jmon/TestRequestHandler.java create mode 100755 parser/src/main/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParser.java create mode 100755 parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParser.java create mode 100755 parser/src/test/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParserTest.java create mode 100755 parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParserTest.java create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_longok create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_nolong create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpoolListHp create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver_long create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_bug1 create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_long create mode 100644 parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_old create mode 100644 remote-robot/README.md create mode 100644 svn-client-wrapper/LICENSE create mode 100644 svn-client-wrapper/README.md create mode 100644 svn-client-wrapper/pom.xml create mode 100755 svn-client-wrapper/src/main/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapper.java create mode 100755 svn-client-wrapper/src/test/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapperTest.java create mode 100644 svn-client-wrapper/src/test/resources/simplelogger.properties diff --git a/.travis.sh b/.travis.sh deleted file mode 100755 index 4424ab4..0000000 --- a/.travis.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -Eex - -if [[ "$PUBLIC_CI" == "true" ]]; then - export PATH=$JAVA_HOME/bin:$PATH -fi - -( while true; do echo . ; sleep 300; done ) & - -mvn "$@" -q -V clean install - diff --git a/README.md b/README.md index e307a01..773867f 100644 --- a/README.md +++ b/README.md @@ -39,4 +39,4 @@ mvn clean install -Psite-archive -N -- -devel/java/github/misc@7856 +devel/java/github/misc@7872 diff --git a/cms-it/pom.xml b/cms-it/pom.xml index 2c1b44f..790cfc4 100644 --- a/cms-it/pom.xml +++ b/cms-it/pom.xml @@ -30,7 +30,7 @@ org.wildfly.plugins wildfly-maven-plugin - 2.0.0.Final + 2.0.2.Final start-server @@ -124,7 +124,7 @@ true - + diff --git a/env-utils/pom.xml b/env-utils/pom.xml index 5d41d50..5dfc4f6 100644 --- a/env-utils/pom.xml +++ b/env-utils/pom.xml @@ -39,13 +39,13 @@ org.apache.commons commons-lang3 - 3.8.1 + 3.12.0 test junit junit - 4.13 + 4.13.2 test diff --git a/gfx-utils/src/main/java/com/github/jjYBdx4IL/utils/gfx/ImageUtils.java b/gfx-utils/src/main/java/com/github/jjYBdx4IL/utils/gfx/ImageUtils.java index 178d49d..f529f07 100644 --- a/gfx-utils/src/main/java/com/github/jjYBdx4IL/utils/gfx/ImageUtils.java +++ b/gfx-utils/src/main/java/com/github/jjYBdx4IL/utils/gfx/ImageUtils.java @@ -42,6 +42,17 @@ */ public class ImageUtils { + /** + * Crop image by drawing parts of its contents into a new image. + */ + public static BufferedImage cropNew(BufferedImage img, int x, int y, int w, int h) { + BufferedImage imgPart = new BufferedImage(w, h, img.getType()); + Graphics2D gr = imgPart.createGraphics(); + gr.drawImage(img, 0, 0, w, h, x, y, x + w, y + h, null); + gr.dispose(); + return imgPart; + } + public static double colorDist(int pixel1, int pixel2) { int p1r = (pixel1 >> 16) & 0xFF; int p1g = (pixel1 >> 8) & 0xFF; diff --git a/io-utils/src/main/java/com/github/jjYBdx4IL/utils/concurrent/CountDownLatchEx.java b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/concurrent/CountDownLatchEx.java new file mode 100755 index 0000000..d5af68b --- /dev/null +++ b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/concurrent/CountDownLatchEx.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.utils.concurrent; + +/** + * A CountDownLatch variant that does not suppress notifies. + */ + +public class CountDownLatchEx { + + protected long counter; + + /** + * The constructor. + */ + public CountDownLatchEx(long count) { + if (count < 1) { + throw new IllegalArgumentException(); + } + counter = count; + } + + /** + * The constructor. + */ + public CountDownLatchEx() { + this(1); + } + + /** + * Wait for notify or countdown to reach 0. Returns when countdown reaches + * zero or on notify. Returns immeditaly if countdown is already at 0. + * + * @return true if countdown reached 0 + */ + public synchronized boolean await() throws InterruptedException { + if (counter <= 0) { + return true; + } + wait(); + return counter <= 0; + } + + /** + * Wait for notify or countdown to reach 0. Returns when countdown reaches + * zero or on notify. Returns immeditaly if countdown is already at 0. + * + * @return true if countdown reached 0 + */ + public synchronized boolean await(long millis) throws InterruptedException { + if (counter <= 0) { + return true; + } + wait(millis); + return counter <= 0; + } + + /** + * Count down and notify once (only) upon reaching 0. + */ + public synchronized void countDown() { + if (--counter == 0) { + notify(); + } + } +} diff --git a/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/IoUtils.java b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/IoUtils.java index 6bf6f3a..f1311a6 100644 --- a/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/IoUtils.java +++ b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/IoUtils.java @@ -68,7 +68,7 @@ public static void safeWriteTo(File dest, String data, Charset cs) throws IOExce } } } - + /** * Same as {@link #safeWriteTo(File, String, Charset)}, but always uses * UTF-8 for writing. @@ -85,6 +85,13 @@ public static void safeWriteTo(File dest, String data) throws IOException { safeWriteTo(dest, data, Charset.forName("UTF-8")); } + /** + * See {@link #safeWriteTo(File, String)}. + */ + public static void safeWriteTo(Path dest, String data) throws IOException { + safeWriteTo(dest.toFile(), data); + } + /** * Same as {@link #safeWriteTo(File, String, Charset)}, but takes an * {@link java.io.InputStream}. diff --git a/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/ZipUtils.java b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/ZipUtils.java index 93719d1..a5cf4ef 100644 --- a/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/ZipUtils.java +++ b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/io/ZipUtils.java @@ -15,40 +15,52 @@ */ package com.github.jjYBdx4IL.utils.io; -//CHECKSTYLE:OFF +import static java.nio.charset.StandardCharsets.UTF_8; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Collections; +import java.util.jar.Manifest; import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; +import java.util.zip.ZipOutputStream; public class ZipUtils { /** - * Beware! This method is not very reliable (compared to "unzip -t" command line utility). + * Beware! This method is not very reliable (compared to "unzip -t" command + * line utility). * - * @param file the zip file to test + * @param file + * the zip file to test * @return true iff no errors were found */ public static boolean test(File file) { - + long count = 0; - + byte[] buf = new byte[4096]; - + try (ZipFile zipFile = new ZipFile(file)) { for (ZipEntry zipEntry : Collections.list(zipFile.entries())) { try (InputStream is = zipFile.getInputStream(zipEntry)) { - while (is.read(buf) != -1) {} + while (is.read(buf) != -1) { + } } zipEntry.getCrc(); zipEntry.getCompressedSize(); @@ -58,54 +70,79 @@ public static boolean test(File file) { } catch (IOException e) { return false; } - + return count > 0; } - + + /** + * See {@link #extractRecreate(InputStream, Pattern, File, int)}. + */ public static void extractRecreate(Path zipFile, String regex, Path destDir, int stripPathComponents) - throws IOException { + throws IOException { extractRecreate(zipFile.toFile(), regex, destDir.toFile(), stripPathComponents); } + + /** + * See {@link #extractRecreate(InputStream, Pattern, File, int)}. + */ public static void extractRecreate(String zipFileLoc, String regex, String destDir, int stripPathComponents) - throws IOException { + throws IOException { extractRecreate(new File(zipFileLoc), regex, new File(destDir), stripPathComponents); } + + /** + * See {@link #extractRecreate(InputStream, Pattern, File, int)}. + */ public static void extractRecreate(File zipFile, String regex, File destDir, int stripPathComponents) - throws IOException { + throws IOException { try (FileInputStream fis = new FileInputStream(zipFile)) { extractRecreate(fis, regex != null ? Pattern.compile(regex) : null, destDir, stripPathComponents); } } + + /** + * See {@link #extractRecreate(InputStream, Pattern, File, int)}. + */ public static void extractRecreate(InputStream is, Pattern regex, Path destDir, int stripPathComponents) - throws IOException { + throws IOException { extractRecreate(is, regex, destDir.toFile(), stripPathComponents); - } + } + /** - * Extracts a zip stream into a destination folder. The destination folder will be recreated. - * This function condenses the probably most typical use case for archive extraction and provides an - * argument to remove leading path components while extracting the archive. + * Extracts a zip stream into a destination folder. The destination folder + * will be recreated. This function condenses the probably most typical use + * case for archive extraction and provides an argument to remove leading + * path components while extracting the archive. * - * Caveat: because this implementation is working on a continuous input stream, it is not suited (performance-wise) - * for random access, ie. extracing single small files out of large archives. + *

Caveat: because this implementation is working on a continuous input + * stream, it is not suited (performance-wise) for random access, ie. + * extracing single small files out of large archives. * - * @param is zip archive data input stream - * @param regex select files and folders to be extracted (example entry path names to match against: - * "some/folder/", "some/file"). Set to null to include everything. - * @param destDir the destination folder. will be erased without warning. - * @param stripPathComponents strip this man path components from the files and folders being extracted. - * Set to 0 to not strip any path components and enable regular behavior. - * Folder entries above that level are silently ignored. Files above that level will lead to an IOException - - * use the regex argument to exclude them. - * @throws IOException on any error, incl. no files extracted, duplicate zip entries + * @param is + * zip archive data input stream + * @param regex + * select files and folders to be extracted (example entry path + * names to match against: "some/folder/", "some/file"). Set to + * null to include everything. + * @param destDir + * the destination folder. will be erased without warning. + * @param stripPathComponents + * strip this many path components from the files and folders + * being extracted. Set to 0 to not strip any path components and + * enable regular behavior. Folder entries above that level are + * silently ignored. Files above that level will lead to an + * IOException - use the regex argument to exclude them. + * @throws IOException + * on any error, incl. no files extracted, duplicate zip entries */ public static void extractRecreate(InputStream is, Pattern regex, File destDir, int stripPathComponents) - throws IOException { + throws IOException { if (destDir.exists()) { FileUtils.deleteDirectory(destDir); } boolean fileExtracted = false; try (ZipInputStream zis = new ZipInputStream(is)) { - ZipEntry ze; + ZipEntry ze; while ((ze = zis.getNextEntry()) != null) { String name = ze.getName(); if (regex != null && !regex.matcher(name).find()) { @@ -119,7 +156,7 @@ public static void extractRecreate(InputStream is, Pattern regex, File destDir, } throw new IOException("archive entry above stripped path: " + name + ", use regex to exclude"); } - name = name.substring(n+1); + name = name.substring(n + 1); if (name.length() == 0) { if (ze.isDirectory()) { continue; @@ -154,7 +191,7 @@ public static void extractRecreate(InputStream is, Pattern regex, File destDir, throw new IOException("no files have been extracted"); } } - + private static int nthIndexOf(String s, int n) { int pos = 0; for (int i = 0; i < n && pos != -1; i++) { @@ -162,4 +199,156 @@ private static int nthIndexOf(String s, int n) { } return pos; } + + /** + * See {@link #zip(OutputStream, Path, String)}. + */ + public static void zip(Path outputFile, Path srcDir) throws IOException { + zip(outputFile, srcDir, null); + } + + /** + * See {@link #zip(OutputStream, Path, String)}. + */ + public static void zip(Path outputFile, Path srcDir, String prefix) throws IOException { + try (OutputStream os = new FileOutputStream(outputFile.toFile())) { + zip(os, srcDir, prefix); + } + } + + /** + * See {@link #zip(OutputStream, Path, String)}. + */ + public static void zip(OutputStream os, Path srcDir) throws IOException { + zip(os, srcDir, null); + } + + /** + * Zip a dir into an output stream, prepending the zip entry names with an optional prefix. + */ + public static void zip(OutputStream os, Path srcDir, String prefix) throws IOException { + if (prefix == null) { + prefix = ""; + } + try (ZipOutputStream zos = new ZipOutputStream(os)) { + Files.walkFileTree(srcDir, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path relPath = srcDir.relativize(dir); + if (relPath.isAbsolute()) { + throw new IllegalStateException(); + } + String dirNorm = relPath.toString().replace(File.separatorChar, '/'); + + ZipEntry ze = new ZipEntry(dirNorm + "/"); + ze.setTime(attrs.lastModifiedTime().toMillis()); + zos.putNextEntry(ze); + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path relPath = srcDir.relativize(file); + if (relPath.isAbsolute()) { + throw new IllegalStateException(); + } + String fileNorm = relPath.toString().replace(File.separatorChar, '/'); + + ZipEntry ze = new ZipEntry(fileNorm); + ze.setTime(attrs.lastModifiedTime().toMillis()); + ze.setSize(attrs.size()); + zos.putNextEntry(ze); + FileUtils.copyFile(file.toFile(), zos); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + }); + } + } + + /** + * See {@link #zipEntryToString(Path, String, Charset)}. Use UTF-8 as charset. + */ + public static String zipEntryToString(File zipFile, String entryName) throws IOException { + return zipEntryToString(zipFile.toPath(), entryName); + } + + /** + * See {@link #zipEntryToString(Path, String, Charset)}. Use UTF-8 as charset. + */ + public static String zipEntryToString(Path zipFile, String entryName) throws IOException { + return zipEntryToString(zipFile, entryName, UTF_8); + } + + /** + * See {@link #zipEntryToString(Path, String, Charset)}. + */ + public static String zipEntryToString(File zipFile, String entryName, Charset cs) throws IOException { + return zipEntryToString(zipFile.toPath(), entryName, cs); + } + + /** + * Return zip file entry as string. + */ + public static String zipEntryToString(Path zipFile, String entryName, Charset cs) + throws IOException { + try (ZipFile zf = new ZipFile(zipFile.toFile())) { + ZipEntry ze = zf.getEntry(entryName); + if (ze == null) { + return null; + } + try (InputStream is = zf.getInputStream(ze)) { + return IOUtils.toString(is, cs); + } + } + } + + /** + * See {@link #getManifest(Path)}. + */ + public static Manifest getManifest(File zipFile) throws IOException { + return getManifest(zipFile.toPath()); + } + + /** + * Read zip (jar/war) file META-INF/MANIFEST.MF. + */ + public static Manifest getManifest(Path zipFile) throws IOException { + try (ZipFile zf = new ZipFile(zipFile.toFile())) { + ZipEntry ze = zf.getEntry("META-INF/MANIFEST.MF"); + if (ze == null) { + return null; + } + try (InputStream is = zf.getInputStream(ze)) { + return new Manifest(is); + } + } + + } } + + + + + + + + + + + + + + + diff --git a/io-utils/src/main/java/com/github/jjYBdx4IL/utils/lang/ArrayUtils.java b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/lang/ArrayUtils.java new file mode 100755 index 0000000..caf9397 --- /dev/null +++ b/io-utils/src/main/java/com/github/jjYBdx4IL/utils/lang/ArrayUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.utils.lang; + +public class ArrayUtils { + + /** + * Equivalent to: {@link #indexOf(byte[], byte[])} != -1. + */ + public static boolean contains(byte[] haystack, byte[] needle) { + return indexOf(haystack, needle) != -1; + } + + /** + * Search for the first byte array sub match in haystack and return its offset. + * + * @return -1 if no match was found or if needle length is zero + */ + public static int indexOf(byte[] haystack, byte[] needle) { + if (needle.length == 0) { + return -1; + } + for (int i = 0; i < haystack.length + 1 - needle.length; i++) { + if (isPartEqual(haystack, i, needle)) { + return i; + } + } + return -1; + } + + private static boolean isPartEqual(byte[] a, int ai, byte[] b) { + for (int i = 0; i < b.length; i++) { + if (a[ai + i] != b[i]) { + return false; + } + } + return true; + } + + private ArrayUtils() { + } + +} \ No newline at end of file diff --git a/io-utils/src/test/java/com/github/jjYBdx4IL/utils/io/ZipUtilsTest.java b/io-utils/src/test/java/com/github/jjYBdx4IL/utils/io/ZipUtilsTest.java index bb6c0ed..ac0141b 100644 --- a/io-utils/src/test/java/com/github/jjYBdx4IL/utils/io/ZipUtilsTest.java +++ b/io-utils/src/test/java/com/github/jjYBdx4IL/utils/io/ZipUtilsTest.java @@ -15,21 +15,19 @@ */ package com.github.jjYBdx4IL.utils.io; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.Arrays; +import static org.junit.Assert.*; import org.apache.commons.io.FileUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Arrays; + public class ZipUtilsTest { private static final File TEST_ZIP; @@ -45,6 +43,27 @@ public class ZipUtilsTest { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + @Test + public void testZipEntryToString() throws Exception { + assertEquals("123", ZipUtils.zipEntryToString(TEST_ZIP, "123").trim()); + assertEquals("test", ZipUtils.zipEntryToString(TEST_ZIP, "parentdir/test").trim()); + } + + @Test + public void testZip() throws IOException { + File destDir = new File(tempFolder.getRoot(), "unpacked"); + Path outFile = tempFolder.getRoot().toPath().resolve("out.zip"); + + ZipUtils.extractRecreate(TEST_ZIP, null, destDir, 0); + assertEquals(2, FindUtils.globFiles(destDir, "**").size()); // 2 files + + ZipUtils.zip(outFile, destDir.toPath()); + + File destDir2 = new File(tempFolder.getRoot(), "unpacked2"); + ZipUtils.extractRecreate(outFile.toFile(), null, destDir2, 0); + assertEquals(2, FindUtils.globFiles(destDir2, "**").size()); // 2 files + } + @Test public void testExtractRecreate() throws IOException { File destDir = new File(tempFolder.getRoot(), "unpacked"); diff --git a/io-utils/src/test/java/com/github/jjYBdx4IL/utils/lang/ArrayUtilsTest.java b/io-utils/src/test/java/com/github/jjYBdx4IL/utils/lang/ArrayUtilsTest.java new file mode 100755 index 0000000..c6665d3 --- /dev/null +++ b/io-utils/src/test/java/com/github/jjYBdx4IL/utils/lang/ArrayUtilsTest.java @@ -0,0 +1,44 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.utils.lang; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class ArrayUtilsTest { + + @Test + public void testIndexOfByteArray() throws Exception { + assertEquals(-1, ArrayUtils.indexOf(new byte[] {}, new byte[] {})); + assertEquals(-1, ArrayUtils.indexOf(new byte[] {1}, new byte[] {})); + assertEquals(-1, ArrayUtils.indexOf(new byte[] {}, new byte[] {1})); + + assertEquals(0, ArrayUtils.indexOf(new byte[] { 1, 2 }, new byte[] { 1 })); + assertEquals(1, ArrayUtils.indexOf(new byte[] { 1, 2 }, new byte[] { 2 })); + assertEquals(-1, ArrayUtils.indexOf(new byte[] { 1, 2 }, new byte[] { 3 })); + + assertEquals(0, ArrayUtils.indexOf(new byte[] { 1, 2 }, new byte[] { 1, 2 })); + assertEquals(-1, ArrayUtils.indexOf(new byte[] { 1, 2 }, new byte[] { 1, 2, 3 })); + + assertEquals(1, ArrayUtils.indexOf(new byte[] { 1, 2, 3 }, new byte[] { 2 })); + assertEquals(1, ArrayUtils.indexOf(new byte[] { 1, 2, 3, 4 }, new byte[] { 2, 3 })); + assertEquals(-1, ArrayUtils.indexOf(new byte[] { 1, 2, 3, 4 }, new byte[] { 2, 4 })); + + assertEquals(0, ArrayUtils.indexOf(new byte[] { 1, 2, 3 }, new byte[] { 1, 2, 3 })); + assertEquals(1, ArrayUtils.indexOf(new byte[] { 1, 2, 3 }, new byte[] { 2, 3 })); + } +} diff --git a/io-utils/src/test/resources/simplelogger.properties b/io-utils/src/test/resources/simplelogger.properties index 1df57bb..1504366 100644 --- a/io-utils/src/test/resources/simplelogger.properties +++ b/io-utils/src/test/resources/simplelogger.properties @@ -1,39 +1,8 @@ -# SLF4J's SimpleLogger configuration file -# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. - -# Default logging detail level for all instances of SimpleLogger. -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, defaults to "info". org.slf4j.simpleLogger.defaultLogLevel=info - -# Logging detail level for a SimpleLogger instance named "xxxxx". -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, the default logging detail level is used. #org.slf4j.simpleLogger.log.com.github.jjYBdx4IL.graphics.examples=debug -#org.slf4j.simpleLogger.log.net.java.dev.jna=trace -#org.slf4j.simpleLogger.log.twitter4j=trace - -# BEWARE! In this case it's better to add -Dorg.slf4j.simpleLogger.defaultLogLevel=debug to your IDE's "test file" goal. - -# Set to true if you want the current date and time to be included in output messages. -# Default is false, and will output the number of milliseconds elapsed since startup. org.slf4j.simpleLogger.showDateTime=true - -# The date and time format to be used in the output messages. -# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. -# If the format is not specified or is invalid, the default format is used. -# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. #org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss - -# Set to true if you want to output the current thread name. -# Defaults to true. #org.slf4j.simpleLogger.showThreadName=true - -# Set to true if you want the Logger instance name to be included in output messages. -# Defaults to true. #org.slf4j.simpleLogger.showLogName=true - -# Set to true if you want the last component of the name to be included in output messages. -# Defaults to false. org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/jmon/LICENSE.other b/jmon/LICENSE.other new file mode 100755 index 0000000..7f213d9 --- /dev/null +++ b/jmon/LICENSE.other @@ -0,0 +1,34 @@ + +Lists of 32 third-party dependencies. + (The Apache Software License, Version 2.0) IO Utils (com.github.jjYBdx4IL.utils:io-utils:1.1-SNAPSHOT - https://github.com/jjYBdx4IL/io-utils) + (The Apache Software License, Version 2.0) Text Utils (com.github.jjYBdx4IL.utils:text-utils:1.2-SNAPSHOT - https://github.com/jjYBdx4IL/text-utils) + (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) + (Apache 2.0) Gson (com.google.code.gson:gson:2.8.6 - https://github.com/google/gson/gson) + (Apache 2.0) error-prone annotations (com.google.errorprone:error_prone_annotations:2.5.1 - http://nexus.sonatype.org/oss-repository-hosting.html/error_prone_parent/error_prone_annotations) + (The Apache Software License, Version 2.0) Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.1 - https://github.com/google/guava/failureaccess) + (Apache License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:30.1.1-jre - https://github.com/google/guava/guava) + (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) + (The Apache Software License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:1.3 - https://github.com/google/j2objc/) + (EDL 1.0) Jakarta Activation (com.sun.activation:jakarta.activation:2.0.0 - https://github.com/eclipse-ee4j/jaf/jakarta.activation) + (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) Jakarta Mail API (com.sun.mail:jakarta.mail:2.0.0 - http://eclipse-ee4j.github.io/mail/jakarta.mail) + (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.4 - http://commons.apache.org/proper/commons-cli/) + (Apache License, Version 2.0) Apache Commons IO (commons-io:commons-io:2.8.0 - https://commons.apache.org/proper/commons-io/) + (The MIT License (MIT)) ClassGraph (io.github.classgraph:classgraph:4.8.105 - https://github.com/classgraph/classgraph) + (Common Development and Distribution License (CDDL) v1.0) JavaBeans Activation Framework (JAF) (javax.activation:activation:1.1 - http://java.sun.com/products/javabeans/jaf/index.jsp) + (CDDL/GPLv2+CE) JavaBeans Activation Framework API jar (javax.activation:javax.activation-api:1.2.0 - http://java.net/all/javax.activation-api/) + (CDDL + GPLv2 with classpath exception) Java Servlet API (javax.servlet:javax.servlet-api:4.0.1 - https://javaee.github.io/servlet-spec/) + (Apache License, Version 2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.12.0 - https://commons.apache.org/proper/commons-lang/) + (Apache License, Version 2.0) Apache Commons Text (org.apache.commons:commons-text:1.4 - http://commons.apache.org/proper/commons-text) + (Bouncy Castle Licence) Bouncy Castle S/MIME API (org.bouncycastle:bcmail-jdk15on:1.68 - http://www.bouncycastle.org/java.html) + (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk15on:1.68 - http://www.bouncycastle.org/java.html) + (Bouncy Castle Licence) Bouncy Castle Provider (org.bouncycastle:bcprov-jdk15on:1.68 - http://www.bouncycastle.org/java.html) + (The MIT License) Checker Qual (org.checkerframework:checker-qual:3.8.0 - https://checkerframework.org) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 2.0) Jetty :: ALPN :: Client (org.eclipse.jetty:jetty-alpn-client:11.0.3 - https://eclipse.org/jetty/jetty-alpn-parent/jetty-alpn-client) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 2.0) Jetty :: Asynchronous HTTP Client (org.eclipse.jetty:jetty-client:11.0.3 - https://eclipse.org/jetty/jetty-client) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 2.0) Jetty :: Http Utility (org.eclipse.jetty:jetty-http:11.0.3 - https://eclipse.org/jetty/jetty-http) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 2.0) Jetty :: IO Utility (org.eclipse.jetty:jetty-io:11.0.3 - https://eclipse.org/jetty/jetty-io) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 2.0) Jetty :: Server Core (org.eclipse.jetty:jetty-server:11.0.3 - https://eclipse.org/jetty/jetty-server) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 2.0) Jetty :: Utilities (org.eclipse.jetty:jetty-util:11.0.3 - https://eclipse.org/jetty/jetty-util) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 1.0) Jetty :: Jakarta Servlet API and Schemas for JPMS and OSGi (org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:5.0.2 - https://eclipse.org/jetty/jetty-jakarta-servlet-api) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.0-alpha1 - http://www.slf4j.org) + (MIT License) SLF4J Simple Binding (org.slf4j:slf4j-simple:2.0.0-alpha1 - http://www.slf4j.org) diff --git a/jmon/README.md b/jmon/README.md new file mode 100644 index 0000000..b4f4501 --- /dev/null +++ b/jmon/README.md @@ -0,0 +1,62 @@ +# JMon - a simple Nagios replacement + +## Installation + +No CA. Trust is based on deploying truststores manually. For that reason, validity period length isn't really an issue. + +Server key: + + server jmon-cfgdir$ keytool -keystore keystore -storepass password -keypass password \ + -alias server -genkeypair -keyalg rsa -keysize 4096 -validity 12000 -dname CN=jmon.server.com + +Put the server cert into the clients' truststores: + + server jmon-cfgdir$ keytool -keystore keystore -storepass password -keypass password \ + -alias server -export -file server.cert + # for all clients: + server jmon-cfgdir$ scp server.cert clientX:jmon-cfgdir/. + clientX jmon-cfgdir$ keytool -keystore truststore -storepass password -keypass password -noprompt \ + -alias server-cert -import -file server.cert + +Generate client keys identifying each client. The CN part of the certificate subject will be treated as the +hostname when submitting passive server check updates. It doesn't need to match your client's real address/domain. + + clientX jmon-cfgdir$ keytool -keystore keystore -storepass password -keypass password \ + -alias client -genkeypair -keyalg rsa -keysize 4096 -validity 12000 -dname CN=clientX.com + +Add each client's certificate to the server's truststore: + + clientX jmon-cfgdir$ keytool -keystore keystore -storepass password -keypass password \ + -alias client -export -file clientX.cert + clientX jmon-cfgdir$ scp clientX.cert server:jmon-cfgdir/. + # re-import on the server: + server jmon-cfgdir$ keytool -keystore truststore -storepass password -keypass password -noprompt \ + -alias clientX-cert -import -file clientX.cert + +## Concurrency Model + +* Concurrency starts after loading the definitions/configuration. +* Passive checks are handled by the RequestHandler. +* Active checks are handled by CheckThread. +* Synchronization is based on ServiceState instances. As long as there is only one CheckThread, this +synchronization is superfluous. +* The NotificationThread works on a ConcurrentHashMap that stores active check problems. The synchronization +is done via the ConcurrentHashMap only. The NotificationThread does not access the ServiceState directly. The +thread working on the ServiceState has to construct the problem report while holding the ServiceState instance +lock. +* HostState currently needs no synchronization because there is only one thing that can happen to it: after having +obtained the remote host's name during passive check submission (RequestHandler), there is an issue with the reported +passive check result. In that case, the HostState will get a status != 0. +* Host states include all of the host's service states. Each host's state get written to "hostname.state" +every N minutes after the first unsaved change (currently the save only happens when either a passive check +is submitted to the same host or the dirty flag on the host state is older than N minutes upon active check state +change). Status updates to passive checks trigger state saves immediately +within the request to provide transactional safety for passive check results that have a longer update interval. +Concurrency is achieved by locking the HostState instance. In order to ensure consistency, that also must imply +a lock over all contained service states, ie. the following lock ordering is hereby introduced: + + HostState -> ServiceState. + +Each ServiceState modification must also get a HostState lock. Beware: the service state lock +can be held without holding the HostState lock as long as the service state does not get updated. This is also +a requirement to avoid externally provoked active check delays from blocking the processing. \ No newline at end of file diff --git a/jmon/bin.xml b/jmon/bin.xml new file mode 100644 index 0000000..0d8f99c --- /dev/null +++ b/jmon/bin.xml @@ -0,0 +1,41 @@ + + + bin + + false + + + + ${project.build.directory} + ${project.artifactId} + + ${project.build.finalName}.jar + libs/* + + + + ${project.build.outputDirectory} + ${project.artifactId} + + ${project.artifactId} + + 0755 + unix + + + ${project.basedir} + ${project.artifactId} + true + + README* + LICENSE* + + 0644 + unix + + + diff --git a/jmon/pom.xml b/jmon/pom.xml new file mode 100644 index 0000000..6c1d07e --- /dev/null +++ b/jmon/pom.xml @@ -0,0 +1,310 @@ + + + 4.0.0 + + + com.github.jjYBdx4IL + release-parent + 1.4-SNAPSHOT + ../release-parent/ + + + com.github.jjYBdx4IL.misc + jmon + 1.0-SNAPSHOT + JMon - simple Nagios replacement + + + UTF-8 + UTF-8 + 16 + 16 + 11.0.3 + ${project.build.directory}/keystore + + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + + true + + + + + maven-enforcer-plugin + + + enforce + + enforce + + verify + + + + + + + + + + No Snapshots Allowed! + true + + + [1.8,) + + + [3.0,) + + + + + + + + org.codehaus.mojo + license-maven-plugin + + + + + maven-antrun-plugin + + + + run + + generate-resources + + + + + + + + + + + + + + + maven-jar-plugin + + + default-jar + + + + true + com.github.jjYBdx4IL.jmon.Main + libs/ + custom + $${artifact.artifactId}-$${artifact.version}$${dashClassifier?}.$${artifact.extension} + + + + properties.xml + README + ${project.artifactId} + + + + + + + + maven-assembly-plugin + + + package + + single + + + + + bin.xml + + zip + dir + + false + + + + + maven-dependency-plugin + + + unpack-wrapper + generate-sources + + copy-dependencies + + + runtime + ${project.build.directory}/libs + + + + + + + maven-checkstyle-plugin + + true + + + + + + + + maven-compiler-plugin + 3.8.1 + + + maven-clean-plugin + 3.0.0 + + + maven-deploy-plugin + 2.8.2 + + + maven-failsafe-plugin + 2.22.1 + + + maven-install-plugin + 2.5.2 + + + maven-jar-plugin + 3.0.0 + + + maven-javadoc-plugin + 3.3.0 + + + maven-resources-plugin + 3.0.0 + + + maven-surefire-plugin + 3.0.0-M5 + + 10 + + ${project.build.directory} + + + + + + + + + + org.eclipse.jetty + jetty-client + ${jetty.version} + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + com.google.code.gson + gson + 2.8.6 + + + com.google.guava + guava + 30.1.1-jre + + + javax.servlet + javax.servlet-api + 4.0.1 + + + org.bouncycastle + bcprov-jdk15on + 1.68 + + + org.bouncycastle + bcmail-jdk15on + 1.68 + + + io.github.classgraph + classgraph + 4.8.105 + + + commons-cli + commons-cli + 1.4 + + + com.sun.mail + jakarta.mail + 2.0.0 + + + com.github.jjYBdx4IL.utils + io-utils + 1.1-SNAPSHOT + + + com.github.jjYBdx4IL.utils + text-utils + 1.2-SNAPSHOT + + + + org.slf4j + slf4j-simple + 2.0.0-alpha1 + + + org.slf4j + slf4j-api + 2.0.0-alpha1 + + + + junit + junit + 4.13.2 + test + + + + + + + org.apache.commons + commons-lang3 + 3.12.0 + + + + diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/CheckThread.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/CheckThread.java new file mode 100755 index 0000000..c0cd3a5 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/CheckThread.java @@ -0,0 +1,131 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; + +import com.github.jjYBdx4IL.jmon.checks.CheckResult; +import com.github.jjYBdx4IL.jmon.dto.ServiceState; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class CheckThread extends Thread { + + private static final Logger LOG = LoggerFactory.getLogger(CheckThread.class); + + private final CountDownLatch stopLatch = new CountDownLatch(1); + private long waitMillis = 100; + private final Model model; + private final TreeSet queue = new TreeSet<>(new TimedExecution.Comparatr()); + private ReporterThread reporterThread = null; + + public CheckThread(Model m) { + super("CheckThread"); + model = m; + } + + public void init() { + model.populateExecutionQueue(queue); + } + + @Override + public void run() { + LOG.debug("check thread started"); + if (queue.isEmpty()) { + LOG.info("no active checks configured, check thread terminates itself."); + return; + } + + try { + waitMillis = queue.first().plannedExecEpochMillis - System.currentTimeMillis(); + while (!stopLatch.await(Math.max(waitMillis, 1L), TimeUnit.MILLISECONDS)) { + doSomeWork(); + waitMillis = queue.first().plannedExecEpochMillis - System.currentTimeMillis(); + } + } catch (InterruptedException e) { + LOG.debug("check thread aborting"); + throw new RuntimeException(e); + } + LOG.debug("check thread terminating regularly"); + } + + private void doSomeWork() { + TimedExecution te = queue.pollFirst(); + ServiceState s = te.state; + synchronized (s) { + CheckResult r = null; + if (!s.def.passive) { + try { + r = te.checkInstance.execute(); + } catch (Exception ex) { + r = new CheckResult(ex.getMessage(), ServiceState.STATUS_CODE_ERROR); + } + } else { + if (s.isTimeout()) { + r = new CheckResult("Timeout", 2); + } + } + + if (r != null) { + synchronized (te.state.hostState) { + final boolean wasProblem = s.status > 1 && (s.def.passive || s.tries >= s.def.tries); + final boolean isProblem = r.status > 1 && (s.def.passive || s.tries >= s.def.tries - 1); + + s.millisSinceEpoch = System.currentTimeMillis(); + s.status = r.status; + s.msg = r.msg; + if (s.status > 0) { + s.tries++; + } else { + s.tries = 0; + } + te.plannedExecEpochMillis = s.nextExec(); + + if (isProblem && !wasProblem) { + LOG.debug("registered as new problem"); + reporterThread.addReport(s); + } else if (!isProblem && wasProblem) { + LOG.debug("problem solved"); + reporterThread.removeReport(s); + } + + if (LOG.isDebugEnabled()) { + LOG.debug(f("%s@%s. status %d, tries %d, msg: %s", te.state.def.name, + te.state.def.hostDef.hostname, te.state.status, te.state.tries, te.state.msg)); + } + + te.state.hostState.save(false); + } + } + } + queue.add(te); + LOG.debug("timed executions scheduled: {}", queue.size()); + } + + public void shutdown() throws InterruptedException { + stopLatch.countDown(); + join(); + } + + public void setReporterThread(ReporterThread reporterThread) { + this.reporterThread = reporterThread; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Config.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Config.java new file mode 100755 index 0000000..ac615bd --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Config.java @@ -0,0 +1,236 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.gson.Gson; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; + +import javax.net.ssl.SSLSocketFactory; + +public class Config { + + @SuppressWarnings("unused") + private static final Logger LOG = LoggerFactory.getLogger(Config.class); + + public static final long GENERIC_MAX_RETRIEVAL_SIZE = 1024 * 1024L; + + public static int port = 443; + public static IProblemReporter reporter = new MailProblemReporter(); + + public static final Gson gson = Utils.createGson(); + public static SSLSocketFactory sslSocketFactory = null; + + // 0 to save on every update + public static long hostSaveIvalMillis = 60 * 1000L; + + public static Path cfgDir = Paths.get(System.getProperty("user.home")); + public static Path ks; + public static Path ts; + + // NOT used in server mode: + public static final Path spoolDir = Paths.get("/var/spool/nagiosresdump"); + public static URL spoolSendToUrl = null; + + public static CommandLine cmd; + + public static final String OPT_HELP = "help"; + public static final String OPT_HELP_CONFIG = "helpConfig"; + public static final String OPT_CFGDIR = "cfgdir"; + public static final String OPT_IGNORENOACTIVECHECKS = "ignoreNoActiveChecks"; + public static final String OPT_ACTION_RUN_SERVER = "runServer"; + public static final String OPT_ACTION_STORE_IN_SPOOL = "storeInSpool"; + public static final String OPT_ACTION_XMIT_SPOOL = "xmitSpool"; + public static final String OPT_SPOOL_SEND_TO = "spoolSendTo"; + + public static enum ACTION { + RUN_SERVER, + STORE_IN_SPOOL, + XMIT_SPOOL + } + + public static ACTION selectedAction = null; + + public static InstanceLock instanceLock = null; + + public static boolean parseCmdLine(String[] args) throws Exception { + Options options = new Options(); + options.addOption("h", OPT_HELP, false, "this help page"); + options.addOption(null, OPT_HELP_CONFIG, false, "config help page"); + options.addOption("c", OPT_CFGDIR, true, "configuration directory"); + options.addOption(null, OPT_IGNORENOACTIVECHECKS, false, "ignore if no active checks are configured"); + options.addOption(null, OPT_ACTION_RUN_SERVER, false, "server mode"); + options.addOption(null, OPT_ACTION_STORE_IN_SPOOL, true, + "store mode - writes status update to spool directory (fmt: \"[012]:serviceName:message\")"); + options.addOption(null, OPT_ACTION_XMIT_SPOOL, false, + "transmit spool mode - sends spooled status updates to server"); + options.addOption(null, OPT_SPOOL_SEND_TO, true, + "name of jmon server where the spooled status updates will be sent"); + CommandLineParser parser = new DefaultParser(); + cmd = parser.parse(options, args); + + if (cmd.hasOption(OPT_HELP)) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("jmon", options); + return false; + } + if (cmd.hasOption(OPT_HELP_CONFIG)) { + Set klazznames = Utils.scanChecks(); + System.out.print(""" + == Host and services config == + + Put host definitions into the directory specified by the --cfgdir option. + For convenience you may name them "fqdn.def" where fqdn is the fully qualified + hostname, ie. "www.google.de", or an IP address. The suffix ".def" is important. + Alternatively, you may also use the optional hostname parameter that is part if the host + definition itself (syntax is JSON (Gson without html escaping)): + + example.com.def: + + { + "hostname": "example.com", // optional, will use filename minus .def by default + // merge services defined in another file (excl. defaults) + "includes": ["arbitrary other file (incl host .defs) given by relative path", ...], + "services": { + "arbitrary service name": { + "passive": true, // default: false + // for passive checks: + "timeout": "1w2d3h4m5s", // default 1h (1 hour) + // for active checks: + "check": "name of the service check, see below", + "conf": "...", // check-specific configuration, see below + "checkIval": "15m", // default 5m (5 minutes) + "retryIval": "1m", // default 1m + "tries": 4 // default 3 + } + }, + "defaults": { // section and entries are optional + // change defaults for this host: + "timeout": "6h", + "checkIval": "30m", + "retryIval": "2m", + "tries" : 5 + } + } + + The "includes" parameter will include the services definition from other files. That + way you can define basic service checks for generic host types. + + == Available Checks ( : ) == + + """); + for (String cn : klazznames) { + try { + Class klazz = Class.forName(cn); + System.out.print(klazz.getSimpleName()); + System.out.print(" : "); + Method m = klazz.getMethod("help"); + if (!m.getReturnType().equals(String.class)) { + throw new NoSuchMethodException("wrong return type"); + } + String s = (String) m.invoke(null); + if (s == null) { + throw new NoSuchMethodException("null returned"); + } + System.out.println(s); + } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + System.out.println("no help text found!"); + } + } + return false; + } + + if (cmd.hasOption(OPT_ACTION_RUN_SERVER)) { + selectedAction = ACTION.RUN_SERVER; + } + if (cmd.hasOption(OPT_ACTION_STORE_IN_SPOOL)) { + if (selectedAction != null) { + System.out.println(f("multiple actions selected: %s", OPT_ACTION_STORE_IN_SPOOL)); + return false; + } + selectedAction = ACTION.STORE_IN_SPOOL; + } + if (cmd.hasOption(OPT_ACTION_XMIT_SPOOL)) { + if (selectedAction != null) { + System.out.println(f("multiple actions selected: %s", OPT_ACTION_XMIT_SPOOL)); + return false; + } + selectedAction = ACTION.XMIT_SPOOL; + } + + if (selectedAction == null) { + System.out.println("ERROR: no operation mode selected."); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("jmon", options); + return false; + } + + if (cmd.hasOption(OPT_CFGDIR)) { + cfgDir = Paths.get(cmd.getOptionValue(OPT_CFGDIR)); + } + ks = cfgDir.resolve("keystore"); + ts = cfgDir.resolve("truststore"); + + sslSocketFactory = Utils.createSSLSocketFactory(ts); + + if (selectedAction.equals(ACTION.XMIT_SPOOL) || selectedAction.equals(ACTION.STORE_IN_SPOOL)) { + checkArgument(Files.exists(spoolDir), "directory missing: %s", spoolDir); + } + + if (cmd.hasOption(OPT_SPOOL_SEND_TO)) { + spoolSendToUrl = new URL(cmd.getOptionValue(OPT_SPOOL_SEND_TO)); + } + + if (selectedAction.equals(ACTION.XMIT_SPOOL)) { + if (spoolSendToUrl == null) { + System.out.println("The selected action requires the " + OPT_SPOOL_SEND_TO + " argument."); + return false; + } + } + + if (selectedAction.equals(ACTION.RUN_SERVER)) { + if (!Files.exists(ks)) { + System.out.println("no keystore found: " + ks.toString()); + return false; + } + if (!Files.exists(ts)) { + System.out.println("no truststore found: " + ts.toString()); + return false; + } + } + + return true; + } + +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IExecModule.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IExecModule.java new file mode 100755 index 0000000..75094c6 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IExecModule.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +public interface IExecModule { + + void exec(); + void shutdown(); +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IProblemReporter.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IProblemReporter.java new file mode 100755 index 0000000..b4784a8 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IProblemReporter.java @@ -0,0 +1,21 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +public interface IProblemReporter { + + void send(String subject, String body) throws Exception; +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IServiceStateReceiver.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IServiceStateReceiver.java new file mode 100755 index 0000000..7f8df19 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IServiceStateReceiver.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import com.github.jjYBdx4IL.jmon.dto.ServiceStateXfer; + +public interface IServiceStateReceiver { + + boolean handle(String hostname, ServiceStateXfer ssx); +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IShutdownHandler.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IShutdownHandler.java new file mode 100755 index 0000000..ff9f375 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/IShutdownHandler.java @@ -0,0 +1,21 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +public interface IShutdownHandler { + + void shutdown(); +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/InstanceLock.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/InstanceLock.java new file mode 100755 index 0000000..183b368 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/InstanceLock.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.commons.lang3.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class InstanceLock implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(InstanceLock.class); + + public static final Path RUN_LOCK = Paths.get("/run/lock"); + public static final Path RUN_LOCK_FILE = RUN_LOCK.resolve("jmonsrv.lock"); + + private final FileOutputStream out; + private FileLock lock = null; + + public InstanceLock() throws IOException, InterruptedException { + Path lockFile; + if (SystemUtils.IS_OS_UNIX && Files.exists(RUN_LOCK)) { + lockFile = RUN_LOCK_FILE; + } else { + lockFile = Config.cfgDir.resolve(".lock"); + } + LOG.debug("acquiring lock file {}", lockFile); + out = new FileOutputStream(lockFile.toFile(), true); + checkNotNull(out); + for (int i = 0; i < 30; i++) { + if (i > 0) { + LOG.debug("failed to acquire lock, retrying..."); + Thread.sleep(1000L); + } + try { + lock = out.getChannel().lock(); + break; + } catch (OverlappingFileLockException ex) { + } + } + if (lock == null) { + out.close(); + throw new IOException("failed to acquire lock, giving up"); + } + LOG.debug("lock acquired: {}", lockFile); + } + + @Override + public void close() throws Exception { + lock.release(); + out.close(); + LOG.debug("lock released"); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/MailProblemReporter.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/MailProblemReporter.java new file mode 100755 index 0000000..f76d4bb --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/MailProblemReporter.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +public class MailProblemReporter implements IProblemReporter { + + @Override + public void send(String subject, String body) throws Exception { + Utils.sendMessage(subject, body); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Main.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Main.java new file mode 100755 index 0000000..63ace71 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Main.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import java.util.concurrent.CountDownLatch; + +public class Main { + + // for testing: + IExecModule m = null; + CountDownLatch startup = new CountDownLatch(1); + CountDownLatch shutdown = new CountDownLatch(1); + + public Main() { + } + + public void run(String[] args) throws Exception { + try { + if (!Config.parseCmdLine(args)) { + startup.countDown(); + return; + } + + if (Config.ACTION.RUN_SERVER.equals(Config.selectedAction)) { + m = new RunServer(); + try { + Config.instanceLock = new InstanceLock(); + try { + m.exec(); + } finally { + startup.countDown(); + m.shutdown(); + } + } finally { + if (Config.instanceLock != null) { + Config.instanceLock.close(); + } + } + } + else if (Config.ACTION.STORE_IN_SPOOL.equals(Config.selectedAction)) { + m = new StoreInSpool(); + try { + m.exec(); + } finally { + startup.countDown(); + } + } else { + startup.countDown(); + } + } finally { + startup.countDown(); + shutdown.countDown(); + } + } + + public static void main(String[] args) throws Exception { + new Main().run(args); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Model.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Model.java new file mode 100755 index 0000000..02464ea --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Model.java @@ -0,0 +1,269 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.github.jjYBdx4IL.jmon.checks.ICheck; +import com.github.jjYBdx4IL.jmon.dto.HostDef; +import com.github.jjYBdx4IL.jmon.dto.HostDef.MergeException; +import com.github.jjYBdx4IL.jmon.dto.HostState; +import com.github.jjYBdx4IL.jmon.dto.ServiceDef; +import com.github.jjYBdx4IL.jmon.dto.ServiceState; +import com.github.jjYBdx4IL.jmon.dto.ServiceStateXfer; +import com.google.gson.JsonSyntaxException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +public class Model implements IServiceStateReceiver, IShutdownHandler { + + private static final Logger LOG = LoggerFactory.getLogger(Model.class); + + private final Map hostDefs = new HashMap<>(); + private final Map hostStates = new HashMap<>(); + + // check name -> ctor + private final Map> checkConstructors = new HashMap<>(); + + private ReporterThread notificationThread = null; + + public long loadedServicesCounter = 0; + + public Model() { + } + + public void setReporterThread(ReporterThread notificationThread) { + this.notificationThread = notificationThread; + } + + public void init() throws Exception { + scan(); + load(); + } + + private void scan() throws ClassNotFoundException { + Set klazznames = Utils.scanChecks(); + + for (String checkCn : klazznames) { + Constructor ctor = null; + Class klazz = Class.forName(checkCn); + for (Constructor ctr : klazz.getDeclaredConstructors()) { + Type[] types = ctr.getGenericParameterTypes(); + if (types.length == 2 && types[0].equals(HostDef.class) && types[1].equals(ServiceDef.class)) { + ctor = ctr; + break; + } + } + checkNotNull(ctor); + + String checkName = klazz.getSimpleName().toLowerCase(Locale.ROOT); + if (checkConstructors.put(checkName, ctor) != null) { + throw new RuntimeException("duplicate check name: " + checkName); + } + } + } + + private void load() throws IOException { + LOG.debug("loading host definitions ({})", Config.cfgDir); + Files.list(Config.cfgDir).filter(p -> !Files.isDirectory(p)).filter(p -> p.toFile().getName().endsWith(".def")) + .forEach(p -> load(p)); + LOG.debug("imported {} service definitions distributed over {} host definitions", + loadedServicesCounter, hostDefs.size()); + checkArgument(!hostDefs.isEmpty(), "no host definitions found"); + checkArgument(loadedServicesCounter > 0, "no service definitions found"); + checkArgument(!hostStates.isEmpty()); + } + + private void load(Path hostdefLoc) { + LOG.debug("loading host definition ({})", hostdefLoc); + String hostname = hostdefLoc.toFile().getName(); + hostname = hostname.substring(0, hostname.length() - 4); + + String json = null; + try { + json = Files.readString(hostdefLoc); + HostDef hostdef = Config.gson.fromJson(json, HostDef.class); + + // default hostname derived from filename: + if (hostdef.hostname == null || hostdef.hostname.isBlank()) { + hostdef.hostname = hostname; + } + + processIncludes(hostdef, new LinkedHashSet(Arrays.asList(hostdefLoc))); + + hostdef.fillInServiceNames(); + hostdef.fillInServiceHostDefs(hostdef); + hostdef.fillInDefaults(); + hostdef.validate(); + LOG.debug("imported hostdef: {}", hostdef); + hostDefs.put(hostdef.hostname, hostdef); + + Path stateLoc = Config.cfgDir.resolve(hostname + ".state"); + HostState hs; + if (Files.exists(stateLoc)) { + hs = Config.gson.fromJson(Files.readString(stateLoc), HostState.class); + } else { + hs = new HostState(hostdef.hostname); + } + pruneAndAugment(hs, hostdef); + hs.hostname = hostdef.hostname; + hs.fillInServiceDefs(hostdef); + hs.fillInHostState(); + hs.services.values().stream().filter(s -> s.status > 1) + .forEach(s -> notificationThread.addReport(s)); + hostStates.put(hostdef.hostname, hs); + + loadedServicesCounter += hostdef.services.size(); + } catch (Exception e) { + if (json != null && e instanceof JsonSyntaxException) { + LOG.error("bad json syntax: {}", json); + } + throw new RuntimeException(e); + } + } + + // included files can be host + private void processIncludes(HostDef hostdef, LinkedHashSet loopDetect) + throws JsonSyntaxException, IOException { + LOG.trace("processIncludes: {}", loopDetect); + if (hostdef.includes == null) { + return; + } + for (String includeLoc : hostdef.includes) { + Path includePath = Config.cfgDir.resolve(includeLoc); + if (loopDetect.contains(includePath)) { + throw new RuntimeException("include loop detected: trying to include \"" + includePath + + "\", current include sequence is: " + loopDetect); + } + HostDef ihd = Config.gson.fromJson(Files.readString(includePath), HostDef.class); + loopDetect.add(includePath); + processIncludes(ihd, loopDetect); + loopDetect.remove(includePath); + + // merge service defs + try { + hostdef.addServices(ihd); + } catch (MergeException e) { + throw new RuntimeException(loopDetect.toString(), e); + } + } + } + + private void pruneAndAugment(HostState hostState, HostDef hostdef) { + // remove stale host state entries + hostState.services.keySet().retainAll(hostdef.services.keySet()); + // augment missing host state entries + hostdef.services.keySet().forEach(s -> { + if (!hostState.services.containsKey(s)) { + hostState.services.put(s, new ServiceState(hostdef.services.get(s), hostState)); + } + }); + } + + // passive check result submission handler (used by RequestHandler) + @Override + public boolean handle(String hostname, ServiceStateXfer ssx) { + ssx.validate(); + + HostDef hd = hostDefs.get(hostname); + if (hd == null) { + LOG.error("bad hostname submitted for passive check: {}", hostname); + return false; + } + HostState hs = hostStates.get(hostname); + checkNotNull(hs); + ServiceDef sdef = hd.services.get(ssx.service); + if (sdef == null) { + LOG.error("bad service name submitted for passive check: {}", ssx.toString()); + return false; + } + + ServiceState serviceState = hs.services.get(ssx.service); + checkNotNull(serviceState); + + synchronized (serviceState) { + synchronized (serviceState.hostState) { + final boolean wasProblem = serviceState.status > 1; + final boolean isProblem = ssx.state > 1; + + serviceState.millisSinceEpoch = System.currentTimeMillis(); + serviceState.status = ssx.state; + serviceState.msg = ssx.msg != null ? ssx.msg : ""; + + serviceState.hostState.save(true); + + if (isProblem && !wasProblem) { + notificationThread.addReport(serviceState); + } else if (!isProblem && wasProblem) { + notificationThread.removeReport(serviceState); + } + } + } + + return true; + } + + public void populateExecutionQueue(TreeSet queue) { + for (Entry he : hostDefs.entrySet()) { + HostDef hostdef = he.getValue(); + HostState hostState = hostStates.get(he.getKey()); + checkNotNull(hostState); + for (Entry se : hostdef.services.entrySet()) { + ServiceDef sd = se.getValue(); + ServiceState serviceState = hostState.services.get(se.getKey()); + checkNotNull(serviceState); + if (sd.passive) { + queue.add(new TimedExecution(null, serviceState)); + } else { + Constructor ctor = checkConstructors.get(sd.check); + try { + ICheck check = (ICheck) ctor.newInstance(hostdef, sd); + queue.add(new TimedExecution(check, serviceState)); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e1) { + throw new RuntimeException(e1); + } + } + } + } + LOG.debug("{} service executions queued", queue.size()); + if (queue.isEmpty() && !Config.cmd.hasOption(Config.OPT_IGNORENOACTIVECHECKS)) { + throw new RuntimeException("no active checks configured"); + } + } + + @Override + public void shutdown() { + hostStates.forEach((k, hs) -> hs.shutdown()); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/ReporterThread.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/ReporterThread.java new file mode 100755 index 0000000..9553c3f --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/ReporterThread.java @@ -0,0 +1,211 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; + +import com.github.jjYBdx4IL.jmon.cmdqueue.AddReportCommand; +import com.github.jjYBdx4IL.jmon.cmdqueue.IReportCommand; +import com.github.jjYBdx4IL.jmon.cmdqueue.RemoveReportCommand; +import com.github.jjYBdx4IL.jmon.cmdqueue.ShutdownCommand; +import com.github.jjYBdx4IL.jmon.dto.ReportingStatus; +import com.github.jjYBdx4IL.jmon.dto.ServiceState; +import com.github.jjYBdx4IL.utils.io.IoUtils; +import com.google.gson.JsonSyntaxException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +public class ReporterThread extends Thread { + + private static final Logger LOG = LoggerFactory.getLogger(ReporterThread.class); + + public static final long MAX_NUMBER_OF_REPORTS_IN_SUMMARY = 50; + public static final long DEFAULT_WAIT_MILLIS = 60 * 1000L; + + private long waitMillis = DEFAULT_WAIT_MILLIS; + private long notificationSendIvalMillis = 3600 * 1000L; + private long retryDelayMillis = 300 * 1000L; + + private ReportingStatus status = null; + private final Path lastReportStatusPath; + + private final Map problemReports = new HashMap<>(); + private final BlockingQueue commandQueue = new LinkedBlockingQueue<>(); + + public ReporterThread(Model m) { + super("ReporterThread"); + lastReportStatusPath = Config.cfgDir.resolve("reporting_state"); + if (Files.exists(lastReportStatusPath)) { + try { + status = Config.gson.fromJson(Files.readString(lastReportStatusPath), + ReportingStatus.class); + } catch (JsonSyntaxException | IOException ex) { + LOG.error("error reading last NotificationStatus: ", ex); + // we'll eventually recover from this by writing a new one + } + } + if (status == null) { + status = new ReportingStatus(); + } + } + + @Override + public void run() { + LOG.debug("reporter thread started"); + try { + boolean done = false; + while (!done) { + IReportCommand command = commandQueue.poll(Math.max(waitMillis, 1L), TimeUnit.MILLISECONDS); + if (command != null) { + if (command instanceof ShutdownCommand) { + done = true; + } + else if (command instanceof AddReportCommand) { + AddReportCommand c = (AddReportCommand) command; + problemReports.put(c.reportId, c.reportContent); + } + else if (command instanceof RemoveReportCommand) { + RemoveReportCommand c = (RemoveReportCommand) command; + problemReports.remove(c.reportId); + } + } + if (!done) { + doSomeReporting(); + } + } + } catch ( + + InterruptedException e) { + throw new RuntimeException(e); + } finally { + LOG.debug("reporter thread quitting"); + } + } + + private String buildProblemReport() { + StringBuilder sb = new StringBuilder(); + + sb.append(f("%d active problems.\n", problemReports.size())); + + if (problemReports.size() > MAX_NUMBER_OF_REPORTS_IN_SUMMARY) { + sb.append("Too many problems. Skipping listing.\n"); + } else { + problemReports.forEach((state, report) -> { + sb.append(report); + sb.append("\n"); + }); + } + + return sb.toString(); + } + + private void doSomeReporting() { + LOG.debug("doSomeReporting()"); + + final long now = System.currentTimeMillis(); + + waitMillis = DEFAULT_WAIT_MILLIS; // restore default + + // force delay for retries + if (status.sendFailure) { + final long nextRetry = status.lastSent + retryDelayMillis; + if (nextRetry > now) { + waitMillis = nextRetry - now; + return; + } + } + + final boolean escalation = !problemReports.isEmpty() && status.lastStatusReported == 0; + final boolean deescalation = problemReports.isEmpty() && status.lastStatusReported != 0; + + // report escalations or de-escalations immediately, unless there was a send failure + if (escalation) { + sendReport(buildProblemReport()); + return; + } + else if (deescalation) { + sendReport(null); + return; + } + else if (problemReports.isEmpty()) { + return; + } + else { // re-report issues + long nextPossibleTime = status.lastSent + + (status.sendFailure ? retryDelayMillis : notificationSendIvalMillis); + if (nextPossibleTime > now) { // last issue report/send try too recent? + waitMillis = nextPossibleTime - now + 100L; + return; + } + + sendReport(buildProblemReport()); + return; + } + } + + private void sendReport(String report) { + try { + if (report == null) { + Config.reporter.send("JMon Recovery Report", "All clear!"); + status.lastContentSent = null; + status.lastStatusReported = 0; + } else { + Config.reporter.send("JMon Error Report", report); + status.lastStatusReported = 2; + } + status.lastContentSent = report; + status.sendFailure = false; + } catch (Exception ex) { + LOG.error("failed to send notification", ex); + status.sendFailure = true; + } + + status.lastSent = System.currentTimeMillis(); + + try { + IoUtils.safeWriteTo(lastReportStatusPath, Config.gson.toJson(status)); + } catch (IOException ex) { + LOG.error("", ex); + } + } + + public void shutdown() throws InterruptedException { + addCommand(new ShutdownCommand()); + join(); + } + + protected void addCommand(IReportCommand command) { + commandQueue.add(command); + } + + public void addReport(ServiceState s) { + commandQueue.add(new AddReportCommand(s.def.id(), s.asReport())); + } + + public void removeReport(ServiceState s) { + commandQueue.add(new RemoveReportCommand(s.def.id())); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/RequestHandler.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/RequestHandler.java new file mode 100755 index 0000000..0819023 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/RequestHandler.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import com.github.jjYBdx4IL.jmon.dto.ServiceStateXfer; +import com.google.gson.JsonSyntaxException; + +import org.bouncycastle.asn1.x500.RDN; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x500.style.BCStyle; +import org.bouncycastle.asn1.x500.style.IETFUtils; +import org.bouncycastle.cert.jcajce.JcaX509CertificateHolder; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.SSLSession; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class RequestHandler extends AbstractHandler { + + private static final Logger LOG = LoggerFactory.getLogger(RequestHandler.class); + public static final String JSON_TYPE = "application/json"; + private final IServiceStateReceiver stateReceiver; + + public RequestHandler(IServiceStateReceiver receiver) { + LOG.debug("RequestHandler()"); + this.stateReceiver = receiver; + } + + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + + if (!"POST".equals(request.getMethod())) { + error(baseRequest, response, "http method not supported"); + return; + } + + SSLSession ssl = (SSLSession) request.getAttribute("org.eclipse.jetty.servlet.request.ssl_session"); + if (ssl == null) { + error(baseRequest, response, "no SSL"); + return; + } + + // we use the remote certificate's CN subject part as the service's + // hostname: + String cn = extractCn((X509Certificate) ssl.getPeerCertificates()[0]); + LOG.debug("authenticated cn: {}", cn); + + ServiceStateXfer serviceState; + try { + serviceState = Config.gson.fromJson(request.getReader(), ServiceStateXfer.class); + } catch (JsonSyntaxException ex) { + LOG.error("{}", cn, ex); + error(baseRequest, response, "json syntax error"); + return; + } + + if (!stateReceiver.handle(cn, serviceState)) { + error(baseRequest, response, null); + return; + } + + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + } + + protected static void error(Request baseRequest, HttpServletResponse response, String publicMessage) + throws IOException { + if (publicMessage != null) { + LOG.error(publicMessage); + } else { + publicMessage = "internal server error"; + } + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + response.setContentType("text/plain; charset=utf-8"); + response.getWriter().println(publicMessage); + baseRequest.setHandled(true); + } + + private String extractCn(X509Certificate peerCert) { + try { + X500Name x500name = new JcaX509CertificateHolder(peerCert).getSubject(); + RDN cn = x500name.getRDNs(BCStyle.CN)[0]; + return IETFUtils.valueToString(cn.getFirst().getValue()); + } catch (CertificateEncodingException ex) { + LOG.error("", ex); + } + return null; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/RunServer.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/RunServer.java new file mode 100755 index 0000000..a96e720 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/RunServer.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +public class RunServer implements IExecModule { + + public final Server server = new Server(); + public final Model model; + public final CheckThread checkThread; + public final ReporterThread reporterThread; + + public RunServer() { + model = new Model(); + checkThread = new CheckThread(model); + reporterThread = new ReporterThread(model); + model.setReporterThread(reporterThread); + checkThread.setReporterThread(reporterThread); + } + + protected void startServer() throws Exception { + HttpConfiguration https = new HttpConfiguration(); + https.addCustomizer(new SecureRequestCustomizer()); + SslContextFactory.Server ssl = new SslContextFactory.Server(); + + ssl.setCertAlias("server"); + ssl.setKeyStorePath(Config.ks.toString()); + ssl.setKeyStorePassword("password"); + + ssl.setNeedClientAuth(true); + ssl.setTrustStorePath(Config.ts.toString()); + ssl.setTrustStorePassword("password"); + + ServerConnector sslConnector = new ServerConnector(server, new SslConnectionFactory(ssl, "http/1.1"), + new HttpConnectionFactory(https)); + sslConnector.setPort(Config.port); + + server.setConnectors(new Connector[] { sslConnector }); + + server.setHandler(new RequestHandler(model)); + + server.start(); + } + + @Override + public void exec() { + try { + model.init(); + checkThread.init(); + checkThread.start(); + reporterThread.start(); + startServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void shutdown() { + try { + server.join(); + checkThread.shutdown(); + reporterThread.shutdown(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/StoreInSpool.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/StoreInSpool.java new file mode 100755 index 0000000..721af74 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/StoreInSpool.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import com.github.jjYBdx4IL.jmon.dto.ServiceStateXfer; +import com.github.jjYBdx4IL.utils.io.IoUtils; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class StoreInSpool implements IExecModule { + + public static final Pattern PAT = Pattern.compile("^(\\d+):([a-zA-Z0-9_-]+):(.*)$"); + + @Override + public void exec() { + String s = Config.cmd.getOptionValue(Config.OPT_ACTION_STORE_IN_SPOOL); + Matcher m = PAT.matcher(s); + if (!m.find()) { + throw new RuntimeException("invalid status message format: " + s); + } + ServiceStateXfer ssx = new ServiceStateXfer(); + ssx.state = Integer.parseInt(m.group(1)); + ssx.service = m.group(2); + ssx.msg = m.group(3); + String json = Config.gson.toJson(ssx); + try { + IoUtils.safeWriteTo(Config.spoolDir.resolve(ssx.service), json); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void shutdown() { + // not used + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/TimedExecution.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/TimedExecution.java new file mode 100755 index 0000000..ffbe290 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/TimedExecution.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import com.github.jjYBdx4IL.jmon.checks.ICheck; +import com.github.jjYBdx4IL.jmon.dto.ServiceState; + +import java.util.Comparator; + +public class TimedExecution { + + public long plannedExecEpochMillis; + public ICheck checkInstance; + public ServiceState state; + + public TimedExecution(ICheck checkInst, ServiceState serviceState) { + checkInstance = checkInst; + state = serviceState; + plannedExecEpochMillis = serviceState.nextExec(); + } + + public static class Comparatr implements Comparator { + + @Override + public int compare(TimedExecution o1, TimedExecution o2) { + return Long.compare(o1.plannedExecEpochMillis, o2.plannedExecEpochMillis); + } + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Utils.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Utils.java new file mode 100755 index 0000000..c06da91 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/Utils.java @@ -0,0 +1,179 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.github.jjYBdx4IL.jmon.checks.ICheck; +import com.github.jjYBdx4IL.utils.time.TimeUtils; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic; +import org.eclipse.jetty.io.ClientConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.lang.reflect.Type; +import java.nio.file.Path; +import java.security.KeyStore; +import java.time.Duration; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManagerFactory; + +import io.github.classgraph.ClassGraph; +import io.github.classgraph.ScanResult; +import jakarta.mail.BodyPart; +import jakarta.mail.Message; +import jakarta.mail.MessagingException; +import jakarta.mail.Multipart; +import jakarta.mail.Session; +import jakarta.mail.Transport; +import jakarta.mail.internet.InternetAddress; +import jakarta.mail.internet.MimeBodyPart; +import jakarta.mail.internet.MimeMessage; +import jakarta.mail.internet.MimeMultipart; +import jakarta.mail.internet.ParseException; + +public class Utils { + + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + public static Gson createGson() { + GsonBuilder b = new GsonBuilder(); + b.disableHtmlEscaping(); + if (LOG.isDebugEnabled()) { + b.setPrettyPrinting(); + } + b.registerTypeAdapter(Duration.class, new DurationDeserializer()); + b.registerTypeAdapter(Duration.class, new DurationSerializer()); + return b.create(); + } + + public static class DurationSerializer implements JsonSerializer { + public JsonElement serialize(Duration src, Type typeOfSrc, JsonSerializationContext context) { + return new JsonPrimitive(TimeUtils.millisToDuration(src.getSeconds() * 1000L)); + } + } + + public static class DurationDeserializer implements JsonDeserializer { + public Duration deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + return Duration.ofSeconds(TimeUtils.durationToMillis(json.getAsJsonPrimitive().getAsString()) / 1000L); + } + } + + public static void sendMessage(String subject, String body) + throws ParseException, MessagingException { + Session session = createSmtpSession(); + MimeMessage msg = new MimeMessage(session); + msg.setRecipient(Message.RecipientType.TO, new InternetAddress(System.getProperty("user.name") + "@localhost")); + msg.setSubject(subject); + Multipart mp = new MimeMultipart(); + BodyPart bp = new MimeBodyPart(); + bp.setText(body); + mp.addBodyPart(bp); + msg.setContent(mp); + try (Transport t = session.getTransport()) { + t.connect(); + t.sendMessage(msg, msg.getAllRecipients()); + } catch (MessagingException ex) { + LOG.error("failed to send {} - {}", subject, body); + LOG.error("", ex); + } + } + + protected static Session createSmtpSession() { + Properties props = new Properties(); + props.put("mail.transport.protocol", "smtp"); + props.put("mail.host", "localhost"); + + Session session = Session.getDefaultInstance(props); + if (LOG.isDebugEnabled()) { + LOG.debug("enabling smtp debugging"); + session.setDebug(true); + } + return session; + } + + public static Set scanChecks() { + LOG.debug("scanning check implementations"); + Set klazznames = new HashSet<>(); + try (ScanResult scanResult = new ClassGraph() + .enableClassInfo() + .scan()) { + scanResult.getClassesImplementing(ICheck.class.getName()) + .forEach(ci -> { + LOG.trace("{}", ci); + klazznames.add(ci.getName()); + }); + } + LOG.debug("check implementations found: {}", klazznames.size()); + checkArgument(!klazznames.isEmpty()); + return klazznames; + } + + public static SSLSocketFactory createSSLSocketFactory(Path trustStore) throws Exception { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + try (InputStream is = new FileInputStream(trustStore.toFile())) { + keyStore.load(is, "password".toCharArray()); + } + + TrustManagerFactory trustManagerFactory = TrustManagerFactory + .getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), null); + + return sslContext.getSocketFactory(); + } + + public static HttpClient getClient(String keyAlias) throws Exception { + SslContextFactory.Client ssl = new SslContextFactory.Client(); + + ssl.setValidatePeerCerts(true); + ssl.setTrustStorePath(Config.ts.toString()); + ssl.setTrustStorePassword("password"); + + ssl.setCertAlias(keyAlias); + ssl.setKeyStorePath(Config.ks.toString()); + ssl.setKeyStorePassword("password"); + + ClientConnector clientConnector = new ClientConnector(); + clientConnector.setSslContextFactory(ssl); + + HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + httpClient.start(); + return httpClient; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/XmitSpool.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/XmitSpool.java new file mode 100755 index 0000000..d84215b --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/XmitSpool.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Request.Content; +import org.eclipse.jetty.client.util.StringRequestContent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +public class XmitSpool implements IExecModule { + + private static final Logger LOG = LoggerFactory.getLogger(XmitSpool.class); + + @Override + public void exec() { + try { + HttpClient c = Utils.getClient("client"); + // exclude temporary files written by IoUtils::safeWriteTo + List files = Files.list(Config.spoolDir).filter(a -> !a.toFile().getName().startsWith(".")) + .collect(Collectors.toList()); + for (Path file : files) { + String json = Files.readString(file); + Content content = new StringRequestContent(RequestHandler.JSON_TYPE, json, UTF_8); + ContentResponse response = c.POST(Config.spoolSendToUrl.toExternalForm()).body(content).send(); + if (response.getStatus() == 200) { + LOG.info("transmitted: {}", file); + Files.delete(file); + } else { + LOG.error("transmission failure: {} - response was: {} - {}", file, response.getStatus(), + response.getReason()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void shutdown() { + // not used + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CertExpiryCheck.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CertExpiryCheck.java new file mode 100755 index 0000000..93fcfe4 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CertExpiryCheck.java @@ -0,0 +1,106 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.checks; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; + +import com.github.jjYBdx4IL.jmon.Config; +import com.github.jjYBdx4IL.jmon.dto.HostDef; +import com.github.jjYBdx4IL.jmon.dto.ServiceDef; + +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.cert.X509Certificate; +import java.util.Date; + +import javax.net.ssl.HttpsURLConnection; + +public class CertExpiryCheck extends CheckBase implements ICheck { + + public static final int WARN_DAYS = 10; + public static final int ERROR_DAYS = 6; + + private final URL url; + private final boolean truststore; + + public CertExpiryCheck(HostDef hostDef, ServiceDef serviceDef) { + super(hostDef, serviceDef); + + Conf conf = null; + if (serviceDef.conf != null) { + conf = Config.gson.fromJson(serviceDef.conf, Conf.class); + } + if (conf != null) { + truststore = conf.truststore; + } else { + truststore = false; + } + + try { + if (conf != null && conf.port != 0) { + url = new URL(f("https://%s:%d", serviceDef.hostDef.hostname, conf.port)); + } else { + url = new URL(f("https://%s", serviceDef.hostDef.hostname)); + } + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + @Override + public CheckResult execute() throws Exception { + HttpsURLConnection conn = (HttpsURLConnection) url.openConnection(); + if (truststore) { + conn.setSSLSocketFactory(Config.sslSocketFactory); + } + try (InputStream is = conn.getInputStream()) { + conn.connect(); + + X509Certificate c = (X509Certificate) conn.getServerCertificates()[0]; + if (c == null) { + throw new Exception("remote cert not found"); + } + + Date notAfter = c.getNotAfter(); + if (notAfter == null) { + throw new Exception("notAfter date not found"); + } + + long daysRemaining = (notAfter.getTime() - System.currentTimeMillis()) / 86400L / 1000L; + int status = 0; + if (daysRemaining < ERROR_DAYS) { + status = 2; + } + else if (daysRemaining < WARN_DAYS) { + status = 1; + } + + return new CheckResult(f("certificate expires in %d days", daysRemaining), status); + } + } + + public static String help() { + return "{\"truststore\":false} : use server truststore.\n" + + " {\"port\":443} : use non-standard remote port.\n" + + " 10 days - warning, 6 days before expiry - error"; + } + + public static class Conf { + public boolean truststore; + public int port; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckBase.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckBase.java new file mode 100755 index 0000000..93d26e4 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckBase.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.checks; + +import com.github.jjYBdx4IL.jmon.dto.HostDef; +import com.github.jjYBdx4IL.jmon.dto.ServiceDef; + +public class CheckBase { + + protected final HostDef hostDef; + protected final ServiceDef serviceDef; + + protected CheckBase(HostDef hostDef, ServiceDef serviceDef) { + this.hostDef = hostDef; + this.serviceDef = serviceDef; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckResult.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckResult.java new file mode 100755 index 0000000..643ca2f --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/CheckResult.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.checks; + +import com.github.jjYBdx4IL.jmon.dto.ServiceState; + +public class CheckResult { + + public String msg; + public int status; + + public CheckResult(String message, int status0) { + msg = message; + status = status0; + } + + public CheckResult() { + msg = ServiceState.STATUS_STR_OK; + status = ServiceState.STATUS_CODE_OK; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/ICheck.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/ICheck.java new file mode 100755 index 0000000..35f5a4e --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/ICheck.java @@ -0,0 +1,21 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.checks; + +public interface ICheck { + + CheckResult execute() throws Exception; +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/UrlCheck.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/UrlCheck.java new file mode 100755 index 0000000..416cab3 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/checks/UrlCheck.java @@ -0,0 +1,114 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.checks; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.github.jjYBdx4IL.jmon.Config; +import com.github.jjYBdx4IL.jmon.dto.HostDef; +import com.github.jjYBdx4IL.jmon.dto.ServiceDef; +import com.github.jjYBdx4IL.utils.io.IoUtils; +import com.github.jjYBdx4IL.utils.lang.ArrayUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; + +import javax.net.ssl.HttpsURLConnection; + +public class UrlCheck extends CheckBase implements ICheck { + + private static final Logger LOG = LoggerFactory.getLogger(UrlCheck.class); + + private final URL url; + private final byte[] expectedContent; + private final boolean truststore; + + public UrlCheck(HostDef hostDef, ServiceDef serviceDef) { + super(hostDef, serviceDef); + + String path = "/"; + String expContent = null; + Conf conf = null; + if (serviceDef.conf != null) { + conf = Config.gson.fromJson(serviceDef.conf, Conf.class); + } + if (conf != null) { + if (conf.path != null) { + path = conf.path; + } + if (conf.content != null && !conf.content.isEmpty()) { + expContent = conf.content; + } + truststore = conf.truststore; + } else { + truststore = false; + } + + if (!path.isEmpty() && path.startsWith("/")) { + path = "/" + path; + } + + try { + if (conf != null && conf.port != 0) { + url = new URL(f("https://%s:%d%s", serviceDef.hostDef.hostname, conf.port, path)); + } else { + url = new URL(f("https://%s%s", serviceDef.hostDef.hostname, path)); + } + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + + expectedContent = expContent != null ? expContent.getBytes(UTF_8) : null; + } + + @Override + public CheckResult execute() throws Exception { + HttpsURLConnection conn = (HttpsURLConnection) url.openConnection(); + if (truststore) { + conn.setSSLSocketFactory(Config.sslSocketFactory); + } + try (InputStream is = conn.getInputStream()) { + byte[] content = IoUtils.toByteArray(is, Config.GENERIC_MAX_RETRIEVAL_SIZE); + + if (expectedContent != null && !ArrayUtils.contains(content, expectedContent)) { + return new CheckResult(f("%s not found in page %s", expectedContent, url.toExternalForm()), 2); + } + + return new CheckResult(); + } catch (Exception ex) { + LOG.warn("", ex); + throw ex; + } + } + + public static String help() { + return "{\"path\":\"/\", \"content\":\"expected content or null\",\n" + + " \"port\":443, - use this remote port\n" + + " \"truststore\":false} - use server truststore?"; + } + + public static class Conf { + public String path; + public String content; + public boolean truststore; + public int port; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/AddReportCommand.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/AddReportCommand.java new file mode 100755 index 0000000..787cc86 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/AddReportCommand.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.cmdqueue; + +public class AddReportCommand implements IReportCommand { + + public final String reportId; + public final String reportContent; + + public AddReportCommand(String reportId, String reportContent) { + this.reportId = reportId; + this.reportContent = reportContent; + } + +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/IReportCommand.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/IReportCommand.java new file mode 100755 index 0000000..941b0cb --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/IReportCommand.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.cmdqueue; + +public interface IReportCommand { + +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/RemoveReportCommand.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/RemoveReportCommand.java new file mode 100755 index 0000000..69029ca --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/RemoveReportCommand.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.cmdqueue; + +public class RemoveReportCommand implements IReportCommand { + + public final String reportId; + + public RemoveReportCommand(String reportId) { + this.reportId = reportId; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/ShutdownCommand.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/ShutdownCommand.java new file mode 100755 index 0000000..8b33869 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/cmdqueue/ShutdownCommand.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.cmdqueue; + +public class ShutdownCommand implements IReportCommand { + +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/Defaults.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/Defaults.java new file mode 100755 index 0000000..c9c7367 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/Defaults.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +import java.time.Duration; + +public class Defaults { + + public Duration timeout = Duration.ofSeconds(3600); + public Duration checkIval = Duration.ofSeconds(300); + public Duration retryIval = Duration.ofSeconds(60); + public int tries = 3; + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Defaults [timeout=").append(timeout).append(", checkIval=").append(checkIval) + .append(", retryIval=").append(retryIval).append(", tries=").append(tries).append("]"); + return builder.toString(); + } + +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostDef.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostDef.java new file mode 100755 index 0000000..56e3127 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostDef.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +// host config +public class HostDef { + + public String hostname; + public Map services = new HashMap<>(); + public String[] includes; + public Defaults defaults = new Defaults(); + + public void validate() { + checkArgument(hostname != null && !hostname.isBlank() && hostname.trim().equals(hostname)); + services.forEach((k,v) -> v.validate()); + } + + public void addServices(HostDef ihd) throws MergeException { + for (Entry e : ihd.services.entrySet()) { + if (services.containsKey(e.getKey())) { + throw new MergeException("service definition " + e.getKey() + " cannot be merged: already exists"); + } + services.put(e.getKey(), e.getValue()); + } + } + + @SuppressWarnings("serial") + public static class MergeException extends Exception { + public MergeException(String msg) { + super(msg); + } + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("HostDef [hostname=").append(hostname).append(", services=").append(services) + .append(", includes=").append(Arrays.toString(includes)).append(", defaults=").append(defaults).append("]"); + return builder.toString(); + } + + public void fillInDefaults() { + services.forEach((k,v) -> v.fillInDefaults(defaults)); + } + + public void fillInServiceNames() { + services.forEach((k,v) -> {v.name = k;}); + } + + public void fillInServiceHostDefs(HostDef hostdef) { + services.forEach((k,v) -> {v.hostDef = hostdef;}); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostState.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostState.java new file mode 100755 index 0000000..a0b2a8b --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/HostState.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +import com.github.jjYBdx4IL.jmon.Config; +import com.github.jjYBdx4IL.jmon.IShutdownHandler; +import com.github.jjYBdx4IL.utils.io.IoUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +public class HostState implements IShutdownHandler { + + private static final Logger LOG = LoggerFactory.getLogger(HostState.class); + public static final String fileExt = ".state"; + + public Map services = new HashMap<>(); + + public transient String hostname; + public transient long lastSaved = System.currentTimeMillis(); + private transient boolean dirty = false; + + public HostState(String name) { + hostname = name; + } + + public void fillInServiceDefs(HostDef hostdef) { + services.forEach((k,v) -> {v.def = hostdef.services.get(k);}); + } + + public void fillInHostState() { + services.forEach((k,v) -> {v.hostState = this;}); + } + + public synchronized void save(boolean force) { + dirty = true; + long now = System.currentTimeMillis(); + if (force || Config.hostSaveIvalMillis == 0 || lastSaved + Config.hostSaveIvalMillis <= now) { + try { + Path file = Config.cfgDir.resolve(hostname + fileExt); + LOG.debug("saving {}", file); + IoUtils.safeWriteTo(file, Config.gson.toJson(this)); + lastSaved = now; + dirty = false; + } catch (IOException e) { + LOG.error("", e); + } + } + } + + @Override + public void shutdown() { + if (dirty) { + save(true); + } + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("HostState [services=").append(services).append("]"); + return builder.toString(); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ReportingStatus.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ReportingStatus.java new file mode 100755 index 0000000..bf831ed --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ReportingStatus.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +public class ReportingStatus { + + public String lastContentSent; + public int lastStatusReported; // currently only either 0 or 2 + public long lastSent; + public boolean sendFailure; + + public ReportingStatus() { + lastContentSent = null; + lastStatusReported = 0; + lastSent = 0; + sendFailure = false; + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceDef.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceDef.java new file mode 100755 index 0000000..30a27f3 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceDef.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; +import static com.google.common.base.Preconditions.checkArgument; + +import java.time.Duration; + +public class ServiceDef { + + public String name; + public String check; // the check to use, ie. "CertExpiryCheck", case-insensitive + public String conf; // check conf (may be null) + public boolean passive = false; + + // for passive checks: + public Duration timeout; + + // for active checks: + public Duration checkIval; + public Duration retryIval; + public Integer tries; + + public HostDef hostDef; + + public void validate() { + checkArgument(name != null && !name.isBlank() && name.trim().equals(name)); + if (passive) { + checkArgument(timeout != null && !timeout.isNegative()); + } else { + checkArgument(check != null); + checkArgument(checkIval != null && !checkIval.isNegative()); + checkArgument(retryIval != null && !retryIval.isNegative()); + checkArgument(tries > 0); + } + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ServiceDef [name=").append(name).append(", check=").append(check).append(", conf=").append(conf) + .append(", passive=").append(passive).append(", timeout=").append(timeout).append(", checkIval=") + .append(checkIval).append(", retryIval=").append(retryIval).append(", tries=").append(tries) + .append("]"); + return builder.toString(); + } + + public void fillInDefaults(Defaults defaults) { + if (timeout == null) { + timeout = defaults.timeout; + } + if (checkIval == null) { + checkIval = defaults.checkIval; + } + if (retryIval == null) { + retryIval = defaults.retryIval; + } + if (tries == null) { + tries = defaults.tries; + } + } + + public String id() { + return f("%s@%s", name, hostDef.hostname); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceState.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceState.java new file mode 100755 index 0000000..1a37116 --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceState.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; +import static com.google.common.base.Preconditions.checkArgument; + +public class ServiceState { + + public static final String STATUS_STR_OK = "OK"; + public static final int STATUS_CODE_OK = 0; + public static final String STATUS_STR_WARN = "WARN"; + public static final int STATUS_CODE_WARN = 1; + public static final String STATUS_STR_ERROR = "ERROR"; + public static final int STATUS_CODE_ERROR = 2; + + public int status = 0; + public String msg = ""; + public long millisSinceEpoch = 0; + public int tries = 0; // ignored for passive checks + + public transient ServiceDef def; + public transient HostState hostState; + + public ServiceState(ServiceDef def, HostState hostState) { + this.def = def; + this.hostState = hostState; + } + + public long nextExec() { + if (def.passive) { + if (status < 2) { + return millisSinceEpoch + def.timeout.getSeconds() * 1000L + 100L; + } else { + return System.currentTimeMillis() + 7 * 24 * 3600 * 1000L; + } + } + + if (status == 0 || tries >= def.tries) { + return millisSinceEpoch + def.checkIval.getSeconds() * 1000L; + } + + return millisSinceEpoch + def.retryIval.getSeconds() * 1000L; + } + + public String asReport() { + return f("%s - %s - %s - %s%n", def.hostDef.hostname, def.name, getStatusString(), msg); + } + + private String getStatusString() { + if (status == STATUS_CODE_OK) { + return STATUS_STR_OK; + } + else if (status == STATUS_CODE_WARN) { + return STATUS_STR_WARN; + } + else { + return STATUS_STR_ERROR; + } + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ServiceState [status=").append(status).append(", msg=").append(msg) + .append(", millisSinceEpoch=").append(millisSinceEpoch).append(", tries=").append(tries).append(", def=") + .append(def).append("]"); + return builder.toString(); + } + + public boolean isTimeout() { + checkArgument(def.passive); + return millisSinceEpoch + def.timeout.getSeconds() * 1000L < System.currentTimeMillis(); + } +} diff --git a/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceStateXfer.java b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceStateXfer.java new file mode 100755 index 0000000..437315f --- /dev/null +++ b/jmon/src/main/java/com/github/jjYBdx4IL/jmon/dto/ServiceStateXfer.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon.dto; + +import static com.google.common.base.Preconditions.checkArgument; + +// for passive service status transmission: +public class ServiceStateXfer { + + public int state; + public String msg; + public String service; + + public void validate() { + checkArgument(state >= 0); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ServiceStateXfer [state=").append(state).append(", msg=").append(msg).append(", service=") + .append(service).append("]"); + return builder.toString(); + } +} diff --git a/jmon/src/main/resources/simplelogger.properties b/jmon/src/main/resources/simplelogger.properties new file mode 100755 index 0000000..493c05a --- /dev/null +++ b/jmon/src/main/resources/simplelogger.properties @@ -0,0 +1,7 @@ +org.slf4j.simpleLogger.defaultLogLevel=info +org.slf4j.simpleLogger.log.com.github.jjYBdx4IL.jmon=debug +org.slf4j.simpleLogger.showDateTime=false +org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z +org.slf4j.simpleLogger.showThreadName=true +org.slf4j.simpleLogger.showLogName=false +org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/jmon/src/test/java/com/github/jjYBdx4IL/jmon/RunServerTest.java b/jmon/src/test/java/com/github/jjYBdx4IL/jmon/RunServerTest.java new file mode 100755 index 0000000..439ec37 --- /dev/null +++ b/jmon/src/test/java/com/github/jjYBdx4IL/jmon/RunServerTest.java @@ -0,0 +1,515 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.github.jjYBdx4IL.jmon.dto.HostState; +import com.github.jjYBdx4IL.jmon.dto.ServiceState; +import com.google.gson.Gson; + +import org.apache.commons.io.FileUtils; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Request.Content; +import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic; +import org.eclipse.jetty.client.util.StringRequestContent; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.io.ClientConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.Thread.UncaughtExceptionHandler; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLHandshakeException; + +public class RunServerTest { + + private static final Logger LOG = LoggerFactory.getLogger(RunServerTest.class); + + public static final String url = "https://localhost:8443/"; + public static final String CLIENT_KEY_ALIAS = "client"; + public static final String CLIENT_KEY_ALIAS_UNKNOWN = "unknownclient"; + public static final String SERVER_KEY_ALIAS = "server"; + + public static final Path tgtDir = Paths.get(System.getProperty("project.build.directory", "./target")); + + private static final Gson gson = Utils.createGson(); + + Path cfgDir; + Path serverKs; + Path serverTs; + Path clientKs; + Path clientTs; + Path sslTestServerKs; + Path sslTestServerTs; + Path sslTestServerExpiredKs; + Path sslTestServerExpiredTs; + static int testCounter = 0; + + public Thread t = null; + public Main main = null; + public RunServer m = null; + public Throwable caught = null; + final BlockingQueue mails = new LinkedBlockingQueue<>(); + + public void keytool(Path store, String alias, String cmdline) throws InterruptedException, IOException { + List args = new ArrayList<>(); + args.add("keytool"); + args.add("-keystore"); + args.add(store.toString()); + args.add("-storepass"); + args.add("password"); + args.add("-alias"); + args.add(alias); + args.add("-noprompt"); + args.addAll(Arrays.asList(cmdline.split("\\s+"))); + assertEquals(0, new ProcessBuilder(args).inheritIO().directory(tgtDir.toFile()).start().waitFor()); + } + + public void setupKeystores() throws Exception { + if (Files.exists(serverKs)) { + return; + } + + // generate (jmon) server key and server keystore + keytool(serverKs, SERVER_KEY_ALIAS, "-genkey -dname CN=localhost -keyalg rsa"); + keytool(serverKs, SERVER_KEY_ALIAS, "-export -file cert"); + keytool(serverTs, "server-cert", "-import -file cert"); + keytool(clientTs, "server-cert", "-import -file cert"); + + // generate client key and client keystore for passive check submissions + keytool(clientKs, CLIENT_KEY_ALIAS, "-genkey -dname CN=client -keyalg rsa"); + keytool(clientKs, CLIENT_KEY_ALIAS, "-export -file cert"); + keytool(serverTs, "client-cert", "-import -file cert"); + + // client unknown to jmon: + keytool(clientKs, CLIENT_KEY_ALIAS_UNKNOWN, "-genkey -dname CN=unknownclient -keyalg rsa"); + + // "remote" ssl test server to run active checks against: + keytool(sslTestServerKs, "testserver", "-genkey -dname CN=localhost -keyalg rsa"); + keytool(sslTestServerKs, "testserver", "-export -file cert"); + keytool(serverTs, "testserver-cert", "-import -file cert"); + + // "remote" ssl test server to run active checks against: + keytool(sslTestServerExpiredKs, "testserver", "-genkey -dname CN=localhost -keyalg rsa -validity 1"); + keytool(sslTestServerExpiredKs, "testserver", "-export -file cert"); + keytool(serverTs, "testserverExpired-cert", "-import -file cert"); + } + + @Before + public void before() throws Exception { + cfgDir = tgtDir.resolve("test" + testCounter++); + serverKs = tgtDir.resolve("keystore"); + serverTs = tgtDir.resolve("truststore"); + clientKs = tgtDir.resolve("keystore_client"); + clientTs = tgtDir.resolve("truststore_client"); + sslTestServerKs = tgtDir.resolve("keystore_testserver"); + ; + sslTestServerTs = null; + sslTestServerExpiredKs = tgtDir.resolve("keystore_testserver_expired"); + ; + sslTestServerExpiredTs = null; + + setupKeystores(); + + FileUtils.deleteDirectory(cfgDir.toFile()); + Files.createDirectories(cfgDir); + for (Path p : new Path[] { serverKs, serverTs }) { + Files.copy(p, cfgDir.resolve(p.toFile().getName())); + } + } + + public void start(String[] args) throws Exception { + TestRequestHandler.reset(); + SslTestServer.start(sslTestServerKs, sslTestServerTs); + + if (args == null) { + args = new String[] { "--runServer", "--cfgdir", cfgDir.toString() }; + } + // force saves on every service state update + Config.hostSaveIvalMillis = 0; + Config.port = 8443; + Config.reporter = new IProblemReporter() { + + @Override + public void send(String subject, String body) throws Exception { + mails.add(subject + "\n\n" + body); + } + }; + mails.clear(); + + final String[] args2 = args; + + main = new Main(); + t = new Thread("main-forked") { + public void run() { + try { + main.run(args2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + }; + caught = null; + t.setUncaughtExceptionHandler(new UncaughtExceptionHandler() { + + @Override + public void uncaughtException(Thread t, Throwable e) { + caught = e; + } + }); + t.start(); + main.startup.await(); + if (main.m != null && main.m instanceof RunServer) { + m = (RunServer) main.m; + } else { + m = null; + } + LOG.info("m = {}", m); + Thread.sleep(1000); + } + + @After + public void after() throws Exception { + + if (m != null) { + int dataSize = 1024 * 1024; + + System.out.println("Used Memory : " + + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / dataSize + " MB"); + System.out.println("Free Memory : " + Runtime.getRuntime().freeMemory() / dataSize + " MB"); + System.out.println("Total Memory : " + Runtime.getRuntime().totalMemory() / dataSize + " MB"); + System.out.println("Max Memory : " + Runtime.getRuntime().maxMemory() / dataSize + " MB"); + + System.out.println("** After GC: **"); + Runtime.getRuntime().gc(); + + System.out.println("Used Memory : " + + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / dataSize + " MB"); + System.out.println("Free Memory : " + Runtime.getRuntime().freeMemory() / dataSize + " MB"); + System.out.println("Total Memory : " + Runtime.getRuntime().totalMemory() / dataSize + " MB"); + System.out.println("Max Memory : " + Runtime.getRuntime().maxMemory() / dataSize + " MB"); + + m.server.stop(); + main.shutdown.await(); + t.join(); + t = null; + m = null; + main = null; + } + + SslTestServer.server.stop(); + } + + @Test + public void testNoHostDefinitionsError() throws Exception { + start(null); + t.join(); + assertTrue(caught.getMessage().contains("no host definitions found")); + } + + @Test + public void testPassiveCheckResultSubmission() throws Exception { + writePassiveHostDef(); + start(new String[] { "--runServer", "--cfgdir", cfgDir.toString(), "--ignoreNoActiveChecks" }); + + // there are no passive check submissions yet, so we immediately get an + // error for the service: + HostState hs = parseState("client"); + assertEquals(1, hs.services.size()); + assertEquals(2, hs.services.values().iterator().next().status); + + assertHttpStatus(200, validPassiveCheckSubmitJson); + hs = parseState("client"); + assertEquals(1, hs.services.size()); + assertEquals(0, hs.services.values().iterator().next().status); + } + + @Test + public void testInvalidPassiveCheckResultSubmission() throws Exception { + writePassiveHostDef(); + start(new String[] { "--runServer", "--cfgdir", cfgDir.toString(), "--ignoreNoActiveChecks" }); + + parseState("client"); + + assertHttpStatus(500, invalidPassiveCheckSubmitJson1); + + try { + parseState("client"); + fail(); + } catch (NoSuchFileException ex) { + } + + assertHttpStatus(500, invalidPassiveCheckSubmitJson2); + + try { + parseState("client"); + fail(); + } catch (NoSuchFileException ex) { + } + } + + @Test + public void testPassiveCheckResultSubmissionSslHandshakeException() throws Exception { + writePassiveHostDef(); + start(new String[] { "--runServer", "--cfgdir", cfgDir.toString(), "--ignoreNoActiveChecks" }); + + try { + fetchResponse(validPassiveCheckSubmitJson, CLIENT_KEY_ALIAS_UNKNOWN); + fail(); + } catch (ExecutionException ex) { + assertTrue(ex.getCause() instanceof SSLHandshakeException); + } + } + + @Test + public void testHelp() throws Exception { + start(new String[] { "--help" }); + } + + @Test + public void testHelpConfig() throws Exception { + start(new String[] { "--helpConfig" }); + } + + @Test + public void testReportingNoMessage() throws Exception { + writeActiveHostDef("urlcheck", f("{truststore:true,port:%d}", SslTestServer.port)); + start(null); + + HostState hs = parseState("localhost"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(0, s.status); + + assertTrue(mails.isEmpty()); + } + + @Test + public void testReportingErrorMessage() throws Exception { + writeActiveHostDef("urlcheck", f("{content=\"Test-Content\",truststore:true,port:%d}", SslTestServer.port), + ",\"tries\":1,\"retryIval\":\"2s\""); + start(null); + + HostState hs = parseState("localhost"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(2, s.status); + + String report = mails.poll(10, TimeUnit.SECONDS); + + assertTrue(report, report.contains("localhost - testservice1 - ERROR - ")); + } + + @Test + public void testReportingRecoveryMessage() throws Exception { + writeActiveHostDef("urlcheck", f("{content=\"Test-Content\",truststore=true,port:%d}", SslTestServer.port), + ",\"tries\":1,\"checkIval\":\"1s\""); + start(null); + + HostState hs = parseState("localhost"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(2, s.status); + + String report = mails.poll(10, TimeUnit.SECONDS); + assertTrue(report, report.contains("localhost - testservice1 - ERROR - ")); + + TestRequestHandler.TEST_CONTENT = "Test-Content"; + + report = mails.poll(10, TimeUnit.SECONDS); + assertTrue(report, report.contains("All clear!")); + } + + @Test + public void testUrlCheck() throws Exception { + writeActiveHostDef("urlcheck", f("{truststore:true,port:%d}", SslTestServer.port)); + start(null); + + HostState hs = parseState("localhost"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(0, s.status); + } + + @Test + public void testCertExpiryGoogleCom() throws Exception { + writeActiveHostDef("www.google.com", "certexpirycheck", null, ""); + start(null); + + HostState hs = parseState("www.google.com"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(0, s.status); + } + + @Test + public void testCertExpiryCheck() throws Exception { + writeActiveHostDef("certexpirycheck", f("{truststore:true,port:%d}", SslTestServer.port)); + start(null); + + HostState hs = parseState("localhost"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(0, s.status); + } + + @Test + public void testCertExpiryCheckError() throws Exception { + sslTestServerKs = sslTestServerExpiredKs; + sslTestServerTs = sslTestServerExpiredTs; + writeActiveHostDef("certexpirycheck", f("{truststore:true,port:%d}", SslTestServer.port)); + start(null); + + HostState hs = parseState("localhost"); + assertEquals(1, hs.services.size()); + ServiceState s = hs.services.values().iterator().next(); + assertEquals(2, s.status); + } + + @Test + public void testSslTestServer() throws Exception { + SslTestServer.start(sslTestServerKs, sslTestServerTs); + + HttpClient c = getJmonClient(); + assertEquals(200, c.GET(f("https://localhost:%d/", SslTestServer.port)).getStatus()); + } + + public void writeActiveHostDef(String checkName, String conf) throws IOException { + writeActiveHostDef(checkName, conf, ""); + } + + public void writeActiveHostDef(String checkName, String conf, String append) throws IOException { + writeActiveHostDef("localhost", checkName, conf, append); + } + + public void writeActiveHostDef(String hostname, String checkName, String conf, String append) throws IOException { + Files.writeString(cfgDir.resolve(f("%s.def", hostname)), f(" {\n" + + " \"services\": {\n" + + " \"testservice1\": {\n" + + " \"check\": \"%s\",\n" + + " \"conf\": %s\n%s" + + " }\n" + + " }\n" + + " }", checkName, gson.toJson(conf), append)); + } + + public static final String invalidPassiveCheckSubmitJson1 = "{\"state\":\"0\",\"msg\":\"\",\"service\":\"\"}"; + public static final String invalidPassiveCheckSubmitJson2 = "{\"state\":\"a\",\"msg\":\"\",\"service\":\"\"}"; + public static final String validPassiveCheckSubmitJson = "{\"state\":0,\"msg\":\"testmsg\",\"service\":\"passivetestservice1\"}"; + + public void writePassiveHostDef() throws IOException { + Files.writeString(cfgDir.resolve("client.def"), f(" {\n" + + " \"services\": {\n" + + " \"passivetestservice1\": {\n" + + " \"passive\": true\n" + + " }\n" + + " }\n" + + " }")); + } + + private void assertHttpStatus(int expectedHttpStatus, String stateMsg) throws Exception { + assertHttpStatus(expectedHttpStatus, stateMsg, CLIENT_KEY_ALIAS); + } + + private void assertHttpStatus(int expectedHttpStatus, String stateMsg, String keyAlias) throws Exception { + ContentResponse response = fetchResponse(stateMsg, keyAlias); + assertEquals(expectedHttpStatus, response.getStatus()); + + } + + private ContentResponse fetchResponse(String stateMsg, String keyAlias) throws Exception { + HttpClient c = getClient(keyAlias); + Content content = new StringRequestContent(RequestHandler.JSON_TYPE, stateMsg, UTF_8); + return c.newRequest(url).method(HttpMethod.POST).body(content).send(); + } + + private HostState parseState(String hostname) throws IOException { + Path stateFile = cfgDir.resolve(hostname + ".state"); + String s = Files.readString(stateFile); + Files.delete(stateFile); + try { + return gson.fromJson(s, HostState.class); + } catch (Exception ex) { + LOG.error("json: {}", s); + throw ex; + } + } + + public HttpClient getClient() throws Exception { + return getClient("client"); + } + + public HttpClient getClient(String keyAlias) throws Exception { + SslContextFactory.Client ssl = new SslContextFactory.Client(); + + ssl.setValidatePeerCerts(true); + ssl.setTrustStorePath(clientTs.toString()); + ssl.setTrustStorePassword("password"); + + ssl.setCertAlias(keyAlias); + ssl.setKeyStorePath(clientKs.toString()); + ssl.setKeyStorePassword("password"); + + ClientConnector clientConnector = new ClientConnector(); + clientConnector.setSslContextFactory(ssl); + + HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + httpClient.start(); + return httpClient; + } + + // client that identifies as if it were JMon itself + public HttpClient getJmonClient() throws Exception { + SslContextFactory.Client ssl = new SslContextFactory.Client(); + + ssl.setValidatePeerCerts(true); + ssl.setTrustStorePath(serverTs.toString()); + ssl.setTrustStorePassword("password"); + + ssl.setCertAlias(SERVER_KEY_ALIAS); + ssl.setKeyStorePath(serverKs.toString()); + ssl.setKeyStorePassword("password"); + + ClientConnector clientConnector = new ClientConnector(); + clientConnector.setSslContextFactory(ssl); + + HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + httpClient.start(); + return httpClient; + } +} diff --git a/jmon/src/test/java/com/github/jjYBdx4IL/jmon/SslTestServer.java b/jmon/src/test/java/com/github/jjYBdx4IL/jmon/SslTestServer.java new file mode 100755 index 0000000..c3c7f68 --- /dev/null +++ b/jmon/src/test/java/com/github/jjYBdx4IL/jmon/SslTestServer.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; + +public class SslTestServer { + + static final Logger LOG = LoggerFactory.getLogger(SslTestServer.class); + + public static final int port = 8441; + + public static Server server = new Server(); + + public static void start(Path ks, Path ts) throws Exception { + HttpConfiguration https = new HttpConfiguration(); + https.addCustomizer(new SecureRequestCustomizer()); + SslContextFactory.Server ssl = new SslContextFactory.Server(); + + ssl.setCertAlias("testserver"); + ssl.setKeyStorePath(ks.toString()); + ssl.setKeyStorePassword("password"); + + ssl.setNeedClientAuth(false); // assume checking publicly visible targets + if (ts != null) { + ssl.setTrustStorePath(ts.toString()); + ssl.setTrustStorePassword("password"); + } + + ServerConnector sslConnector = new ServerConnector(server, new SslConnectionFactory(ssl, "http/1.1"), + new HttpConnectionFactory(https)); + sslConnector.setPort(port); + + server.setConnectors(new Connector[] { sslConnector }); + + server.setHandler(new TestRequestHandler()); + + server.start(); + } +} diff --git a/jmon/src/test/java/com/github/jjYBdx4IL/jmon/TestRequestHandler.java b/jmon/src/test/java/com/github/jjYBdx4IL/jmon/TestRequestHandler.java new file mode 100755 index 0000000..13e294f --- /dev/null +++ b/jmon/src/test/java/com/github/jjYBdx4IL/jmon/TestRequestHandler.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.jmon; + +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintWriter; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public final class TestRequestHandler extends AbstractHandler { + + private static final Logger LOG = LoggerFactory.getLogger(TestRequestHandler.class); + + public static String TEST_CONTENT = ""; + + public static void reset() { + TEST_CONTENT = ""; + } + + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + + LOG.info("{}", request); + LOG.info(" target: {}", target); + + response.setContentType("text/html; charset=utf-8"); + response.setStatus(HttpServletResponse.SC_OK); + + PrintWriter out = response.getWriter(); + + if ("GET".equals(request.getMethod())) { + out.print("

ehlo

"); + out.print(TEST_CONTENT); + } else { + throw new RuntimeException("unsupported method: " + request.getMethod()); + } + + baseRequest.setHandled(true); + } +} \ No newline at end of file diff --git a/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/IgnoreTestExceptionsRule1Test.java b/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/IgnoreTestExceptionsRule1Test.java index 1345563..45cffc4 100644 --- a/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/IgnoreTestExceptionsRule1Test.java +++ b/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/IgnoreTestExceptionsRule1Test.java @@ -19,8 +19,6 @@ import static org.junit.Assert.*; -import com.github.jjYBdx4IL.utils.junit4.IgnoreTestExceptionsRule; - import org.junit.Rule; import org.junit.Test; diff --git a/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/RetryRuleFailExpectedTest.java b/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/RetryRuleFailExpectedTest.java index 9b21294..9c30ca7 100644 --- a/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/RetryRuleFailExpectedTest.java +++ b/junit4-utils/src/test/java/com/github/jjYBdx4IL/utils/junit4/RetryRuleFailExpectedTest.java @@ -17,8 +17,6 @@ import static org.junit.Assert.*; -import com.github.jjYBdx4IL.utils.junit4.RetryRule; - import org.junit.Rule; import org.junit.Test; diff --git a/logging-utils/pom.xml b/logging-utils/pom.xml index 6158cb4..749447b 100644 --- a/logging-utils/pom.xml +++ b/logging-utils/pom.xml @@ -18,6 +18,17 @@ logging helpers 2014 + + + + maven-surefire-plugin + + 5 + + + + + org.eclipse.jetty @@ -28,13 +39,13 @@ junit junit - 4.12 + 4.13.2 test org.slf4j slf4j-simple - 1.7.25 + 1.7.30 test diff --git a/logging-utils/src/test/java/com/github/jjYBdx4IL/utils/logging/JavaUtilLoggingUtilsTest.java b/logging-utils/src/test/java/com/github/jjYBdx4IL/utils/logging/JavaUtilLoggingUtilsTest.java index a30b6b4..d6d8853 100644 --- a/logging-utils/src/test/java/com/github/jjYBdx4IL/utils/logging/JavaUtilLoggingUtilsTest.java +++ b/logging-utils/src/test/java/com/github/jjYBdx4IL/utils/logging/JavaUtilLoggingUtilsTest.java @@ -63,6 +63,7 @@ public void testSetJavaNetURLConsoleLoggingLevel() throws Exception { } countDownLatch.await(); + Thread.sleep(10L); assertTrue(systemErrRule.getLog().contains(token)); } diff --git a/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTest.java b/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTest.java index ae03c66..9cc5cfd 100644 --- a/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTest.java +++ b/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTest.java @@ -15,13 +15,12 @@ */ package com.github.jjYBdx4IL.utils.math; -//CHECKSTYLE:OFF -import java.util.Arrays; -import java.util.Collections; +import static org.junit.Assert.assertEquals; -import static org.junit.Assert.*; import org.junit.Test; -import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; /** * @@ -29,8 +28,6 @@ */ public class ShuffleTest extends ShuffleTestBase { - private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(ShuffleTest.class); - @Test public void testShuffle2() { Integer[] input = new Integer[3]; @@ -48,7 +45,6 @@ public void testShuffle2() { */ @Test public void testShuffleQuality() { - LOG.info("testShuffleQuality"); // perfect quality (transition matrix shows even distribution) float delta = computeDelta(new ShuffleRunner() { @@ -62,7 +58,6 @@ public void shuffle(Integer[] t) { @Test public void testShuffle2Quality() { - LOG.info("testShuffle2Quality"); // not so good quality (transition matrix shows a larger weight on diagonal elements, which means // that shuffle2 tends to leave array elements at their position. float delta = computeDelta(new ShuffleRunner() { @@ -77,7 +72,6 @@ public void shuffle(Integer[] t) { @Test public void testArraysShuffleQuality() { - LOG.info("testArraysShuffleQuality"); // perfect quality (transition matrix shows even distribution) float delta = computeDelta(new ShuffleRunner() { diff --git a/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTestBase.java b/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTestBase.java index b05da38..8076e7d 100644 --- a/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTestBase.java +++ b/math-utils/src/test/java/com/github/jjYBdx4IL/utils/math/ShuffleTestBase.java @@ -72,11 +72,11 @@ protected float computeDelta(ShuffleRunner runner, int arraySize, float loopCount++; } while (loopCount < 2 || deltaVariation > maxDeltaVariation); long duration = System.currentTimeMillis() - start; - LOG.info("================="); - LOG.info(String.format("delta: %f, shuffle(T[]) calls per second: %d", delta, + LOG.debug("================="); + LOG.debug(String.format("delta: %f, shuffle(T[]) calls per second: %d", delta, loopCount * iterationsPerLoop * 1000L / duration)); for (long[] ia : counts) { - LOG.info(Arrays.toString(ia)); + LOG.debug(Arrays.toString(ia)); } return delta; diff --git a/net-utils/pom.xml b/net-utils/pom.xml index 566502f..680e092 100644 --- a/net-utils/pom.xml +++ b/net-utils/pom.xml @@ -32,7 +32,7 @@ org.apache.commons commons-lang3 - 3.8.1 + 3.12.0 diff --git a/net-utils/src/main/java/com/github/jjYBdx4IL/utils/net/NetIoUtils.java b/net-utils/src/main/java/com/github/jjYBdx4IL/utils/net/NetIoUtils.java index 347f282..4ec7397 100644 --- a/net-utils/src/main/java/com/github/jjYBdx4IL/utils/net/NetIoUtils.java +++ b/net-utils/src/main/java/com/github/jjYBdx4IL/utils/net/NetIoUtils.java @@ -161,7 +161,6 @@ private static Charset getCharset(HttpEntity entity) throws IOException { private static Charset getCharset(String contentType) throws IOException { String[] values = contentType.split(";"); // values.length should be 2 - String charset = ""; for (String value : values) { value = value.trim(); diff --git a/net-utils/src/test/java/com/github/jjYBdx4IL/utils/net/AddressUtilsTest.java b/net-utils/src/test/java/com/github/jjYBdx4IL/utils/net/AddressUtilsTest.java index 3f982e9..a30e975 100644 --- a/net-utils/src/test/java/com/github/jjYBdx4IL/utils/net/AddressUtilsTest.java +++ b/net-utils/src/test/java/com/github/jjYBdx4IL/utils/net/AddressUtilsTest.java @@ -15,32 +15,29 @@ */ package com.github.jjYBdx4IL.utils.net; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + //CHECKSTYLE:OFF import com.github.jjYBdx4IL.utils.env.CI; +import org.junit.Assume; +import org.junit.Ignore; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.InetAddress; import java.net.UnknownHostException; -import static org.junit.Assert.*; - -import org.apache.commons.lang.StringUtils; -import org.junit.Assume; - /** * * @author Github jjYBdx4IL Projects */ public class AddressUtilsTest { - private static final Logger LOG = LoggerFactory.getLogger(AddressUtilsTest.class); - public AddressUtilsTest() { } + @Ignore @Test public void testGetNonLocalHostIPAddress() throws Exception { // don't run this test where we can't control host configurations diff --git a/osdapp/src/main/java/com/github/jjYBdx4IL/utils/osdapp/ZFSPoller.java b/osdapp/src/main/java/com/github/jjYBdx4IL/utils/osdapp/ZFSPoller.java index 1904295..5f410d5 100644 --- a/osdapp/src/main/java/com/github/jjYBdx4IL/utils/osdapp/ZFSPoller.java +++ b/osdapp/src/main/java/com/github/jjYBdx4IL/utils/osdapp/ZFSPoller.java @@ -15,9 +15,11 @@ */ package com.github.jjYBdx4IL.utils.osdapp; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.github.jjYBdx4IL.parser.ParseException; -import com.github.jjYBdx4IL.parser.linux.ZFSStatusParser; -import com.github.jjYBdx4IL.parser.linux.ZFSStatusParser.Result; +import com.github.jjYBdx4IL.parser.linux.ZfsStatusParser; +import com.github.jjYBdx4IL.parser.linux.ZfsStatusParser.Result; import com.github.jjYBdx4IL.utils.net.WakeOnLAN; import org.apache.commons.io.IOUtils; @@ -60,7 +62,7 @@ private void poll() { pb.redirectErrorStream(true); pb.redirectOutput(ProcessBuilder.Redirect.PIPE); Process p = pb.start(); - String zpoolStatusCommandOutput = IOUtils.toString(p.getInputStream()); + String zpoolStatusCommandOutput = IOUtils.toString(p.getInputStream(), UTF_8); p.waitFor(); if (p.exitValue() != 0) { throw new IOException("zpool status command returned bad exit code " + p.exitValue() @@ -71,7 +73,7 @@ private void poll() { LOG.trace(zpoolStatusCommandOutput); } - result = ZFSStatusParser.parse(zpoolStatusCommandOutput); + result = ZfsStatusParser.parse(zpoolStatusCommandOutput); if (LOG.isDebugEnabled()) { LOG.debug(result.toString()); } diff --git a/osdapp/src/test/java/com/github/jjYBdx4IL/utils/osdapp/OSDAppIT.java b/osdapp/src/test/java/com/github/jjYBdx4IL/utils/osdapp/OSDAppIT.java index 1e2afcd..633bc42 100644 --- a/osdapp/src/test/java/com/github/jjYBdx4IL/utils/osdapp/OSDAppIT.java +++ b/osdapp/src/test/java/com/github/jjYBdx4IL/utils/osdapp/OSDAppIT.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; -import com.github.jjYBdx4IL.parser.linux.ZFSStatusParserTest; +import com.github.jjYBdx4IL.parser.linux.ZfsStatusParserTest; import com.github.jjYBdx4IL.utils.env.Maven; import com.github.jjYBdx4IL.utils.gfx.ImageUtils; import com.github.jjYBdx4IL.utils.io.FindUtils; @@ -198,7 +198,7 @@ protected int runOSD( } sudoFile.setExecutable(true); try (OutputStream os = new FileOutputStream(zpoolStatusOutput)) { - IOUtils.copy(ZFSStatusParserTest.class.getResourceAsStream(zfsRes), os); + IOUtils.copy(ZfsStatusParserTest.class.getResourceAsStream(zfsRes), os); } File diskStandbyScriptFile = new File(targetDir, "disk-standby.sh"); diff --git a/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParser.java b/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParser.java new file mode 100755 index 0000000..a54b5f5 --- /dev/null +++ b/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParser.java @@ -0,0 +1,124 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.parser.linux; + +import com.github.jjYBdx4IL.parser.ParseException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SmartCtlParser { + + protected static final Pattern PAT_DEV_SERIAL = Pattern.compile("\\nSerial Number:\\s+(\\S.*)"); + protected static final Pattern PAT_PWRONHRS = Pattern.compile("\\n\\s*9\\s+Power_On_Hours\\s+.*?\\s+(\\d+)\\n"); + protected static final Pattern PAT_shortTestRegex = Pattern + .compile("\\n#\\s*\\d+\\s+Short offline\\s+Completed without error\\s+00\\%\\s+(\\d+)\\s+-"); + protected static final Pattern PAT_longTestRegex = Pattern + .compile("\\n#\\s*\\d+\\s+Extended offline\\s+Completed without error\\s+00\\%\\s+(\\d+)\\s+-"); + + /** + * Parses output of smartctl -a /dev/sdX. + */ + public static Result parse(String input) throws ParseException { + Long shortTestAge = null; + Long longTestAge = null; + + final String infoSectionStr = getSection(input, "\n\n=== START OF INFORMATION SECTION ==="); + + Matcher m = PAT_DEV_SERIAL.matcher(infoSectionStr); + if (!m.find()) { + throw new ParseException("cannot find device serial"); + } + final String devSerial = m.group(1); + + // extract the device's internal time reference (power on hours) ... + + // + //SMART Attributes Data Structure revision number : 16 + // Vendor Specific SMART Attributes with Thresholds : + //ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + // 1 Raw_Read_Error_Rate 0x002f 200 200 051 Pre - fail Always - 0 + //... + // 9 Power_On_Hours 0x0032 100 100 000 Old_age Always - 10 + //... + // + + final String attrsSectionStr = getSection(input, "\n\nSMART Attributes Data Structure revision number"); + if (attrsSectionStr.isEmpty()) { + throw new ParseException("cannot find SMART Attributes section for " + devSerial); + } + + m = PAT_PWRONHRS.matcher(attrsSectionStr); + if (!m.find()) { + throw new ParseException("cannot find Power_On_Hours SMART Attribute for " + devSerial); + } + + final long pwrOnHrs = Long.parseLong(m.group(1)); + + // ... and compare it against last successful completions of both short and long self-tests: + + // + //SMART Self-test log structure revision number 1 + //Num Test_Description Status Remaining LifeTime(hours) LBA_of_first_error + //# 1 Short offline Completed without error 00 % 0 - + // + final String selfTestLogStr = getSection(input, "\n\nSMART Self-test log structure"); + if (selfTestLogStr.isEmpty()) { + throw new ParseException("cannot find Self-test log structure output for " + devSerial); + } + + m = PAT_shortTestRegex.matcher(selfTestLogStr); + if (m.find()) { + shortTestAge = pwrOnHrs - Long.parseLong(m.group(1)); + } + + m = PAT_longTestRegex.matcher(selfTestLogStr); + if (m.find()) { + longTestAge = pwrOnHrs - Long.parseLong(m.group(1)); + } + + return new Result(devSerial, pwrOnHrs, shortTestAge, longTestAge); + } + + protected static String getSection(String input, String start) { + int pos = input.indexOf(start); + if (pos == -1) { + return ""; + } + int endpos = input.indexOf("\n\n", pos + start.length()); + if (endpos == -1) { + return input.substring(pos); + } + return input.substring(pos, endpos); + } + + public static class Result { + public final String devSerial; + public final long pwrOnHrs; + public final Long shortTestAgeHrs; + public final Long longTestAgeHrs; + + /** + * The parse result. + */ + public Result(String devSerial, long pwrOnHrs, Long shortTestAgeHrs, Long longTestAgeHrs) { + this.devSerial = devSerial; + this.pwrOnHrs = pwrOnHrs; + this.shortTestAgeHrs = shortTestAgeHrs; + this.longTestAgeHrs = longTestAgeHrs; + } + } +} diff --git a/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParser.java b/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParser.java index 7d12b9a..7b04da1 100644 --- a/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParser.java +++ b/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParser.java @@ -15,68 +15,85 @@ */ package com.github.jjYBdx4IL.parser.linux; -//CHECKSTYLE:OFF -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; import com.github.jjYBdx4IL.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Locale; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** + * ZFS status output parser. * * @author Github jjYBdx4IL Projects */ -public class ZFSStatusParser { +public class ZfsStatusParser { - private static final Logger LOG = LoggerFactory.getLogger(ZFSStatusParser.class); + private static final Logger LOG = LoggerFactory.getLogger(ZfsStatusParser.class); private static final Pattern poolLinePattern = Pattern.compile("^ pool:"); private static final Pattern stateLinePattern = Pattern.compile("^ state:\\s*(\\S+)$"); private static final Pattern scrubOrScanLinePattern = Pattern.compile("^( scan| scrub):"); - private static final Pattern scrubOrScanActivityLinePattern = Pattern.compile("^( scan| scrub):.* in progress[, ]"); + private static final Pattern scrubOrScanActivityLinePattern = Pattern + .compile("^( scan| scrub):.* in progress[, ]"); + private static final Pattern inProgressSincePattern = Pattern + .compile(" in progress since [a-zA-Z]+ ([a-zA-Z]+ \\d+ \\d+:\\d+:\\d+ \\d+)"); private static final Pattern configLinePattern = Pattern.compile("^config:"); - private static final Pattern configWhitelistLinePattern - = Pattern.compile("^(\\s*|\\s+NAME\\s+STATE\\s+READ\\s+WRITE\\s+CKSUM\\s*|.*\\s+0\\s+0\\s+0\\s*)$"); + private static final Pattern configWhitelistLinePattern = Pattern.compile( + "^(\\s*|\\s+NAME\\s+STATE\\s+READ\\s+WRITE\\s+CKSUM\\s*|.*\\s+0\\s+0\\s+0\\s*(?:\\s+\\(resilvering\\))?)$"); private static final Pattern errorsLinePattern = Pattern.compile("^errors:"); + public static final DateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss yyyy", Locale.ROOT); + + static { + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + } + /** - * Transforms the output of the zpool status command into an alert/error level. - *
    - *
  • 0: OK (pool state: "ONLINE") - *
  • 1: degrading (pool state: "ONLINE"): not yet degraded, but single devices show correctable (due to - * redundancy) CRC, read or write errors without being offline. - *
  • 2: pool state: "DEGRADED" - *
  • 3: pool state: "UNAVAIL" or everything else - *
+ * Transforms the output of the zpool status command into an alert/error + * level.
  • 0: OK (pool state: "ONLINE")
  • 1: degrading (pool + * state: "ONLINE"): not yet degraded, but single devices show correctable + * (due to redundancy) CRC, read or write errors without being offline. + *
  • 2: pool state: "DEGRADED"
  • 3: pool state: "UNAVAIL" or everything + * else
* - * @param zpoolStatusCmdOutput the console output of the "zpool status" command + * @param zpoolStatusCmdOutput + * the console output of the "zpool status" command * @return the parsed {@link Result} - * @throws com.github.jjYBdx4IL.parser.ParseException if the input is not correct/not understoof by the parser + * @throws com.github.jjYBdx4IL.parser.ParseException + * if the input is not correct/not understoof by the parser */ public static Result parse(String zpoolStatusCmdOutput) throws ParseException { int numPools = 0; - // 0: " pool:" + // 0: " pool:" // 1: " state:" // 2:ignore everything until "scrub:" or "scan:" // 3: ignore everything until "config:" - // 4: look for read/write/checksum errors until "errors:" or " pool:" - // 5: ignore everything until EOF or " pool:" (->1) + // 4: look for read/write/checksum errors until "errors:" or " pool:" + // 5: ignore everything until EOF or " pool:" (->1) int state = 0; AlertLevel alertLevel = AlertLevel.OK; int lineNumber = 0; boolean scrubOrScanActivityPresent = false; + Instant oldestInProgressSince = null; for (String l : zpoolStatusCmdOutput.split("\r?\n")) { lineNumber++; if (LOG.isTraceEnabled()) { - LOG.trace(String.format("line %02d, state %d: %s", lineNumber, state, l)); + LOG.trace(f("line %02d, state %d: %s", lineNumber, state, l)); } AlertLevel level = AlertLevel.OK; switch (state) { case 0: if (!poolLinePattern.matcher(l).find()) { - throw new ParseException(createExceptionMessage(zpoolStatusCmdOutput, lineNumber, "zpool name")); + throw new ParseException( + createExceptionMessage(zpoolStatusCmdOutput, lineNumber, "zpool name")); } numPools++; state++; @@ -84,12 +101,14 @@ public static Result parse(String zpoolStatusCmdOutput) throws ParseException { case 1: Matcher m = stateLinePattern.matcher(l); if (!m.find()) { - throw new ParseException(createExceptionMessage(zpoolStatusCmdOutput, lineNumber, "zpool state")); + throw new ParseException( + createExceptionMessage(zpoolStatusCmdOutput, lineNumber, "zpool state")); } try { level = PoolState.valueOf(m.group(1)).getAlertLevel(); } catch (IllegalArgumentException ex) { - throw new ParseException(createExceptionMessage(zpoolStatusCmdOutput, lineNumber, "zpool state, unknown state: " + m.group(1))); + throw new ParseException(createExceptionMessage(zpoolStatusCmdOutput, lineNumber, + "zpool state, unknown state: " + m.group(1))); } state++; break; @@ -100,6 +119,17 @@ public static Result parse(String zpoolStatusCmdOutput) throws ParseException { if (scrubOrScanActivityLinePattern.matcher(l).find()) { scrubOrScanActivityPresent = true; } + m = inProgressSincePattern.matcher(l); + if (m.find()) { + try { + Instant inProgressSince = sdf.parse(m.group(1)).toInstant(); + if (oldestInProgressSince == null || inProgressSince.isBefore(oldestInProgressSince)) { + oldestInProgressSince = inProgressSince; + } + } catch (java.text.ParseException e) { + throw new ParseException("failed to parse date: " + l); + } + } break; case 3: if (configLinePattern.matcher(l).find()) { @@ -122,6 +152,8 @@ public static Result parse(String zpoolStatusCmdOutput) throws ParseException { state = 1; } break; + default: + throw new IllegalStateException(); } alertLevel = level.worseThan(alertLevel) ? level : alertLevel; } @@ -140,10 +172,14 @@ public static Result parse(String zpoolStatusCmdOutput) throws ParseException { if (state == 4) { throw new ParseException(createExceptionMessage(zpoolStatusCmdOutput, lineNumber, "zpool errors")); } - return new Result(alertLevel, numPools, scrubOrScanActivityPresent); + if (oldestInProgressSince != null && !scrubOrScanActivityPresent) { + throw new IllegalStateException(); + } + return new Result(alertLevel, numPools, scrubOrScanActivityPresent, oldestInProgressSince); } - private static String createExceptionMessage(String zpoolStatusCmdOutput, int errorLineNumber, String errorMessage) { + private static String createExceptionMessage(String zpoolStatusCmdOutput, int errorLineNumber, + String errorMessage) { StringBuilder sb = new StringBuilder(); int lineNumber = 0; for (String l : zpoolStatusCmdOutput.split(System.lineSeparator())) { @@ -164,12 +200,13 @@ private static String createExceptionMessage(String zpoolStatusCmdOutput, int er return sb.toString(); } - private ZFSStatusParser() { + private ZfsStatusParser() { } public enum AlertLevel { OK(0), DEGRADING(1), DEGRADED(2), FAILURE(3); + private final int numericLevel; AlertLevel(int numericLevel) { @@ -192,11 +229,13 @@ protected static AlertLevel getByNumericLevel(int numericLevel) { } throw new IllegalArgumentException("invalid numeric level: " + numericLevel); } - }; + } public static class Result { /** + * the alert level. + * * @return the alertLevel */ public AlertLevel getAlertLevel() { @@ -204,72 +243,102 @@ public AlertLevel getAlertLevel() { } /** + * number of pools. + * * @return the numPools */ public int getNumPools() { return numPools; } + /** + * is any activity going on? + * + *

This is somewhat redundant with {@link #getInProgressSince()}. + * + * @return the scrubOrScanActivityPresent + */ + public boolean isScrubOrScanActivityPresent() { + return scrubOrScanActivityPresent; + } + + /** + * For newer ZFS versions this returns the oldest start time of any + * ongoing operation (resilver/scrub/scan). + * + * @return in progress since-instant + */ + public Instant getInProgressSince() { + return inProgressSince; + } + private final AlertLevel alertLevel; private final int numPools; private final boolean scrubOrScanActivityPresent; + private final Instant inProgressSince; - public Result(AlertLevel alertLevel, int numPools, boolean scrubOrScanActivityPresent) { + /** + * Constructor. + */ + public Result(AlertLevel alertLevel, int numPools, boolean scrubOrScanActivityPresent, + Instant inProgressSince) { this.alertLevel = alertLevel; this.numPools = numPools; this.scrubOrScanActivityPresent = scrubOrScanActivityPresent; + this.inProgressSince = inProgressSince; + } + + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Result [alertLevel=").append(alertLevel).append(", numPools=").append(numPools) + .append(", scrubOrScanActivityPresent=").append(scrubOrScanActivityPresent).append(", inProgressSince=") + .append(inProgressSince).append("]"); + return builder.toString(); } @Override public int hashCode() { - int hash = 3; - hash = 71 * hash + Objects.hashCode(this.alertLevel); - hash = 71 * hash + this.numPools; - hash = 71 * hash + (this.scrubOrScanActivityPresent ? 1 : 0); - return hash; + final int prime = 31; + int result = 1; + result = prime * result + ((alertLevel == null) ? 0 : alertLevel.hashCode()); + result = prime * result + ((inProgressSince == null) ? 0 : inProgressSince.hashCode()); + result = prime * result + numPools; + result = prime * result + (scrubOrScanActivityPresent ? 1231 : 1237); + return result; } @Override public boolean equals(Object obj) { + if (this == obj) { + return true; + } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } - final Result other = (Result) obj; - if (this.alertLevel != other.alertLevel) { + Result other = (Result) obj; + if (alertLevel != other.alertLevel) { + return false; + } + if (inProgressSince == null) { + if (other.inProgressSince != null) { + return false; + } + } else if (!inProgressSince.equals(other.inProgressSince)) { return false; } - if (this.numPools != other.numPools) { + if (numPools != other.numPools) { return false; } - if (this.scrubOrScanActivityPresent != other.scrubOrScanActivityPresent) { + if (scrubOrScanActivityPresent != other.scrubOrScanActivityPresent) { return false; } return true; } - - /** - * @return the scrubOrScanActivityPresent - */ - public boolean isScrubOrScanActivityPresent() { - return scrubOrScanActivityPresent; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Result ["); - builder.append("alertLevel="); - builder.append(alertLevel); - builder.append(", numPools="); - builder.append(numPools); - builder.append(", scrubOrScanActivityPresent="); - builder.append(scrubOrScanActivityPresent); - builder.append("]"); - return builder.toString(); - } } private enum PoolState { diff --git a/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParser.java b/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParser.java new file mode 100755 index 0000000..e7f908f --- /dev/null +++ b/parser/src/main/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParser.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.parser.linux; + +import com.github.jjYBdx4IL.parser.ParseException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; + +public class ZfsPoolListParser { + + /** + * Parse output of zpool list -H -p. + */ + public static List parse(String input) throws ParseException { + try (Scanner s = new Scanner(input)) { + List r = new ArrayList<>(); + while (s.hasNextLine()) { + r.add(new Result(s.nextLine())); + } + return r; + } + } + + public static class Result { + public final String name; + public final long size; + public final long alloc; + public final long free; + public final String ckpoint; + public final String expandsz; + public final String frag; + public final String cap; + public final String dedup; + public final String health; + public final String altroot; + public final float freePct; + + /** + * The parse result. + */ + public Result(String line) throws ParseException { + String[] parts = line.split("\\t"); + if (parts.length != 11) { + throw new ParseException("bad format: " + line); + } + name = parts[0]; + size = Long.parseLong(parts[1]); + alloc = Long.parseLong(parts[2]); + free = Long.parseLong(parts[3]); + ckpoint = parts[4]; + expandsz = parts[5]; + frag = parts[6]; + cap = parts[7]; + dedup = parts[8]; + health = parts[9]; + altroot = parts[10]; + freePct = size == 0 ? 0.f : (100.f * free / size); + } + } +} diff --git a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcNetDevParserTest.java b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcNetDevParserTest.java index 5a883bd..6a4b0e0 100644 --- a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcNetDevParserTest.java +++ b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcNetDevParserTest.java @@ -15,12 +15,8 @@ */ package com.github.jjYBdx4IL.parser.linux; -//CHECKSTYLE:OFF import static org.junit.Assert.assertEquals; -import com.github.jjYBdx4IL.parser.linux.ProcNetDevData; -import com.github.jjYBdx4IL.parser.linux.ProcNetDevParser; - import org.junit.Test; /** diff --git a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatDataTest.java b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatDataTest.java index 06e225d..4c15781 100644 --- a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatDataTest.java +++ b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatDataTest.java @@ -15,17 +15,14 @@ */ package com.github.jjYBdx4IL.parser.linux; -//CHECKSTYLE:OFF -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; - import static org.junit.Assert.assertEquals; -import com.github.jjYBdx4IL.parser.linux.ProcPidStatData; - import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; + /** * * @author Github jjYBdx4IL Projects diff --git a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatusParserTest.java b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatusParserTest.java index 698cf6e..f24917c 100644 --- a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatusParserTest.java +++ b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ProcPidStatusParserTest.java @@ -15,17 +15,14 @@ */ package com.github.jjYBdx4IL.parser.linux; -//CHECKSTYLE:OFF -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; - import static org.junit.Assert.assertEquals; -import com.github.jjYBdx4IL.parser.linux.ProcPidStatusParser; - import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; + /** * * @author Github jjYBdx4IL Projects diff --git a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParserTest.java b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParserTest.java new file mode 100755 index 0000000..eaab7ee --- /dev/null +++ b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/SmartCtlParserTest.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.parser.linux; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import com.github.jjYBdx4IL.parser.linux.SmartCtlParser.Result; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +public class SmartCtlParserTest { + + @Test + public void testName() throws Exception { + String input = IOUtils.toString( + ZfsStatusParserTest.class.getResourceAsStream("smart_a_output_longok"), UTF_8); + Result r = SmartCtlParser.parse(input); + assertEquals("WD-WCC7K0YKSETV", r.devSerial); + assertEquals(1019, r.pwrOnHrs); + assertEquals(Long.valueOf(19), r.shortTestAgeHrs); + assertEquals(Long.valueOf(1), r.longTestAgeHrs); + + input = IOUtils.toString( + ZfsStatusParserTest.class.getResourceAsStream("smart_a_output_nolong"), UTF_8); + r = SmartCtlParser.parse(input); + assertEquals("WD-WCC7K2ATSZXJ", r.devSerial); + assertEquals(10, r.pwrOnHrs); + assertEquals(Long.valueOf(10), r.shortTestAgeHrs); + assertNull(r.longTestAgeHrs); + } +} diff --git a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParserTest.java b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParserTest.java index 1b66471..b7fb6b3 100644 --- a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParserTest.java +++ b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZFSStatusParserTest.java @@ -15,11 +15,14 @@ */ package com.github.jjYBdx4IL.parser.linux; +import static com.github.jjYBdx4IL.utils.text.StringUtil.f; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import com.github.jjYBdx4IL.parser.ParseException; -import com.github.jjYBdx4IL.parser.linux.ZFSStatusParser.AlertLevel; -import com.github.jjYBdx4IL.parser.linux.ZFSStatusParser.Result; +import com.github.jjYBdx4IL.parser.linux.ZfsStatusParser.AlertLevel; +import com.github.jjYBdx4IL.parser.linux.ZfsStatusParser.Result; +import com.github.jjYBdx4IL.utils.time.TimeUtils; import org.apache.commons.io.IOUtils; import org.junit.Test; @@ -27,49 +30,88 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.time.Instant; //CHECKSTYLE:OFF /** * * @author Github jjYBdx4IL Projects */ -public class ZFSStatusParserTest { +public class ZfsStatusParserTest { - private static final Logger LOG = LoggerFactory.getLogger(ZFSStatusParserTest.class); + private static final Logger LOG = LoggerFactory.getLogger(ZfsStatusParserTest.class); + + private static class Res { + public final String filename; + public final int expPrio; + public final int expPools; + public final int expActivity; + public final Instant since; + + Res(String fn, int expPrio, int expPools, int expActivity, Instant since) { + this.filename = fn; + this.expPrio = expPrio; + this.expActivity = expActivity; + this.expPools = expPools; + this.since = since; + } + } + + @Test + public void testParse2() throws IOException, ParseException, java.text.ParseException { + Res[] expectedResults = new Res[] { + new Res("zpool_output_resilver", 0, 2, 1, TimeUtils.parseISO8601("2020-12-13T00:24:31Z").toInstant()), + new Res("zpool_output_resilver_long", 0, 2, 1, TimeUtils.parseISO8601("2020-12-09T00:24:31Z").toInstant()), + new Res("zpool_output_scrub", 0, 2, 1, TimeUtils.parseISO8601("2020-12-13T00:24:31Z").toInstant()), + new Res("zpool_output_scrub_bug1", 0, 2, 0, null), + new Res("zpool_output_scrub_long", 0, 2, 1, TimeUtils.parseISO8601("2020-12-09T00:24:31Z").toInstant()), + new Res("zpool_output_scrub_old", 0, 2, 1, TimeUtils.parseISO8601("2020-12-13T00:24:31Z").toInstant()) + }; + + for (Res exp : expectedResults) { + LOG.debug(exp.filename); + try { + Result actualResult = parse(exp.filename); + Result expResult = new Result(AlertLevel.getByNumericLevel(exp.expPrio), exp.expPools, + exp.expActivity == 1, exp.since); + assertEquals(exp.filename, expResult, actualResult); + } catch (ParseException ex) { + LOG.info("", ex); + assertEquals(exp.filename, exp.expPrio, -1); + } + } + } @Test public void testParse() throws IOException, ParseException { int[][] expectedResults = { // file num, error prio, numpools, activity - {1, 1, 1, 0}, - {2, 0, 1, 0}, - {3, 2, 1, 0}, - {4, 3, 1, 0}, - {5, 2, 1, 0}, - {6, 2, 1, 0}, - {7, 1, 1, 0}, - {8, 2, 1, 0}, - {9, 2, 2, 0}, - {10, 1, 1, 0}, - {11, 2, 1, 1}, - {12, -1, 1, 0}, - {13, -1, 1, 0}, - {14, -1, 1, 0}, - {15, 2, 3, 0}, - {16, 0, 1, 1}, - {17, 3, 3, 0}, + { 1, 1, 1, 0 }, + { 2, 0, 1, 0 }, + { 3, 2, 1, 0 }, + { 4, 3, 1, 0 }, + { 5, 2, 1, 0 }, + { 6, 2, 1, 0 }, + { 7, 1, 1, 0 }, + { 8, 2, 1, 0 }, + { 9, 2, 2, 0 }, + { 10, 1, 1, 0 }, + { 11, 2, 1, 1 }, + { 12, -1, 1, 0 }, + { 13, -1, 1, 0 }, + { 14, -1, 1, 0 }, + { 15, 2, 3, 0 }, + { 16, 0, 1, 1 }, + { 17, 3, 3, 0 }, }; for (int[] expectedResult : expectedResults) { - String inputFileName = String.format("zpool_status_%d.txt", expectedResult[0]); + String inputFileName = f("zpool_status_%d.txt", expectedResult[0]); LOG.debug(inputFileName); - @SuppressWarnings("deprecation") - String zpoolStatusCmdOutput = IOUtils.toString( - ZFSStatusParserTest.class.getResourceAsStream(inputFileName)); try { - Result actualResult = ZFSStatusParser.parse(zpoolStatusCmdOutput); + Result actualResult = parse(inputFileName); Result expResult = new Result(AlertLevel.getByNumericLevel(expectedResult[1]), expectedResult[2], - expectedResult[3] == 1); + expectedResult[3] == 1, null); assertEquals(inputFileName, expResult, actualResult); } catch (ParseException ex) { assertEquals(inputFileName, expectedResult[1], -1); @@ -78,4 +120,10 @@ public void testParse() throws IOException, ParseException { } } + private Result parse(String res) throws IOException, ParseException { + String zpoolStatusCmdOutput = IOUtils.toString( + ZfsStatusParserTest.class.getResourceAsStream(res), UTF_8); + return ZfsStatusParser.parse(zpoolStatusCmdOutput); + } + } diff --git a/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParserTest.java b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParserTest.java new file mode 100755 index 0000000..9fa30cb --- /dev/null +++ b/parser/src/test/java/com/github/jjYBdx4IL/parser/linux/ZfsPoolListParserTest.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2017 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.parser.linux; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +import com.github.jjYBdx4IL.parser.linux.ZfsPoolListParser.Result; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import java.util.List; + +public class ZfsPoolListParserTest { + + @Test + public void testName() throws Exception { + String input = IOUtils.toString( + ZfsStatusParserTest.class.getResourceAsStream("zpoolListHp"), UTF_8); + List r = ZfsPoolListParser.parse(input); + assertEquals(2, r.size()); + assertEquals("bpool", r.get(0).name); + assertEquals(265662464, r.get(0).free); + assertEquals("rpool", r.get(1).name); + assertEquals(19.f, r.get(1).freePct, 1.f); + } +} diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_longok b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_longok new file mode 100644 index 0000000..52f557b --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_longok @@ -0,0 +1,99 @@ +smartctl 6.6 2017-11-05 r4594 [x86_64-linux-4.19.0-13-amd64] (local build) +Copyright (C) 2002-17, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Western Digital Red +Device Model: WDC WD40EFRX-68N32N0 +Serial Number: WD-WCC7K0YKSETV +LU WWN Device Id: 5 0014ee 2bdd71059 +Firmware Version: 82.00A82 +User Capacity: 4,000,787,030,016 bytes [4.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5400 rpm +Form Factor: 3.5 inches +Device is: In smartctl database [for details use: -P show] +ATA Version is: ACS-3 T13/2161-D revision 5 +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Wed Dec 23 09:54:01 2020 CET +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +General SMART Values: +Offline data collection status: (0x00) Offline data collection activity + was never started. + Auto Offline Data Collection: Disabled. +Self-test execution status: ( 0) The previous self-test routine completed + without error or no self-test has ever + been run. +Total time to complete Offline +data collection: (45300) seconds. +Offline data collection +capabilities: (0x7b) SMART execute Offline immediate. + Auto Offline data collection on/off support. + Suspend Offline collection upon new + command. + Offline surface scan supported. + Self-test supported. + Conveyance Self-test supported. + Selective Self-test supported. +SMART capabilities: (0x0003) Saves SMART data before entering + power-saving mode. + Supports SMART auto save timer. +Error logging capability: (0x01) Error logging supported. + General Purpose Logging supported. +Short self-test routine +recommended polling time: ( 2) minutes. +Extended self-test routine +recommended polling time: ( 481) minutes. +Conveyance self-test routine +recommended polling time: ( 5) minutes. +SCT capabilities: (0x303d) SCT Status supported. + SCT Error Recovery Control supported. + SCT Feature Control supported. + SCT Data Table supported. + +SMART Attributes Data Structure revision number: 16 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x002f 200 200 051 Pre-fail Always - 0 + 3 Spin_Up_Time 0x0027 100 253 021 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 000 Old_age Always - 3 + 5 Reallocated_Sector_Ct 0x0033 200 200 140 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x002e 200 200 000 Old_age Always - 0 + 9 Power_On_Hours 0x0032 100 100 000 Old_age Always - 1019 + 10 Spin_Retry_Count 0x0032 100 253 000 Old_age Always - 0 + 11 Calibration_Retry_Count 0x0032 100 253 000 Old_age Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 000 Old_age Always - 3 +192 Power-Off_Retract_Count 0x0032 200 200 000 Old_age Always - 0 +193 Load_Cycle_Count 0x0032 200 200 000 Old_age Always - 7 +194 Temperature_Celsius 0x0022 120 114 000 Old_age Always - 30 +196 Reallocated_Event_Count 0x0032 200 200 000 Old_age Always - 0 +197 Current_Pending_Sector 0x0032 200 200 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0030 100 253 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x0032 200 200 000 Old_age Always - 0 +200 Multi_Zone_Error_Rate 0x0008 200 200 000 Old_age Offline - 0 + +SMART Error Log Version: 1 +No Errors Logged + +SMART Self-test log structure revision number 1 +Num Test_Description Status Remaining LifeTime(hours) LBA_of_first_error +# 1 Extended offline Completed without error 00% 1018 - +# 2 Short offline Completed without error 00% 1000 - +# 3 Extended offline Completed without error 00% 18 - +# 4 Short offline Completed without error 00% 0 - + +SMART Selective self-test log data structure revision number 1 + SPAN MIN_LBA MAX_LBA CURRENT_TEST_STATUS + 1 0 0 Not_testing + 2 0 0 Not_testing + 3 0 0 Not_testing + 4 0 0 Not_testing + 5 0 0 Not_testing +Selective self-test flags (0x0): + After scanning selected spans, do NOT read-scan remainder of disk. +If Selective self-test is pending on power-up, resume after 0 minute delay. + diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_nolong b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_nolong new file mode 100644 index 0000000..3f363c4 --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/smart_a_output_nolong @@ -0,0 +1,95 @@ +smartctl 6.6 2017-11-05 r4594 [x86_64-linux-4.19.0-13-amd64] (local build) +Copyright (C) 2002-17, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Western Digital Red +Device Model: WDC WD40EFRX-68N32N0 +Serial Number: WD-WCC7K2ATSZXJ +LU WWN Device Id: 5 0014ee 2688a8e78 +Firmware Version: 82.00A82 +User Capacity: 4,000,787,030,016 bytes [4.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5400 rpm +Form Factor: 3.5 inches +Device is: In smartctl database [for details use: -P show] +ATA Version is: ACS-3 T13/2161-D revision 5 +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Wed Dec 23 01:30:02 2020 CET +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +General SMART Values: +Offline data collection status: (0x00) Offline data collection activity + was never started. + Auto Offline Data Collection: Disabled. +Self-test execution status: ( 249) Self-test routine in progress... + 90% of test remaining. +Total time to complete Offline +data collection: (44700) seconds. +Offline data collection +capabilities: (0x7b) SMART execute Offline immediate. + Auto Offline data collection on/off support. + Suspend Offline collection upon new + command. + Offline surface scan supported. + Self-test supported. + Conveyance Self-test supported. + Selective Self-test supported. +SMART capabilities: (0x0003) Saves SMART data before entering + power-saving mode. + Supports SMART auto save timer. +Error logging capability: (0x01) Error logging supported. + General Purpose Logging supported. +Short self-test routine +recommended polling time: ( 2) minutes. +Extended self-test routine +recommended polling time: ( 474) minutes. +Conveyance self-test routine +recommended polling time: ( 5) minutes. +SCT capabilities: (0x303d) SCT Status supported. + SCT Error Recovery Control supported. + SCT Feature Control supported. + SCT Data Table supported. + +SMART Attributes Data Structure revision number: 16 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x002f 200 200 051 Pre-fail Always - 0 + 3 Spin_Up_Time 0x0027 100 253 021 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 000 Old_age Always - 3 + 5 Reallocated_Sector_Ct 0x0033 200 200 140 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x002e 100 253 000 Old_age Always - 0 + 9 Power_On_Hours 0x0032 100 100 000 Old_age Always - 10 + 10 Spin_Retry_Count 0x0032 100 253 000 Old_age Always - 0 + 11 Calibration_Retry_Count 0x0032 100 253 000 Old_age Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 000 Old_age Always - 3 +192 Power-Off_Retract_Count 0x0032 200 200 000 Old_age Always - 0 +193 Load_Cycle_Count 0x0032 200 200 000 Old_age Always - 3 +194 Temperature_Celsius 0x0022 117 114 000 Old_age Always - 33 +196 Reallocated_Event_Count 0x0032 200 200 000 Old_age Always - 0 +197 Current_Pending_Sector 0x0032 200 200 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0030 100 253 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x0032 200 200 000 Old_age Always - 0 +200 Multi_Zone_Error_Rate 0x0008 100 253 000 Old_age Offline - 0 + +SMART Error Log Version: 1 +No Errors Logged + +SMART Self-test log structure revision number 1 +Num Test_Description Status Remaining LifeTime(hours) LBA_of_first_error +# 1 Short offline Completed without error 00% 0 - + +SMART Selective self-test log data structure revision number 1 + SPAN MIN_LBA MAX_LBA CURRENT_TEST_STATUS + 1 0 0 Not_testing + 2 0 0 Not_testing + 3 0 0 Not_testing + 4 0 0 Not_testing + 5 0 0 Not_testing +Selective self-test flags (0x0): + After scanning selected spans, do NOT read-scan remainder of disk. +If Selective self-test is pending on power-up, resume after 0 minute delay. + diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpoolListHp b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpoolListHp new file mode 100644 index 0000000..31b56dd --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpoolListHp @@ -0,0 +1,2 @@ +bpool 503316480 237654016 265662464 - - 14 47 1.00 ONLINE - +rpool 3985729650688 3257638158336 728091492352 - - 15 81 2.55 ONLINE - diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver new file mode 100644 index 0000000..ed14cd7 --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver @@ -0,0 +1,39 @@ + pool: bpool + state: ONLINE +status: Some supported features are not enabled on the pool. The pool can + still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(5) for details. + scan: scrub repaired 0B in 0 days 00:00:04 with 0 errors on Sun Dec 13 00:24:31 2020 +config: + + NAME STATE READ WRITE CKSUM + bpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part3 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part3 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K2ATSZXJ-part3 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K0YKSETV-part3 ONLINE 0 0 0 + +errors: No known data errors + + pool: rpool + state: ONLINE +status: One or more devices is currently being resilvered. The pool will + continue to function, possibly in a degraded state. +action: Wait for the resilver to complete. + scan: resilver in progress since Sun Dec 13 00:24:31 2020 + 1.39T scanned at 409M/s, 451G issued at 130M/s, 1.39T total + 905G resilvered, 31.74% done, 0 days 02:07:40 to go +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part4 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part4 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K2ATSZXJ-part4 ONLINE 0 0 0 (resilvering) + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K0YKSETV-part4 ONLINE 0 0 0 (resilvering) + +errors: No known data errors diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver_long b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver_long new file mode 100644 index 0000000..135b64c --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_resilver_long @@ -0,0 +1,39 @@ + pool: bpool + state: ONLINE +status: Some supported features are not enabled on the pool. The pool can + still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(5) for details. + scan: scrub repaired 0B in 0 days 00:00:04 with 0 errors on Sun Dec 13 00:24:31 2020 +config: + + NAME STATE READ WRITE CKSUM + bpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part3 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part3 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K2ATSZXJ-part3 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K0YKSETV-part3 ONLINE 0 0 0 + +errors: No known data errors + + pool: rpool + state: ONLINE +status: One or more devices is currently being resilvered. The pool will + continue to function, possibly in a degraded state. +action: Wait for the resilver to complete. + scan: resilver in progress since Sun Dec 09 00:24:31 2020 + 1.39T scanned at 409M/s, 451G issued at 130M/s, 1.39T total + 905G resilvered, 31.74% done, 0 days 02:07:40 to go +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part4 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part4 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K2ATSZXJ-part4 ONLINE 0 0 0 (resilvering) + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K0YKSETV-part4 ONLINE 0 0 0 (resilvering) + +errors: No known data errors diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub new file mode 100644 index 0000000..43c5dea --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub @@ -0,0 +1,32 @@ + pool: bpool + state: ONLINE +status: Some supported features are not enabled on the pool. The pool can + still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(5) for details. + scan: scrub repaired 0B in 0 days 00:00:11 with 0 errors on Sun Dec 13 00:24:13 2020 +config: + + NAME STATE READ WRITE CKSUM + bpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part3 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part3 ONLINE 0 0 0 + +errors: No known data errors + + pool: rpool + state: ONLINE + scan: scrub in progress since Sun Dec 13 00:24:31 2020 + 526G scanned at 126M/s, 390G issued at 93.9M/s, 526G total + 0B repaired, 74.25% done, 0 days 00:24:36 to go +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part4 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part4 ONLINE 0 0 0 + +errors: No known data errors diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_bug1 b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_bug1 new file mode 100644 index 0000000..4492cd7 --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_bug1 @@ -0,0 +1,30 @@ + pool: bpool + state: ONLINE +status: Some supported features are not enabled on the pool. The pool can + still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(5) for details. + scan: scrub repaired 0B in 0 days 00:00:03 with 0 errors on Tue Jan 5 06:24:05 2021 +config: + + NAME STATE READ WRITE CKSUM + bpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K2ATSZXJ-part3 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K0YKSETV-part3 ONLINE 0 0 0 + +errors: No known data errors + + pool: rpool + state: ONLINE + scan: scrub repaired 0B in 0 days 03:26:28 with 0 errors on Tue Jan 5 09:50:30 2021 +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K2ATSZXJ-part4 ONLINE 0 0 0 + ata-WDC_WD40EFRX-68N32N0_WD-WCC7K0YKSETV-part4 ONLINE 0 0 0 + +errors: No known data errors diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_long b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_long new file mode 100644 index 0000000..2b16ac0 --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_long @@ -0,0 +1,32 @@ + pool: bpool + state: ONLINE +status: Some supported features are not enabled on the pool. The pool can + still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(5) for details. + scan: scrub repaired 0B in 0 days 00:00:11 with 0 errors on Sun Dec 13 00:24:13 2020 +config: + + NAME STATE READ WRITE CKSUM + bpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part3 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part3 ONLINE 0 0 0 + +errors: No known data errors + + pool: rpool + state: ONLINE + scan: scrub in progress since Sun Dec 09 00:24:31 2020 + 526G scanned at 126M/s, 390G issued at 93.9M/s, 526G total + 0B repaired, 74.25% done, 0 days 00:24:36 to go +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part4 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part4 ONLINE 0 0 0 + +errors: No known data errors diff --git a/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_old b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_old new file mode 100644 index 0000000..7b6a708 --- /dev/null +++ b/parser/src/test/resources/com/github/jjYBdx4IL/parser/linux/zpool_output_scrub_old @@ -0,0 +1,32 @@ + pool: bpool + state: ONLINE +status: Some supported features are not enabled on the pool. The pool can + still be used, but some features are unavailable. +action: Enable all features using 'zpool upgrade'. Once this is done, + the pool may no longer be accessible by software that does not support + the features. See zpool-features(5) for details. + scan: scrub repaired 0B in 0 days 00:00:11 with 0 errors on Sun Dec 02 00:24:13 2020 +config: + + NAME STATE READ WRITE CKSUM + bpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part3 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part3 ONLINE 0 0 0 + +errors: No known data errors + + pool: rpool + state: ONLINE + scan: scrub in progress since Sun Dec 13 00:24:31 2020 + 526G scanned at 126M/s, 390G issued at 93.9M/s, 526G total + 0B repaired, 74.25% done, 0 days 00:24:36 to go +config: + + NAME STATE READ WRITE CKSUM + rpool ONLINE 0 0 0 + mirror-0 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D809L5JH-part4 ONLINE 0 0 0 + ata-WDC_WD40EFAX-68JH4N0_WD-WX12D80N5E9E-part4 ONLINE 0 0 0 + +errors: No known data errors diff --git a/pom.xml b/pom.xml index 036ec37..1772e6b 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ h2-frontend io-utils jersey-client-utils + jmon jna-utils jsoup-utils junit4-utils @@ -93,6 +94,7 @@ selenium-test-utils solr-webapp solr-utils + svn-client-wrapper swing-utils text-utils vmmgmt-utils diff --git a/proc-utils/src/main/java/com/github/jjYBdx4IL/utils/proc/ProcRunner.java b/proc-utils/src/main/java/com/github/jjYBdx4IL/utils/proc/ProcRunner.java index df65005..183e720 100644 --- a/proc-utils/src/main/java/com/github/jjYBdx4IL/utils/proc/ProcRunner.java +++ b/proc-utils/src/main/java/com/github/jjYBdx4IL/utils/proc/ProcRunner.java @@ -41,7 +41,7 @@ * * @author jjYBdx4IL */ -public class ProcRunner { +public class ProcRunner implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ProcRunner.class); private static final long DEFAULT_TIMEOUT = 0L; // no timeout by default @@ -103,6 +103,23 @@ public ProcRunner(Path exe, String... command) { public Charset getConsoleEncoding() { return consoleEncoding; } + + public ProcRunner rootLocaleUtc() { + environment().put("LC_ALL", "C"); + environment().put("LANG", "C"); + environment().put("TZ", "UTC"); + return this; + } + + public ProcRunner prependPath(Path element) { + String s = environment().get("PATH"); + if (s == null || s.trim().isEmpty()) { + environment().put("PATH", element.toAbsolutePath().toString()); + } else { + environment().put("PATH", element.toAbsolutePath().toString() + File.pathSeparator + s); + } + return this; + } /** * Interpret program output in the given encoding. Default is the JVM's default @@ -211,6 +228,11 @@ public void run() { return this; } + @Override + public void close() throws InterruptedException, IOException { + kill(); + } + public int kill() throws InterruptedException, IOException { return waitFor(-1, null); } diff --git a/release-parent/pom.xml b/release-parent/pom.xml index 9659153..ae73118 100644 --- a/release-parent/pom.xml +++ b/release-parent/pom.xml @@ -89,7 +89,7 @@ org.owasp dependency-check-maven - false + true true @@ -308,11 +308,11 @@ org.owasp dependency-check-maven - 6.1.6 - - false - true - + 6.2.0 + + true + true + com.googlecode.maven-download-plugin @@ -406,7 +406,7 @@ maven-compiler-plugin - 3.8.0 + 3.8.1 maven-clean-plugin diff --git a/remote-robot/README.md b/remote-robot/README.md new file mode 100644 index 0000000..a9a343a --- /dev/null +++ b/remote-robot/README.md @@ -0,0 +1,5 @@ +# remote-robot + +The launch4j plugin still requires 32 bit support. On Ubuntu: + + apt install gcc-multilib diff --git a/svn-client-wrapper/LICENSE b/svn-client-wrapper/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/svn-client-wrapper/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/svn-client-wrapper/README.md b/svn-client-wrapper/README.md new file mode 100644 index 0000000..c64c058 --- /dev/null +++ b/svn-client-wrapper/README.md @@ -0,0 +1,4 @@ +# xml-dom4j-utils + +[![Build Status](https://travis-ci.org/jjYBdx4IL/misc.png?branch=master)](https://travis-ci.org/jjYBdx4IL/misc) + diff --git a/svn-client-wrapper/pom.xml b/svn-client-wrapper/pom.xml new file mode 100644 index 0000000..a2d4355 --- /dev/null +++ b/svn-client-wrapper/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + + com.github.jjYBdx4IL + release-parent + 1.4-SNAPSHOT + ../release-parent/ + + + com.github.jjYBdx4IL.utils + svn-sclient-wrapper + 1.0-SNAPSHOT + SVN Client Wrapper + using svn command line client to access SVN + 2021 + + + + + com.github.jjYBdx4IL.utils + env-utils + 1.1-SNAPSHOT + + + com.github.jjYBdx4IL.utils + proc-utils + 1.2-SNAPSHOT + + + org.slf4j + slf4j-api + 1.7.30 + + + + junit + junit + 4.13.2 + test + + + org.slf4j + slf4j-simple + 1.7.30 + test + + + diff --git a/svn-client-wrapper/src/main/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapper.java b/svn-client-wrapper/src/main/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapper.java new file mode 100755 index 0000000..4798f1a --- /dev/null +++ b/svn-client-wrapper/src/main/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapper.java @@ -0,0 +1,257 @@ +/* + * Copyright © 2021 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.utils.svncw; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.github.jjYBdx4IL.utils.env.WindowsUtils; +import com.github.jjYBdx4IL.utils.proc.ProcRunner; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SvnClientWrapper { + + private static final Logger LOG = LoggerFactory.getLogger(SvnClientWrapper.class); + + protected final String svnExe; + + /** + * Can be used to determine if the given path is part of a subversion + * checkout (!= managed by SVN). This method does not call any subversion + * client, it simply traverses parent directories and tries to find and + * parse the .svn/format file. + * + * @return -1 if startDir is not part of a subversion checkout + */ + public static int getSvnCoFormat(Path startDir) throws IOException { + Path root = getSvnCoRoot(startDir); + try { + return root == null ? -1 + : Integer.parseInt(FileUtils.readFileToString(root.resolve(".svn/format").toFile(), UTF_8).trim()); + } catch (NumberFormatException ex) { + throw new IOException(ex); + } + } + + /** + * This method does not call any subversion client, it simply traverses + * parent directories and tries to find and parse the + * .svn/format file. + * + * @return the path containing the .svn/format file. null if + * not found. + */ + public static Path getSvnCoRoot(Path startDir) throws IOException { + Path current = startDir.toAbsolutePath(); + do { + Path p = current.resolve(".svn/format"); + if (Files.isRegularFile(p)) { + try { + Integer.parseInt(FileUtils.readFileToString(p.toFile(), UTF_8).trim()); + return current; + } catch (NumberFormatException ex) { + throw new IOException(ex); + } + } + current = current.getParent(); + } + while (current != null); + return null; + } + + /** + * Shorthand for {@link #getSvnCoFormat(Path)} != -1. + */ + public static boolean isSvnCheckout(Path dir) throws IOException { + return getSvnCoFormat(dir) != -1; + } + + /** + * Construct svn client wrapper. Will use cygwin on Windows if installed. + * Otherwise reverts to svn or svn.exe on system path. + */ + public SvnClientWrapper() { + if (SystemUtils.IS_OS_WINDOWS) { + String cygPath = WindowsUtils.getCygwinInstallationPath(); + if (cygPath != null) { + svnExe = cygPath + "\\bin\\svn.exe"; + } else { + svnExe = "svn.exe"; + } + } else { + svnExe = "svn"; + } + LOG.debug("SVN_EXE = {}", svnExe); + } + + /** + * Creates a runnable ProcRunner for the svn executable. + */ + public ProcRunner createRunner(Path workingDirectory, List svnArgs) throws IOException { + File wd = workingDirectory.toFile(); + if (!wd.exists() || !wd.isDirectory()) { + throw new IOException("working directory does not exist or isn't a directory: " + wd); + } + List cmd = new ArrayList<>(); + cmd.add(svnExe); + cmd.addAll(svnArgs); + if (LOG.isDebugEnabled()) { + LOG.debug("cmd: {} ({})", StringUtils.join(cmd, " "), wd.getCanonicalPath()); + } + ProcRunner pr = new ProcRunner(cmd); + pr.rootLocaleUtc(); + pr.setWorkDir(wd); + return pr; + } + + /** + * Get svn info output as a map. + */ + public Map getSvnInfo(Path pathSpec) throws IOException, InterruptedException { + ProcRunner pr = createRunner(pathSpec, Arrays.asList("info", ".")); + int rc = pr.run(); + if (LOG.isTraceEnabled()) { + LOG.trace("rc = {}, output: {}", rc, pr.getOutputBlob()); + } + if (rc != 0) { + throw new IOException("svn info failed: " + pr.getOutputBlob()); + } + Pattern p = Pattern.compile("^(\\S+[^:]+):\\s(.+)$"); + Map map = new HashMap<>(); + for (String s : pr.getOutputLines()) { + Matcher m = p.matcher(s); + if (m.find()) { + map.put(m.group(1), m.group(2)); + } + } + return map; + } + + /** + * Get output lines from svn status. + */ + public ConcurrentLinkedQueue getSvnStatus(Path pathSpec) throws IOException, InterruptedException { + ProcRunner pr = createRunner(pathSpec, Arrays.asList("status", ".")); + int rc = pr.run(); + if (LOG.isTraceEnabled()) { + LOG.trace("rc = {}, output: {}", rc, pr.getOutputBlob()); + } + if (rc != 0) { + throw new IOException("svn status failed: " + pr.getOutputBlob()); + } + return pr.getOutputLines(); + } + + /** + * Returns the first line of the svn --version command. Null if the command + * failed. Can be used to check whether the svn client is available. + */ + public String version() throws IOException, InterruptedException { + ProcRunner pr = createRunner(Paths.get(System.getProperty("user.home")), Arrays.asList("--version")); + int rc = pr.run(); + if (LOG.isTraceEnabled()) { + LOG.trace("rc = {}, output: {}", rc, pr.getOutputBlob()); + } + if (rc != 0) { + return null; + } + return pr.getOutputLines().iterator().next(); + } + + public boolean isAvailable() throws IOException, InterruptedException { + return version() != null; + } + + /** + * Return the svn executable as being used by the wrapper. + */ + public String exe() { + return svnExe; + } + + /** + * Get host part of svn URL returned by svn info. + */ + public String getSvnHost(Path pathSpec) throws IOException, InterruptedException { + String url = getSvnInfo(pathSpec).get("URL"); + Pattern p = Pattern.compile("^([^/]*//[^/]+)(?:|/.*)"); + Matcher m = p.matcher(url); + if (m.find()) { + return m.group(1); + } + return null; + } + + /** + * Returns the root path of the svn checkout. Uses information dumped by svn + * info. + */ + public Path getSvnRoot(Path someCoDir) throws IOException, InterruptedException { + someCoDir = someCoDir.toAbsolutePath().normalize(); + Map info = getSvnInfo(someCoDir); + String rel = info.get("Relative URL"); + if (rel == null || rel.length() < 2) { + throw new IOException("relative url not found: " + someCoDir); + } + if (rel.equals("^/")) { + return someCoDir; + } + int n = Paths.get(rel.substring(2)).getNameCount(); + Path p = someCoDir.toAbsolutePath(); + for (int i = 0; i < n; i++) { + p = p.getParent(); + } + return p; + } + + /** + * Check if svn status output contains anything. + */ + public boolean checkClean(Path pathSpec) throws IOException, InterruptedException { + return getSvnStatus(pathSpec).size() == 0; + } + + /** + * Returns last changed revision, appends " (dirty)" if svn status output + * isn't empty. + */ + public String getSvnLcRevDirty(Path pathSpec) throws IOException, InterruptedException { + boolean isDirty = !checkClean(pathSpec); + String rev = getSvnInfo(pathSpec).get("Last Changed Rev"); + if (rev == null || rev.trim().isEmpty()) { + throw new IOException("last changed rev not found"); + } + return isDirty ? rev + " (dirty)" : rev; + } +} diff --git a/svn-client-wrapper/src/test/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapperTest.java b/svn-client-wrapper/src/test/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapperTest.java new file mode 100755 index 0000000..edb1be3 --- /dev/null +++ b/svn-client-wrapper/src/test/java/com/github/jjYBdx4IL/utils/svncw/SvnClientWrapperTest.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2021 jjYBdx4IL (https://github.com/jjYBdx4IL) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.jjYBdx4IL.utils.svncw; + +import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Map; + +public class SvnClientWrapperTest { + + SvnClientWrapper scw = new SvnClientWrapper(); + + @Before + public void before() throws IOException, InterruptedException { + assumeTrue(SvnClientWrapper.isSvnCheckout(Paths.get("."))); + } + + @Test + public void testGetSvnInfo() throws Exception { + Map r = scw.getSvnInfo(Paths.get(".")); + assertTrue(r.containsKey("URL")); + } + + @Test + public void testGetSvnRoot() throws Exception { + assertEquals(SvnClientWrapper.getSvnCoRoot(Paths.get(".")), scw.getSvnRoot(Paths.get("."))); + } +} diff --git a/svn-client-wrapper/src/test/resources/simplelogger.properties b/svn-client-wrapper/src/test/resources/simplelogger.properties new file mode 100644 index 0000000..56ea85a --- /dev/null +++ b/svn-client-wrapper/src/test/resources/simplelogger.properties @@ -0,0 +1,8 @@ +org.slf4j.simpleLogger.defaultLogLevel=info +org.slf4j.simpleLogger.log.com.github.jjYBdx4IL.utils.svncw=debug +org.slf4j.simpleLogger.showDateTime=false +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z +#org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss +#org.slf4j.simpleLogger.showThreadName=true +#org.slf4j.simpleLogger.showLogName=true +org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/swing-utils/src/test/resources/simplelogger.properties b/swing-utils/src/test/resources/simplelogger.properties index 93327b2..eeec2aa 100644 --- a/swing-utils/src/test/resources/simplelogger.properties +++ b/swing-utils/src/test/resources/simplelogger.properties @@ -1,37 +1,8 @@ -# SLF4J's SimpleLogger configuration file -# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. - -# Default logging detail level for all instances of SimpleLogger. -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, defaults to "info". org.slf4j.simpleLogger.defaultLogLevel=info - -# Logging detail level for a SimpleLogger instance named "xxxxx". -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, the default logging detail level is used. -org.slf4j.simpleLogger.log.com.github.jjYBdx4IL.utils.awt=trace - -# BEWARE! In this case it's better to add -Dorg.slf4j.simpleLogger.defaultLogLevel=debug to your IDE's "test file" goal. - -# Set to true if you want the current date and time to be included in output messages. -# Default is false, and will output the number of milliseconds elapsed since startup. -org.slf4j.simpleLogger.showDateTime=true - -# The date and time format to be used in the output messages. -# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. -# If the format is not specified or is invalid, the default format is used. -# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +org.slf4j.simpleLogger.log.com.github.jjYBdx4IL.utils.awt=info +#org.slf4j.simpleLogger.showDateTime=true #org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss - -# Set to true if you want to output the current thread name. -# Defaults to true. #org.slf4j.simpleLogger.showThreadName=true - -# Set to true if you want the Logger instance name to be included in output messages. -# Defaults to true. #org.slf4j.simpleLogger.showLogName=true - -# Set to true if you want the last component of the name to be included in output messages. -# Defaults to false. org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/text-utils/pom.xml b/text-utils/pom.xml index 12ec022..bca6bbe 100644 --- a/text-utils/pom.xml +++ b/text-utils/pom.xml @@ -49,19 +49,19 @@ junit junit - 4.12 + 4.13.2 test org.slf4j slf4j-simple - 1.7.25 + 1.7.30 test org.apache.commons commons-lang3 - 3.7 + 3.12.0 test diff --git a/text-utils/src/test/java/com/github/jjYBdx4IL/utils/text/EtaTest.java b/text-utils/src/test/java/com/github/jjYBdx4IL/utils/text/EtaTest.java index fcb8a36..23dd400 100644 --- a/text-utils/src/test/java/com/github/jjYBdx4IL/utils/text/EtaTest.java +++ b/text-utils/src/test/java/com/github/jjYBdx4IL/utils/text/EtaTest.java @@ -26,12 +26,12 @@ public class EtaTest { @Test public void test() throws InterruptedException { Eta eta = new Eta<>(0, 10, 300); - LOG.info(eta.toString(0)); + LOG.debug(eta.toString(0)); for(int i=0; i<9; i++) { Thread.sleep(200l); String etaStr = eta.toStringPeriodical(i+1); if (etaStr != null) { - LOG.info(etaStr); + LOG.debug(etaStr); } } } diff --git a/text-utils/src/test/java/com/github/jjYBdx4IL/utils/time/TimeUsageTest.java b/text-utils/src/test/java/com/github/jjYBdx4IL/utils/time/TimeUsageTest.java index 0c91ae7..049f7c5 100644 --- a/text-utils/src/test/java/com/github/jjYBdx4IL/utils/time/TimeUsageTest.java +++ b/text-utils/src/test/java/com/github/jjYBdx4IL/utils/time/TimeUsageTest.java @@ -32,6 +32,6 @@ public void test() throws Exception { a.startSub("two"); a.startSub("two"); a.close(); - LOG.info(a.toString()); + LOG.debug(a.toString()); } } diff --git a/wildfly-client/src/main/java/com/github/jjYBdx4IL/wildfly/WildFlyClient.java b/wildfly-client/src/main/java/com/github/jjYBdx4IL/wildfly/WildFlyClient.java index ed8fad8..cfad193 100644 --- a/wildfly-client/src/main/java/com/github/jjYBdx4IL/wildfly/WildFlyClient.java +++ b/wildfly-client/src/main/java/com/github/jjYBdx4IL/wildfly/WildFlyClient.java @@ -307,8 +307,8 @@ public void uploadDeploy(String deploymentName, String fileLoc) throws Exception * Upload and deploy (add+enable). Skips upload and deployment if remote * checksum matches. */ - public void uploadDeployIfChanged(String deploymentName, String fileLoc) throws Exception { - final String localSha1 = IoUtils.getDigest(fileLoc, "SHA-1").toLowerCase(Locale.ROOT); + public boolean uploadDeployIfChanged(String deploymentName, File war) throws Exception { + final String localSha1 = IoUtils.getDigest(war, "SHA-1").toLowerCase(Locale.ROOT); boolean needsRemoval = false; if (exec(f("/deployment=%s:read-resource", deploymentName))) { needsRemoval = true; @@ -322,12 +322,12 @@ public void uploadDeployIfChanged(String deploymentName, String fileLoc) throws String deployedSha1 = Hex.encodeHexString(Base64.getDecoder().decode(deployedHash), true); if (localSha1.equals(deployedSha1)) { LOG.info("deployment content hash unchanged, skipping"); - return; + return false; } } } - String uploadHash = upload(fileLoc); + String uploadHash = upload(war); String uploadSha1 = Hex.encodeHexString(Base64.getDecoder().decode(uploadHash), true); if (!localSha1.equals(uploadSha1)) { throw new IOException("checksum mismatch for upload: upload=" + uploadSha1 + ", local=" + localSha1); @@ -338,6 +338,7 @@ public void uploadDeployIfChanged(String deploymentName, String fileLoc) throws } assertPostCmd(f("/deployment=%s:add(enabled=true,content=[{\"hash\":{\"BYTES_VALUE\":\"%s\"}}])", deploymentName, uploadHash)); + return true; } /**