From bc3945d4a19a4bf02b99df447e6aa0a3832d14ba Mon Sep 17 00:00:00 2001 From: Gilles Barbier Date: Thu, 12 Sep 2024 00:13:27 +0200 Subject: [PATCH] Global Event Listener (#259) * Revert Kotlin version to 2.0.10 due to publishing issues New Features: * Expose JsonPath API * Add hasContext method to Task.kt * eventListener are now global, with auto-refresh Breaking Changes: * Remove default values for storage. Previously default values were set for local developement. we think now it can be misleading when going to production. * Normalize setters on all config builders * Refactor field names for consistency in PoliciesConfig * Updated field names to use consistent terminology by removing "In" and improving readability. (timeoutSeconds, shutdownGracePeriodSeconds, retentionTimeMinutes, retentionSizeMB, messageTTLSeconds, etc..) * Change 'user' to 'username' in MySQLConfig, PostgresConfig and RedisConfig * Transport definition in config * Update InfiniticClient and InfiniticWorker constructor, to only take a config object * Update the 'from***' static method for Clients and Workers Improvements: * More reliable client deletion when topic is closing --- .vscode/launch.json | 15 + buildSrc/src/main/kotlin/Ci.kt | 2 +- buildSrc/src/main/kotlin/Plugins.kt | 2 +- infinitic-cache/build.gradle.kts | 2 + .../caches/caffeine/CaffeineCachedKeySet.kt | 5 +- .../caches/caffeine/CaffeineCachedKeyValue.kt | 4 +- .../infinitic/cache/caches/caffeine/setup.kt | 4 +- .../io/infinitic/cache/config/CacheConfig.kt | 51 +- ...ffeineConfig.kt => CaffeineCacheConfig.kt} | 21 +- .../caffeine/CaffeineCachedKeySetTests.kt | 4 +- .../caffeine/CaffeineCachedKeyValueTests.kt | 4 +- .../infinitic/cache/config/CacheConfigImpl.kt | 2 +- .../cache/config/CacheConfigTests.kt | 49 +- .../cache/config/CaffeineConfigTests.kt | 4 +- .../io/infinitic/clients/InfiniticClient.kt | 115 ++- ...ientConfig.kt => InfiniticClientConfig.kt} | 27 +- ...e.kt => InfiniticClientConfigInterface.kt} | 4 +- .../clients/dispatcher/ClientDispatcher.kt | 91 +- .../clients/dispatcher/ResponseFlow.kt | 60 ++ .../infinitic/clients/InfiniticClientTests.kt | 48 +- .../clients/dispatcher/ResponseFlowTest.kt | 138 +++ .../io/infinitic/clients/samples/timeouts.kt | 2 +- .../io/infinitic/clients/scaffold/response.kt | 87 ++ .../events/config/EventListenerConfig.kt | 90 -- .../events/messages/CloudEventTests.kt | 62 +- infinitic-common/build.gradle.kts | 5 +- .../infinitic/cloudEvents/SelectionConfig.kt | 43 + .../logs/loggersName.kt | 23 +- .../infinitic/common/data/MillisDuration.kt | 2 +- .../common/exceptions/thisShouldNotHappen.kt | 11 +- .../ExecutorRegistryInterface.kt} | 31 +- .../common/transport/InfiniticConsumer.kt | 7 +- .../common/transport/InfiniticResources.kt | 12 +- .../io/infinitic/common/transport/Topic.kt | 2 +- .../{ => logged}/LoggedInfiniticConsumer.kt | 19 +- .../{ => logged}/LoggedInfiniticProducer.kt | 4 +- .../transport/{ => logged}/loggedInfinitic.kt | 2 +- .../io/infinitic/common/utils/ClassUtil.kt | 6 +- .../kotlin/io/infinitic/common/utils/merge.kt | 14 +- .../common/workers/config/RetryPolicy.kt | 51 +- .../registry/RegisteredEventListener.kt | 31 - .../registry/RegisteredServiceTagEngine.kt | 27 - .../registry/RegisteredWorkflowExecutor.kt | 137 --- .../registry/RegisteredWorkflowStateEngine.kt | 27 - .../registry/RegisteredWorkflowTagEngine.kt | 27 - .../common/workflows/executors/Parser.kt | 37 + .../main/kotlin/io/infinitic/tasks/Task.kt | 3 + .../kotlin/io/infinitic/tasks/TaskContext.kt | 2 - .../kotlin/io/infinitic/tasks/WithRetry.kt | 3 +- .../kotlin/io/infinitic/tasks/WithTimeout.kt | 7 +- ...0.15.1.avsc => clientEnvelope-0.16.0.avsc} | 0 ...5.1.avsc => delegatedTaskData-0.16.0.avsc} | 0 ....avsc => serviceEventEnvelope-0.16.0.avsc} | 0 ...sc => serviceExecutorEnvelope-0.16.0.avsc} | 0 ....1.avsc => serviceTagEnvelope-0.16.0.avsc} | 0 ...1.avsc => workflowCmdEnvelope-0.16.0.avsc} | 0 ...vsc => workflowEngineEnvelope-0.16.0.avsc} | 0 ...avsc => workflowEventEnvelope-0.16.0.avsc} | 0 ...-0.15.1.avsc => workflowState-0.16.0.avsc} | 0 ...1.avsc => workflowTagEnvelope-0.16.0.avsc} | 0 ...vsc => workflowTaskParameters-0.16.0.avsc} | 0 ...sc => workflowTaskReturnValue-0.16.0.avsc} | 0 infinitic-common/src/main/resources/versions | 2 +- .../io/infinitic/serDe/SerDeJavaTest.java | 7 - .../infinitic/common/utils/ClassUtilTests.kt | 10 +- .../registry/RegisteredWorkflowTests.kt | 190 ---- .../serDe/kotlin/SerDeKotlinTests.kt | 3 - .../panels/infrastructure/AllServicesState.kt | 2 +- .../infrastructure/AllWorkflowsState.kt | 2 +- infinitic-storage/build.gradle.kts | 2 +- .../storage/compression/CompressionConfig.kt | 15 +- .../storage/config/InMemoryConfig.kt | 5 +- .../storage/config/InMemoryStorageConfig.kt | 67 ++ .../infinitic/storage/config/MySQLConfig.kt | 203 ++-- .../storage/config/MySQLStorageConfig.kt | 118 +++ .../storage/config/PostgresConfig.kt | 199 ++-- .../storage/config/PostgresStorageConfig.kt | 117 +++ .../infinitic/storage/config/RedisConfig.kt | 121 +-- .../storage/config/RedisStorageConfig.kt | 105 +++ .../infinitic/storage/config/StorageConfig.kt | 129 +-- .../inMemory/InMemoryKeySetStorage.kt | 2 +- .../inMemory/InMemoryKeyValueStorage.kt | 2 +- .../mysql/MySQLKeySetStorage.kt | 2 +- .../mysql/MySQLKeyValueStorage.kt | 2 +- .../postgres/PostgresKeySetStorage.kt | 2 +- .../postgres/PostgresKeyValueStorage.kt | 2 +- .../redis/RedisKeySetStorage.kt | 2 +- .../redis/RedisKeyValueStorage.kt | 2 +- .../config/MySQLStorageConfigTest.java | 172 ++++ .../storage/config/PostgresConfigTest.java | 172 ++++ .../storage/config/RedisConfigTest.java | 139 +++ .../storage/config/InMemoryConfigTests.kt | 33 +- .../storage/config/MySQLConfigTests.kt | 168 +++- .../storage/config/PostgresConfigTests.kt | 167 +++- .../storage/config/RedisConfigTests.kt | 126 ++- .../storage/config/StorageConfigImpl.kt | 2 +- .../storage/config/StorageConfigInterface.kt | 0 .../storage/config/StorageConfigTests.kt | 126 --- .../infinitic/storage/config/StorageTests.kt | 66 +- .../inMemory/InMemoryKeySetStorageTests.kt | 3 +- .../inMemory/InMemoryKeyValueStorageTests.kt | 3 +- .../mysql/MySQLKeySetStorageTests.kt | 9 +- .../mysql/MySQLKeyValueStorageTests.kt | 9 +- .../databases/mysql/MySqlConfigTests.kt | 108 --- .../databases/postgres/PostgresConfigTests.kt | 107 --- .../postgres/PostgresKeySetStorageTests.kt | 6 +- .../postgres/PostgresKeyValueStorageTests.kt | 6 +- .../redis/RedisKeySetStorageTests.kt | 1 - .../redis/RedisKeyValueStorageTests.kt | 1 - .../infinitic/tasks/executor/TaskExecutor.kt | 62 +- .../tasks/executor/task/TaskContextImpl.kt | 2 - .../tasks/executor/TaskExecutorTests.kt | 104 +- .../tasks/executor/samples/SimpleService.kt | 6 +- .../src/test/kotlin/io/infinitic/Test.kt | 50 +- .../kotlin/io/infinitic/scaffolds/engine.kt | 4 +- .../tests/branches/BranchesWorkflowTests.kt | 6 +- .../tests/channels/ChannelWorkflowTests.kt | 10 +- .../tests/timeouts/TimeoutsWorkflow.kt | 2 +- .../tests/timeouts/TimeoutsWorkflowTests.kt | 21 +- .../kotlin/io/infinitic/utils/UtilService.kt | 4 +- .../test/kotlin/io/infinitic/utils/retries.kt | 4 +- .../kotlin/io/infinitic/utils/timeouts.kt | 4 +- infinitic-tests/src/test/resources/pulsar.yml | 19 +- .../src/test/resources/register.yml | 162 +++- .../test/resources/simplelogger.properties | 2 + .../io/infinitic/inMemory/InMemoryChannels.kt | 76 +- .../inMemory/InMemoryInfiniticConsumer.kt | 34 +- .../inMemory/InMemoryInfiniticProducer.kt | 4 +- .../inMemory/InMemoryInfiniticResources.kt | 44 + .../pulsar/PulsarInfiniticConsumer.kt | 68 +- .../pulsar/PulsarInfiniticResources.kt | 24 +- .../pulsar/admin/PulsarInfiniticAdmin.kt | 175 ++-- .../infinitic/pulsar/config/PulsarConfig.kt | 239 ++--- .../config/auth/AuthenticationOAuth2Config.kt | 9 +- .../config/auth/AuthenticationSaslConfig.kt | 11 +- .../pulsar/config/policies/PoliciesConfig.kt | 40 +- .../io/infinitic/pulsar/consumers/Consumer.kt | 103 +- .../pulsar/consumers/ConsumerConfig.kt | 44 +- .../pulsar/resources/PulsarResources.kt | 29 +- .../pulsar/resources/PulsarSubscription.kt | 8 +- .../io/infinitic/pulsar/resources/Topics.kt | 2 +- .../pulsar/admin/PulsarInfiniticAdminTests.kt | 16 +- .../pulsar/consumers/ConsumerTests.kt | 95 +- .../pulsar/resources/PulsarResourcesTest.kt | 16 +- .../config/InMemoryTransportConfig.kt | 51 + .../transport/config/PulsarTransportConfig.kt | 212 +++++ .../transport/config/TransportConfig.kt | 110 +-- .../config/TransportConfigInterface.kt | 10 +- .../transport}/config/PulsarConfigTests.kt | 51 +- infinitic-utils/build.gradle.kts | 5 + .../kotlin/io/infinitic/config/loaders.kt | 76 ++ .../io/infinitic/properties/properties.kt | 22 +- .../io/infinitic/workers/InfiniticWorker.kt | 768 +++++++++------ .../workers/InfiniticWorkerBuilder.kt | 132 +++ .../workers/config/ConfigGetterInterface.kt | 47 + .../workers/config/EventListenerConfig.kt | 203 ++++ .../workers/config/InfiniticWorkerConfig.kt | 108 +++ .../InfiniticWorkerConfigInterface.kt} | 18 +- .../{register => }/config/LogsConfig.kt | 4 +- .../infinitic/workers/config/ServiceConfig.kt | 84 ++ .../workers/config/ServiceExecutorConfig.kt | 202 ++++ .../workers/config/ServiceTagEngineConfig.kt | 102 ++ .../infinitic/workers/config/WorkerConfig.kt | 168 ---- .../workers/config/WorkflowConfig.kt | 86 ++ .../workers/config/WorkflowExecutorConfig.kt | 256 +++++ .../config/WorkflowStateEngineConfig.kt | 100 ++ .../workers/config/WorkflowTagEngineConfig.kt | 103 ++ .../workers/register/InfiniticRegister.kt | 172 ---- .../workers/register/InfiniticRegisterImpl.kt | 403 -------- .../workers/register/config/ServiceConfig.kt | 137 --- .../register/config/ServiceConfigDefault.kt | 94 -- .../workers/register/config/WorkflowConfig.kt | 186 ---- .../register/config/WorkflowConfigDefault.kt | 108 --- .../workers/register/config/default.kt | 54 -- .../workers/registry/ExecutorRegistry.kt | 135 +++ .../workers/InfiniticWorkerTests.java | 68 -- .../workers/JavaInfiniticWorkerTests.java | 123 +++ .../infinitic/workers/InfiniticWorkerTests.kt | 417 ++++++++ .../config/ConfigGetterInterfaceTests.kt | 135 +++ .../config/EventListenerConfigTests.kt | 199 ++++ .../workers/config/RetryPolicyTests.kt | 79 -- .../workers/config/ServiceConfigTests.kt | 87 +- .../config/ServiceExecutorConfigTests.kt | 231 +++++ .../config/ServiceTagEngineConfigTests.kt | 113 +++ .../workers/config/WorkerConfigTests.kt | 265 ------ .../workers/config/WorkflowConfigTests.kt | 108 ++- .../config/WorkflowExecutorConfigTests.kt | 426 +++++++++ .../config/WorkflowStateEngineConfigTests.kt | 115 +++ .../config/WorkflowTagEngineConfigTests.kt | 114 +++ .../register/InfiniticRegisterTests.kt | 887 ------------------ .../io/infinitic/workers/samples/ServiceA.kt | 4 +- .../io/infinitic/workers/samples/WorkflowA.kt | 4 +- .../io/infinitic/workers/scafold/start.kt | 214 +++++ .../services/exceptionInInitializerError.yml | 18 +- .../config/services/incompatibleListener.yml | 19 +- .../services/incompatibleServiceName.yml | 15 +- .../resources/config/services/instance.yml | 15 +- .../config/services/instanceWithListener.yml | 21 +- .../services/invocationTargetException.yml | 38 - .../resources/config/services/unknown.yml | 15 +- .../config/services/unknownListener.yml | 21 +- .../workflows/exceptionInInitializerError.yml | 15 +- .../workflows/incompatibleWorkflowName.yml | 15 +- .../workflows/incompatibleWorkflowsName.yml | 19 +- .../resources/config/workflows/instance.yml | 15 +- .../workflows/invocationTargetException.yml | 15 +- .../config/workflows/notAWorkflow.yml | 15 +- .../resources/config/workflows/unknown.yml | 15 +- .../test/resources/simplelogger.properties | 41 + .../workflows/engine/WorkflowStateEngine.kt | 2 +- .../config/WorkflowStateEngineConfig.kt | 63 -- .../storage/LoggedWorkflowStateStorage.kt | 2 +- .../workflows/tag/WorkflowTagEngine.kt | 8 +- .../tag/config/WorkflowTagEngineConfig.kt | 63 -- .../workflowTask/WorkflowTaskImpl.kt | 2 + .../workflowTask/workflowProperties.kt | 67 -- .../workflows/workflowTask/PropertiesTests.kt | 1 + publish.gradle.kts | 3 +- 218 files changed, 8182 insertions(+), 5799 deletions(-) create mode 100644 .vscode/launch.json rename infinitic-cache/src/main/kotlin/io/infinitic/cache/config/{CaffeineConfig.kt => CaffeineCacheConfig.kt} (79%) rename infinitic-client/src/main/kotlin/io/infinitic/clients/config/{ClientConfig.kt => InfiniticClientConfig.kt} (69%) rename infinitic-client/src/main/kotlin/io/infinitic/clients/config/{ClientConfigInterface.kt => InfiniticClientConfigInterface.kt} (91%) create mode 100644 infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ResponseFlow.kt create mode 100644 infinitic-client/src/test/kotlin/io/infinitic/clients/dispatcher/ResponseFlowTest.kt create mode 100644 infinitic-client/src/test/kotlin/io/infinitic/clients/scaffold/response.kt delete mode 100644 infinitic-cloudevents/src/main/kotlin/io/infinitic/events/config/EventListenerConfig.kt create mode 100644 infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/SelectionConfig.kt rename infinitic-common/src/main/kotlin/io/infinitic/{common => cloudEvents}/logs/loggersName.kt (65%) rename infinitic-common/src/main/kotlin/io/infinitic/common/{workers/registry/WorkerRegistry.kt => registry/ExecutorRegistryInterface.kt} (56%) rename infinitic-transport/src/main/kotlin/io/infinitic/transport/config/Transport.kt => infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticResources.kt (81%) rename infinitic-common/src/main/kotlin/io/infinitic/common/transport/{ => logged}/LoggedInfiniticConsumer.kt (89%) rename infinitic-common/src/main/kotlin/io/infinitic/common/transport/{ => logged}/LoggedInfiniticProducer.kt (93%) rename infinitic-common/src/main/kotlin/io/infinitic/common/transport/{ => logged}/loggedInfinitic.kt (98%) delete mode 100644 infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredEventListener.kt delete mode 100644 infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceTagEngine.kt delete mode 100644 infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowExecutor.kt delete mode 100644 infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowStateEngine.kt delete mode 100644 infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTagEngine.kt rename infinitic-common/src/main/resources/schemas/{clientEnvelope-0.15.1.avsc => clientEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{delegatedTaskData-0.15.1.avsc => delegatedTaskData-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{serviceEventEnvelope-0.15.1.avsc => serviceEventEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{serviceExecutorEnvelope-0.15.1.avsc => serviceExecutorEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{serviceTagEnvelope-0.15.1.avsc => serviceTagEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowCmdEnvelope-0.15.1.avsc => workflowCmdEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowEngineEnvelope-0.15.1.avsc => workflowEngineEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowEventEnvelope-0.15.1.avsc => workflowEventEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowState-0.15.1.avsc => workflowState-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowTagEnvelope-0.15.1.avsc => workflowTagEnvelope-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowTaskParameters-0.15.1.avsc => workflowTaskParameters-0.16.0.avsc} (100%) rename infinitic-common/src/main/resources/schemas/{workflowTaskReturnValue-0.15.1.avsc => workflowTaskReturnValue-0.16.0.avsc} (100%) delete mode 100644 infinitic-common/src/test/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTests.kt create mode 100644 infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryStorageConfig.kt create mode 100644 infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLStorageConfig.kt create mode 100644 infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresStorageConfig.kt create mode 100644 infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisStorageConfig.kt rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/inMemory/InMemoryKeySetStorage.kt (97%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/inMemory/InMemoryKeyValueStorage.kt (97%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/mysql/MySQLKeySetStorage.kt (98%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/mysql/MySQLKeyValueStorage.kt (98%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/postgres/PostgresKeySetStorage.kt (98%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/postgres/PostgresKeyValueStorage.kt (98%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/redis/RedisKeySetStorage.kt (97%) rename infinitic-storage/src/main/kotlin/io/infinitic/storage/{storages => databases}/redis/RedisKeyValueStorage.kt (97%) create mode 100644 infinitic-storage/src/test/java/io/infinitic/storage/config/MySQLStorageConfigTest.java create mode 100644 infinitic-storage/src/test/java/io/infinitic/storage/config/PostgresConfigTest.java create mode 100644 infinitic-storage/src/test/java/io/infinitic/storage/config/RedisConfigTest.java rename infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfigInterface.kt => infinitic-storage/src/test/kotlin/io/infinitic/storage/config/InMemoryConfigTests.kt (60%) rename infinitic-storage/src/{main => test}/kotlin/io/infinitic/storage/config/StorageConfigInterface.kt (100%) delete mode 100644 infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigTests.kt delete mode 100644 infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySqlConfigTests.kt delete mode 100644 infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresConfigTests.kt create mode 100644 infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticResources.kt rename infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceExecutor.kt => infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticResources.kt (65%) create mode 100644 infinitic-transport/src/main/kotlin/io/infinitic/transport/config/InMemoryTransportConfig.kt create mode 100644 infinitic-transport/src/main/kotlin/io/infinitic/transport/config/PulsarTransportConfig.kt rename {infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar => infinitic-transport/src/test/kotlin/io/infinitic/transport}/config/PulsarConfigTests.kt (67%) create mode 100644 infinitic-utils/src/main/kotlin/io/infinitic/config/loaders.kt rename infinitic-task-tag/src/main/kotlin/io/infinitic/tasks/tag/config/ServiceTagEngineConfig.kt => infinitic-utils/src/main/kotlin/io/infinitic/properties/properties.kt (74%) create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorkerBuilder.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ConfigGetterInterface.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/EventListenerConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfig.kt rename infinitic-worker/src/main/kotlin/io/infinitic/workers/{register/config/RegisterConfigInterface.kt => config/InfiniticWorkerConfigInterface.kt} (81%) rename infinitic-worker/src/main/kotlin/io/infinitic/workers/{register => }/config/LogsConfig.kt (94%) create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceExecutorConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceTagEngineConfig.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowExecutorConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfig.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfig.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegister.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegisterImpl.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfig.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfigDefault.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfig.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfigDefault.kt delete mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/default.kt create mode 100644 infinitic-worker/src/main/kotlin/io/infinitic/workers/registry/ExecutorRegistry.kt delete mode 100644 infinitic-worker/src/test/java/io/infinitic/workers/InfiniticWorkerTests.java create mode 100644 infinitic-worker/src/test/java/io/infinitic/workers/JavaInfiniticWorkerTests.java create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ConfigGetterInterfaceTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/EventListenerConfigTests.kt delete mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/RetryPolicyTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceExecutorConfigTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceTagEngineConfigTests.kt delete mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkerConfigTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowExecutorConfigTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfigTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfigTests.kt delete mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/register/InfiniticRegisterTests.kt create mode 100644 infinitic-worker/src/test/kotlin/io/infinitic/workers/scafold/start.kt delete mode 100644 infinitic-worker/src/test/resources/config/services/invocationTargetException.yml create mode 100644 infinitic-worker/src/test/resources/simplelogger.properties delete mode 100644 infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/config/WorkflowStateEngineConfig.kt delete mode 100644 infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/config/WorkflowTagEngineConfig.kt delete mode 100644 infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/workflowProperties.kt diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..4df042203 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "kotlin", + "request": "launch", + "name": "Kotlin Launch", + "projectRoot": "${workspaceFolder}", + "mainClass": "path.to.your.MainClassKt" + } + ] +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/Ci.kt b/buildSrc/src/main/kotlin/Ci.kt index ead090600..ea908cec4 100644 --- a/buildSrc/src/main/kotlin/Ci.kt +++ b/buildSrc/src/main/kotlin/Ci.kt @@ -25,7 +25,7 @@ object Ci { private const val SNAPSHOT = "-SNAPSHOT" // base version number - private const val BASE = "0.15.1" + private const val BASE = "0.16.0" // GitHub run number private val githubRunNumber = System.getenv("GITHUB_RUN_NUMBER") diff --git a/buildSrc/src/main/kotlin/Plugins.kt b/buildSrc/src/main/kotlin/Plugins.kt index f0d4c4dfa..57b28439b 100644 --- a/buildSrc/src/main/kotlin/Plugins.kt +++ b/buildSrc/src/main/kotlin/Plugins.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -const val kotlinVersion = "2.0.20" +const val kotlinVersion = "2.0.0" @Suppress("ConstPropertyName") object Plugins { diff --git a/infinitic-cache/build.gradle.kts b/infinitic-cache/build.gradle.kts index bf6a3f5ba..8b2bf1174 100644 --- a/infinitic-cache/build.gradle.kts +++ b/infinitic-cache/build.gradle.kts @@ -21,6 +21,8 @@ * Licensor: infinitic.io */ dependencies { + implementation(project(":infinitic-utils")) + implementation(Libs.Caffeine.caffeine) testImplementation(Libs.Hoplite.yaml) diff --git a/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySet.kt b/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySet.kt index cebc9d286..bc52282d1 100644 --- a/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySet.kt +++ b/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySet.kt @@ -25,11 +25,12 @@ package io.infinitic.cache.caches.caffeine import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine import io.infinitic.cache.Flushable -import io.infinitic.cache.config.CaffeineConfig +import io.infinitic.cache.config.CaffeineCacheConfig import io.infinitic.cache.data.Bytes import io.infinitic.cache.keySet.CachedKeySet -internal class CaffeineCachedKeySet(config: CaffeineConfig) : CachedKeySet, Flushable { +internal class CaffeineCachedKeySet(config: CaffeineCacheConfig) : CachedKeySet, + Flushable { private val caffeine: Cache> = Caffeine.newBuilder().setup(config).build() diff --git a/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValue.kt b/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValue.kt index 2b09ce2e8..a9fa0514e 100644 --- a/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValue.kt +++ b/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValue.kt @@ -25,10 +25,10 @@ package io.infinitic.cache.caches.caffeine import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine import io.infinitic.cache.Flushable -import io.infinitic.cache.config.CaffeineConfig +import io.infinitic.cache.config.CaffeineCacheConfig import io.infinitic.cache.keyValue.CachedKeyValue -internal class CaffeineCachedKeyValue(config: CaffeineConfig) : CachedKeyValue, +internal class CaffeineCachedKeyValue(config: CaffeineCacheConfig) : CachedKeyValue, Flushable { private var caffeine: Cache = Caffeine.newBuilder().setup(config).build() diff --git a/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/setup.kt b/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/setup.kt index 1a5b32397..3967e757e 100644 --- a/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/setup.kt +++ b/infinitic-cache/src/main/kotlin/io/infinitic/cache/caches/caffeine/setup.kt @@ -25,13 +25,13 @@ package io.infinitic.cache.caches.caffeine import com.github.benmanes.caffeine.cache.RemovalCause import io.github.oshai.kotlinlogging.KotlinLogging import io.infinitic.cache.config.CacheConfig -import io.infinitic.cache.config.CaffeineConfig +import io.infinitic.cache.config.CaffeineCacheConfig import java.util.concurrent.TimeUnit import com.github.benmanes.caffeine.cache.Caffeine as CaffeineCache internal val logger = KotlinLogging.logger(CacheConfig::class.java.name) -internal fun CaffeineCache.setup(config: CaffeineConfig): CaffeineCache { +internal fun CaffeineCache.setup(config: CaffeineCacheConfig): CaffeineCache { config.maximumSize?.let { maximumSize(it) } config.expireAfterAccess?.let { expireAfterAccess(it, TimeUnit.SECONDS) } diff --git a/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CacheConfig.kt b/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CacheConfig.kt index 8078bac28..fd954953d 100644 --- a/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CacheConfig.kt +++ b/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CacheConfig.kt @@ -22,43 +22,36 @@ */ package io.infinitic.cache.config -import io.infinitic.cache.caches.caffeine.CaffeineCachedKeySet -import io.infinitic.cache.caches.caffeine.CaffeineCachedKeyValue +import io.infinitic.cache.config.CaffeineCacheConfig.CaffeineConfigBuilder import io.infinitic.cache.keySet.CachedKeySet import io.infinitic.cache.keyValue.CachedKeyValue +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString -data class CacheConfig( - internal val caffeine: CaffeineConfig? = null -) { +sealed interface CacheConfig { + val keySet: CachedKeySet? + val keyValue: CachedKeyValue? + val type: String companion object { @JvmStatic - fun from(caffeineConfig: CaffeineConfig) = CacheConfig(caffeine = caffeineConfig) - } - - val type: CacheType by lazy { - when { - caffeine != null -> CacheType.CAFFEINE - else -> CacheType.NONE - } - } + fun builder() = CaffeineConfigBuilder() - val keySet: CachedKeySet? by lazy { - when { - caffeine != null -> CaffeineCachedKeySet(caffeine) - else -> null - } - } + /** Create CacheConfig from files in file system */ + @JvmStatic + fun fromYamlFile(vararg files: String): CacheConfig = + loadFromYamlFile(*files) - val keyValue: CachedKeyValue? by lazy { - when { - caffeine != null -> CaffeineCachedKeyValue(caffeine) - else -> null - } - } + /** Create CacheConfig from files in resources directory */ + @JvmStatic + fun fromYamlResource(vararg resources: String): CacheConfig = + loadFromYamlResource(*resources) - enum class CacheType { - NONE, - CAFFEINE + /** Create CacheConfig from yaml strings */ + @JvmStatic + fun fromYamlString(vararg yamls: String): CacheConfig = + loadFromYamlString(*yamls) } } + diff --git a/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CaffeineConfig.kt b/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CaffeineCacheConfig.kt similarity index 79% rename from infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CaffeineConfig.kt rename to infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CaffeineCacheConfig.kt index 556df8557..24b50939b 100644 --- a/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CaffeineConfig.kt +++ b/infinitic-cache/src/main/kotlin/io/infinitic/cache/config/CaffeineCacheConfig.kt @@ -22,12 +22,25 @@ */ package io.infinitic.cache.config +import io.infinitic.cache.caches.caffeine.CaffeineCachedKeySet +import io.infinitic.cache.caches.caffeine.CaffeineCachedKeyValue +import io.infinitic.cache.keySet.CachedKeySet +import io.infinitic.cache.keyValue.CachedKeyValue + @Suppress("unused") -data class CaffeineConfig( +data class CaffeineCacheConfig( val maximumSize: Long? = 10000, // 10k units val expireAfterAccess: Long? = 3600, // 1 hour val expireAfterWrite: Long? = 3600 // 1 hour -) { +) : CacheConfig { + + override val type = "caffeine" + + override val keySet: CachedKeySet by lazy { CaffeineCachedKeySet(this) } + + override val keyValue: CachedKeyValue by lazy { CaffeineCachedKeyValue(this) } + + init { maximumSize?.let { require(it > 0) { "maximumSize MUST be > 0" } } expireAfterAccess?.let { require(it >= 0) { "expireAfterAccess MUST be >= 0" } } @@ -43,7 +56,7 @@ data class CaffeineConfig( * Caffeine builder (Useful for Java user) */ class CaffeineConfigBuilder { - private val default = CaffeineConfig() + private val default = CaffeineCacheConfig() private var maximumSize = default.maximumSize private var expireAfterAccess = default.expireAfterAccess private var expireAfterWrite = default.expireAfterWrite @@ -52,7 +65,7 @@ data class CaffeineConfig( fun setExpireAfterAccess(expAftAccess: Long) = apply { this.expireAfterAccess = expAftAccess } fun setExpireAfterWrite(expAftWrite: Long) = apply { this.expireAfterWrite = expAftWrite } - fun build() = CaffeineConfig( + fun build() = CaffeineCacheConfig( maximumSize = maximumSize, expireAfterAccess = expireAfterAccess, expireAfterWrite = expireAfterWrite, diff --git a/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySetTests.kt b/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySetTests.kt index f0afce811..7523e5a09 100644 --- a/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySetTests.kt +++ b/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeySetTests.kt @@ -22,14 +22,14 @@ */ package io.infinitic.cache.caches.caffeine -import io.infinitic.cache.config.CaffeineConfig +import io.infinitic.cache.config.CaffeineCacheConfig import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe class CaffeineCachedKeySetTests : StringSpec( { - val storage = CaffeineCachedKeySet(CaffeineConfig()) + val storage = CaffeineCachedKeySet(CaffeineCacheConfig()) beforeTest { storage.set("key", setOf("foo".toByteArray(), "bar".toByteArray())) } diff --git a/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValueTests.kt b/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValueTests.kt index 38bd13217..a076f01c8 100644 --- a/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValueTests.kt +++ b/infinitic-cache/src/test/kotlin/io/infinitic/cache/caches/caffeine/CaffeineCachedKeyValueTests.kt @@ -22,7 +22,7 @@ */ package io.infinitic.cache.caches.caffeine -import io.infinitic.cache.config.CaffeineConfig +import io.infinitic.cache.config.CaffeineCacheConfig import io.kotest.assertions.throwables.shouldNotThrowAny import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe @@ -30,7 +30,7 @@ import io.kotest.matchers.shouldBe class CaffeineCachedKeyValueTests : StringSpec( { - val storage = CaffeineCachedKeyValue(CaffeineConfig()) + val storage = CaffeineCachedKeyValue(CaffeineCacheConfig()) val known = "foo" val unknown = "bar" diff --git a/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigImpl.kt b/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigImpl.kt index 09d6e0c9c..7be7ff9ed 100644 --- a/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigImpl.kt +++ b/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigImpl.kt @@ -23,5 +23,5 @@ package io.infinitic.cache.config internal data class CacheConfigImpl( - override val cache: CacheConfig = CacheConfig() + override val cache: CacheConfig? = null ) : CacheConfigInterface diff --git a/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigTests.kt b/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigTests.kt index 93b7648d3..cb77dc02a 100644 --- a/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigTests.kt +++ b/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CacheConfigTests.kt @@ -35,31 +35,20 @@ class CacheConfigTests : "default cache should be None" { val config = loadConfigFromYaml("nothing:") - config shouldBe CacheConfigImpl(cache = CacheConfig()) - config.cache.type shouldBe CacheConfig.CacheType.NONE - config.cache.keyValue shouldBe null - config.cache.keySet shouldBe null + config shouldBe CacheConfigImpl(cache = null) } - "cache without type should be None" { - val config = loadConfigFromYaml("cache:") + "can set null cache " { + val config = loadConfigFromYaml("cache: null") - config shouldBe CacheConfigImpl(cache = CacheConfig()) - config.cache.type shouldBe CacheConfig.CacheType.NONE - config.cache.keyValue shouldBe null - config.cache.keySet shouldBe null + config shouldBe CacheConfigImpl(cache = null) } - "can choose Caffeine cache" { - val config = loadConfigFromYaml( - """ -cache: - caffeine: - """, - ) - config shouldBe CacheConfigImpl(cache = CacheConfig(caffeine = CaffeineConfig())) - config.cache.type shouldBe CacheConfig.CacheType.CAFFEINE - config.cache.keyValue!!::class shouldBe CaffeineCachedKeyValue::class + "default cache should be Caffeine" { + val config = loadConfigFromYaml("cache:") + + config shouldBe CacheConfigImpl(cache = CaffeineCacheConfig()) + config.cache?.keyValue!!::class shouldBe CaffeineCachedKeyValue::class config.cache.keySet!!::class shouldBe CaffeineCachedKeySet::class } @@ -67,25 +56,19 @@ cache: val config = loadConfigFromYaml( """ cache: - caffeine: - maximumSize: 100 - expireAfterAccess: 42 - expireAfterWrite: 64 + maximumSize: 100 + expireAfterAccess: 42 + expireAfterWrite: 64 """, ) config shouldBe CacheConfigImpl( - cache = CacheConfig( - caffeine = CaffeineConfig( - 100, - 42, - 64, - ), + cache = CaffeineCacheConfig( + 100, + 42, + 64, ), ) - config.cache.type shouldBe CacheConfig.CacheType.CAFFEINE - config.cache.keyValue!!::class shouldBe CaffeineCachedKeyValue::class - config.cache.keySet!!::class shouldBe CaffeineCachedKeySet::class } }, ) diff --git a/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CaffeineConfigTests.kt b/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CaffeineConfigTests.kt index 557143061..998f32c91 100644 --- a/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CaffeineConfigTests.kt +++ b/infinitic-cache/src/test/kotlin/io/infinitic/cache/config/CaffeineConfigTests.kt @@ -28,11 +28,11 @@ import io.kotest.matchers.shouldBe class CaffeineConfigTests : StringSpec( { "Can create CaffeineConfig through builder" { - val caffeineConfig = CaffeineConfig + val caffeineCacheConfig = CaffeineCacheConfig .builder() .build() - caffeineConfig shouldBe CaffeineConfig() + caffeineCacheConfig shouldBe CaffeineCacheConfig() } }, ) diff --git a/infinitic-client/src/main/kotlin/io/infinitic/clients/InfiniticClient.kt b/infinitic-client/src/main/kotlin/io/infinitic/clients/InfiniticClient.kt index b3af03748..e2cfd6444 100644 --- a/infinitic-client/src/main/kotlin/io/infinitic/clients/InfiniticClient.kt +++ b/infinitic-client/src/main/kotlin/io/infinitic/clients/InfiniticClient.kt @@ -23,13 +23,10 @@ package io.infinitic.clients import io.github.oshai.kotlinlogging.KotlinLogging -import io.infinitic.autoclose.addAutoCloseResource -import io.infinitic.autoclose.autoClose -import io.infinitic.clients.config.ClientConfig -import io.infinitic.clients.config.ClientConfigInterface +import io.infinitic.clients.config.InfiniticClientConfig +import io.infinitic.clients.config.InfiniticClientConfigInterface import io.infinitic.clients.dispatcher.ClientDispatcher import io.infinitic.common.clients.messages.ClientMessage -import io.infinitic.common.data.MillisInstant import io.infinitic.common.data.methods.MethodReturnValue import io.infinitic.common.proxies.ExistingWorkflowProxyHandler import io.infinitic.common.proxies.NewWorkflowProxyHandler @@ -38,21 +35,23 @@ import io.infinitic.common.proxies.RequestByWorkflowId import io.infinitic.common.proxies.RequestByWorkflowTag import io.infinitic.common.tasks.data.ServiceName import io.infinitic.common.tasks.data.TaskId -import io.infinitic.common.transport.InfiniticConsumer -import io.infinitic.common.transport.InfiniticProducer -import io.infinitic.common.transport.LoggedInfiniticConsumer -import io.infinitic.common.transport.LoggedInfiniticProducer +import io.infinitic.common.transport.logged.LoggedInfiniticConsumer +import io.infinitic.common.transport.logged.LoggedInfiniticProducer import io.infinitic.common.utils.annotatedName import io.infinitic.common.workflows.data.workflowMethods.WorkflowMethodId import io.infinitic.common.workflows.data.workflows.WorkflowMeta import io.infinitic.common.workflows.data.workflows.WorkflowTag import io.infinitic.exceptions.clients.InvalidIdTagSelectionException import io.infinitic.exceptions.clients.InvalidStubException -import io.infinitic.transport.config.TransportConfig +import io.infinitic.properties.isLazyInitialized import io.infinitic.workflows.DeferredStatus import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.cancel +import kotlinx.coroutines.job +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout import org.jetbrains.annotations.TestOnly import java.lang.reflect.Proxy import java.util.concurrent.CompletableFuture @@ -60,31 +59,69 @@ import java.util.concurrent.atomic.AtomicBoolean @Suppress("unused") class InfiniticClient( - consumer: InfiniticConsumer, - producer: InfiniticProducer + val config: InfiniticClientConfigInterface ) : InfiniticClientInterface { + private val resources by lazy { + config.transport.resources + } + private val consumer by lazy { + LoggedInfiniticConsumer(logger, config.transport.consumer) + } + private val producer by lazy { + LoggedInfiniticProducer(logger, config.transport.producer).apply { + config.name?.let { name = it } + } + } + + private val shutdownGracePeriodSeconds = config.transport.shutdownGracePeriodSeconds + private var isClosed: AtomicBoolean = AtomicBoolean(false) - // Scope used to consuming messages + // Scope used to asynchronously send message, and also to consumes messages private val clientScope = CoroutineScope(Dispatchers.IO) - private val dispatcher = ClientDispatcher(clientScope, consumer, producer) + private val dispatcher by lazy { ClientDispatcher(clientScope, consumer, producer) } override val name by lazy { producer.name } /** Get last Deferred created by the call of a stub */ override val lastDeferred get() = dispatcher.getLastDeferred() - /** Close client if interrupted */ - init { - Runtime.getRuntime().addShutdownHook(Thread { close() }) - } - override fun close() { if (isClosed.compareAndSet(false, true)) { + logger.info { "Closing client..." } clientScope.cancel() - autoClose() + runBlocking { + try { + withTimeout((shutdownGracePeriodSeconds * 1000).toLong()) { + clientScope.coroutineContext.job.join() + } + } catch (e: TimeoutCancellationException) { + logger.warn { + "The grace period (${shutdownGracePeriodSeconds}s) allotted when closing the client was insufficient." + + "Some ongoing messages may not have been sent properly." + } + } finally { + deleteClientTopics() + config.transport.close() + } + } + logger.info { "Client closed." } + } + } + + /** + * Deletes the topics associated with the client + * (Do NOT delete the client DLQ topic to allow manual inspection of failed messages) + */ + private suspend fun deleteClientTopics() { + if (::consumer.isLazyInitialized) { + resources.deleteTopicForClient(name).getOrElse { + logger.warn(it) { "Unable to delete topic for client $name, please delete it manually." } + }?.let { + logger.info { "Client topic $it deleted." } + } } } @@ -190,8 +227,7 @@ class InfiniticClient( } @TestOnly - internal suspend fun handle(message: ClientMessage, publishTime: MillisInstant) = - dispatcher.handle(message, publishTime) + internal suspend fun handle(message: ClientMessage) = dispatcher.responseFlow.emit(message) private fun getProxyHandler(stub: Any): ProxyHandler<*> { val exception by lazy { InvalidStubException("$stub") } @@ -234,42 +270,19 @@ class InfiniticClient( private val logger = KotlinLogging.logger {} - /** Create InfiniticClient from config */ - @JvmStatic - fun fromConfig(config: ClientConfigInterface): InfiniticClient = with(config) { - // Create TransportConfig - val transportConfig = TransportConfig(transport, pulsar, shutdownGracePeriodInSeconds) - - // Get Infinitic Consumer - val consumer = LoggedInfiniticConsumer(logger, transportConfig.consumer) - - // Get Infinitic Producer - val producer = LoggedInfiniticProducer(logger, transportConfig.producer) - - // apply name if it exists - name?.let { producer.name = it } - - // Create Infinitic Client - InfiniticClient(consumer, producer).also { - // close consumer with the client - it.addAutoCloseResource(consumer) - } - } - - /** Create InfiniticClient with config from resources directory */ @JvmStatic - fun fromConfigResource(vararg resources: String): InfiniticClient = - fromConfig(ClientConfig.fromResource(*resources)) + fun fromYamlResource(vararg resources: String) = + InfiniticClient(InfiniticClientConfig.fromYamlResource(*resources)) /** Create InfiniticClient with config from system file */ @JvmStatic - fun fromConfigFile(vararg files: String): InfiniticClient = - fromConfig(ClientConfig.fromFile(*files)) + fun fromYamlFile(vararg files: String) = + InfiniticClient(InfiniticClientConfig.fromYamlFile(*files)) /** Create InfiniticClient with config from yaml strings */ @JvmStatic - fun fromConfigYaml(vararg yamls: String): InfiniticClient = - fromConfig(ClientConfig.fromYaml(*yamls)) + fun fromYamlString(vararg yamls: String) = + InfiniticClient(InfiniticClientConfig.fromYamlString(*yamls)) } } diff --git a/infinitic-client/src/main/kotlin/io/infinitic/clients/config/ClientConfig.kt b/infinitic-client/src/main/kotlin/io/infinitic/clients/config/InfiniticClientConfig.kt similarity index 69% rename from infinitic-client/src/main/kotlin/io/infinitic/clients/config/ClientConfig.kt rename to infinitic-client/src/main/kotlin/io/infinitic/clients/config/InfiniticClientConfig.kt index 3d79ae781..8f5b4c657 100644 --- a/infinitic-client/src/main/kotlin/io/infinitic/clients/config/ClientConfig.kt +++ b/infinitic-client/src/main/kotlin/io/infinitic/clients/config/InfiniticClientConfig.kt @@ -25,37 +25,30 @@ package io.infinitic.clients.config import io.infinitic.common.config.loadConfigFromFile import io.infinitic.common.config.loadConfigFromResource import io.infinitic.common.config.loadConfigFromYaml -import io.infinitic.pulsar.config.PulsarConfig -import io.infinitic.transport.config.Transport +import io.infinitic.transport.config.TransportConfig -data class ClientConfig( +data class InfiniticClientConfig( /** Client name */ override val name: String? = null, /** Transport configuration */ - override val transport: Transport = Transport.pulsar, - - /** Pulsar configuration */ - override val pulsar: PulsarConfig? = null, - - /** Shutdown Grace Period */ - override val shutdownGracePeriodInSeconds: Double = 10.0 -) : ClientConfigInterface { + override val transport: TransportConfig, +) : InfiniticClientConfigInterface { companion object { /** Create ClientConfig from file in file system */ @JvmStatic - fun fromFile(vararg files: String): ClientConfig = - loadConfigFromFile(*files) + fun fromYamlFile(vararg files: String): InfiniticClientConfig = + loadConfigFromFile(*files) /** Create ClientConfig from file in resources directory */ @JvmStatic - fun fromResource(vararg resources: String): ClientConfig = - loadConfigFromResource(*resources) + fun fromYamlResource(vararg resources: String): InfiniticClientConfig = + loadConfigFromResource(*resources) /** Create ClientConfig from yaml strings */ @JvmStatic - fun fromYaml(vararg yamls: String): ClientConfig = - loadConfigFromYaml(*yamls) + fun fromYamlString(vararg yamls: String): InfiniticClientConfig = + loadConfigFromYaml(*yamls) } } diff --git a/infinitic-client/src/main/kotlin/io/infinitic/clients/config/ClientConfigInterface.kt b/infinitic-client/src/main/kotlin/io/infinitic/clients/config/InfiniticClientConfigInterface.kt similarity index 91% rename from infinitic-client/src/main/kotlin/io/infinitic/clients/config/ClientConfigInterface.kt rename to infinitic-client/src/main/kotlin/io/infinitic/clients/config/InfiniticClientConfigInterface.kt index ac1a38358..ca59069ce 100644 --- a/infinitic-client/src/main/kotlin/io/infinitic/clients/config/ClientConfigInterface.kt +++ b/infinitic-client/src/main/kotlin/io/infinitic/clients/config/InfiniticClientConfigInterface.kt @@ -24,7 +24,7 @@ package io.infinitic.clients.config import io.infinitic.transport.config.TransportConfigInterface -interface ClientConfigInterface : TransportConfigInterface { - /** Client name */ +interface InfiniticClientConfigInterface : TransportConfigInterface { + /** (Optional) Client name */ val name: String? } diff --git a/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ClientDispatcher.kt b/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ClientDispatcher.kt index 8818808b3..4169c633d 100644 --- a/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ClientDispatcher.kt +++ b/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ClientDispatcher.kt @@ -37,7 +37,6 @@ import io.infinitic.common.clients.messages.MethodUnknown import io.infinitic.common.clients.messages.WorkflowIdsByTag import io.infinitic.common.clients.messages.interfaces.MethodMessage import io.infinitic.common.data.MillisDuration -import io.infinitic.common.data.MillisInstant import io.infinitic.common.data.methods.MethodName import io.infinitic.common.data.methods.MethodReturnValue import io.infinitic.common.data.methods.decodeReturnValue @@ -97,10 +96,8 @@ import io.infinitic.exceptions.clients.MultipleCustomIdException import io.infinitic.workflows.DeferredStatus import io.infinitic.workflows.SendChannel import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.flow.first import kotlinx.coroutines.future.future -import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.coroutines.runBlocking import org.jetbrains.annotations.TestOnly import java.lang.reflect.Method import java.util.concurrent.CompletableFuture @@ -121,19 +118,24 @@ internal class ClientDispatcher( private val clientRequester by lazy { ClientRequester(clientName = ClientName.from(emitterName)) } // flag telling if the client consumer loop is initialized - private val isClientConsumerInitialized: AtomicBoolean = AtomicBoolean(false) - - // Flow used to receive messages - private val responseFlow = MutableSharedFlow(replay = 0) - - private suspend fun T.sendTo(topic: Topic) = with(producer) { sendTo(topic) } - - private fun T.sendToAsync(topic: Topic) = clientScope.future { sendTo(topic) } + private val hasClientConsumerStarted: AtomicBoolean = AtomicBoolean(false) + + /** + * A mutable shared flow that buffers the latest `Result` of `ClientMessage`. + * This flow is used to publish and observe the outcome of client messages received by the dispatcher. + * + * The flow maintains a replay cache of 1 element, ensuring that the most recent value + * is always available to new subscribers immediately upon subscription, + * in particular the last exception if the consuming process failed + */ + internal val responseFlow = ResponseFlow() + + private suspend fun T.sendTo(topic: Topic) = with(producer) { + sendTo(topic) + } - // a message received by the client is sent to responseFlow - @Suppress("UNUSED_PARAMETER") - internal suspend fun handle(message: ClientMessage, publishTime: MillisInstant) { - responseFlow.emit(message) + private fun T.sendToAsync(topic: Topic) = clientScope.future { + sendTo(topic) } // Utility to get access to last deferred @@ -208,7 +210,7 @@ internal class ClientDispatcher( ?: Long.MAX_VALUE // lazily starts client consumer if not already started and waits - val waiting = waitForAsync(timeout) { + val waiting = awaitAsync(timeout) { it is MethodMessage && it.workflowId == workflowId && it.workflowMethodId == methodId } @@ -378,7 +380,6 @@ internal class ClientDispatcher( else -> thisShouldNotHappen() } - fun retryTaskAsync( workflowName: WorkflowName, requestBy: RequestBy, @@ -423,7 +424,7 @@ internal class ClientDispatcher( workflowTag: WorkflowTag ): Set { // lazily starts client consumer if not already started and waits - val waiting = waitForAsync { + val waiting = awaitAsync { (it is WorkflowIdsByTag) && (it.workflowName == workflowName) && (it.workflowTag == workflowTag) @@ -435,6 +436,7 @@ internal class ClientDispatcher( emitterName = emitterName, emittedAt = null, ) + // synchronously sent the message to get errors msg.sendToAsync(WorkflowTagEngineTopic).join() @@ -721,37 +723,46 @@ internal class ClientDispatcher( return deferredSend } - private fun ProxyHandler<*>.getTimeout(): MillisDuration? = timeoutInMillisDuration.getOrElse { throw IllegalStateException("Unable to retrieve Timeout info when dispatching $method", it) } - private fun waitForAsync( + private fun startListeningAsync(): CompletableFuture = clientScope.future { + try { + consumer.start( + subscription = MainSubscription(ClientTopic), + entity = emitterName.toString(), + handler = { message, _ -> responseFlow.emit(message) }, + beforeDlq = null, + concurrency = 1, + ) + } catch (e: Exception) { + // all subsequent calls to await will fail and trigger this exception + responseFlow.emitThrowable(e) + throw e + } + } + + private fun awaitAsync( timeout: Long = Long.MAX_VALUE, predicate: suspend (ClientMessage) -> Boolean - ): CompletableFuture { - // lazily starts client consumer if not already started - if (isClientConsumerInitialized.compareAndSet(false, true)) { - clientScope.future { - consumer.start( - subscription = MainSubscription(ClientTopic), - entity = emitterName.toString(), - handler = ::handle, - beforeDlq = null, - concurrency = 1, - ) - }.join() - } + ): CompletableFuture = clientScope.future { + await(timeout, predicate) + } - // wait for the first message that matches the predicate - return clientScope.future { - withTimeoutOrNull(timeout) { - responseFlow.first { predicate(it) } - } + private fun await( + timeout: Long = Long.MAX_VALUE, + predicate: suspend (ClientMessage) -> Boolean + ): ClientMessage? = runBlocking { + if (hasClientConsumerStarted.compareAndSet(false, true)) { + // asynchronously starts client consumer if not already started + startListeningAsync() } - } + // immediately wait for the message that matches the predicate + responseFlow.first(timeout) { predicate(it) } + } companion object { @TestOnly diff --git a/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ResponseFlow.kt b/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ResponseFlow.kt new file mode 100644 index 000000000..351a1da77 --- /dev/null +++ b/infinitic-client/src/main/kotlin/io/infinitic/clients/dispatcher/ResponseFlow.kt @@ -0,0 +1,60 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.clients.dispatcher + +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withTimeoutOrNull + +internal class ResponseFlow { + private val responseFlow = MutableSharedFlow>(replay = 1) + .apply { tryEmit(Result.success(null)) } + + suspend fun emit(value: T) { + responseFlow.emit(Result.success(value)) + } + + suspend fun emitThrowable(e: Throwable) { + responseFlow.emit(Result.failure(e)) + } + + suspend fun first(timeout: Long = Long.MAX_VALUE) = first(timeout) { true } + + suspend fun first(timeout: Long = Long.MAX_VALUE, predicate: suspend (T) -> Boolean): T? = + withTimeoutOrNull(timeout) { + var isFirst = true + responseFlow.first { result -> + when (isFirst) { + true -> { + isFirst = false + // throw the exception + result.getOrThrow() + // ignore the first value if not an exception + false + } + + false -> predicate(result.getOrThrow()!!) + } + }.getOrThrow()!! + } +} diff --git a/infinitic-client/src/test/kotlin/io/infinitic/clients/InfiniticClientTests.kt b/infinitic-client/src/test/kotlin/io/infinitic/clients/InfiniticClientTests.kt index a0395a88f..60be5f12a 100644 --- a/infinitic-client/src/test/kotlin/io/infinitic/clients/InfiniticClientTests.kt +++ b/infinitic-client/src/test/kotlin/io/infinitic/clients/InfiniticClientTests.kt @@ -22,6 +22,7 @@ */ package io.infinitic.clients +import io.infinitic.clients.config.InfiniticClientConfig import io.infinitic.clients.deferred.ExistingDeferredWorkflow import io.infinitic.clients.samples.FakeClass import io.infinitic.clients.samples.FakeInterface @@ -35,7 +36,6 @@ import io.infinitic.common.clients.data.ClientName import io.infinitic.common.clients.messages.MethodCompleted import io.infinitic.common.clients.messages.WorkflowIdsByTag import io.infinitic.common.data.MillisDuration -import io.infinitic.common.data.MillisInstant import io.infinitic.common.data.methods.MethodArgs import io.infinitic.common.data.methods.MethodName import io.infinitic.common.data.methods.MethodParameterTypes @@ -52,8 +52,6 @@ import io.infinitic.common.tasks.executors.messages.ServiceExecutorMessage import io.infinitic.common.tasks.tags.messages.CompleteDelegatedTask import io.infinitic.common.tasks.tags.messages.ServiceTagMessage import io.infinitic.common.transport.ClientTopic -import io.infinitic.common.transport.InfiniticConsumer -import io.infinitic.common.transport.InfiniticProducer import io.infinitic.common.transport.MainSubscription import io.infinitic.common.transport.ServiceTagEngineTopic import io.infinitic.common.transport.Subscription @@ -87,6 +85,9 @@ import io.infinitic.common.workflows.tags.messages.WorkflowTagEngineMessage import io.infinitic.exceptions.WorkflowTimedOutException import io.infinitic.exceptions.clients.InvalidChannelUsageException import io.infinitic.exceptions.clients.InvalidStubException +import io.infinitic.inMemory.InMemoryInfiniticConsumer +import io.infinitic.inMemory.InMemoryInfiniticProducer +import io.infinitic.transport.config.InMemoryTransportConfig import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe @@ -99,9 +100,8 @@ import io.mockk.mockk import io.mockk.slot import java.util.concurrent.CopyOnWriteArrayList -private val taskTagSlots = CopyOnWriteArrayList() // multithreading update -private val workflowTagSlots = - CopyOnWriteArrayList() // multithreading update +private val taskTagSlots = CopyOnWriteArrayList() +private val workflowTagSlots = CopyOnWriteArrayList() private val taskSlot = slot() private val workflowCmdSlot = slot() private val delaySlot = slot() @@ -118,7 +118,7 @@ private fun tagResponse() { workflowIds = setOf(WorkflowId(), WorkflowId()), emitterName = EmitterName("mockk"), ) - later { client.handle(workflowIdsByTag, MillisInstant.now()) } + later { client.handle(workflowIdsByTag) } } } } @@ -133,22 +133,36 @@ private fun engineResponse() { methodReturnValue = MethodReturnValue.from("success", null), emitterName = EmitterName("mockk"), ) - later { client.handle(methodCompleted, MillisInstant.now()) } + later { client.handle(methodCompleted) } } } -private val producer = mockk { +internal val mockedProducer = mockk { every { name } returns "$clientNameTest" - coEvery { capture(taskTagSlots).sendTo(ServiceTagEngineTopic) } answers { } - coEvery { capture(workflowTagSlots).sendTo(WorkflowTagEngineTopic) } answers { tagResponse() } - coEvery { capture(workflowCmdSlot).sendTo(WorkflowStateCmdTopic) } answers { engineResponse() } + coEvery { + internalSendTo(capture(taskTagSlots), ServiceTagEngineTopic) + } answers { } + coEvery { + internalSendTo(capture(workflowTagSlots), WorkflowTagEngineTopic) + } answers { tagResponse() } + coEvery { + internalSendTo(capture(workflowCmdSlot), WorkflowStateCmdTopic) + } answers { engineResponse() } } -private val consumer = mockk { +internal val mockedConsumer = mockk { coEvery { start(any>(), "$clientNameTest", any(), any(), any()) } just Runs } -private val client = InfiniticClient(consumer, producer) +internal val mockedTransport = mockk { + every { consumer } returns mockedConsumer + every { producer } returns mockedProducer + every { shutdownGracePeriodSeconds } returns 5.0 +} + +internal val infiniticClientConfig = InfiniticClientConfig(transport = mockedTransport) + +private val client = InfiniticClient(infiniticClientConfig) internal class InfiniticClientTests : StringSpec( { @@ -193,7 +207,7 @@ internal class InfiniticClientTests : StringSpec( // when asynchronously dispatching a workflow, the consumer should not be started coVerify(exactly = 0) { - consumer.start( + mockedConsumer.start( MainSubscription(ClientTopic), "$clientNameTest", any(), @@ -408,7 +422,7 @@ internal class InfiniticClientTests : StringSpec( // when waiting for a workflow, the consumer should be started coVerify { - consumer.start( + mockedConsumer.start( MainSubscription(ClientTopic), "$clientNameTest", any(), @@ -422,7 +436,7 @@ internal class InfiniticClientTests : StringSpec( // the consumer should be started only once coVerify(exactly = 1) { - consumer.start( + mockedConsumer.start( MainSubscription(ClientTopic), "$clientNameTest", any(), diff --git a/infinitic-client/src/test/kotlin/io/infinitic/clients/dispatcher/ResponseFlowTest.kt b/infinitic-client/src/test/kotlin/io/infinitic/clients/dispatcher/ResponseFlowTest.kt new file mode 100644 index 000000000..9836e4d5a --- /dev/null +++ b/infinitic-client/src/test/kotlin/io/infinitic/clients/dispatcher/ResponseFlowTest.kt @@ -0,0 +1,138 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.clients.dispatcher + +import io.infinitic.common.fixtures.later +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch + +internal class ResponseFlowTest : StringSpec( + { + "First get null if timeout" { + val response = ResponseFlow() + + response.first(100) shouldBe null + } + + "First does not return element emitted before first call" { + val response = ResponseFlow() + + response.emit("foo") + response.first(100) shouldBe null + } + + "First returns element emitted after first() call" { + val response = ResponseFlow() + + later(10) { + response.emit("foo") + } + response.first() shouldBe "foo" + } + + "First returns element filtered" { + val response = ResponseFlow() + + later(10) { + response.emit("foo1") + response.emit("foo2") + response.emit("foo3") + response.emit("foo4") + response.emit("foo5") + } + response.first { it == "foo3" } shouldBe "foo3" + } + + "Can call multiple first() in parallel" { + val response = ResponseFlow() + + later(10) { + response.emit("foo1") + response.emit("foo2") + response.emit("foo3") + response.emit("foo4") + response.emit("foo5") + } + coroutineScope { + launch { + response.first { it == "foo2" } shouldBe "foo2" + } + launch { + response.first { it == "foo5" } shouldBe "foo5" + } + } + } + + "can not retrieve the same element" { + val response = ResponseFlow() + + later(10) { + response.emit("foo1") + response.emit("foo2") + response.emit("foo3") + response.emit("foo4") + response.emit("foo5") + } + response.first { it == "foo2" } shouldBe "foo2" + // foo2 has been emitted before the next call + response.first(100) { it == "foo2" } shouldBe null + } + + "can retrieve the same element in parallel" { + val response = ResponseFlow() + + later(10) { + response.emit("foo1") + response.emit("foo2") + response.emit("foo3") + response.emit("foo4") + response.emit("foo5") + } + coroutineScope { + launch { + response.first { it == "foo2" } shouldBe "foo2" + } + launch { + response.first { it == "foo2" } shouldBe "foo2" + } + } + } + + "Once an exception is thrown, we can not retrieve anything anymore" { + val response = ResponseFlow() + val e = RuntimeException() + + response.emitThrowable(e) + + try { + response.first() + } catch (t: Throwable) { + t shouldBe e + } + shouldThrow { response.first() } shouldBe e + } + }, +) diff --git a/infinitic-client/src/test/kotlin/io/infinitic/clients/samples/timeouts.kt b/infinitic-client/src/test/kotlin/io/infinitic/clients/samples/timeouts.kt index e9cba5428..b35921145 100644 --- a/infinitic-client/src/test/kotlin/io/infinitic/clients/samples/timeouts.kt +++ b/infinitic-client/src/test/kotlin/io/infinitic/clients/samples/timeouts.kt @@ -26,5 +26,5 @@ package io.infinitic.clients.samples import io.infinitic.tasks.WithTimeout class After50MilliSeconds : WithTimeout { - override fun getTimeoutInSeconds() = 0.05 + override fun getTimeoutSeconds() = 0.05 } diff --git a/infinitic-client/src/test/kotlin/io/infinitic/clients/scaffold/response.kt b/infinitic-client/src/test/kotlin/io/infinitic/clients/scaffold/response.kt new file mode 100644 index 000000000..c4b33e931 --- /dev/null +++ b/infinitic-client/src/test/kotlin/io/infinitic/clients/scaffold/response.kt @@ -0,0 +1,87 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.clients.scaffold + +import io.infinitic.clients.dispatcher.ResponseFlow +import io.kotest.common.runBlocking +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.future.future +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.util.concurrent.CompletableFuture +import java.util.concurrent.atomic.AtomicBoolean + + +private class Client { + private val scope = CoroutineScope(Dispatchers.IO) + private val responseFlow = ResponseFlow() + private val isStarted = AtomicBoolean(false) + var msg = 0 + + private fun startAsync(): CompletableFuture = scope.future { + try { + while (isActive) { + msg++ + delay(5) + responseFlow.emit(msg.toString()) + if (msg == 150) throw Exception("Breaking!") + } + } catch (e: Exception) { + responseFlow.emitThrowable(e) + throw e + } + } + + fun await(timeout: Long = Long.MAX_VALUE, predicate: (String) -> Boolean): String? = runBlocking { + if (isStarted.compareAndSet(false, true)) { + startAsync() + } + responseFlow.first(timeout) { predicate(it) } + } +} + +suspend fun main() { + val client = Client() + println("continue") + client.await { it.toLong() == 1L }.also { println("Found $it") } + coroutineScope { + launch { + client.await { it.toLong() >= 100 }.also { println("Found $it") } + } + launch { + try { + client.await { it.toLong() >= 200 }.also { println("Found $it") } + } catch (e: Exception) { + println("Found $e") + } + } + launch { + client.await { it.toLong() >= 30000 }.also { println("Found $it") } + } + } + + println("joined") +} diff --git a/infinitic-cloudevents/src/main/kotlin/io/infinitic/events/config/EventListenerConfig.kt b/infinitic-cloudevents/src/main/kotlin/io/infinitic/events/config/EventListenerConfig.kt deleted file mode 100644 index 4ba8ab828..000000000 --- a/infinitic-cloudevents/src/main/kotlin/io/infinitic/events/config/EventListenerConfig.kt +++ /dev/null @@ -1,90 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.events.config - -import io.infinitic.cloudEvents.CloudEventListener -import io.infinitic.common.utils.getInstance - -@Suppress("unused") -data class EventListenerConfig( - var `class`: String? = null, - var concurrency: Int? = null, - var subscriptionName: String? = null, -) { - var isDefined = true - - val instance: CloudEventListener? - get() = `class`?.getInstance()?.getOrThrow() as CloudEventListener? - - init { - `class`?.let { - require(it.isNotEmpty()) { error("'class' must not be empty") } - val instance = it.getInstance().getOrThrow() - require(instance is CloudEventListener) { - error("Class '$`class`' must implement '${CloudEventListener::class.java.name}'") - } - } - - concurrency?.let { - require(it >= 0) { - error("'${::concurrency.name}' must be an integer >= 0") - } - } - - subscriptionName?.let { - require(it.isNotEmpty()) { error("'${::subscriptionName.name}' must not be empty") } - } - } - - companion object { - @JvmStatic - fun builder() = EventListenerConfigBuilder() - } - - /** - * EventListenerConfig builder (Useful for Java user) - */ - class EventListenerConfigBuilder { - private val default = EventListenerConfig() - private var `class` = default.`class` - private var concurrency = default.concurrency - private var subscriptionName = default.subscriptionName - - fun `class`(`class`: String) = - apply { this.`class` = `class` } - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun subscriptionName(subscriptionName: String) = - apply { this.subscriptionName = subscriptionName } - - fun build() = EventListenerConfig( - `class`, - concurrency, - subscriptionName, - ) - } - - private fun error(txt: String) = "eventListener: $txt" -} diff --git a/infinitic-cloudevents/src/test/kotlin/io/infinitic/events/messages/CloudEventTests.kt b/infinitic-cloudevents/src/test/kotlin/io/infinitic/events/messages/CloudEventTests.kt index 9d09c0124..79a0de30e 100644 --- a/infinitic-cloudevents/src/test/kotlin/io/infinitic/events/messages/CloudEventTests.kt +++ b/infinitic-cloudevents/src/test/kotlin/io/infinitic/events/messages/CloudEventTests.kt @@ -85,8 +85,10 @@ import io.infinitic.common.workflows.engine.messages.WorkflowCmdMessage import io.infinitic.common.workflows.engine.messages.WorkflowCompletedEvent import io.infinitic.common.workflows.engine.messages.WorkflowEventMessage import io.infinitic.common.workflows.engine.messages.WorkflowStateEngineMessage +import io.infinitic.storage.config.InMemoryStorageConfig +import io.infinitic.transport.config.InMemoryTransportConfig import io.infinitic.workers.InfiniticWorker -import io.infinitic.workers.config.WorkerConfig +import io.infinitic.workers.config.EventListenerConfig import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe import io.mockk.Runs @@ -101,32 +103,39 @@ import net.bytebuddy.utility.RandomString import java.net.URI import kotlin.reflect.full.isSubclassOf -private const val serviceConfig = """ -transport: inMemory -storage: inMemory -""" -private val workerConfig = WorkerConfig.fromYaml(serviceConfig) private val events = mutableListOf() private val eventListener = mockk { every { onEvent(capture(events)) } just Runs } -private val worker = InfiniticWorker.fromConfig(workerConfig).apply { - registerServiceEventListener("ServiceA", 2, eventListener, null) - registerWorkflowEventListener("WorkflowA", 2, eventListener, null) - startAsync() -} + +val transport = InMemoryTransportConfig() + +private val worker = InfiniticWorker.builder() + .setTransport(transport) + .setStorage(InMemoryStorageConfig.builder().build()) + .setEventListener( + EventListenerConfig.builder() + .setListener(eventListener) + .setRefreshDelaySeconds(0.0) + .setConcurrency(2), + ) + .build() private suspend fun T.sendToTopic(topic: Topic) { - with(worker.producer) { sendTo(topic) } + with(transport.producer) { sendTo(topic) } // wait a bit to let listener do its work - delay(200) + // and the listener to discover new services and workflows + delay(100) } suspend fun main() { + worker.startAsync() + ServiceExecutorMessage::class.sealedSubclasses.forEach { events.clear() val message = TestFactory.random(it, mapOf("serviceName" to ServiceName("ServiceA"))) message.sendToTopic(ServiceExecutorTopic) + events.firstOrNull()?.let { event -> val json = String(JsonFormat().serialize(event)) println(message) @@ -180,20 +189,24 @@ suspend fun main() { } } - + worker.close() } internal class CloudEventTests : StringSpec( { - beforeTest { - events.clear() + beforeSpec { + worker.startAsync() } afterSpec { worker.close() } + beforeEach { + events.clear() + } + ServiceExecutorMessage::class.sealedSubclasses.forEach { "Check ${it.simpleName} event envelope from Service Executor topic" { val message = TestFactory.random(it, mapOf("serviceName" to ServiceName("ServiceA"))) @@ -202,7 +215,7 @@ internal class CloudEventTests : events.size shouldBe 1 val event = events.first() event.id shouldBe message.messageId.toString() - event.source shouldBe URI("inmemory/services/ServiceA") + event.source shouldBe URI("inMemory/services/ServiceA") event.dataContentType shouldBe "application/json" event.subject shouldBe message.taskId.toString() event.type shouldBe when (it) { @@ -220,7 +233,7 @@ internal class CloudEventTests : events.size shouldBe 1 val event = events.first() event.id shouldBe message.messageId.toString() - event.source shouldBe URI("inmemory/services/ServiceA") + event.source shouldBe URI("inMemory/services/ServiceA") event.dataContentType shouldBe "application/json" event.subject shouldBe message.taskId.toString() event.type shouldBe when (it) { @@ -250,7 +263,7 @@ internal class CloudEventTests : events.size shouldBe 1 val event = events.first() - event.source shouldBe URI("inmemory/services/executor/WorkflowA") + event.source shouldBe URI("inMemory/services/executor/WorkflowA") event.subject shouldBe message.taskId.toString() event.type shouldBe when (it) { ExecuteTask::class -> "infinitic.task.start" @@ -275,7 +288,7 @@ internal class CloudEventTests : events.size shouldBe 1 val event = events.first() - event.source shouldBe URI("inmemory/services/executor/WorkflowA") + event.source shouldBe URI("inMemory/services/executor/WorkflowA") event.subject shouldBe message.taskId.toString() event.type shouldBe when (it) { TaskStartedEvent::class -> "infinitic.task.started" @@ -313,7 +326,7 @@ internal class CloudEventTests : if (events.size == 1) { val event = events.first() event.id shouldBe message.messageId.toString() - event.source shouldBe URI("inmemory/workflows/WorkflowA") + event.source shouldBe URI("inMemory/workflows/WorkflowA") event.dataContentType shouldBe "application/json" event.subject shouldBe message.workflowId.toString() event.type shouldBe type @@ -353,7 +366,8 @@ internal class CloudEventTests : "Check ${it.simpleName} event envelope from engine topic" { val message = TestFactory.random( it, - mapOf("workflowName" to WorkflowName("WorkflowA"), + mapOf( + "workflowName" to WorkflowName("WorkflowA"), ), ) message.sendToTopic(WorkflowStateEngineTopic) @@ -376,7 +390,7 @@ internal class CloudEventTests : if (events.size == 1) { val event = events.first() event.id shouldBe message.messageId.toString() - event.source shouldBe URI("inmemory/workflows/WorkflowA") + event.source shouldBe URI("inMemory/workflows/WorkflowA") event.dataContentType shouldBe "application/json" event.subject shouldBe message.workflowId.toString() event.type shouldBe type @@ -414,7 +428,7 @@ internal class CloudEventTests : if (events.size == 1) { val event = events.first() event.id shouldBe message.messageId.toString() - event.source shouldBe URI("inmemory/workflows/WorkflowA") + event.source shouldBe URI("inMemory/workflows/WorkflowA") event.dataContentType shouldBe "application/json" event.subject shouldBe message.workflowId.toString() event.type shouldBe type diff --git a/infinitic-common/build.gradle.kts b/infinitic-common/build.gradle.kts index dc5ff7c33..3dd7166fc 100644 --- a/infinitic-common/build.gradle.kts +++ b/infinitic-common/build.gradle.kts @@ -25,9 +25,10 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { id("java-test-fixtures") } dependencies { - implementation(Libs.Serialization.json) - implementation(Libs.JsonPath.jayway) + // JsonPath is used on client API + api(Libs.JsonPath.jayway) + implementation(Libs.Serialization.json) implementation(Libs.Mockk.mockk) implementation(Libs.Hoplite.core) implementation(Libs.Hoplite.yaml) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/SelectionConfig.kt b/infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/SelectionConfig.kt new file mode 100644 index 000000000..7b7062efb --- /dev/null +++ b/infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/SelectionConfig.kt @@ -0,0 +1,43 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.cloudEvents + +@Suppress("unused") +data class SelectionConfig( + val allow: List? = null, + val disallow: List = listOf() +) { + init { + allow?.forEach { + require(it.isNotEmpty()) { error("'${::allow.name}' must not contain empty element") } + } + disallow.forEach { + require(it.isNotEmpty()) { error("'${::disallow.name}' must not contain empty element") } + } + } + + fun isIncluded(name: String) = + (allow == null || allow.contains(name)) && !disallow.contains(name) + + private fun error(txt: String) = "eventListener: $txt" +} diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/logs/loggersName.kt b/infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/logs/loggersName.kt similarity index 65% rename from infinitic-common/src/main/kotlin/io/infinitic/common/logs/loggersName.kt rename to infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/logs/loggersName.kt index f9c7c6c4a..fd18bb39b 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/logs/loggersName.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/cloudEvents/logs/loggersName.kt @@ -20,20 +20,19 @@ * * Licensor: infinitic.io */ -package io.infinitic.common.logs +package io.infinitic.cloudEvents.logs private const val INFINITIC_PREFIX = "io.infinitic" - private const val CLOUD_EVENTS = "$INFINITIC_PREFIX.cloudEvents" -const val WORKFLOW_STATE_ENGINE = "WorkflowStateEngine" -const val WORKFLOW_TAG_ENGINE = "WorkflowTagEngine" -const val WORKFLOW_EXECUTOR = "WorkflowExecutor" -const val SERVICE_TAG_ENGINE = "ServiceTagEngine" -const val SERVICE_EXECUTOR = "ServiceExecutor" +private const val WORKFLOW_STATE_ENGINE = "WorkflowStateEngine" +private const val WORKFLOW_TAG_ENGINE = "WorkflowTagEngine" +private const val WORKFLOW_EXECUTOR = "WorkflowExecutor" +private const val SERVICE_TAG_ENGINE = "ServiceTagEngine" +private const val SERVICE_EXECUTOR = "ServiceExecutor" -const val WORKFLOW_STATE_ENGINE_CLOUD_EVENTS = "$CLOUD_EVENTS.$WORKFLOW_STATE_ENGINE" -const val WORKFLOW_TAG_ENGINE_CLOUD_EVENTS = "$CLOUD_EVENTS.$WORKFLOW_TAG_ENGINE" -const val WORKFLOW_EXECUTOR_CLOUD_EVENTS = "$CLOUD_EVENTS.$WORKFLOW_EXECUTOR" -const val SERVICE_TAG_ENGINE_CLOUD_EVENTS = "$CLOUD_EVENTS.$SERVICE_TAG_ENGINE" -const val SERVICE_EXECUTOR_CLOUD_EVENTS = "$CLOUD_EVENTS.$SERVICE_EXECUTOR" +const val CLOUD_EVENTS_WORKFLOW_STATE_ENGINE = "$CLOUD_EVENTS.$WORKFLOW_STATE_ENGINE" +const val CLOUD_EVENTS_WORKFLOW_TAG_ENGINE = "$CLOUD_EVENTS.$WORKFLOW_TAG_ENGINE" +const val CLOUD_EVENTS_WORKFLOW_EXECUTOR = "$CLOUD_EVENTS.$WORKFLOW_EXECUTOR" +const val CLOUD_EVENTS_SERVICE_TAG_ENGINE = "$CLOUD_EVENTS.$SERVICE_TAG_ENGINE" +const val CLOUD_EVENTS_SERVICE_EXECUTOR = "$CLOUD_EVENTS.$SERVICE_EXECUTOR" diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/data/MillisDuration.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/data/MillisDuration.kt index 3c006af17..904b22e5f 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/data/MillisDuration.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/data/MillisDuration.kt @@ -34,7 +34,7 @@ import kotlinx.serialization.json.JsonPrimitive @Serializable(with = MillisDurationSerializer::class) data class MillisDuration(val millis: Long) : Comparable, JsonAble { - override fun toString() = "${String.format("%.3fs", millis / 1000.0)}s" + override fun toString() = String.format("%.3fs", millis / 1000.0) override fun toJson() = JsonPrimitive(millis) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/exceptions/thisShouldNotHappen.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/exceptions/thisShouldNotHappen.kt index ac9ab61ac..eb2164f94 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/exceptions/thisShouldNotHappen.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/exceptions/thisShouldNotHappen.kt @@ -25,8 +25,9 @@ package io.infinitic.common.exceptions fun thisShouldNotHappen(reason: String? = null): Nothing = throw RuntimeException( "this should not happen${ - when (reason) { - null -> "" - else -> ": $reason" - } - }") + when (reason) { + null -> "" + else -> ": $reason" + } + }", + ) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/WorkerRegistry.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/registry/ExecutorRegistryInterface.kt similarity index 56% rename from infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/WorkerRegistry.kt rename to infinitic-common/src/main/kotlin/io/infinitic/common/registry/ExecutorRegistryInterface.kt index 4339e37c7..3f1306e74 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/WorkerRegistry.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/registry/ExecutorRegistryInterface.kt @@ -20,26 +20,23 @@ * * Licensor: infinitic.io */ -package io.infinitic.common.workers.registry +package io.infinitic.common.registry import io.infinitic.common.tasks.data.ServiceName +import io.infinitic.common.workflows.data.workflowTasks.WorkflowTaskParameters import io.infinitic.common.workflows.data.workflows.WorkflowName -import org.jetbrains.annotations.TestOnly +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout +import io.infinitic.workflows.Workflow +import io.infinitic.workflows.WorkflowCheckMode -class WorkerRegistry { - val serviceTagEngines = mutableMapOf() - val serviceExecutors = mutableMapOf() - val serviceEventListeners = mutableMapOf() +interface ExecutorRegistryInterface { + fun getServiceExecutorInstance(serviceName: ServiceName): Any + fun getServiceExecutorWithTimeout(serviceName: ServiceName): WithTimeout? + fun getServiceExecutorWithRetry(serviceName: ServiceName): WithRetry? - val workflowTagEngines = mutableMapOf() - val workflowStateEngines = mutableMapOf() - val workflowExecutors = mutableMapOf() - val workflowEventListeners = mutableMapOf() - - @TestOnly - fun flush() { - serviceTagEngines.values.forEach { it.storage.flush() } - workflowTagEngines.values.forEach { it.storage.flush() } - workflowStateEngines.values.forEach { it.storage.flush() } - } + fun getWorkflowExecutorInstance(workflowTaskParameters: WorkflowTaskParameters): Workflow + fun getWorkflowExecutorWithTimeout(workflowName: WorkflowName): WithTimeout? + fun getWorkflowExecutorWithRetry(workflowName: WorkflowName): WithRetry? + fun getWorkflowExecutorCheckMode(workflowName: WorkflowName): WorkflowCheckMode? } diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticConsumer.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticConsumer.kt index 3742c5607..d69ac634d 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticConsumer.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticConsumer.kt @@ -22,15 +22,10 @@ */ package io.infinitic.common.transport -import io.github.oshai.kotlinlogging.KLogger import io.infinitic.common.data.MillisInstant import io.infinitic.common.messages.Message -interface InfiniticConsumer : AutoCloseable { - - var workerLogger: KLogger - - fun join() +interface InfiniticConsumer { suspend fun start( subscription: Subscription, diff --git a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/Transport.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticResources.kt similarity index 81% rename from infinitic-transport/src/main/kotlin/io/infinitic/transport/config/Transport.kt rename to infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticResources.kt index 545af6834..6c83741ef 100644 --- a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/Transport.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/InfiniticResources.kt @@ -20,10 +20,12 @@ * * Licensor: infinitic.io */ -package io.infinitic.transport.config +package io.infinitic.common.transport -@Suppress("EnumEntryName") -enum class Transport { - pulsar, - inMemory +interface InfiniticResources { + suspend fun getServices(): Set + + suspend fun getWorkflows(): Set + + suspend fun deleteTopicForClient(clientName: String): Result } diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/Topic.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/Topic.kt index c739f94ef..394e1a8c7 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/Topic.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/Topic.kt @@ -133,7 +133,7 @@ data object WorkflowExecutorEventTopic : WorkflowTopic.isTimer get() = when (this) { diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/LoggedInfiniticConsumer.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/LoggedInfiniticConsumer.kt similarity index 89% rename from infinitic-common/src/main/kotlin/io/infinitic/common/transport/LoggedInfiniticConsumer.kt rename to infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/LoggedInfiniticConsumer.kt index 759d3131c..cf5e2ce30 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/LoggedInfiniticConsumer.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/LoggedInfiniticConsumer.kt @@ -20,11 +20,13 @@ * * Licensor: infinitic.io */ -package io.infinitic.common.transport +package io.infinitic.common.transport.logged import io.github.oshai.kotlinlogging.KLogger import io.infinitic.common.data.MillisInstant import io.infinitic.common.messages.Message +import io.infinitic.common.transport.InfiniticConsumer +import io.infinitic.common.transport.Subscription class LoggedInfiniticConsumer( private val logger: KLogger, @@ -61,19 +63,4 @@ class LoggedInfiniticConsumer( concurrency, ) } - - override var workerLogger: KLogger - get() = consumer.workerLogger - set(value) { - consumer.workerLogger = value - } - - override fun close() { - consumer.close() - } - - - override fun join() { - consumer.join() - } } diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/LoggedInfiniticProducer.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/LoggedInfiniticProducer.kt similarity index 93% rename from infinitic-common/src/main/kotlin/io/infinitic/common/transport/LoggedInfiniticProducer.kt rename to infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/LoggedInfiniticProducer.kt index 98a6b08df..65e82e2db 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/LoggedInfiniticProducer.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/LoggedInfiniticProducer.kt @@ -20,11 +20,13 @@ * * Licensor: infinitic.io */ -package io.infinitic.common.transport +package io.infinitic.common.transport.logged import io.github.oshai.kotlinlogging.KLogger import io.infinitic.common.data.MillisDuration import io.infinitic.common.messages.Message +import io.infinitic.common.transport.InfiniticProducer +import io.infinitic.common.transport.Topic class LoggedInfiniticProducer( private val logger: KLogger, diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/loggedInfinitic.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/loggedInfinitic.kt similarity index 98% rename from infinitic-common/src/main/kotlin/io/infinitic/common/transport/loggedInfinitic.kt rename to infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/loggedInfinitic.kt index 17c767162..d5bebac5c 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/transport/loggedInfinitic.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/transport/logged/loggedInfinitic.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.common.transport +package io.infinitic.common.transport.logged import io.infinitic.common.clients.messages.ClientMessage import io.infinitic.common.clients.messages.interfaces.MethodMessage diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/utils/ClassUtil.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/utils/ClassUtil.kt index c80e02280..e032166b3 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/utils/ClassUtil.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/utils/ClassUtil.kt @@ -404,7 +404,7 @@ internal val Class.timeoutInMillis: Result? private fun Class.findTimeoutInMillis(): Result? { // if this is not a WithTimeout interface, return null - if (!hasMethodImplemented(WithTimeout::getTimeoutInSeconds.javaMethod!!)) + if (!hasMethodImplemented(WithTimeout::getTimeoutSeconds.javaMethod!!)) return null // this is not an interface, we can build an instance and call the method @@ -429,7 +429,7 @@ private fun Class.findTimeoutInMillis(): Result? { private fun mock(klass: Class): T { val mock = mockkClass(klass.kotlin) when (mock is WithTimeout) { - true -> every { mock.getTimeoutInSeconds() } answers { callOriginal() } + true -> every { mock.getTimeoutSeconds() } answers { callOriginal() } false -> Unit } return mock @@ -450,7 +450,7 @@ private val String.constructorError: String private val String.instanceError: String get() = "Error during class '$this' instantiation" -internal fun Class.getInstance( +fun Class.getInstance( noEmptyConstructor: String = (this.name).noEmptyConstructor, constructorError: String = (this.name).constructorError, instanceError: String = (this.name).instanceError, diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/utils/merge.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/utils/merge.kt index fbae61eaa..99ef2f9be 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/utils/merge.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/utils/merge.kt @@ -23,18 +23,22 @@ package io.infinitic.common.utils +import io.infinitic.common.exceptions.thisShouldNotHappen import kotlin.reflect.KParameter import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.primaryConstructor -@JvmName("mergeNullableRight") -inline infix fun T.merge(default: T?): T { - if (default == null) return this +@JvmName("mergeNullableLeft") +inline infix fun T?.merge(default: T?): T? { + if (this == null) return default return this merge default } -inline infix fun T.merge(default: T): T { - val constructor = T::class.constructors.first() +inline infix fun T.merge(default: T?): T { + if (default == null) return this + + val constructor = T::class.primaryConstructor ?: thisShouldNotHappen() val params = constructor.parameters.associateBy({ it.name!! }, { it }) val args = mutableMapOf() diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/config/RetryPolicy.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/config/RetryPolicy.kt index 63b7d50c9..f548c81a3 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/config/RetryPolicy.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/config/RetryPolicy.kt @@ -32,8 +32,6 @@ import kotlin.random.Random sealed class RetryPolicy(open val maximumRetries: Int, open val ignoredExceptions: List) : WithRetry { - var isDefined = true - val ignoredClasses: List> by lazy { ignoredExceptions.map { klass -> klass.getClass().getOrThrow().also { @@ -66,6 +64,13 @@ sealed class RetryPolicy(open val maximumRetries: Int, open val ignoredException protected abstract fun getSecondsBeforeRetry(attempt: Int): Double? } +@Suppress("ClassName") +data object UNSET_RETRY_POLICY : RetryPolicy(0, listOf()) { + override fun check() {} + override fun getSecondsBeforeRetry(attempt: Int) = -Double.MAX_VALUE +} + +@Suppress("unused") data class ExponentialBackoffRetryPolicy( val minimumSeconds: Double = 1.0, val maximumSeconds: Double = 1000 * minimumSeconds, @@ -95,4 +100,46 @@ data class ExponentialBackoffRetryPolicy( override fun getSecondsBeforeRetry(attempt: Int): Double = min(maximumSeconds, minimumSeconds * backoffCoefficient.pow(attempt)) * (1 + randomFactor * (2 * Random.nextDouble() - 1)) + + companion object { + @JvmStatic + fun builder() = ExponentialBackoffRetryPolicyBuilder() + } + + class ExponentialBackoffRetryPolicyBuilder() { + private val default = ExponentialBackoffRetryPolicy() + private var minimumSeconds = default.minimumSeconds + private var maximumSeconds = default.maximumSeconds + private var backoffCoefficient = default.backoffCoefficient + private var randomFactor = default.randomFactor + private var maximumRetries = default.maximumRetries + private var ignoredExceptions = default.ignoredExceptions as MutableList + + fun build() = ExponentialBackoffRetryPolicy( + minimumSeconds, + maximumSeconds, + backoffCoefficient, + randomFactor, + maximumRetries, + ignoredExceptions, + ) + + fun setMinimumSeconds(minimumSeconds: Double) = + apply { this.minimumSeconds = minimumSeconds } + + fun setMaximumSeconds(maximumSeconds: Double) = + apply { this.maximumSeconds = maximumSeconds } + + fun setBackoffCoefficient(backoffCoefficient: Double) = + apply { this.backoffCoefficient = backoffCoefficient } + + fun setRandomFactor(randomFactor: Double) = + apply { this.randomFactor = randomFactor } + + fun setMaximumRetries(maximumRetries: Int) = + apply { this.maximumRetries = maximumRetries } + + fun addIgnoredException(exception: Class) = + apply { ignoredExceptions.add(exception.name) } + } } diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredEventListener.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredEventListener.kt deleted file mode 100644 index 443dea4b5..000000000 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredEventListener.kt +++ /dev/null @@ -1,31 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.common.workers.registry - -import io.infinitic.cloudEvents.CloudEventListener - -data class RegisteredEventListener( - val eventListener: CloudEventListener, - val concurrency: Int, - val subscriptionName: String? -) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceTagEngine.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceTagEngine.kt deleted file mode 100644 index 68e2e09bc..000000000 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceTagEngine.kt +++ /dev/null @@ -1,27 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.common.workers.registry - -import io.infinitic.common.tasks.tags.storage.TaskTagStorage - -data class RegisteredServiceTagEngine(val concurrency: Int, val storage: TaskTagStorage) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowExecutor.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowExecutor.kt deleted file mode 100644 index 70699cccd..000000000 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowExecutor.kt +++ /dev/null @@ -1,137 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.common.workers.registry - -import io.infinitic.common.exceptions.thisShouldNotHappen -import io.infinitic.common.workers.config.WorkflowVersion -import io.infinitic.common.workflows.WorkflowContext -import io.infinitic.common.workflows.data.workflowTasks.WorkflowTaskParameters -import io.infinitic.common.workflows.data.workflows.WorkflowName -import io.infinitic.common.workflows.emptyWorkflowContext -import io.infinitic.exceptions.workflows.UnknownWorkflowVersionException -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout -import io.infinitic.workflows.Workflow -import io.infinitic.workflows.WorkflowCheckMode - -typealias WorkflowFactories = List<() -> Workflow> -typealias WorkflowFactory = () -> Workflow - -data class RegisteredWorkflowExecutor( - val workflowName: WorkflowName, - val factories: WorkflowFactories, - val concurrency: Int, - val withTimeout: WithTimeout?, - val withRetry: WithRetry?, - val checkMode: WorkflowCheckMode? -) { - private var classFactoryInstanceByVersion: Map, WorkflowFactory, Workflow>> - - var classes: List> - - init { - require(factories.isNotEmpty()) { "List of factory must not be empty for workflow '$workflowName'" } - - // this is needed in case the workflow properties use the workflow context - Workflow.setContext(emptyWorkflowContext) - // store the list of classes generated by `factories`. - classes = factories.mapIndexed { index, factory -> - try { - factory()::class.java - } catch (e: Exception) { - throw IllegalArgumentException( - "Error in factory #$index while registering workflow $workflowName", - e, - ) - } - } - - // check that each class is associated with a different version - val classVersionMap = classes.associateWith { WorkflowVersion.from(it) } - classVersionMap.values - .groupingBy { it } - .eachCount() - .filter { it.value > 1 } - .keys - .forEach { version -> - val list = classVersionMap.filter { version == it.value }.keys.joinToString { it.name } - - throw IllegalArgumentException( - "$list have same version $version for workflow $workflowName", - ) - } - - val instances = factories.map { it() } - - // for each version, we store the Triple, () -> Workflow> - classFactoryInstanceByVersion = classes.mapIndexed { index, item -> - Triple(item, factories[index], instances[index]) - }.associateBy { WorkflowVersion.from(it.first) } - } - - private val lastVersion by lazy { - classFactoryInstanceByVersion.keys.maxOrNull() ?: thisShouldNotHappen() - } - - fun getInstance(workflowTaskParameters: WorkflowTaskParameters): Workflow = - with(workflowTaskParameters) { - // set WorkflowContext before Workflow instance creation - Workflow.setContext( - WorkflowContext( - workflowName = workflowName.toString(), - workflowId = workflowId.toString(), - methodName = workflowMethod.methodName.toString(), - methodId = workflowMethod.workflowMethodId.toString(), - meta = workflowMeta.map, - tags = workflowTags.map { it.tag }.toSet(), - ), - ) - - getInstanceByVersion(workflowVersion) - } - - fun getInstanceByVersion(workflowVersion: WorkflowVersion?): Workflow { - val (klass, factory, instance) = getClassFactoryByVersion(workflowVersion) - return factory().also { - // checking that the factory does not create an instance whose class is different from the initial call - if (!klass.isInstance(it)) throw IllegalArgumentException( - "The workflow factory has generated an instance of the class ${it::class}, " + - "which does not match the expected class $klass. Check your workflow factory implementation.", - ) - // checking that the instance created is different from the initial instance created - if (it === instance) throw IllegalArgumentException( - "The workflow factory has returned the same object instance ($it) twice. " + - "It should always return a new object instance when called. Check your workflow factory implementation.", - ) - } - } - - private fun getClassFactoryByVersion(workflowVersion: WorkflowVersion?): Triple, () -> Workflow, Workflow> { - val version = workflowVersion ?: lastVersion - - return classFactoryInstanceByVersion[version] ?: throw UnknownWorkflowVersionException( - workflowName, - version, - ) - } -} diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowStateEngine.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowStateEngine.kt deleted file mode 100644 index f43937aee..000000000 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowStateEngine.kt +++ /dev/null @@ -1,27 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.common.workers.registry - -import io.infinitic.common.workflows.engine.storage.WorkflowStateStorage - -data class RegisteredWorkflowStateEngine(val concurrency: Int, val storage: WorkflowStateStorage) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTagEngine.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTagEngine.kt deleted file mode 100644 index 7c8034bfd..000000000 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTagEngine.kt +++ /dev/null @@ -1,27 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.common.workers.registry - -import io.infinitic.common.workflows.tags.storage.WorkflowTagStorage - -data class RegisteredWorkflowTagEngine(val concurrency: Int, var storage: WorkflowTagStorage) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workflows/executors/Parser.kt b/infinitic-common/src/main/kotlin/io/infinitic/common/workflows/executors/Parser.kt index 537e41044..94aec0b37 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workflows/executors/Parser.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/common/workflows/executors/Parser.kt @@ -23,13 +23,24 @@ package io.infinitic.common.workflows.executors import com.fasterxml.jackson.annotation.JsonView +import io.github.oshai.kotlinlogging.KLogger +import io.infinitic.annotations.Ignore import io.infinitic.common.exceptions.thisShouldNotHappen +import io.infinitic.common.workflows.data.properties.PropertyHash import io.infinitic.common.workflows.data.properties.PropertyName import io.infinitic.common.workflows.data.properties.PropertyValue +import io.infinitic.workflows.Channel +import io.infinitic.workflows.Workflow +import org.slf4j.Logger +import java.lang.reflect.Proxy import java.security.InvalidParameterException import kotlin.reflect.KProperty1 +import kotlin.reflect.full.createType import kotlin.reflect.full.findAnnotations +import kotlin.reflect.full.hasAnnotation +import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.starProjectedType import kotlin.reflect.jvm.javaField fun setPropertiesToObject(obj: T, values: Map) { @@ -60,6 +71,32 @@ fun getPropertiesFromObject( }, ) +fun Workflow.setProperties( + propertiesHashValue: Map, + propertiesNameHash: Map +) { + val properties = propertiesNameHash.mapValues { + propertiesHashValue[it.value] + ?: thisShouldNotHappen("unknown hash ${it.value} in $propertiesHashValue") + } + + setPropertiesToObject(this, properties) +} + +fun Workflow.getProperties() = + getPropertiesFromObject(this) { + // excludes Channels + !it.first.returnType.isSubtypeOf(Channel::class.starProjectedType) && + // excludes Proxies (tasks and workflows) and null + !(it.second?.let { Proxy.isProxyClass(it::class.java) } ?: true) && + // exclude SLF4J loggers + !it.first.returnType.isSubtypeOf(Logger::class.createType()) && + // exclude KotlinLogging loggers + !it.first.returnType.isSubtypeOf(KLogger::class.createType()) && + // exclude Ignore annotation + !it.first.hasAnnotation() + } + private val KProperty1<*, *>.jsonViewClass get(): Class<*>? { val jsonViewAnnotation = findAnnotations(JsonView::class) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/tasks/Task.kt b/infinitic-common/src/main/kotlin/io/infinitic/tasks/Task.kt index a0ab6ec50..d9661f9c6 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/tasks/Task.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/tasks/Task.kt @@ -28,6 +28,9 @@ import io.infinitic.common.tasks.executors.errors.ExecutionError object Task { private val context: ThreadLocal = ThreadLocal.withInitial { null } + @JvmStatic + fun hasContext() = context.get() != null + @JvmStatic fun setContext(c: TaskContext) { context.set(c) diff --git a/infinitic-common/src/main/kotlin/io/infinitic/tasks/TaskContext.kt b/infinitic-common/src/main/kotlin/io/infinitic/tasks/TaskContext.kt index db642dbef..e254ef011 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/tasks/TaskContext.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/tasks/TaskContext.kt @@ -30,14 +30,12 @@ import io.infinitic.common.tasks.data.TaskRetryIndex import io.infinitic.common.tasks.data.TaskRetrySequence import io.infinitic.common.tasks.executors.errors.ExecutionError import io.infinitic.common.workers.config.WorkflowVersion -import io.infinitic.common.workers.registry.WorkerRegistry import io.infinitic.common.workflows.data.workflows.WorkflowId import io.infinitic.common.workflows.data.workflows.WorkflowName interface TaskContext { val client: InfiniticClientInterface val workerName: String - val workerRegistry: WorkerRegistry val serviceName: ServiceName val taskId: TaskId val taskName: MethodName diff --git a/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithRetry.kt b/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithRetry.kt index e65e1b568..a7c44797c 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithRetry.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithRetry.kt @@ -26,8 +26,7 @@ fun interface WithRetry { fun getSecondsBeforeRetry(retry: Int, e: Exception): Double? companion object { - @JvmStatic - val NONE: WithTimeout = WithTimeout { null } + val UNSET = WithRetry { _, _ -> null } } } diff --git a/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithTimeout.kt b/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithTimeout.kt index ec78769a3..3683fe56b 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithTimeout.kt +++ b/infinitic-common/src/main/kotlin/io/infinitic/tasks/WithTimeout.kt @@ -25,17 +25,16 @@ package io.infinitic.tasks import kotlin.math.max fun interface WithTimeout { - fun getTimeoutInSeconds(): Double? + fun getTimeoutSeconds(): Double? companion object { - @JvmStatic - val NONE: WithTimeout = WithTimeout { null } + val UNSET = WithTimeout { null } } } val WithTimeout.millis: Result get() = try { - Result.success(getTimeoutInSeconds()?.let { max(0L, (it * 1000).toLong()) }) + Result.success(getTimeoutSeconds()?.let { max(0L, (it * 1000).toLong()) }) } catch (e: Exception) { Result.failure(e) } diff --git a/infinitic-common/src/main/resources/schemas/clientEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/clientEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/clientEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/clientEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/delegatedTaskData-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/delegatedTaskData-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/delegatedTaskData-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/delegatedTaskData-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/serviceEventEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/serviceEventEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/serviceEventEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/serviceEventEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/serviceExecutorEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/serviceExecutorEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/serviceExecutorEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/serviceExecutorEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/serviceTagEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/serviceTagEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/serviceTagEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/serviceTagEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowCmdEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowCmdEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowCmdEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowCmdEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowEngineEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowEngineEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowEngineEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowEngineEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowEventEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowEventEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowEventEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowEventEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowState-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowState-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowState-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowState-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowTagEnvelope-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowTagEnvelope-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowTagEnvelope-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowTagEnvelope-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowTaskParameters-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowTaskParameters-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowTaskParameters-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowTaskParameters-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/schemas/workflowTaskReturnValue-0.15.1.avsc b/infinitic-common/src/main/resources/schemas/workflowTaskReturnValue-0.16.0.avsc similarity index 100% rename from infinitic-common/src/main/resources/schemas/workflowTaskReturnValue-0.15.1.avsc rename to infinitic-common/src/main/resources/schemas/workflowTaskReturnValue-0.16.0.avsc diff --git a/infinitic-common/src/main/resources/versions b/infinitic-common/src/main/resources/versions index 0c296379a..d2ccd76b0 100644 --- a/infinitic-common/src/main/resources/versions +++ b/infinitic-common/src/main/resources/versions @@ -32,4 +32,4 @@ 0.14.0 0.14.1 0.15.0 -0.15.1 +0.16.0 diff --git a/infinitic-common/src/test/java/io/infinitic/serDe/SerDeJavaTest.java b/infinitic-common/src/test/java/io/infinitic/serDe/SerDeJavaTest.java index 6e2581b32..cc7421f09 100644 --- a/infinitic-common/src/test/java/io/infinitic/serDe/SerDeJavaTest.java +++ b/infinitic-common/src/test/java/io/infinitic/serDe/SerDeJavaTest.java @@ -43,7 +43,6 @@ public void nullObjShouldBeSerializableDeserializable() { public void simpleObjectShouldBeSerializableDeserializable() { Pojo1 val1 = new Pojo1("42", 42, JType.TYPE_1); SerializedData data = SerializedData.encode(val1, null, null); - System.out.println(data); Assertions.assertEquals(val1, data.decode(null, null)); } @@ -63,9 +62,7 @@ public void objectShouldBeDeserializableEvenWithLessProperties() { Pojo1 val1 = new Pojo1("42", 42, JType.TYPE_1); Pojo2 val2 = new Pojo2("42", 42); SerializedData original = SerializedData.encode(val2, Pojo2.class, null); - System.out.println(original.toJsonString()); SerializedData data = original.copy(original.toJsonString().replace("Pojo2", "Pojo1").getBytes(), original.getDataType(), original.getMeta()); - System.out.println(data.toJsonString()); Assertions.assertEquals(val1, data.decode(Pojo1.class, null)); } @@ -76,7 +73,6 @@ public void testPolymorphicDeserialize() { PojoWrapper val1 = new PojoWrapper(pojo, pojo); SerializedData data = SerializedData.encode(val1, null, null); - System.out.println(data.toJsonString()); Assertions.assertEquals(val1, data.decode(null, null)); } @@ -90,9 +86,6 @@ public void ListOfSimpleObjectShouldBeSerializableDeserializableWithoutType() { pojos.add(pojob); SerializedData data = SerializedData.encode(pojos, null, null); - System.out.println(SerializedData.encode(pojoa, null, null)); - System.out.println(data); - Assertions.assertEquals(pojos, data.decode(null, null)); } diff --git a/infinitic-common/src/test/kotlin/io/infinitic/common/utils/ClassUtilTests.kt b/infinitic-common/src/test/kotlin/io/infinitic/common/utils/ClassUtilTests.kt index 927f721af..5d39df88c 100644 --- a/infinitic-common/src/test/kotlin/io/infinitic/common/utils/ClassUtilTests.kt +++ b/infinitic-common/src/test/kotlin/io/infinitic/common/utils/ClassUtilTests.kt @@ -99,12 +99,12 @@ class ClassUtilTests : StringSpec( } "can read timeout from interface with default" { - TrueBar::getTimeoutInSeconds.javaMethod!!.getMillisDuration(TrueBar::class.java) + TrueBar::getTimeoutSeconds.javaMethod!!.getMillisDuration(TrueBar::class.java) .getOrThrow() shouldBe MillisDuration(1000L) } "can not read timeout from interface without default" { - Bar::getTimeoutInSeconds.javaMethod!!.getMillisDuration(Bar::class.java) + Bar::getTimeoutSeconds.javaMethod!!.getMillisDuration(Bar::class.java) .getOrThrow() shouldBe null } @@ -266,12 +266,12 @@ private interface Bar : WithTimeout { private class BarImpl : Bar { override fun foo() {} - override fun getTimeoutInSeconds() = 1.0 + override fun getTimeoutSeconds() = 1.0 } private interface TrueBar : WithTimeout { fun foo() - override fun getTimeoutInSeconds() = 1.0 + override fun getTimeoutSeconds() = 1.0 } private class TrueBarImpl : TrueBar { @@ -281,5 +281,5 @@ private class TrueBarImpl : TrueBar { } class After10MilliSeconds : WithTimeout { - override fun getTimeoutInSeconds() = 0.01 + override fun getTimeoutSeconds() = 0.01 } diff --git a/infinitic-common/src/test/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTests.kt b/infinitic-common/src/test/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTests.kt deleted file mode 100644 index 841f27af0..000000000 --- a/infinitic-common/src/test/kotlin/io/infinitic/common/workers/registry/RegisteredWorkflowTests.kt +++ /dev/null @@ -1,190 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -@file:Suppress("ClassName") - -package io.infinitic.common.workers.registry - -import io.infinitic.common.workers.config.WorkflowVersion -import io.infinitic.common.workflows.data.workflows.WorkflowName -import io.infinitic.exceptions.workflows.UnknownWorkflowVersionException -import io.infinitic.workflows.Workflow -import io.kotest.assertions.throwables.shouldNotThrowAny -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.shouldBe -import io.kotest.matchers.string.shouldContain - -@Suppress("unused") -class RegisteredWorkflowTests : - StringSpec( - { - class MyWorkflow : Workflow() - class MyWorkflow_0 : Workflow() - class MyWorkflow_1 : Workflow() - class MyWorkflow_2 : Workflow() - class MyWorkflow_WithContext : Workflow() { - val id = workflowId - } - - "List of classes must not be empty" { - val e = shouldThrow { - RegisteredWorkflowExecutor(WorkflowName("foo"), listOf(), 42, null, null, null) - } - e.message shouldContain "List of factory must not be empty for workflow 'foo'" - } - - "Exception when workflows have same version" { - val e = shouldThrow { - RegisteredWorkflowExecutor( - WorkflowName("foo"), - listOf({ MyWorkflow() }, { MyWorkflow_0() }), - 42, - null, - null, - null, - ) - } - e.message shouldContain "have same version" - } - - "Exception when requesting unknown version" { - val e = shouldThrow { - val w = MyWorkflow() - val r = RegisteredWorkflowExecutor( - WorkflowName("foo"), - listOf { w }, - 42, - null, - null, - null, - ) - r.getInstanceByVersion(WorkflowVersion(1)) - } - e.message shouldContain "Unknown version '1' for Workflow 'foo'" - } - - "Exception when factory creates singleton" { - val e = shouldThrow { - val w = MyWorkflow() - val r = RegisteredWorkflowExecutor( - WorkflowName("foo"), - listOf { w }, - 42, - null, - null, - null, - ) - r.getInstanceByVersion(WorkflowVersion(0)) - } - e.message shouldContain "The workflow factory has returned the same object instance" - } - - "Exception when factory returns different classes" { - val e = shouldThrow { - val w0 = MyWorkflow() - val w1 = MyWorkflow_0() - var flag = true - val r = RegisteredWorkflowExecutor( - WorkflowName("foo"), - listOf { - when (flag) { - true -> w0.also { flag = !flag } - else -> w1 - } - }, - 42, - null, - null, - null, - ) - r.getInstanceByVersion(WorkflowVersion(0)) - } - e.message shouldContain "which does not match the expected class" - } - - "Get instance with single class" { - val rw = RegisteredWorkflowExecutor( - WorkflowName("foo"), listOf { MyWorkflow() }, 42, null, null, null, - ) - // get explicit version 0 - rw.getInstanceByVersion(WorkflowVersion(0))::class.java shouldBe MyWorkflow::class.java - // get default version - rw.getInstanceByVersion(null)::class.java shouldBe MyWorkflow::class.java - // get unknown version - val e = shouldThrow { - rw.getInstanceByVersion(WorkflowVersion(1)) - } - e.message shouldContain "Unknown version '1'" - } - - "Get instance with single class with version" { - val rw = - RegisteredWorkflowExecutor( - WorkflowName("foo"), listOf { MyWorkflow_2() }, 42, null, null, null, - ) - // get explicit version 0 - rw.getInstanceByVersion(WorkflowVersion(2))::class.java shouldBe MyWorkflow_2::class.java - // get default version - rw.getInstanceByVersion(null)::class.java shouldBe MyWorkflow_2::class.java - // get unknown version - val e = shouldThrow { - rw.getInstanceByVersion(WorkflowVersion(1)) - } - e.message shouldContain "Unknown version '1'" - } - - "Get instance with multiple classes" { - val rw = RegisteredWorkflowExecutor( - WorkflowName("foo"), - listOf({ MyWorkflow() }, { MyWorkflow_2() }), - 42, - null, - null, - null, - ) - // get explicit version 0 - rw.getInstanceByVersion(WorkflowVersion(0))::class.java shouldBe MyWorkflow::class.java - // get explicit version 2 - rw.getInstanceByVersion(WorkflowVersion(2))::class.java shouldBe MyWorkflow_2::class.java - // get default version - rw.getInstanceByVersion(null)::class.java shouldBe MyWorkflow_2::class.java - // get unknown version - val e = shouldThrow { - rw.getInstanceByVersion( - WorkflowVersion( - 1, - ), - ) - } - e.message shouldContain "Unknown version '1'" - } - - "Get instance with single class using context" { - shouldNotThrowAny { - RegisteredWorkflowExecutor( - WorkflowName("foo"), listOf { MyWorkflow_WithContext() }, 42, null, null, null, - ) - } - } - }, - ) diff --git a/infinitic-common/src/test/kotlin/io/infinitic/serDe/kotlin/SerDeKotlinTests.kt b/infinitic-common/src/test/kotlin/io/infinitic/serDe/kotlin/SerDeKotlinTests.kt index 7b91b388c..3f669f2ab 100644 --- a/infinitic-common/src/test/kotlin/io/infinitic/serDe/kotlin/SerDeKotlinTests.kt +++ b/infinitic-common/src/test/kotlin/io/infinitic/serDe/kotlin/SerDeKotlinTests.kt @@ -180,7 +180,6 @@ class SerDeTests : StringSpec( "Instant should be serializable / deserializable (with type)" { val val1: Instant = Instant.now() val val2 = SerializedData.encode(val1, Instant::class.java, null) - .also { println(it); println(Instant::class.java) } .decode(Instant::class.java, null) val2 shouldBe val1 @@ -331,7 +330,6 @@ class SerDeTests : StringSpec( "List of Objects wit serializer should be serializable / deserializable (without type)" { val val1 = listOf(Obj1("42", 42, Type.TYPE_1), Obj1("24", 24, Type.TYPE_2)) val ser = SerializedData.encode(val1, null, null) - println(ser) val val2 = ser.decode(null, null) val2 shouldBe val1 @@ -340,7 +338,6 @@ class SerDeTests : StringSpec( "List of Objects wit serializer should be serializable / deserializable (with type)" { val val1 = listOf(Obj1("42", 42, Type.TYPE_1), Obj1("24", 24, Type.TYPE_2)) val ser = SerializedData.encode(val1, typeOf>().javaType, null) - println(ser) val val2 = ser.decode(typeOf>().javaType, null) val2 shouldBe val1 diff --git a/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllServicesState.kt b/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllServicesState.kt index 52c3f979a..3b5f14c87 100644 --- a/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllServicesState.kt +++ b/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllServicesState.kt @@ -38,7 +38,7 @@ data class AllServicesState( override fun create(names: JobNames, stats: JobStats) = AllServicesState(names = names, stats = stats) - override suspend fun getNames() = Infinitic.pulsarResources.getServicesName() + override suspend fun getNames() = Infinitic.pulsarResources.getServiceNames() override suspend fun getPartitionedStats(name: String): Result { val topic = with(Infinitic.pulsarResources) { ServiceExecutorTopic.fullName(name) } diff --git a/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllWorkflowsState.kt b/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllWorkflowsState.kt index 219c82312..52e6fdba9 100644 --- a/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllWorkflowsState.kt +++ b/infinitic-dashboard/src/main/kotlin/io/infinitic/dashboard/panels/infrastructure/AllWorkflowsState.kt @@ -38,7 +38,7 @@ data class AllWorkflowsState( override fun create(names: JobNames, stats: JobStats) = AllWorkflowsState(names = names, stats = stats) - override suspend fun getNames() = Infinitic.pulsarResources.getWorkflowsName() + override suspend fun getNames() = Infinitic.pulsarResources.getWorkflowNames() override suspend fun getPartitionedStats(name: String): Result { val topic = diff --git a/infinitic-storage/build.gradle.kts b/infinitic-storage/build.gradle.kts index 11b139f32..568ec5067 100644 --- a/infinitic-storage/build.gradle.kts +++ b/infinitic-storage/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { // Cache implementation(project(":infinitic-cache")) + implementation(project(":infinitic-utils")) implementation(Libs.Hoplite.core) implementation("com.zaxxer:HikariCP:5.0.1") @@ -43,7 +44,6 @@ dependencies { testImplementation(Libs.TestContainers.testcontainers) testImplementation(Libs.TestContainers.mysql) testImplementation(Libs.TestContainers.postgresql) - testImplementation(Libs.Hoplite.yaml) } apply("../publish.gradle.kts") diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/compression/CompressionConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/compression/CompressionConfig.kt index feb325a92..e2c872a89 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/compression/CompressionConfig.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/compression/CompressionConfig.kt @@ -57,13 +57,12 @@ enum class CompressionConfig { // have the signature of a compression type. // As such signature is generally only a few bytes, it should not be that rare. // That's why below we return the data if we have an error during the decompression. - val type = - try { - CompressorStreamFactory.detect(input) - } catch (e: CompressorException) { - // no compressor type found, return original - return data - } + val type = try { + CompressorStreamFactory.detect(input) + } catch (e: CompressorException) { + // no compressor type found, return original + return data + } val out = ByteArrayOutputStream() // decompress @@ -74,7 +73,7 @@ enum class CompressionConfig { } catch (e: Exception) { // see comment above logger.info(e) { - "Error when decompressing data with '$type' algorithm, fallback to not decompressing" + "Error occurred while decompressing data using the '$type' algorithm. Returning original data." } return data.also { out.close() } } diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryConfig.kt index 2158f6cf2..1c73be885 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryConfig.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryConfig.kt @@ -25,7 +25,10 @@ package io.infinitic.storage.config import io.infinitic.storage.data.Bytes import java.util.concurrent.ConcurrentHashMap -data class InMemoryConfig(private val type: String = "unused") { +data class InMemoryConfig( + val unused: String? = null, +) { + companion object { val pools = ConcurrentHashMap() diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryStorageConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryStorageConfig.kt new file mode 100644 index 000000000..f3f134fb5 --- /dev/null +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/InMemoryStorageConfig.kt @@ -0,0 +1,67 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.storage.config + +import io.infinitic.cache.config.CacheConfig +import io.infinitic.storage.compression.CompressionConfig +import io.infinitic.storage.databases.inMemory.InMemoryKeySetStorage +import io.infinitic.storage.databases.inMemory.InMemoryKeyValueStorage +import io.infinitic.storage.keySet.KeySetStorage +import io.infinitic.storage.keyValue.KeyValueStorage + +data class InMemoryStorageConfig( + internal val inMemory: InMemoryConfig, + override var compression: CompressionConfig? = null, + override var cache: CacheConfig? = null +) : StorageConfig() { + override val dbKeySet: KeySetStorage by lazy { + InMemoryKeySetStorage.from(inMemory) + } + override val type = "inMemory" + + override val dbKeyValue: KeyValueStorage by lazy { + InMemoryKeyValueStorage.from(inMemory) + } + + companion object { + @JvmStatic + fun builder() = InMemoryConfigBuilder() + } + + /** + * InMemoryStorageConfig builder + */ + class InMemoryConfigBuilder : StorageConfigBuilder { + private var compression: CompressionConfig? = null + private var cache: CacheConfig? = null + + fun setCompression(compression: CompressionConfig) = apply { this.compression = compression } + fun setCache(cache: CacheConfig) = apply { this.cache = cache } + + override fun build() = InMemoryStorageConfig( + compression = compression, + cache = cache, + inMemory = InMemoryConfig(), + ) + } +} diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLConfig.kt index 00b656ef4..ad0bb5815 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLConfig.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLConfig.kt @@ -22,60 +22,122 @@ */ package io.infinitic.storage.config -import com.sksamuel.hoplite.Secret import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import java.util.concurrent.ConcurrentHashMap -@Suppress("unused") -data class MySQLConfig( - val host: String = "127.0.0.1", - val port: Int = 3306, - val user: String = "root", - val password: Secret? = null, - val database: String = DEFAULT_DATABASE, - val keySetTable: String = DEFAULT_KEY_SET_TABLE, - val keyValueTable: String = DEFAULT_KEY_VALUE_TABLE, - val maximumPoolSize: Int? = null, - val minimumIdle: Int? = null, - val idleTimeout: Long? = null, // milli seconds - val connectionTimeout: Long? = null, // milli seconds - val maxLifetime: Long? = null // milli seconds -) { +/** + * Configuration for MySQL database connection. + * + * @property host The host address of the MySQL server. + * @property port The port number on which the MySQL server is listening. + * @property username The username for connecting to the database. + * @property password The password for connecting to the database, if applicable. + * @property database The name of the database to connect to. Optional, default is "infinitic". + * @property keySetTable The name of the table used for key sets. Optional, default is "key_set_storage". + * @property keyValueTable The name of the table used for key-value pairs. Optional, default is "key_value_storage". + * @property maximumPoolSize The maximum size of the database connection pool. Optional, default is HikariCP driver's default. + * @property minimumIdle The minimum number of idle connections in the pool. Optional, default is HikariCP driver's default. + * @property idleTimeout The maximum amount of time a connection is allowed to sit idle in the pool (in milliseconds). Optional, default is HikariCP driver's default. + * @property connectionTimeout The maximum time that a connection attempt will wait for a connection to be provided (in milliseconds). Optional, default is HikariCP driver's default. + * @property maxLifetime The maximum lifetime of a connection in the pool (in milliseconds). Optional, default is HikariCP driver's default. + * + * Example for local development: + * + * ```kotlin + * MySQLConfig("localhost", 3306, "root", null) + * ``` + * + * ```java + * MySQLConfig.builder() + * .setHost("localhost") + * .setPort(3306) + * .setUser("root") + * .setPassword(null) + * .build(); + * ``` + */ +interface MySQLConfigInterface { + val host: String + val port: Int + val username: String + val password: String? + val database: String + val keySetTable: String + val keyValueTable: String + val maximumPoolSize: Int? + val minimumIdle: Int? + val idleTimeout: Long? + val connectionTimeout: Long? + val maxLifetime: Long? +} + +data class MySQLConfig( + override val host: String, + override val port: Int, + override val username: String, + override val password: String?, + override val database: String = DEFAULT_DATABASE, + override val keySetTable: String = DEFAULT_KEY_SET_TABLE, + override val keyValueTable: String = DEFAULT_KEY_VALUE_TABLE, + override val maximumPoolSize: Int? = null, + override val minimumIdle: Int? = null, + override val idleTimeout: Long? = null, // milli seconds + override val connectionTimeout: Long? = null, // milli seconds + override val maxLifetime: Long? = null // milli seconds +) : MySQLConfigInterface { private val jdbcUrl = "jdbc:mysql://$host:$port/$database" private val jdbcUrlDefault = "jdbc:mysql://$host:$port/" private val driverClassName = "com.mysql.cj.jdbc.Driver" init { maximumPoolSize?.let { - require(it > 0) { "maximumPoolSize must be strictly positive" } + require(it > 0) { "Invalid value for '${::maximumPoolSize.name}': $it. The value must be > 0." } } minimumIdle?.let { - require(it >= 0) { "minimumIdle must be positive" } + require(it >= 0) { "Invalid value for '${::minimumIdle.name}': $it. The value must be >= 0." } } idleTimeout?.let { - require(it > 0) { "idleTimeout must be strictly positive" } + require(it > 0) { "Invalid value for '${::idleTimeout.name}': $it. The value must be > 0." } } connectionTimeout?.let { - require(it > 0) { "connectionTimeout must be strictly positive" } + require(it > 0) { "Invalid value for '${::connectionTimeout.name}': $it. The value must be > 0." } } maxLifetime?.let { - require(it > 0) { "maxLifetime must be strictly positive" } + require(it > 0) { "Invalid value for '${::maxLifetime.name}': $it. The value must be > 0." } } - require(keySetTable.isValidTableName()) { "'$keySetTable' is not a valid MySQL table name" } - require(keyValueTable.isValidTableName()) { "'$keyValueTable' is not a valid MySQL table name" } + require(database.isValidDatabaseName()) { + "Invalid value for '${::database.name}': '$database' is not a valid MySQL database name" + } + require(keySetTable.isValidTableName()) { + "Invalid value for '${::keySetTable.name}': '$keySetTable' is not a valid MySQL table name" + } + require(keyValueTable.isValidTableName()) { + "Invalid value for '${::keyValueTable.name}': '$keyValueTable' is not a valid MySQL table name" + } } - companion object { - @JvmStatic - fun builder() = MySQLConfigBuilder() + /** + * Returns a string representation of the `MySQLConfig` object with an obfuscated password property. + * The optional properties are included only if they have non-null values. + */ + override fun toString() = + "${this::class.java.simpleName}(host='$host', port=$port, username='$username', password='******', " + + "database=$database, keySetTable=$keySetTable, keyValueTable=$keyValueTable" + + (maximumPoolSize?.let { ", maximumPoolSize=$it" } ?: "") + + (minimumIdle?.let { ", minimumIdle=$it" } ?: "") + + (idleTimeout?.let { ", idleTimeout=$it" } ?: "") + + (connectionTimeout?.let { ", connectionTimeout=$it" } ?: "") + + (maxLifetime?.let { ", maxLifetime=$it" } ?: "") + + ")" + companion object { private val pools = ConcurrentHashMap() - private const val DEFAULT_KEY_VALUE_TABLE = "key_value_storage" - private const val DEFAULT_KEY_SET_TABLE = "key_set_storage" - private const val DEFAULT_DATABASE = "infinitic" + internal const val DEFAULT_KEY_VALUE_TABLE = "key_value_storage" + internal const val DEFAULT_KEY_SET_TABLE = "key_set_storage" + internal const val DEFAULT_DATABASE = "infinitic" } fun close() { @@ -91,17 +153,19 @@ data class MySQLConfig( HikariDataSource(hikariConfig) } - private val hikariConfig = HikariConfig().apply { - val config = this@MySQLConfig - jdbcUrl = config.jdbcUrl - driverClassName = config.driverClassName - username = config.user - password = config.password?.value - config.maximumPoolSize?.let { maximumPoolSize = it } - config.minimumIdle?.let { minimumIdle = it } - config.idleTimeout?.let { idleTimeout = it } - config.connectionTimeout?.let { connectionTimeout = it } - config.maxLifetime?.let { maxLifetime = it } + private val hikariConfig by lazy { + HikariConfig().apply { + val config = this@MySQLConfig + jdbcUrl = config.jdbcUrl + driverClassName = config.driverClassName + username = config.username + password = config.password + config.maximumPoolSize?.let { maximumPoolSize = it } + config.minimumIdle?.let { minimumIdle = it } + config.idleTimeout?.let { idleTimeout = it } + config.connectionTimeout?.let { connectionTimeout = it } + config.maxLifetime?.let { maxLifetime = it } + } } private fun HikariDataSource.databaseExists(databaseName: String): Boolean = @@ -136,11 +200,16 @@ data class MySQLConfig( // use a default source jdbcUrl = config.jdbcUrlDefault driverClassName = config.driverClassName - username = config.user - password = config.password?.value + username = config.username + password = config.password }, ) + private fun String.isValidDatabaseName(): Boolean { + val regex = "^[a-zA-Z_][a-zA-Z0-9_]{0,63}$".toRegex() + return isNotEmpty() && matches(regex) + } + private fun String.isValidTableName(): Boolean { // Check length if (length > 64) { @@ -160,54 +229,4 @@ data class MySQLConfig( // Okay if it passed all checks return true } - - /** - * MySQLConfig builder (Useful for Java user) - */ - class MySQLConfigBuilder { - private val default = MySQLConfig() - - private var host = default.host - private var port = default.port - private var user = default.user - private var password = default.password - private var database = default.database - private var keySetTable = default.keySetTable - private var keyValueTable = default.keyValueTable - private var maximumPoolSize = default.maximumPoolSize - private var minimumIdle = default.minimumIdle - private var idleTimeout = default.idleTimeout - private var connectionTimeout = default.connectionTimeout - private var maxLifetime = default.maxLifetime - - fun setHost(host: String) = apply { this.host = host } - fun setPort(port: Int) = apply { this.port = port } - fun setUser(user: String) = apply { this.user = user } - fun setPassword(password: Secret?) = apply { this.password = password } - fun setDatabase(database: String) = apply { this.database = database } - fun setKeySetTable(keySetTable: String) = apply { this.keySetTable = keySetTable } - fun setKeyValueTable(keyValueTable: String) = apply { this.keyValueTable = keyValueTable } - fun setMaximumPoolSize(maximumPoolSize: Int?) = apply { this.maximumPoolSize = maximumPoolSize } - fun setMinimumIdle(minimumIdle: Int?) = apply { this.minimumIdle = minimumIdle } - fun setIdleTimeout(idleTimeout: Long?) = apply { this.idleTimeout = idleTimeout } - fun setConnectionTimeout(connTimeout: Long?) = apply { this.connectionTimeout = connTimeout } - fun setMaxLifetime(maxLifetime: Long?) = apply { this.maxLifetime = maxLifetime } - - fun build(): MySQLConfig { - return MySQLConfig( - host = host, - port = port, - user = user, - password = password, - database = database, - keySetTable = keySetTable, - keyValueTable = keyValueTable, - maximumPoolSize = maximumPoolSize, - minimumIdle = minimumIdle, - idleTimeout = idleTimeout, - connectionTimeout = connectionTimeout, - maxLifetime = maxLifetime, - ) - } - } } diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLStorageConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLStorageConfig.kt new file mode 100644 index 000000000..701394fc7 --- /dev/null +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/MySQLStorageConfig.kt @@ -0,0 +1,118 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.storage.config + +import io.infinitic.cache.config.CacheConfig +import io.infinitic.storage.compression.CompressionConfig +import io.infinitic.storage.config.MySQLConfig.Companion.DEFAULT_DATABASE +import io.infinitic.storage.config.MySQLConfig.Companion.DEFAULT_KEY_SET_TABLE +import io.infinitic.storage.config.MySQLConfig.Companion.DEFAULT_KEY_VALUE_TABLE +import io.infinitic.storage.databases.mysql.MySQLKeySetStorage +import io.infinitic.storage.databases.mysql.MySQLKeyValueStorage +import io.infinitic.storage.keySet.KeySetStorage +import io.infinitic.storage.keyValue.KeyValueStorage + + +data class MySQLStorageConfig( + internal val mysql: MySQLConfig, + override var compression: CompressionConfig? = null, + override var cache: CacheConfig? = null +) : StorageConfig(), MySQLConfigInterface by mysql { + + override val type = "mysql" + + override val dbKeyValue: KeyValueStorage by lazy { + MySQLKeyValueStorage.from(mysql) + } + + override val dbKeySet: KeySetStorage by lazy { + MySQLKeySetStorage.from(mysql) + } + + companion object { + @JvmStatic + fun builder() = MySQLStorageConfigBuilder() + } + + /** + * MySQLStorageConfig builder + */ + class MySQLStorageConfigBuilder: StorageConfigBuilder { + private var compression: CompressionConfig? = null + private var cache: CacheConfig? = null + private var host: String? = null + private var port: Int? = null + private var username: String? = null + private var password: String? = null + private var database: String = DEFAULT_DATABASE + private var keySetTable: String = DEFAULT_KEY_SET_TABLE + private var keyValueTable: String = DEFAULT_KEY_VALUE_TABLE + private var maximumPoolSize: Int? = null + private var minimumIdle: Int? = null + private var idleTimeout: Long? = null + private var connectionTimeout: Long? = null + private var maxLifetime: Long? = null + + fun setCompression(compression: CompressionConfig) = apply { this.compression = compression } + fun setCache(cache: CacheConfig) = apply { this.cache = cache } + fun setHost(host: String) = apply { this.host = host } + fun setPort(port: Int) = apply { this.port = port } + fun setUserName(user: String) = apply { this.username = user } + fun setPassword(password: String) = apply { this.password = password } + fun setDatabase(database: String) = apply { this.database = database } + fun setKeySetTable(keySetTable: String) = apply { this.keySetTable = keySetTable } + fun setKeyValueTable(keyValueTable: String) = apply { this.keyValueTable = keyValueTable } + fun setMaximumPoolSize(maximumPoolSize: Int) = apply { this.maximumPoolSize = maximumPoolSize } + fun setMinimumIdle(minimumIdle: Int) = apply { this.minimumIdle = minimumIdle } + fun setIdleTimeout(idleTimeout: Long) = apply { this.idleTimeout = idleTimeout } + fun setConnectionTimeout(connTimeout: Long) = apply { this.connectionTimeout = connTimeout } + fun setMaxLifetime(maxLifetime: Long) = apply { this.maxLifetime = maxLifetime } + + override fun build(): MySQLStorageConfig { + require(host != null) { "${MySQLConfig::host.name} must not be null" } + require(port != null) { "${MySQLConfig::port.name} must not be null" } + require(username != null) { "${MySQLConfig::username.name} must not be null" } + + val mySqlConfig = MySQLConfig( + host = host!!, + port = port!!, + username = username!!, + password = password, + database = database, + keySetTable = keySetTable, + keyValueTable = keyValueTable, + maximumPoolSize = maximumPoolSize, + minimumIdle = minimumIdle, + idleTimeout = idleTimeout, + connectionTimeout = connectionTimeout, + maxLifetime = maxLifetime, + ) + + return MySQLStorageConfig( + compression = compression, + cache = cache, + mysql = mySqlConfig, + ) + } + } +} diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresConfig.kt index 6ba560217..ad77914d1 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresConfig.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresConfig.kt @@ -22,26 +22,72 @@ */ package io.infinitic.storage.config -import com.sksamuel.hoplite.Secret import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource import java.util.concurrent.ConcurrentHashMap +/** + * Configuration for PostgresSQL database connection. + * + * @property host The host address of the PostgresSQL server. + * @property port The port number on which the PostgresSQL server is listening. + * @property username The username for connecting to the database. + * @property password The password for connecting to the database, if applicable. + * @property database The name of the database to connect to. Optional, default is "infinitic". + * @property keySetTable The name of the table used for key sets. Optional, default is "key_set_storage". + * @property keyValueTable The name of the table used for key-value pairs. Optional, default is "key_value_storage". + * @property maximumPoolSize The maximum size of the database connection pool. Optional, default is HikariCP driver's default. + * @property minimumIdle The minimum number of idle connections in the pool. Optional, default is HikariCP driver's default. + * @property idleTimeout The maximum amount of time a connection is allowed to sit idle in the pool (in milliseconds). Optional, default is HikariCP driver's default. + * @property connectionTimeout The maximum time that a connection attempt will wait for a connection to be provided (in milliseconds). Optional, default is HikariCP driver's default. + * @property maxLifetime The maximum lifetime of a connection in the pool (in milliseconds). Optional, default is HikariCP driver's default. + * + * Example for local development: + * + * ```kotlin + * PostgresConfig("localhost", 5432, "postgres", null) + * ``` + * + * ```java + * PostgresConfig.builder() + * .setHost("localhost") + * .setPort(5432) + * .setUser("postgres") + * .setPassword(null) + * .build(); + * ``` + */ + +interface PostgresConfigInterface { + val host: String + val port: Int + val username: String + val password: String? + val database: String + val keySetTable: String + val keyValueTable: String + val maximumPoolSize: Int? + val minimumIdle: Int? + val idleTimeout: Long? + val connectionTimeout: Long? + val maxLifetime: Long? +} + @Suppress("unused") data class PostgresConfig( - val host: String = "127.0.0.1", - val port: Int = 5432, - val user: String = "postgres", - val password: Secret? = null, - val database: String = DEFAULT_DATABASE, - val keySetTable: String = DEFAULT_KEY_SET_TABLE, - val keyValueTable: String = DEFAULT_KEY_VALUE_TABLE, - val maximumPoolSize: Int? = null, - val minimumIdle: Int? = null, - val idleTimeout: Long? = null, // milli seconds - val connectionTimeout: Long? = null, // milli seconds - val maxLifetime: Long? = null // milli seconds -) { + override val host: String, + override val port: Int, + override val username: String, + override val password: String? = null, + override val database: String = DEFAULT_DATABASE, + override val keySetTable: String = DEFAULT_KEY_SET_TABLE, + override val keyValueTable: String = DEFAULT_KEY_VALUE_TABLE, + override val maximumPoolSize: Int? = null, + override val minimumIdle: Int? = null, + override val idleTimeout: Long? = null, // milli seconds + override val connectionTimeout: Long? = null, // milli seconds + override val maxLifetime: Long? = null // milli seconds +) : PostgresConfigInterface { private val jdbcUrl = "jdbc:postgresql://$host:$port/$database" private val jdbcUrlDefault = "jdbc:postgresql://$host:$port/postgres" @@ -49,33 +95,52 @@ data class PostgresConfig( init { maximumPoolSize?.let { - require(it > 0) { "maximumPoolSize must be strictly positive" } + require(it > 0) { "Invalid value for '${::maximumPoolSize.name}': $it. The value must be > 0." } } minimumIdle?.let { - require(it >= 0) { "minimumIdle must be positive" } + require(it >= 0) { "Invalid value for '${::minimumIdle.name}': $it. The value must be >= 0." } } idleTimeout?.let { - require(it > 0) { "idleTimeout must be strictly positive" } + require(it > 0) { "Invalid value for '${::idleTimeout.name}': $it. The value must be > 0." } } connectionTimeout?.let { - require(it > 0) { "connectionTimeout must be strictly positive" } + require(it > 0) { "Invalid value for '${::connectionTimeout.name}': $it. The value must be > 0." } } maxLifetime?.let { - require(it > 0) { "maxLifetime must be strictly positive" } + require(it > 0) { "Invalid value for '${::maxLifetime.name}': $it. The value must be > 0." } } - require(keySetTable.isValidTableName()) { "'$keySetTable' is not a valid PostgresSQL table name" } - require(keyValueTable.isValidTableName()) { "'$keyValueTable' is not a valid PostgresSQL table name" } + require(database.isValidDatabaseName()) { + "Invalid value for '${::database.name}': '$database' is not a valid MySQL database name" + } + require(keySetTable.isValidTableName()) { + "Invalid value for '${::keySetTable.name}': '$keySetTable' is not a valid MySQL table name" + } + require(keyValueTable.isValidTableName()) { + "Invalid value for '${::keyValueTable.name}': '$keyValueTable' is not a valid MySQL table name" + } } - companion object { - @JvmStatic - fun builder() = PostgresConfigBuilder() + /** + * Returns a string representation of the `PostgresConfig` object with an obfuscated password property. + * The optional properties are included only if they have non-null values. + */ + override fun toString() = + "${this::class.java.simpleName}(host='$host', port=$port, username='$username', password='******', " + + "database=$database, keySetTable=$keySetTable, keyValueTable=$keyValueTable" + + (maximumPoolSize?.let { ", maximumPoolSize=$it" } ?: "") + + (minimumIdle?.let { ", minimumIdle=$it" } ?: "") + + (idleTimeout?.let { ", idleTimeout=$it" } ?: "") + + (connectionTimeout?.let { ", connectionTimeout=$it" } ?: "") + + (maxLifetime?.let { ", maxLifetime=$it" } ?: "") + + ")" + companion object { private val pools = ConcurrentHashMap() - private const val DEFAULT_KEY_VALUE_TABLE = "key_value_storage" - private const val DEFAULT_KEY_SET_TABLE = "key_set_storage" - private const val DEFAULT_DATABASE = "infinitic" + + internal const val DEFAULT_KEY_VALUE_TABLE = "key_value_storage" + internal const val DEFAULT_KEY_SET_TABLE = "key_set_storage" + internal const val DEFAULT_DATABASE = "infinitic" } fun close() { @@ -90,17 +155,19 @@ data class PostgresConfig( HikariDataSource(hikariConfig) } - private val hikariConfig = HikariConfig().apply { - val config = this@PostgresConfig - jdbcUrl = config.jdbcUrl - driverClassName = config.driverClassName - username = config.user - password = config.password?.value - config.maximumPoolSize?.let { maximumPoolSize = it } - config.minimumIdle?.let { minimumIdle = it } - config.idleTimeout?.let { idleTimeout = it } - config.connectionTimeout?.let { connectionTimeout = it } - config.maxLifetime?.let { maxLifetime = it } + private val hikariConfig by lazy { + HikariConfig().apply { + val config = this@PostgresConfig + jdbcUrl = config.jdbcUrl + driverClassName = config.driverClassName + username = config.username + password = config.password + config.maximumPoolSize?.let { maximumPoolSize = it } + config.minimumIdle?.let { minimumIdle = it } + config.idleTimeout?.let { idleTimeout = it } + config.connectionTimeout?.let { connectionTimeout = it } + config.maxLifetime?.let { maxLifetime = it } + } } private fun HikariDataSource.databaseExists(databaseName: String): Boolean = @@ -134,11 +201,16 @@ data class PostgresConfig( // use a default source jdbcUrl = this@PostgresConfig.jdbcUrlDefault driverClassName = this@PostgresConfig.driverClassName - username = this@PostgresConfig.user - password = this@PostgresConfig.password?.value + username = this@PostgresConfig.username + password = this@PostgresConfig.password }, ) + private fun String.isValidDatabaseName(): Boolean { + val regex = "^[a-zA-Z_][a-zA-Z0-9_\$]{0,62}$".toRegex() + return isNotEmpty() && matches(regex) + } + private fun String.isValidTableName(): Boolean { // Check length // Note that since Postgres uses bytes and Kotlin uses UTF-16 characters, @@ -160,51 +232,4 @@ data class PostgresConfig( // Okay if it passed all checks return true } - - /** - * PostgresConfig builder (Useful for Java user) - */ - class PostgresConfigBuilder { - private val default = PostgresConfig() - private var host = default.host - private var port = default.port - private var user = default.user - private var password = default.password - private var database = default.database - private var keySetTable = default.keySetTable - private var keyValueTable = default.keyValueTable - private var maximumPoolSize = default.maximumPoolSize - private var minimumIdle = default.minimumIdle - private var idleTimeout = default.idleTimeout - private var connectionTimeout = default.connectionTimeout - private var maxLifetime = default.maxLifetime - - fun setHost(host: String) = apply { this.host = host } - fun setPort(port: Int) = apply { this.port = port } - fun setUser(user: String) = apply { this.user = user } - fun setPassword(password: Secret?) = apply { this.password = password } - fun setDatabase(database: String) = apply { this.database = database } - fun setKeySetTable(keySetTable: String) = apply { this.keySetTable = keySetTable } - fun setKeyValueTable(keyValueTable: String) = apply { this.keyValueTable = keyValueTable } - fun setMaximumPoolSize(maximumPoolSize: Int?) = apply { this.maximumPoolSize = maximumPoolSize } - fun setMinimumIdle(minimumIdle: Int?) = apply { this.minimumIdle = minimumIdle } - fun setIdleTimeout(idleTimeout: Long?) = apply { this.idleTimeout = idleTimeout } - fun setConnectionTimeout(connTimeout: Long?) = apply { this.connectionTimeout = connTimeout } - fun setMaxLifetime(maxLifetime: Long?) = apply { this.maxLifetime = maxLifetime } - - fun build() = PostgresConfig( - host = host, - port = port, - user = user, - password = password, - database = database, - keySetTable = keySetTable, - keyValueTable = keyValueTable, - maximumPoolSize = maximumPoolSize, - minimumIdle = minimumIdle, - idleTimeout = idleTimeout, - connectionTimeout = connectionTimeout, - maxLifetime = maxLifetime, - ) - } } diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresStorageConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresStorageConfig.kt new file mode 100644 index 000000000..2ffe90d0e --- /dev/null +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/PostgresStorageConfig.kt @@ -0,0 +1,117 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.storage.config + +import io.infinitic.cache.config.CacheConfig +import io.infinitic.storage.compression.CompressionConfig +import io.infinitic.storage.config.PostgresConfig.Companion.DEFAULT_DATABASE +import io.infinitic.storage.config.PostgresConfig.Companion.DEFAULT_KEY_SET_TABLE +import io.infinitic.storage.config.PostgresConfig.Companion.DEFAULT_KEY_VALUE_TABLE +import io.infinitic.storage.databases.postgres.PostgresKeySetStorage +import io.infinitic.storage.databases.postgres.PostgresKeyValueStorage +import io.infinitic.storage.keySet.KeySetStorage +import io.infinitic.storage.keyValue.KeyValueStorage + +data class PostgresStorageConfig( + internal val postgres: PostgresConfig, + override var compression: CompressionConfig? = null, + override var cache: CacheConfig? = null +) : StorageConfig(), PostgresConfigInterface by postgres { + + override val type = "postgres" + + override val dbKeyValue: KeyValueStorage by lazy { + PostgresKeyValueStorage.from(postgres) + } + + override val dbKeySet: KeySetStorage by lazy { + PostgresKeySetStorage.from(postgres) + } + + companion object { + @JvmStatic + fun builder() = PostgresConfigBuilder() + } + + /** + * PostgresStorageConfig builder + */ + class PostgresConfigBuilder: StorageConfigBuilder { + private var compression: CompressionConfig? = null + private var cache: CacheConfig? = null + private var host: String? = null + private var port: Int? = null + private var username: String? = null + private var password: String? = null + private var database: String = DEFAULT_DATABASE + private var keySetTable: String = DEFAULT_KEY_SET_TABLE + private var keyValueTable: String = DEFAULT_KEY_VALUE_TABLE + private var maximumPoolSize: Int? = null + private var minimumIdle: Int? = null + private var idleTimeout: Long? = null + private var connectionTimeout: Long? = null + private var maxLifetime: Long? = null + + fun setCompression(compression: CompressionConfig) = apply { this.compression = compression } + fun setCache(cache: CacheConfig) = apply { this.cache = cache } + fun setHost(host: String) = apply { this.host = host } + fun setPort(port: Int) = apply { this.port = port } + fun setUserName(user: String) = apply { this.username = user } + fun setPassword(password: String) = apply { this.password = password } + fun setDatabase(database: String) = apply { this.database = database } + fun setKeySetTable(keySetTable: String) = apply { this.keySetTable = keySetTable } + fun setKeyValueTable(keyValueTable: String) = apply { this.keyValueTable = keyValueTable } + fun setMaximumPoolSize(maximumPoolSize: Int) = apply { this.maximumPoolSize = maximumPoolSize } + fun setMinimumIdle(minimumIdle: Int) = apply { this.minimumIdle = minimumIdle } + fun setIdleTimeout(idleTimeout: Long) = apply { this.idleTimeout = idleTimeout } + fun setConnectionTimeout(connTimeout: Long) = apply { this.connectionTimeout = connTimeout } + fun setMaxLifetime(maxLifetime: Long) = apply { this.maxLifetime = maxLifetime } + + override fun build(): PostgresStorageConfig { + require(host != null) { "${PostgresConfig::host.name} must not be null" } + require(port != null) { "${PostgresConfig::port.name} must not be null" } + require(username != null) { "${PostgresConfig::username.name} must not be null" } + + val postgresConfig = PostgresConfig( + host = host!!, + port = port!!, + username = username!!, + password = password, + database = database, + keySetTable = keySetTable, + keyValueTable = keyValueTable, + maximumPoolSize = maximumPoolSize, + minimumIdle = minimumIdle, + idleTimeout = idleTimeout, + connectionTimeout = connectionTimeout, + maxLifetime = maxLifetime, + ) + + return PostgresStorageConfig( + compression = compression, + cache = cache, + postgres = postgresConfig, + ) + } + } +} diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisConfig.kt index c256f9802..e606b93eb 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisConfig.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisConfig.kt @@ -22,31 +22,81 @@ */ package io.infinitic.storage.config -import com.sksamuel.hoplite.Secret import redis.clients.jedis.JedisPool import redis.clients.jedis.JedisPoolConfig import redis.clients.jedis.Protocol import java.util.concurrent.ConcurrentHashMap +/** + * RedisConfig is a data class that represents the configuration for connecting to a Redis server. + * + * @property host The Redis server host address. + * @property port The Redis server port number. + * @property username The username to authenticate with Redis server (optional). + * @property password The password to authenticate with Redis server (optional). + * @property database The Redis database index (default is 0). + * @property timeout The timeout in milliseconds for socket connection timeout (default is 2000 milliseconds). + * @property ssl A flag indicating whether to use SSL for the connection (default is false). + * @property poolConfig The configuration for the connection pool (default is PoolConfig with default values). + * + * Example for local development: + * + * ```kotlin + * RedisConfig("localhost", 6379) + * ``` + * + * ```java + * RedisConfig.builder() + * .setHost("localhost") + * .setPort(6379) + * .build(); + * ``` + */ + +interface RedisConfigInterface { + val host: String + val port: Int + val username: String? + val password: String? + val database: Int + val timeout: Int + val ssl: Boolean + val poolConfig: RedisConfig.PoolConfig +} + @Suppress("unused") data class RedisConfig( - val host: String = Protocol.DEFAULT_HOST, - var port: Int = Protocol.DEFAULT_PORT, - var timeout: Int = Protocol.DEFAULT_TIMEOUT, - var user: String? = null, - var password: Secret? = null, - var database: Int = Protocol.DEFAULT_DATABASE, - var ssl: Boolean = false, - var poolConfig: PoolConfig = PoolConfig() -) { + override val host: String, + override val port: Int, + override val username: String? = null, + override val password: String? = null, + override val database: Int = Protocol.DEFAULT_DATABASE, + override val timeout: Int = Protocol.DEFAULT_TIMEOUT, + override val ssl: Boolean = false, + override val poolConfig: PoolConfig = PoolConfig() +) : RedisConfigInterface { companion object { - @JvmStatic - fun builder() = RedisConfigBuilder() - private val pools = ConcurrentHashMap() } + init { + require(host.isNotBlank()) { "Invalid value for '${::host.name}': $host. The value must not be blank." } + require(port > 0) { "Invalid value for '${::port.name}': $port. The value must be > 0." } + require(database >= 0) { "Invalid value for '${::database.name}': $database. The value must be >= 0." } + require(timeout > 0) { "Invalid value for '${::timeout.name}': $timeout. The value must be > 0." } + } + + /** + * Returns a string representation of the `PostgresConfig` object with an obfuscated password property. + * The optional properties are included only if they have non-null values. + */ + override fun toString() = + "${this::class.java.simpleName}(host='$host', port=$port" + + (username?.let { ", username='$it'" } ?: "") + + (password?.let { ", password='******'" } ?: "") + + ", database=$database, timeout=$timeout, ssl=$ssl, poolConfig=$poolConfig)" + fun close() { pools[this]?.close() pools.remove(this) @@ -59,18 +109,9 @@ data class RedisConfig( it.minIdle = poolConfig.minIdle } ): JedisPool = pools.getOrPut(this) { - when (password?.value.isNullOrEmpty()) { - true -> JedisPool(jedisPoolConfig, host, port, database) - false -> JedisPool( - jedisPoolConfig, - host, - port, - timeout, - user, - password?.value, - database, - ssl, - ) + when (password.isNullOrEmpty()) { + true -> JedisPool(jedisPoolConfig, host, port, timeout, ssl) + false -> JedisPool(jedisPoolConfig, host, port, timeout, username, password, database, ssl) } } @@ -96,34 +137,4 @@ data class RedisConfig( fun build() = PoolConfig(maxTotal, maxIdle, minIdle) } } - - /** - * RedisConfig builder (Useful for Java user) - */ - class RedisConfigBuilder { - private val default = RedisConfig() - private var host = default.host - private var port = default.port - private var timeout = default.timeout - private var user = default.user - private var password = default.password - private var database = default.database - private var ssl = default.ssl - private var poolConfig = default.poolConfig - - fun setHost(host: String) = apply { this.host = host } - fun setPort(port: Int) = apply { this.port = port } - fun setTimeout(timeout: Int) = apply { this.timeout = timeout } - fun setUser(user: String?) = apply { this.user = user } - fun setPassword(password: Secret?) = apply { this.password = password } - fun setDatabase(database: Int) = apply { this.database = database } - fun setSsl(ssl: Boolean) = apply { this.ssl = ssl } - fun setPoolConfig(poolConfig: PoolConfig) = apply { this.poolConfig = poolConfig } - fun setPoolConfig(poolConfigBuilder: PoolConfig.PoolConfigBuilder) = - apply { this.poolConfig = poolConfigBuilder.build() } - - fun build() = RedisConfig(host, port, timeout, user, password, database, ssl, poolConfig) - } } - - diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisStorageConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisStorageConfig.kt new file mode 100644 index 000000000..f35c00cac --- /dev/null +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/RedisStorageConfig.kt @@ -0,0 +1,105 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.storage.config + +import io.infinitic.cache.config.CacheConfig +import io.infinitic.storage.compression.CompressionConfig +import io.infinitic.storage.config.RedisConfig.PoolConfig +import io.infinitic.storage.databases.redis.RedisKeySetStorage +import io.infinitic.storage.databases.redis.RedisKeyValueStorage +import io.infinitic.storage.keySet.KeySetStorage +import io.infinitic.storage.keyValue.KeyValueStorage +import redis.clients.jedis.Protocol + +data class RedisStorageConfig( + internal val redis: RedisConfig, + override var compression: CompressionConfig? = null, + override var cache: CacheConfig? = null +) : StorageConfig(), RedisConfigInterface by redis { + + override val type = "redis" + + override val dbKeyValue: KeyValueStorage by lazy { + RedisKeyValueStorage.from(redis) + } + + override val dbKeySet: KeySetStorage by lazy { + RedisKeySetStorage.from(redis) + } + + companion object { + @JvmStatic + fun builder() = RedisStorageConfigBuilder() + } + + /** + * RedisStorageConfig builder + */ + class RedisStorageConfigBuilder : StorageConfigBuilder { + private var compression: CompressionConfig? = null + private var cache: CacheConfig? = null + private var host: String? = null + private var port: Int? = null + private var username: String? = null + private var password: String? = null + private var database: Int = Protocol.DEFAULT_DATABASE + private var timeout: Int = Protocol.DEFAULT_TIMEOUT + private var ssl: Boolean? = null + private var poolConfig: PoolConfig? = null + + fun setCompression(compression: CompressionConfig) = apply { this.compression = compression } + fun setCache(cache: CacheConfig) = apply { this.cache = cache } + fun setHost(host: String) = apply { this.host = host } + fun setPort(port: Int) = apply { this.port = port } + fun setUserName(user: String) = apply { this.username = user } + fun setPassword(password: String) = apply { this.password = password } + fun setDatabase(database: Int) = apply { this.database = database } + fun setTimeout(timeout: Int) = apply { this.timeout = timeout } + fun setSsl(ssl: Boolean) = apply { this.ssl = ssl } + fun setPoolConfig(poolConfig: PoolConfig) = apply { this.poolConfig = poolConfig } + fun setPoolConfig(poolConfigBuilder: PoolConfig.PoolConfigBuilder) = + apply { this.poolConfig = poolConfigBuilder.build() } + + override fun build(): RedisStorageConfig { + require(host != null) { "${RedisConfig::host.name} must not be null" } + require(port != null) { "${RedisConfig::port.name} must not be null" } + + val redisConfig = RedisConfig( + host!!, + port!!, + username, + password, + database, + timeout, + ssl ?: false, + poolConfig ?: PoolConfig(), + ) + + return RedisStorageConfig( + compression = compression, + cache = cache, + redis = redisConfig, + ) + } + } +} diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/StorageConfig.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/StorageConfig.kt index 49e034154..23a3648d7 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/StorageConfig.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/StorageConfig.kt @@ -23,109 +23,38 @@ package io.infinitic.storage.config import io.infinitic.cache.config.CacheConfig +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString import io.infinitic.storage.compression.CompressionConfig import io.infinitic.storage.keySet.CachedKeySetStorage import io.infinitic.storage.keySet.KeySetStorage import io.infinitic.storage.keyValue.CachedKeyValueStorage import io.infinitic.storage.keyValue.CompressedKeyValueStorage import io.infinitic.storage.keyValue.KeyValueStorage -import io.infinitic.storage.storages.inMemory.InMemoryKeySetStorage -import io.infinitic.storage.storages.inMemory.InMemoryKeyValueStorage -import io.infinitic.storage.storages.mysql.MySQLKeySetStorage -import io.infinitic.storage.storages.mysql.MySQLKeyValueStorage -import io.infinitic.storage.storages.postgres.PostgresKeySetStorage -import io.infinitic.storage.storages.postgres.PostgresKeyValueStorage -import io.infinitic.storage.storages.redis.RedisKeySetStorage -import io.infinitic.storage.storages.redis.RedisKeyValueStorage -@Suppress("unused") -data class StorageConfig( - private var inMemory: InMemoryConfig? = null, - private val redis: RedisConfig? = null, - private val mysql: MySQLConfig? = null, - private val postgres: PostgresConfig? = null, - var compression: CompressionConfig? = null, - var cache: CacheConfig? = null -) { - init { - val nonNul = listOfNotNull(inMemory, redis, mysql, postgres) +@Suppress("MemberVisibilityCanBePrivate", "unused") +sealed class StorageConfig { + abstract var compression: CompressionConfig? + abstract var cache: CacheConfig? - if (nonNul.isEmpty()) { - // default storage is inMemory - inMemory = InMemoryConfig() - } else { - require(nonNul.count() == 1) { "Storage should have only one definition: ${nonNul.joinToString { it::class.java.simpleName }}" } - } - } + abstract val dbKeyValue: KeyValueStorage + abstract val dbKeySet: KeySetStorage - companion object { - @JvmStatic - fun builder() = StorageConfigBuilder() - } + abstract val type: String - fun close() { - when { - inMemory != null -> inMemory!!.close() - redis != null -> redis.close() - mysql != null -> mysql.close() - postgres != null -> postgres.close() - else -> thisShouldNotHappen() - } - } + fun compression(compression: CompressionConfig) = + apply { this.compression = compression } - val type by lazy { - when { - inMemory != null -> StorageType.IN_MEMORY - redis != null -> StorageType.REDIS - mysql != null -> StorageType.MYSQL - postgres != null -> StorageType.POSTGRES - else -> thisShouldNotHappen() - } - } - - val keySet: KeySetStorage by lazy { - when { - inMemory != null -> InMemoryKeySetStorage.from(inMemory!!) - redis != null -> RedisKeySetStorage.from(redis) - mysql != null -> MySQLKeySetStorage.from(mysql) - postgres != null -> PostgresKeySetStorage.from(postgres) - else -> thisShouldNotHappen() - }.withCache() - } + fun cache(cache: CacheConfig) = + apply { this.cache = cache } val keyValue: KeyValueStorage by lazy { - when { - inMemory != null -> InMemoryKeyValueStorage.from(inMemory!!) - redis != null -> RedisKeyValueStorage.from(redis) - mysql != null -> MySQLKeyValueStorage.from(mysql) - postgres != null -> PostgresKeyValueStorage.from(postgres) - else -> thisShouldNotHappen() - }.let { CompressedKeyValueStorage(compression, it) }.withCache() - } - - /** - * StorageConfig builder (Useful for Java user) - */ - class StorageConfigBuilder { - private var inMemory: InMemoryConfig? = null - private var redis: RedisConfig? = null - private var mysql: MySQLConfig? = null - private var postgres: PostgresConfig? = null - private var compression: CompressionConfig? = null - private var cache: CacheConfig? = null - - fun inMemory(inMemory: InMemoryConfig) = apply { this.inMemory = inMemory } - fun redis(redis: RedisConfig) = apply { this.redis = redis } - fun mysql(mysql: MySQLConfig) = apply { this.mysql = mysql } - fun postgres(postgres: PostgresConfig) = apply { this.postgres = postgres } - fun compression(compression: CompressionConfig) = apply { this.compression = compression } - fun cache(cache: CacheConfig) = apply { this.cache = cache } - - fun build() = StorageConfig(inMemory, redis, mysql, postgres, compression, cache) + CompressedKeyValueStorage(compression, dbKeyValue).withCache() } - private fun thisShouldNotHappen(): Nothing { - throw RuntimeException("This should not happen") + val keySet: KeySetStorage by lazy { + dbKeySet.withCache() } private fun KeyValueStorage.withCache() = when { @@ -138,10 +67,24 @@ data class StorageConfig( else -> CachedKeySetStorage(cache!!.keySet!!, this) } - enum class StorageType { - IN_MEMORY, - REDIS, - POSTGRES, - MYSQL + companion object { + /** Create StorageConfig from files in file system */ + @JvmStatic + fun fromYamlFile(vararg files: String): StorageConfig = + loadFromYamlFile(*files) + + /** Create StorageConfig from files in resources directory */ + @JvmStatic + fun fromYamlResource(vararg resources: String): StorageConfig = + loadFromYamlResource(*resources) + + /** Create StorageConfig from yaml strings */ + @JvmStatic + fun fromYamlString(vararg yamls: String): StorageConfig = + loadFromYamlString(*yamls) + } + + interface StorageConfigBuilder { + fun build(): StorageConfig } } diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/inMemory/InMemoryKeySetStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorage.kt similarity index 97% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/inMemory/InMemoryKeySetStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorage.kt index 250133fea..0bbc8fe87 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/inMemory/InMemoryKeySetStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.inMemory +package io.infinitic.storage.databases.inMemory import io.infinitic.storage.config.InMemoryConfig import io.infinitic.storage.data.Bytes diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/inMemory/InMemoryKeyValueStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorage.kt similarity index 97% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/inMemory/InMemoryKeyValueStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorage.kt index 2e08e10ca..e6762c69c 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/inMemory/InMemoryKeyValueStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.inMemory +package io.infinitic.storage.databases.inMemory import io.infinitic.storage.config.InMemoryConfig import io.infinitic.storage.keyValue.KeyValueStorage diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/mysql/MySQLKeySetStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorage.kt similarity index 98% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/mysql/MySQLKeySetStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorage.kt index 948045492..f33c59778 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/mysql/MySQLKeySetStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.mysql +package io.infinitic.storage.databases.mysql import com.zaxxer.hikari.HikariDataSource import io.infinitic.storage.config.MySQLConfig diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/mysql/MySQLKeyValueStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorage.kt similarity index 98% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/mysql/MySQLKeyValueStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorage.kt index 8681d0d3f..f40c3ba5b 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/mysql/MySQLKeyValueStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.mysql +package io.infinitic.storage.databases.mysql import com.zaxxer.hikari.HikariDataSource import io.infinitic.storage.config.MySQLConfig diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/postgres/PostgresKeySetStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorage.kt similarity index 98% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/postgres/PostgresKeySetStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorage.kt index 959132f7e..907efdaec 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/postgres/PostgresKeySetStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.postgres +package io.infinitic.storage.databases.postgres import com.zaxxer.hikari.HikariDataSource import io.infinitic.storage.config.PostgresConfig diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/postgres/PostgresKeyValueStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorage.kt similarity index 98% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/postgres/PostgresKeyValueStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorage.kt index 996d7fa7b..04f368a62 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/postgres/PostgresKeyValueStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.postgres +package io.infinitic.storage.databases.postgres import com.zaxxer.hikari.HikariDataSource import io.infinitic.storage.config.PostgresConfig diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/redis/RedisKeySetStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorage.kt similarity index 97% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/redis/RedisKeySetStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorage.kt index 29e2f0416..a4a4e2277 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/redis/RedisKeySetStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.redis +package io.infinitic.storage.databases.redis import io.infinitic.storage.config.RedisConfig import io.infinitic.storage.keySet.KeySetStorage diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/redis/RedisKeyValueStorage.kt b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorage.kt similarity index 97% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/redis/RedisKeyValueStorage.kt rename to infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorage.kt index 00a00d814..0e7c0592d 100644 --- a/infinitic-storage/src/main/kotlin/io/infinitic/storage/storages/redis/RedisKeyValueStorage.kt +++ b/infinitic-storage/src/main/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorage.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.storage.storages.redis +package io.infinitic.storage.databases.redis import io.infinitic.storage.config.RedisConfig import io.infinitic.storage.keyValue.KeyValueStorage diff --git a/infinitic-storage/src/test/java/io/infinitic/storage/config/MySQLStorageConfigTest.java b/infinitic-storage/src/test/java/io/infinitic/storage/config/MySQLStorageConfigTest.java new file mode 100644 index 000000000..76755b5ee --- /dev/null +++ b/infinitic-storage/src/test/java/io/infinitic/storage/config/MySQLStorageConfigTest.java @@ -0,0 +1,172 @@ +/** + * "Commons Clause" License Condition v1.0 + *

+ * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + *

+ * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + *

+ * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + *

+ * Software: Infinitic + *

+ * License: MIT License (https://opensource.org/licenses/MIT) + *

+ * Licensor: infinitic.io + */ + +package io.infinitic.storage.config; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class MySQLStorageConfigTest { + MySQLStorageConfig.MySQLStorageConfigBuilder builder; + + @BeforeEach + void setUp() { + builder = MySQLStorageConfig.builder() + .setHost("localhost") + .setPort(3306) + .setUserName("root") + .setPassword("password"); + } + + @Test + void testDefaultParameters() { + MySQLStorageConfig config = builder.build(); + + assertEquals("localhost", config.getHost()); + assertEquals(3306, config.getPort()); + assertEquals("root", config.getUsername()); + assertEquals("password", config.getPassword()); + assertEquals("infinitic", config.getDatabase()); + assertEquals("key_set_storage", config.getKeySetTable()); + assertEquals("key_value_storage", config.getKeyValueTable()); + } + + @Test + void testMandatoryHostParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> MySQLStorageConfig.builder() + .setHost("localhost") + .setUserName("root") + .build() + ); + assertTrue(e.getMessage().contains("port")); + } + + @Test + void testMandatoryPortParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> MySQLStorageConfig.builder() + .setHost("localhost") + .setUserName("root") + .build() + ); + assertTrue(e.getMessage().contains("port")); + } + + @Test + void testMandatoryUserParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> MySQLStorageConfig.builder() + .setHost("localhost") + .setPort(3306) + .build() + ); + assertTrue(e.getMessage().contains("user")); + } + + @Test + void testOptionalParameters() { + MySQLStorageConfig config = builder + .setConnectionTimeout(1L) + .setIdleTimeout(2L) + .setMaxLifetime(3L) + .setMinimumIdle(4) + .setMaximumPoolSize(5) + .build(); + + assertEquals(1L, config.getConnectionTimeout()); + assertEquals(2L, config.getIdleTimeout()); + assertEquals(3L, config.getMaxLifetime()); + assertEquals(4, config.getMinimumIdle()); + assertEquals(5, config.getMaximumPoolSize()); + } + + @Test + void testInvalidDatabaseName() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setDatabase("d-b").build() + ); + } + + @Test + void testInvalidKeyValueTableName() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setKeyValueTable("k-s").build() + ); + } + + @Test + void testInvalidKeySetTableName() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setKeySetTable("k-s").build() + ); + } + + @Test + void testInvalidConnectionTimeout() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setConnectionTimeout(-1).build() + ); + } + + @Test + void testInvalidIdleTimeout() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setIdleTimeout(0).build() + ); + } + + @Test + void testInvalidMaxLifetime() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setMaxLifetime(0).build() + ); + } + + @Test + void testInvalidMinimumIdle() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setMinimumIdle(-1).build() + ); + } + + @Test + void testInvalidMaximumPoolSize() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setMaximumPoolSize(0).build() + ); + } +} diff --git a/infinitic-storage/src/test/java/io/infinitic/storage/config/PostgresConfigTest.java b/infinitic-storage/src/test/java/io/infinitic/storage/config/PostgresConfigTest.java new file mode 100644 index 000000000..fbb7d76b3 --- /dev/null +++ b/infinitic-storage/src/test/java/io/infinitic/storage/config/PostgresConfigTest.java @@ -0,0 +1,172 @@ +/** + * "Commons Clause" License Condition v1.0 + *

+ * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + *

+ * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + *

+ * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + *

+ * Software: Infinitic + *

+ * License: MIT License (https://opensource.org/licenses/MIT) + *

+ * Licensor: infinitic.io + */ + +package io.infinitic.storage.config; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class PostgresConfigTest { + PostgresStorageConfig.PostgresConfigBuilder builder; + + @BeforeEach + void setUp() { + builder = PostgresStorageConfig.builder() + .setHost("localhost") + .setPort(5432) + .setUserName("postgres") + .setPassword("password"); + } + + @Test + void testDefaultParameters() { + PostgresStorageConfig config = builder.build(); + + assertEquals("localhost", config.getHost()); + assertEquals(5432, config.getPort()); + assertEquals("postgres", config.getUsername()); + assertEquals("password", config.getPassword()); + assertEquals("infinitic", config.getDatabase()); + assertEquals("key_set_storage", config.getKeySetTable()); + assertEquals("key_value_storage", config.getKeyValueTable()); + } + + @Test + void testMandatoryHostParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> PostgresStorageConfig.builder() + .setHost("localhost") + .setUserName("root") + .build() + ); + assertTrue(e.getMessage().contains("port")); + } + + @Test + void testMandatoryPortParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> PostgresStorageConfig.builder() + .setHost("localhost") + .setUserName("root") + .build() + ); + assertTrue(e.getMessage().contains("port")); + } + + @Test + void testMandatoryUserParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> PostgresStorageConfig.builder() + .setHost("localhost") + .setPort(3306) + .build() + ); + assertTrue(e.getMessage().contains("user")); + } + + @Test + void testOptionalParameters() { + PostgresStorageConfig config = builder + .setConnectionTimeout(1L) + .setIdleTimeout(2L) + .setMaxLifetime(3L) + .setMinimumIdle(4) + .setMaximumPoolSize(5) + .build(); + + assertEquals(1L, config.getConnectionTimeout()); + assertEquals(2L, config.getIdleTimeout()); + assertEquals(3L, config.getMaxLifetime()); + assertEquals(4, config.getMinimumIdle()); + assertEquals(5, config.getMaximumPoolSize()); + } + + @Test + void testInvalidDatabaseName() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setDatabase("d-b").build() + ); + } + + @Test + void testInvalidKeyValueTableName() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setKeyValueTable("k-s").build() + ); + } + + @Test + void testInvalidKeySetTableName() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setKeySetTable("k-s").build() + ); + } + + @Test + void testInvalidConnectionTimeout() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setConnectionTimeout(0).build() + ); + } + + @Test + void testInvalidIdleTimeout() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setIdleTimeout(0).build() + ); + } + + @Test + void testInvalidMaxLifetime() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setMaxLifetime(0).build() + ); + } + + @Test + void testInvalidMinimumIdle() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setMinimumIdle(-1).build() + ); + } + + @Test + void testInvalidMaximumPoolSize() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setMaximumPoolSize(0).build() + ); + } +} diff --git a/infinitic-storage/src/test/java/io/infinitic/storage/config/RedisConfigTest.java b/infinitic-storage/src/test/java/io/infinitic/storage/config/RedisConfigTest.java new file mode 100644 index 000000000..bd0a815ac --- /dev/null +++ b/infinitic-storage/src/test/java/io/infinitic/storage/config/RedisConfigTest.java @@ -0,0 +1,139 @@ +/** + * "Commons Clause" License Condition v1.0 + *

+ * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + *

+ * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + *

+ * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + *

+ * Software: Infinitic + *

+ * License: MIT License (https://opensource.org/licenses/MIT) + *

+ * Licensor: infinitic.io + */ + +package io.infinitic.storage.config; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import redis.clients.jedis.Protocol; + +import static org.junit.jupiter.api.Assertions.*; + +class RedisConfigTest { + RedisStorageConfig.RedisStorageConfigBuilder builder; + + @BeforeEach + void setUp() { + builder = RedisStorageConfig.builder() + .setHost("localhost") + .setPort(6379); + } + + @Test + void testDefaultParameters() { + RedisStorageConfig config = builder.build(); + + assertEquals("localhost", config.getHost()); + assertEquals(6379, config.getPort()); + assertNull(config.getUsername()); + assertNull(config.getPassword()); + assertEquals(Protocol.DEFAULT_DATABASE, config.getDatabase()); + assertEquals(Protocol.DEFAULT_TIMEOUT, config.getTimeout()); + assertFalse(config.getSsl()); + + RedisConfig.PoolConfig poolConfig = config.getPoolConfig(); + assertEquals(-1, poolConfig.getMaxTotal()); + assertEquals(8, poolConfig.getMaxIdle()); + assertEquals(0, poolConfig.getMinIdle()); + } + + @Test + void testMandatoryHostParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> RedisStorageConfig.builder() + .setPort(3306) + .build() + ); + assertTrue(e.getMessage().contains("host")); + } + + @Test + void testMandatoryPortParameters() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> RedisStorageConfig.builder() + .setHost("localhost") + .build() + ); + assertTrue(e.getMessage().contains("port")); + } + + @Test + void testOptionalParameters() { + RedisStorageConfig config = builder + .setUserName("user") + .setPassword("password") + .setDatabase(1) + .setTimeout(42) + .setSsl(true) + .setPoolConfig(RedisConfig.PoolConfig.builder() + .setMaxIdle(1) + .setMaxTotal(2) + .setMinIdle(3) + .build()) + .build(); + + assertEquals("user", config.getUsername()); + assertEquals("password", config.getPassword()); + assertEquals(1, config.getDatabase()); + assertEquals(42, config.getTimeout()); + assertTrue(config.getSsl()); + RedisConfig.PoolConfig poolConfig = config.getPoolConfig(); + assertEquals(1, poolConfig.getMaxIdle()); + assertEquals(2, poolConfig.getMaxTotal()); + assertEquals(3, poolConfig.getMinIdle()); + } + + @Test + void testInvalidHost() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setHost(" ").build() + ); + } + + @Test + void testInvalidPort() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setPort(0).build() + ); + } + + @Test + void testInvalidDatabase() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setDatabase(-1).build() + ); + } + + @Test + void testInvalidTimeout() { + assertThrows( + IllegalArgumentException.class, + () -> builder.setTimeout(0).build() + ); + } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfigInterface.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/InMemoryConfigTests.kt similarity index 60% rename from infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfigInterface.kt rename to infinitic-storage/src/test/kotlin/io/infinitic/storage/config/InMemoryConfigTests.kt index 4daf8a23d..4f94754cc 100644 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfigInterface.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/InMemoryConfigTests.kt @@ -20,14 +20,31 @@ * * Licensor: infinitic.io */ -package io.infinitic.workers.config -import io.infinitic.clients.config.ClientConfigInterface -import io.infinitic.storage.config.StorageConfigInterface -import io.infinitic.workers.register.config.RegisterConfigInterface +package io.infinitic.storage.config -interface WorkerConfigInterface : - RegisterConfigInterface, - StorageConfigInterface, - ClientConfigInterface +import io.infinitic.common.config.loadConfigFromYaml +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +class InMemoryConfigTests : + StringSpec( + { + "can use InMemoryConfig" { + val config = shouldNotThrowAny { + loadConfigFromYaml( + """ +storage: + inMemory: + """, + ) + } + config.storage shouldBe InMemoryStorageConfig( + inMemory = InMemoryConfig(), + compression = null, + cache = null, + ) + } + }, + ) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/MySQLConfigTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/MySQLConfigTests.kt index 1d08d3081..90fdc9894 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/MySQLConfigTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/MySQLConfigTests.kt @@ -22,25 +22,175 @@ */ package io.infinitic.storage.config +import com.sksamuel.hoplite.ConfigException +import io.infinitic.common.config.loadConfigFromYaml +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain class MySQLConfigTests : StringSpec( { - "Can create MySQLConfig through builder" { - val mysqlConfig = MySQLConfig - .builder() - .build() + lateinit var config: MySQLConfig - mysqlConfig shouldBe MySQLConfig() + beforeEach { + config = MySQLConfig("localhost", 3306, "root", null) } "Check MySQLConfig default values do not change to ensure backward compatibility" { - val mysqlConfig = MySQLConfig() + config.database shouldBe "infinitic" + config.keySetTable shouldBe "key_set_storage" + config.keyValueTable shouldBe "key_value_storage" + } + + "should throw for invalid database name" { + val e = shouldThrow { + config.copy(database = "invalid-name") + } + e.message shouldContain "database" + } + + "should throw for invalid key-set table name" { + val e = shouldThrow { + config.copy(keySetTable = "invalid-table") + } + e.message shouldContain "keySetTable" + } + + "should throw for invalid key-value table name" { + val e = shouldThrow { + config.copy(keyValueTable = "invalid-table") + } + e.message shouldContain "keyValueTable" + } + + "should throw for invalid maximumPoolSize" { + val e = shouldThrow { + config.copy(maximumPoolSize = 0) + } + e.message shouldContain "maximumPoolSize" + } + + "should throw for invalid minimumIdle" { + val e = shouldThrow { + config.copy(minimumIdle = -1) + } + e.message shouldContain "minimumIdle" + } + + "should throw for invalid idleTimeout" { + val e = shouldThrow { + config.copy(idleTimeout = 0) + } + e.message shouldContain "idleTimeout" + } + + "should throw for invalid connectionTimeout" { + val e = shouldThrow { + config.copy(connectionTimeout = 0) + } + e.message shouldContain "connectionTimeout" + } - mysqlConfig.database shouldBe "infinitic" - mysqlConfig.keySetTable shouldBe "key_set_storage" - mysqlConfig.keyValueTable shouldBe "key_value_storage" + "should throw for invalid maxLifetime" { + val e = shouldThrow { + config.copy(maxLifetime = 0) + } + e.message shouldContain "maxLifetime" + } + + "toString() should obfuscate password" { + config.toString() shouldBe "MySQLConfig(host='${config.host}', port=${config.port}, username='${config.username}', password='******', " + + "database=${config.database}, keySetTable=${config.keySetTable}, keyValueTable=${config.keyValueTable})" + } + + "Can not load from yaml with no host" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + mysql: + host: localhost + username: root + """, + ) + } + } + + "Can not load from yaml with no port" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + mysql: + port: 3306 + username: root + """, + ) + } + } + + "Can not load from yaml with no user" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + mysql: + host: localhost + port: 3306 + """, + ) + } + } + + val yaml = """ +storage: + mysql: + host: localhost + port: 3306 + username: root""" + + "can load from yaml with only mandatory parameters" { + val storageConfig = loadConfigFromYaml(yaml) + + storageConfig.storage shouldBe MySQLStorageConfig( + mysql = MySQLConfig("localhost", 3306, "root", null), + compression = null, + cache = null, + ) + } + + "can load from yaml with optional parameters" { + val storageConfig = loadConfigFromYaml( + yaml + """ + password: pass + database: azerty + keySetTable: keySet + keyValueTable: keyVal + maximumPoolSize: 1 + minimumIdle: 2 + idleTimeout: 3 + connectionTimeout: 4 + maxLifetime: 5 + """, + ) + + (storageConfig.storage as MySQLStorageConfig).mysql shouldBe + MySQLConfig( + "localhost", + 3306, + "root", + "pass", + "azerty", + "keySet", + "keyVal", + 1, + 2, + 3, + 4, + 5, + ) } }, ) + diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/PostgresConfigTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/PostgresConfigTests.kt index da832a7ce..c67ed9695 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/PostgresConfigTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/PostgresConfigTests.kt @@ -22,25 +22,174 @@ */ package io.infinitic.storage.config +import com.sksamuel.hoplite.ConfigException +import io.infinitic.common.config.loadConfigFromYaml +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain class PostgresConfigTests : StringSpec( { - "Can create PostgresConfig through builder" { - val postgresConfig = PostgresConfig - .builder() - .build() + lateinit var config: PostgresConfig - postgresConfig shouldBe PostgresConfig() + beforeEach { + config = PostgresConfig("localhost", 3306, "root", null) } "Check PostgresConfig default values do not change to ensure backward compatibility" { - val postgresConfig = PostgresConfig() + config.database shouldBe "infinitic" + config.keySetTable shouldBe "key_set_storage" + config.keyValueTable shouldBe "key_value_storage" + } + + "should throw for invalid database name" { + val e = shouldThrow { + config.copy(database = "invalid-name") + } + e.message shouldContain "database" + } + + "should throw for invalid key-set table name" { + val e = shouldThrow { + config.copy(keySetTable = "invalid-table") + } + e.message shouldContain "keySetTable" + } + + "should throw for invalid key-value table name" { + val e = shouldThrow { + config.copy(keyValueTable = "invalid-table") + } + e.message shouldContain "keyValueTable" + } + + "should throw for invalid maximumPoolSize" { + val e = shouldThrow { + config.copy(maximumPoolSize = 0) + } + e.message shouldContain "maximumPoolSize" + } + + "should throw for invalid minimumIdle" { + val e = shouldThrow { + config.copy(minimumIdle = -1) + } + e.message shouldContain "minimumIdle" + } + + "should throw for invalid idleTimeout" { + val e = shouldThrow { + config.copy(idleTimeout = 0) + } + e.message shouldContain "idleTimeout" + } + + "should throw for invalid connectionTimeout" { + val e = shouldThrow { + config.copy(connectionTimeout = 0) + } + e.message shouldContain "connectionTimeout" + } + + "should throw for invalid maxLifetime" { + val e = shouldThrow { + config.copy(maxLifetime = 0) + } + e.message shouldContain "maxLifetime" + } + + "toString() should obfuscate password" { + config.toString() shouldBe "PostgresConfig(host='${config.host}', port=${config.port}, username='${config.username}', password='******', " + + "database=${config.database}, keySetTable=${config.keySetTable}, keyValueTable=${config.keyValueTable})" + } + + "Can not load from yaml with no host" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + postgres: + host: localhost + username: root + """, + ) + } + } + + "Can not load from yaml with no port" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + postgres: + port: 3306 + username: root + """, + ) + } + } + + "Can not load from yaml with no user" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + postgres: + host: localhost + port: 3306 + """, + ) + } + } + + val yaml = """ +storage: + postgres: + host: localhost + port: 3306 + username: root""" + + "can load from yaml with only mandatory parameters" { + val storageConfig = loadConfigFromYaml(yaml) + + storageConfig.storage shouldBe PostgresStorageConfig( + postgres = PostgresConfig("localhost", 3306, "root", null), + compression = null, + cache = null, + ) + } + + "can load from yaml with optional parameters" { + val storageConfig = loadConfigFromYaml( + yaml + """ + password: pass + database: azerty + keySetTable: keySet + keyValueTable: keyVal + maximumPoolSize: 1 + minimumIdle: 2 + idleTimeout: 3 + connectionTimeout: 4 + maxLifetime: 5 + """, + ) - postgresConfig.database shouldBe "infinitic" - postgresConfig.keySetTable shouldBe "key_set_storage" - postgresConfig.keyValueTable shouldBe "key_value_storage" + (storageConfig.storage as PostgresStorageConfig).postgres shouldBe + PostgresConfig( + "localhost", + 3306, + "root", + "pass", + "azerty", + "keySet", + "keyVal", + 1, + 2, + 3, + 4, + 5, + ) } }, ) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/RedisConfigTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/RedisConfigTests.kt index 86c420ae3..978036a45 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/RedisConfigTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/RedisConfigTests.kt @@ -22,31 +22,131 @@ */ package io.infinitic.storage.config +import com.sksamuel.hoplite.ConfigException +import io.infinitic.common.config.loadConfigFromYaml +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain class RedisConfigTests : StringSpec( { - "Can create RedisConfig through builder" { - val redisConfig = RedisConfig - .builder() - .build() + lateinit var config: RedisConfig - redisConfig shouldBe RedisConfig() + beforeEach { + config = RedisConfig(host = "localhost", port = 6379) } - "Can create PoolConfig through builder" { - val poolConfig = RedisConfig.PoolConfig - .builder() - .build() + "Check RedisConfig default values do not change to ensure backward compatibility" { + config.database shouldBe 0 + config.ssl shouldBe false + } - poolConfig shouldBe RedisConfig.PoolConfig() + "should throw for invalid host" { + val e = shouldThrow { + config.copy(host = " ") + } + e.message shouldContain "host" } - "Check RedisConfig default values do not change to ensure backward compatibility" { - val redisConfig = RedisConfig() + "should throw for invalid port" { + val e = shouldThrow { + config.copy(port = 0) + } + e.message shouldContain "port" + } + + "should throw for invalid database" { + val e = shouldThrow { + config.copy(database = -1) + } + e.message shouldContain "database" + } + + "should throw for invalid timeout" { + val e = shouldThrow { + config.copy(timeout = 0) + } + e.message shouldContain "timeout" + } + + "toString() should obfuscate password" { + with(config.copy(username = "admin", password = "")) { + toString() shouldBe "RedisConfig(host='$host', port=$port, username='$username', password='******'" + + ", database=$database, timeout=$timeout, ssl=$ssl, poolConfig=$poolConfig)" + } + } + + "Can not load from yaml with no host" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + redis: + port: 6379 + """, + ) + } + } + + "Can not load from yaml with no port" { + shouldThrow { + loadConfigFromYaml( + """ +storage: + redis: + host: localhost + """, + ) + } + } + + val yaml = """ +storage: + redis: + host: localhost + port: 6379""" + + "can load from yaml with only mandatory parameters" { + val storageConfig = loadConfigFromYaml(yaml) + + storageConfig.storage shouldBe RedisStorageConfig( + redis = RedisConfig("localhost", 6379), + cache = null, + compression = null, + ) + } + + "can load from yaml with optional parameters" { + val storageConfig = loadConfigFromYaml( + yaml + """ + username: root + password: pass + database: 1 + timeout: 2 + ssl: true + poolConfig: + maxTotal: 5 + maxIdle: 6 + minIdle: 7 + """, + ) - redisConfig.database shouldBe 0 + (storageConfig.storage as RedisStorageConfig).redis shouldBe + RedisConfig( + "localhost", + 6379, + "root", + "pass", + 1, + 2, + true, + RedisConfig.PoolConfig( + maxTotal = 5, + maxIdle = 6, + minIdle = 7, + ), + ) } }, ) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigImpl.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigImpl.kt index d596d6b69..221d092f9 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigImpl.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigImpl.kt @@ -23,5 +23,5 @@ package io.infinitic.storage.config internal data class StorageConfigImpl( - override val storage: StorageConfig = StorageConfig() + override val storage: StorageConfig ) : StorageConfigInterface diff --git a/infinitic-storage/src/main/kotlin/io/infinitic/storage/config/StorageConfigInterface.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigInterface.kt similarity index 100% rename from infinitic-storage/src/main/kotlin/io/infinitic/storage/config/StorageConfigInterface.kt rename to infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigInterface.kt diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigTests.kt deleted file mode 100644 index 8f9330ca5..000000000 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageConfigTests.kt +++ /dev/null @@ -1,126 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.storage.config - -import com.sksamuel.hoplite.ConfigException -import com.sksamuel.hoplite.ConfigLoaderBuilder -import com.sksamuel.hoplite.yaml.YamlPropertySource -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.shouldBe -import io.kotest.matchers.string.shouldContain - -class StorageConfigTests : - StringSpec( - { - "Can create StorageConfig through builder" { - val storageConfig = StorageConfig - .builder() - .build() - - storageConfig shouldBe StorageConfig() - } - - "default storage should be inMemory" { - val config = loadConfigFromYaml("nothing:") - - config shouldBe StorageConfigImpl(storage = StorageConfig(inMemory = InMemoryConfig())) - } - - "default storage should not be compressed" { - val config = loadConfigFromYaml("nothing:") - - config shouldBe StorageConfigImpl(storage = StorageConfig(compression = null)) - } - - "storage without type should default" { - val default1 = loadConfigFromYaml("nothing:") - val default2 = loadConfigFromYaml("storage:") - - default1 shouldBe default2 - } - - "can choose inMemory storage" { - val config = loadConfigFromYaml( - """ -storage: - inMemory: - """, - ) - - config shouldBe StorageConfigImpl(storage = StorageConfig(inMemory = InMemoryConfig())) - } - - "can choose Redis storage" { - val config = loadConfigFromYaml( - """ -storage: - redis: - """, - ) - - config shouldBe StorageConfigImpl(storage = StorageConfig(redis = RedisConfig())) - } - - "can choose MySQL storage" { - val config = loadConfigFromYaml( - """ -storage: - mysql: - """, - ) - - config shouldBe StorageConfigImpl(storage = StorageConfig(mysql = MySQLConfig())) - } - - "can choose Postgres storage" { - val config = loadConfigFromYaml( - """ -storage: - postgres: - """, - ) - - config shouldBe StorageConfigImpl(storage = StorageConfig(postgres = PostgresConfig())) - } - - "can not have multiple definition in storage" { - val e = shouldThrow { - loadConfigFromYaml( - """ -storage: - redis: - mysql: - """, - ) - } - e.message shouldContain ("Storage should have only one definition") - } - }, - ) - -internal inline fun loadConfigFromYaml(yaml: String): T = - ConfigLoaderBuilder.default() - .also { builder -> builder.addSource(YamlPropertySource(yaml)) } - .build() - .loadConfigOrThrow() diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageTests.kt index 31e21f71e..c23dc3549 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/config/StorageTests.kt @@ -22,17 +22,16 @@ */ package io.infinitic.storage.config -import com.sksamuel.hoplite.Secret import io.infinitic.storage.DockerOnly +import io.infinitic.storage.databases.inMemory.InMemoryKeySetStorage +import io.infinitic.storage.databases.inMemory.InMemoryKeyValueStorage +import io.infinitic.storage.databases.mysql.MySQLKeySetStorage +import io.infinitic.storage.databases.mysql.MySQLKeyValueStorage +import io.infinitic.storage.databases.postgres.PostgresKeySetStorage +import io.infinitic.storage.databases.postgres.PostgresKeyValueStorage +import io.infinitic.storage.databases.redis.RedisKeySetStorage +import io.infinitic.storage.databases.redis.RedisKeyValueStorage import io.infinitic.storage.keyValue.CompressedKeyValueStorage -import io.infinitic.storage.storages.inMemory.InMemoryKeySetStorage -import io.infinitic.storage.storages.inMemory.InMemoryKeyValueStorage -import io.infinitic.storage.storages.mysql.MySQLKeySetStorage -import io.infinitic.storage.storages.mysql.MySQLKeyValueStorage -import io.infinitic.storage.storages.postgres.PostgresKeySetStorage -import io.infinitic.storage.storages.postgres.PostgresKeyValueStorage -import io.infinitic.storage.storages.redis.RedisKeySetStorage -import io.infinitic.storage.storages.redis.RedisKeyValueStorage import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe import org.testcontainers.containers.MySQLContainer @@ -41,45 +40,43 @@ import org.testcontainers.containers.PostgreSQLContainer class StorageTests : StringSpec( { - "default storage should be inMemory" { - val storage = StorageConfig() - - storage shouldBe StorageConfig(inMemory = InMemoryConfig()) - } - "properties of InMemory" { - val config = StorageConfig(inMemory = InMemoryConfig("test"), compression = null) - - config.type shouldBe StorageConfig.StorageType.IN_MEMORY + val config = InMemoryStorageConfig( + inMemory = InMemoryConfig(), + compression = null, + cache = null, + ) // config.keySet should be InMemoryKeySetStorage() config.keySet::class shouldBe InMemoryKeySetStorage::class (config.keySet as InMemoryKeySetStorage).storage shouldBe - InMemoryKeySetStorage.from(InMemoryConfig("test")).storage + InMemoryKeySetStorage.from(InMemoryConfig()).storage // config.keyValue should be CompressedKeyValueStorage(InMemoryKeyValueStorage()) config.keyValue::class shouldBe CompressedKeyValueStorage::class (config.keyValue as CompressedKeyValueStorage).storage::class shouldBe InMemoryKeyValueStorage::class ((config.keyValue as CompressedKeyValueStorage).storage as InMemoryKeyValueStorage) - .storage shouldBe InMemoryKeyValueStorage.from(InMemoryConfig("test")).storage + .storage shouldBe InMemoryKeyValueStorage.from(InMemoryConfig()).storage } "properties of Redis" { - val config = StorageConfig(redis = RedisConfig(), compression = null) - - config.type shouldBe StorageConfig.StorageType.REDIS + val config = RedisStorageConfig( + redis = RedisConfig(host = "localhost", port = 6379), + compression = null, + cache = null, + ) // config.keySet should be RedisKeySetStorage(pool) config.keySet::class shouldBe RedisKeySetStorage::class - (config.keySet as RedisKeySetStorage).pool shouldBe RedisKeySetStorage.from(RedisConfig()).pool + (config.keySet as RedisKeySetStorage).pool shouldBe RedisKeySetStorage.from(config.redis).pool // config.keyValue should be CompressedKeyValueStorage(RedisKeyValueStorage(pool)) config.keyValue::class shouldBe CompressedKeyValueStorage::class (config.keyValue as CompressedKeyValueStorage).storage::class shouldBe RedisKeyValueStorage::class ((config.keyValue as CompressedKeyValueStorage).storage as RedisKeyValueStorage) - .pool shouldBe RedisKeyValueStorage.from(RedisConfig()).pool + .pool shouldBe RedisKeyValueStorage.from(config.redis).pool } "properties of MySQL".config(enabledIf = { DockerOnly.shouldRun }) { @@ -95,14 +92,11 @@ class StorageTests : val mysql = MySQLConfig( host = mysqlServer.host, port = mysqlServer.firstMappedPort, - user = mysqlServer.username, - password = Secret(mysqlServer.password), - database = mysqlServer.databaseName, - ) + username = mysqlServer.username, + password = mysqlServer.password, + ).copy(database = mysqlServer.databaseName) - val config = StorageConfig(mysql = mysql, compression = null) - - config.type shouldBe StorageConfig.StorageType.MYSQL + val config = MySQLStorageConfig(mysql = mysql, compression = null) // config.keySet should be MySQLKeySetStorage(pool) config.keySet::class shouldBe MySQLKeySetStorage::class @@ -132,14 +126,12 @@ class StorageTests : val postgresConfig = PostgresConfig( host = postgresServer.host, port = postgresServer.firstMappedPort, - user = postgresServer.username, - password = Secret(postgresServer.password), + username = postgresServer.username, + password = postgresServer.password, database = postgresServer.databaseName, ) - val config = StorageConfig(postgres = postgresConfig, compression = null) - - config.type shouldBe StorageConfig.StorageType.POSTGRES + val config = PostgresStorageConfig(postgres = postgresConfig, compression = null) // config.keySet should be PostgresKeySetStorage(pool) config.keySet::class shouldBe PostgresKeySetStorage::class diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorageTests.kt index 0385b8a1a..609896c8d 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeySetStorageTests.kt @@ -24,14 +24,13 @@ package io.infinitic.storage.databases.inMemory import io.infinitic.storage.config.InMemoryConfig import io.infinitic.storage.data.Bytes -import io.infinitic.storage.storages.inMemory.InMemoryKeySetStorage import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe class InMemoryKeySetStorageTests : StringSpec( { - val config = InMemoryConfig("test") + val config = InMemoryConfig() val storage = InMemoryKeySetStorage.from(config) beforeTest { storage.add("foo", "bar".toByteArray()) } diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorageTests.kt index 04e28a355..38f3aee14 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/inMemory/InMemoryKeyValueStorageTests.kt @@ -23,14 +23,13 @@ package io.infinitic.storage.databases.inMemory import io.infinitic.storage.config.InMemoryConfig -import io.infinitic.storage.storages.inMemory.InMemoryKeyValueStorage import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe class InMemoryKeyValueStorageTests : StringSpec( { - val config = InMemoryConfig("test") + val config = InMemoryConfig() val storage = InMemoryKeyValueStorage.from(config) beforeTest { storage.put("foo", "bar".toByteArray()) } diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorageTests.kt index d10aa9666..a1da61156 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeySetStorageTests.kt @@ -22,11 +22,9 @@ */ package io.infinitic.storage.databases.mysql -import com.sksamuel.hoplite.Secret import io.infinitic.storage.DockerOnly import io.infinitic.storage.config.MySQLConfig import io.infinitic.storage.data.Bytes -import io.infinitic.storage.storages.mysql.MySQLKeySetStorage import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe @@ -48,10 +46,9 @@ class MySQLKeySetStorageTests : val config = MySQLConfig( host = mysqlServer.host, port = mysqlServer.firstMappedPort, - user = mysqlServer.username, - password = Secret(mysqlServer.password), - database = mysqlServer.databaseName, - ) + username = mysqlServer.username, + password = mysqlServer.password, + ).copy(database = mysqlServer.databaseName) val storage = MySQLKeySetStorage.from(config) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorageTests.kt index d9571b554..180052e9c 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySQLKeyValueStorageTests.kt @@ -22,10 +22,8 @@ */ package io.infinitic.storage.databases.mysql -import com.sksamuel.hoplite.Secret import io.infinitic.storage.DockerOnly import io.infinitic.storage.config.MySQLConfig -import io.infinitic.storage.storages.mysql.MySQLKeyValueStorage import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe @@ -47,10 +45,9 @@ class MySQLKeyValueStorageTests : val config = MySQLConfig( host = mysqlServer.host, port = mysqlServer.firstMappedPort, - user = mysqlServer.username, - password = Secret(mysqlServer.password), - database = mysqlServer.databaseName, - ) + username = mysqlServer.username, + password = mysqlServer.password, + ).copy(database = mysqlServer.databaseName) val storage = MySQLKeyValueStorage.from(config) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySqlConfigTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySqlConfigTests.kt deleted file mode 100644 index 3ae46739e..000000000 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/mysql/MySqlConfigTests.kt +++ /dev/null @@ -1,108 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ - -package io.infinitic.storage.databases.mysql - -import com.sksamuel.hoplite.ConfigException -import io.infinitic.storage.config.StorageConfigImpl -import io.infinitic.storage.config.loadConfigFromYaml -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec - -class MySqlConfigTests : - StringSpec( - { - - "maximumPoolSize can not be 0" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - mysql: - maximumPoolSize: 0 - """, - ) - } - } - - "minimumIdle can not be negative" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - mysql: - minimumIdle: -1 - """, - ) - } - } - - "idleTimeout can not be 0" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - mysql: - idleTimeout: 0 - """, - ) - } - } - - "connectionTimeout can not be 0" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - mysql: - connectionTimeout: 0 - """, - ) - } - } - - "checking keyValueTable" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - mysql: - keyValueTable: "_invalid" - """, - ) - } - } - - "checking keySetTable" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - mysql: - keySetTable: "_invalid" - """, - ) - } - } - }, - ) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresConfigTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresConfigTests.kt deleted file mode 100644 index 9e5dcf479..000000000 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresConfigTests.kt +++ /dev/null @@ -1,107 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ - -package io.infinitic.storage.databases.postgres - -import com.sksamuel.hoplite.ConfigException -import io.infinitic.storage.config.StorageConfigImpl -import io.infinitic.storage.config.loadConfigFromYaml -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec - -class PostgresConfigTests : StringSpec( - { - - "maximumPoolSize can not be 0" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - postgres: - maximumPoolSize: 0 - """, - ) - } - } - - "minimumIdle can not be negative" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - postgres: - minimumIdle: -1 - """, - ) - } - } - - "idleTimeout can not be 0" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - postgres: - idleTimeout: 0 - """, - ) - } - } - - "connectionTimeout can not be 0" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - postgres: - connectionTimeout: 0 - """, - ) - } - } - - "checking keyValueTable" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - postgres: - keyValueTable: "0invalid" - """, - ) - } - } - - "checking keySetTable" { - shouldThrow { - loadConfigFromYaml( - """ -storage: - postgres: - keySetTable: "0invalid" - """, - ) - } - } - }, -) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorageTests.kt index 168e8b555..2da78c506 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeySetStorageTests.kt @@ -22,11 +22,9 @@ */ package io.infinitic.storage.databases.postgres -import com.sksamuel.hoplite.Secret import io.infinitic.storage.DockerOnly import io.infinitic.storage.config.PostgresConfig import io.infinitic.storage.data.Bytes -import io.infinitic.storage.storages.postgres.PostgresKeySetStorage import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe @@ -48,8 +46,8 @@ class PostgresKeySetStorageTests : val config = PostgresConfig( host = postgresServer.host, port = postgresServer.firstMappedPort, - user = postgresServer.username, - password = Secret(postgresServer.password), + username = postgresServer.username, + password = postgresServer.password, database = postgresServer.databaseName, ) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorageTests.kt index 06b9c7799..ff1df963c 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/postgres/PostgresKeyValueStorageTests.kt @@ -22,10 +22,8 @@ */ package io.infinitic.storage.databases.postgres -import com.sksamuel.hoplite.Secret import io.infinitic.storage.DockerOnly import io.infinitic.storage.config.PostgresConfig -import io.infinitic.storage.storages.postgres.PostgresKeyValueStorage import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe @@ -47,8 +45,8 @@ class PostgresKeyValueStorageTests : val config = PostgresConfig( host = postgresServer.host, port = postgresServer.firstMappedPort, - user = postgresServer.username, - password = Secret(postgresServer.password), + username = postgresServer.username, + password = postgresServer.password, database = postgresServer.databaseName, ) diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorageTests.kt index 69717ad1f..0299598f3 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeySetStorageTests.kt @@ -25,7 +25,6 @@ package io.infinitic.storage.databases.redis import io.infinitic.storage.DockerOnly import io.infinitic.storage.config.RedisConfig import io.infinitic.storage.data.Bytes -import io.infinitic.storage.storages.redis.RedisKeySetStorage import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe diff --git a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorageTests.kt b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorageTests.kt index 207988f90..bb45fbf23 100644 --- a/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorageTests.kt +++ b/infinitic-storage/src/test/kotlin/io/infinitic/storage/databases/redis/RedisKeyValueStorageTests.kt @@ -24,7 +24,6 @@ package io.infinitic.storage.databases.redis import io.infinitic.storage.DockerOnly import io.infinitic.storage.config.RedisConfig -import io.infinitic.storage.storages.redis.RedisKeyValueStorage import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe diff --git a/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/TaskExecutor.kt b/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/TaskExecutor.kt index c63e02732..687f5ecb6 100644 --- a/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/TaskExecutor.kt +++ b/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/TaskExecutor.kt @@ -33,6 +33,7 @@ import io.infinitic.common.data.methods.MethodParameterTypes import io.infinitic.common.data.methods.deserializeArgs import io.infinitic.common.data.methods.encodeReturnValue import io.infinitic.common.emitters.EmitterName +import io.infinitic.common.registry.ExecutorRegistryInterface import io.infinitic.common.requester.WorkflowRequester import io.infinitic.common.requester.workflowId import io.infinitic.common.requester.workflowName @@ -52,7 +53,6 @@ import io.infinitic.common.utils.getMethodPerNameAndParameters import io.infinitic.common.utils.isDelegated import io.infinitic.common.utils.withRetry import io.infinitic.common.utils.withTimeout -import io.infinitic.common.workers.registry.WorkerRegistry import io.infinitic.common.workflows.data.workflowTasks.WorkflowTaskParameters import io.infinitic.common.workflows.data.workflows.WorkflowName import io.infinitic.exceptions.DeferredException @@ -74,7 +74,7 @@ import java.util.concurrent.TimeoutException import kotlin.reflect.jvm.javaMethod class TaskExecutor( - private val workerRegistry: WorkerRegistry, + private val registry: ExecutorRegistryInterface, private val producer: InfiniticProducer, private val client: InfiniticClientInterface ) { @@ -107,7 +107,6 @@ class TaskExecutor( val taskContext = TaskContextImpl( workerName = producer.name, - workerRegistry = workerRegistry, serviceName = msg.serviceName, taskId = msg.taskId, taskName = msg.methodName, @@ -260,9 +259,8 @@ class TaskExecutor( methodParameterTypes: MethodParameterTypes?, methodArgs: MethodArgs ): Triple> { - val registeredServiceExecutor = workerRegistry.serviceExecutors[serviceName]!! - val serviceInstance = registeredServiceExecutor.factory() + val serviceInstance = registry.getServiceExecutorInstance(serviceName) val serviceMethod = serviceInstance::class.java.getMethodPerNameAndParameters( "$methodName", methodParameterTypes?.types, @@ -270,21 +268,28 @@ class TaskExecutor( ) val serviceArgs = serviceMethod.deserializeArgs(methodArgs) - this.withTimeout = - // use withTimeout from registry, if it exists - registeredServiceExecutor.withTimeout - // else use @Timeout annotation, or WithTimeout interface - ?: serviceMethod.withTimeout.getOrThrow() - // else use default value - ?: TASK_WITH_TIMEOUT_DEFAULT + this.withTimeout = when (val wt = registry.getServiceExecutorWithTimeout(serviceName)) { + WithTimeout.UNSET -> + //use @Timeout annotation, or WithTimeout interface + serviceMethod.withTimeout.getOrThrow() + // else use default value + ?: TASK_WITH_TIMEOUT_DEFAULT + + else -> wt + } // use withRetry from registry, if it exists - this.withRetry = registeredServiceExecutor.withRetry - // else use @Timeout annotation, or WithTimeout interface - ?: serviceMethod.withRetry.getOrThrow() - // else use default value + this.withRetry = when (val wr = registry.getServiceExecutorWithRetry(serviceName)) { + WithRetry.UNSET -> + // use @Retry annotation, or WithRetry interface + serviceMethod.withRetry.getOrThrow() + // else use default value ?: TASK_WITH_RETRY_DEFAULT + else -> wr + } + + // check is this method has the @Async annotation this.isDelegated = serviceMethod.isDelegated @@ -301,10 +306,8 @@ class TaskExecutor( val workflowTaskParameters = serviceArgs[0] as WorkflowTaskParameters - // workflow registered in worker - val registeredWorkflowExecutor = workerRegistry.workflowExecutors[workflowName]!! // workflow instance - val workflowInstance = registeredWorkflowExecutor.getInstance(workflowTaskParameters) + val workflowInstance = registry.getWorkflowExecutorInstance(workflowTaskParameters) // method of the workflow instance val workflowMethod = with(workflowTaskParameters) { workflowInstance::class.java.getMethodPerNameAndParameters( @@ -315,7 +318,7 @@ class TaskExecutor( } // get checkMode from registry - val checkMode = registeredWorkflowExecutor.checkMode + val checkMode = registry.getWorkflowExecutorCheckMode(workflowName) // else use CheckMode method annotation on method or class ?: workflowMethod.checkMode // else use default value @@ -328,21 +331,30 @@ class TaskExecutor( } // use withTimeout from registry, if it exists - this.withTimeout = registeredWorkflowExecutor.withTimeout + this.withTimeout = when (val wt = registry.getWorkflowExecutorWithTimeout(workflowName)) { + WithTimeout.UNSET -> // HERE WE ARE LOOKING FOR THE TIMEOUT OF THE WORKFLOW TASK // NOT OF THE WORKFLOW ITSELF, THAT'S WHY WE DO NOT LOOK FOR // THE @Timeout ANNOTATION OR THE WithTimeout INTERFACE // THAT HAS A DIFFERENT MEANING IN WORKFLOWS // else use default value - ?: WORKFLOW_TASK_WITH_TIMEOUT_DEFAULT + WORKFLOW_TASK_WITH_TIMEOUT_DEFAULT + + else -> wt + } // use withRetry from registry, if it exists - this.withRetry = registeredWorkflowExecutor.withRetry + this.withRetry = when (val wr = registry.getWorkflowExecutorWithRetry(workflowName)) { + WithRetry.UNSET -> // else use @Retry annotation, or WithRetry interface - ?: workflowMethod.withRetry.getOrThrow() - // else use default value + workflowMethod.withRetry.getOrThrow() + // else use default value ?: WORKFLOW_TASK_WITH_RETRY_DEFAULT + else -> wr + } + + return Triple(serviceInstance, serviceMethod, serviceArgs) } diff --git a/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/task/TaskContextImpl.kt b/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/task/TaskContextImpl.kt index 56311f3f0..d1e63b82a 100644 --- a/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/task/TaskContextImpl.kt +++ b/infinitic-task-executor/src/main/kotlin/io/infinitic/tasks/executor/task/TaskContextImpl.kt @@ -30,7 +30,6 @@ import io.infinitic.common.tasks.data.TaskRetryIndex import io.infinitic.common.tasks.data.TaskRetrySequence import io.infinitic.common.tasks.executors.errors.ExecutionError import io.infinitic.common.workers.config.WorkflowVersion -import io.infinitic.common.workers.registry.WorkerRegistry import io.infinitic.common.workflows.data.workflows.WorkflowId import io.infinitic.common.workflows.data.workflows.WorkflowName import io.infinitic.tasks.TaskContext @@ -39,7 +38,6 @@ import io.infinitic.tasks.WithTimeout data class TaskContextImpl( override val workerName: String, - override val workerRegistry: WorkerRegistry, override val serviceName: ServiceName, override val taskId: TaskId, override val taskName: MethodName, diff --git a/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/TaskExecutorTests.kt b/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/TaskExecutorTests.kt index 2bba2a8e8..3d9380fd7 100644 --- a/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/TaskExecutorTests.kt +++ b/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/TaskExecutorTests.kt @@ -35,6 +35,7 @@ import io.infinitic.common.data.methods.MethodReturnValue import io.infinitic.common.emitters.EmitterName import io.infinitic.common.fixtures.TestFactory import io.infinitic.common.fixtures.methodParametersFrom +import io.infinitic.common.registry.ExecutorRegistryInterface import io.infinitic.common.requester.ClientRequester import io.infinitic.common.tasks.data.ServiceName import io.infinitic.common.tasks.data.TaskId @@ -56,12 +57,12 @@ import io.infinitic.common.transport.ServiceExecutorEventTopic import io.infinitic.common.transport.ServiceExecutorTopic import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy import io.infinitic.common.workers.data.WorkerName -import io.infinitic.common.workers.registry.RegisteredServiceExecutor -import io.infinitic.common.workers.registry.WorkerRegistry import io.infinitic.exceptions.tasks.ClassNotFoundException import io.infinitic.exceptions.tasks.NoMethodFoundWithParameterCountException import io.infinitic.exceptions.tasks.NoMethodFoundWithParameterTypesException import io.infinitic.exceptions.tasks.TooManyMethodsFoundWithParameterCountException +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout import io.infinitic.tasks.executor.samples.RetryImpl import io.infinitic.tasks.executor.samples.ServiceImplService import io.infinitic.tasks.executor.samples.ServiceWithBuggyRetryInClass @@ -97,7 +98,7 @@ class TaskExecutorTests : val taskEventSlot = CopyOnWriteArrayList() // mocks - val workerRegistry = mockk() + val registry = mockk() val client = mockk() val producer = mockk { every { name } returns "$testWorkerName" @@ -112,20 +113,20 @@ class TaskExecutorTests : } returns Unit } - var taskExecutor = TaskExecutor(workerRegistry, producer, client) - - val service = RegisteredServiceExecutor(1, { ServiceImplService() }, null, null) + var taskExecutor = TaskExecutor(registry, producer, client) // ensure slots are emptied between each test - beforeTest { - clearMocks(workerRegistry) + beforeEach { + clearMocks(registry) afterSlot.clear() taskExecutorSlot.clear() taskEventSlot.clear() } "Task executed should send TaskStarted and TaskCompleted events" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(3, 3) val types = listOf(Int::class.java.name, Int::class.java.name) // with @@ -148,7 +149,9 @@ class TaskExecutorTests : } "Should be able to run an explicit method with 2 parameters" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(3, "3") val types = listOf(Int::class.java.name, String::class.java.name) // with @@ -168,7 +171,9 @@ class TaskExecutorTests : } "Should be able to run an explicit method with 2 parameters without parameterTypes" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(4, "3") val types = null val msg = getExecuteTask("other", input, types) @@ -187,7 +192,9 @@ class TaskExecutorTests : } "Throwable should not be caught on task" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf() val types = listOf() val msg = getExecuteTask("withThrowable", input, types) @@ -204,12 +211,13 @@ class TaskExecutorTests : // Note that the Throwable sent (and not caught) during the test has the side effect // to cancel the coroutineScope of producerAsync, that's why we recreate it after the test // in a real case, the Throwable would kill the worker - taskExecutor = TaskExecutor(workerRegistry, producer, client) + taskExecutor = TaskExecutor(registry, producer, client) } "Should throw ClassNotFoundException when trying to process an unknown task" { - every { workerRegistry.serviceExecutors[testServiceName] } throws - ClassNotFoundException("task") + every { registry.getServiceExecutorInstance(testServiceName) } throws ClassNotFoundException( + "task", + ) val input = arrayOf(2, "3") val types = listOf(Int::class.java.name, String::class.java.name) val msg = getExecuteTask("unknown", input, types) @@ -228,7 +236,9 @@ class TaskExecutorTests : } "Should throw NoMethodFoundWithParameterTypesException when trying to process an unknown method" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(2, "3") val types = listOf(Int::class.java.name, String::class.java.name) // with @@ -248,7 +258,9 @@ class TaskExecutorTests : } "Should throw NoMethodFoundWithParameterCount when trying to process an unknown method without parameterTypes" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(2, "3") // with val msg = getExecuteTask("unknown", input, null) @@ -267,7 +279,9 @@ class TaskExecutorTests : } "Should throw TooManyMethodsFoundWithParameterCount when trying to process an unknown method without parameterTypes" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceImplService() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(2, "3") // with val msg = getExecuteTask("handle", input, null) @@ -286,8 +300,9 @@ class TaskExecutorTests : } "Should retry with correct exception with Retry interface" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy(factory = { SimpleServiceWithRetry() }) + every { registry.getServiceExecutorInstance(testServiceName) } returns SimpleServiceWithRetry() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET // with val msg = getExecuteTask("handle", arrayOf(2, "3"), null) // when @@ -310,8 +325,9 @@ class TaskExecutorTests : } "Should retry with correct exception with Retry annotation on method" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy(factory = { ServiceWithRetryInMethod() }) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithRetryInMethod() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET // with val msg = getExecuteTask("handle", arrayOf(2, "3"), null) // when @@ -335,8 +351,9 @@ class TaskExecutorTests : } "Should retry with correct exception with Retry annotation on class" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy(factory = { ServiceWithRetryInClass() }) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithRetryInClass() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET // with val msg = getExecuteTask("handle", arrayOf(2, "3"), null) // when @@ -360,9 +377,9 @@ class TaskExecutorTests : } "Should not retry if error in getSecondsBeforeRetry" { - every { workerRegistry.serviceExecutors[testServiceName] } returns service.copy( - factory = { ServiceWithBuggyRetryInClass() }, - ) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithBuggyRetryInClass() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET // with val msg = getExecuteTask("handle", arrayOf(2, "3"), null) // when @@ -380,8 +397,9 @@ class TaskExecutorTests : } "Should be able to access context from task" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy(factory = { ServiceWithContext() }) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithContext() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns WithRetry.UNSET val input = arrayOf(2, "3") // with val msg = getExecuteTask("handle", input, null) @@ -403,12 +421,10 @@ class TaskExecutorTests : } "Should throw TimeoutException with timeout from Registry" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy( - factory = { ServiceWithRegisteredTimeout() }, - withTimeout = { 0.1 }, - withRetry = ExponentialBackoffRetryPolicy(maximumRetries = 0), - ) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithRegisteredTimeout() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns { 0.1 } + every { registry.getServiceExecutorWithRetry(testServiceName) } returns + ExponentialBackoffRetryPolicy(maximumRetries = 0) val types = listOf(Int::class.java.name, String::class.java.name) // with val msg = getExecuteTask("handle", arrayOf(2, "3"), types) @@ -427,11 +443,10 @@ class TaskExecutorTests : } "Should throw TimeoutException with timeout from method Annotation" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy( - factory = { ServiceWithTimeoutOnMethod() }, - withRetry = ExponentialBackoffRetryPolicy(maximumRetries = 0), - ) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithTimeoutOnMethod() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns + ExponentialBackoffRetryPolicy(maximumRetries = 0) val input = arrayOf(2, "3") val types = listOf(Int::class.java.name, String::class.java.name) // with @@ -451,11 +466,10 @@ class TaskExecutorTests : } "Should throw TimeoutException with timeout from class Annotation" { - every { workerRegistry.serviceExecutors[testServiceName] } returns - service.copy( - factory = { ServiceWithTimeoutOnClass() }, - withRetry = ExponentialBackoffRetryPolicy(maximumRetries = 0), - ) + every { registry.getServiceExecutorInstance(testServiceName) } returns ServiceWithTimeoutOnClass() + every { registry.getServiceExecutorWithTimeout(testServiceName) } returns WithTimeout.UNSET + every { registry.getServiceExecutorWithRetry(testServiceName) } returns + ExponentialBackoffRetryPolicy(maximumRetries = 0) val input = arrayOf(2, "3") val types = listOf(Int::class.java.name, String::class.java.name) // with diff --git a/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/samples/SimpleService.kt b/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/samples/SimpleService.kt index ece679d31..78ecb4a40 100644 --- a/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/samples/SimpleService.kt +++ b/infinitic-task-executor/src/test/kotlin/io/infinitic/tasks/executor/samples/SimpleService.kt @@ -94,7 +94,7 @@ internal class ServiceWithTimeout : WithTimeout { return (i * j.toInt() * Task.retrySequence).toString() } - override fun getTimeoutInSeconds(): Double = TIMEOUT + override fun getTimeoutSeconds(): Double = TIMEOUT } @Timeout(TimeoutImpl::class) @@ -146,9 +146,9 @@ internal class TimeoutImpl : WithTimeout { const val TIMEOUT = 0.1 } - override fun getTimeoutInSeconds() = TIMEOUT + override fun getTimeoutSeconds() = TIMEOUT } internal class BuggyTimeoutImpl : WithTimeout { - override fun getTimeoutInSeconds(): Double = throw IllegalArgumentException() + override fun getTimeoutSeconds(): Double = throw IllegalArgumentException() } diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/Test.kt b/infinitic-tests/src/test/kotlin/io/infinitic/Test.kt index 3a905c9ac..7887920e4 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/Test.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/Test.kt @@ -24,12 +24,12 @@ package io.infinitic import io.infinitic.common.fixtures.DockerOnly import io.infinitic.common.workflows.data.workflows.WorkflowId -import io.infinitic.common.workflows.data.workflows.WorkflowName import io.infinitic.common.workflows.engine.state.WorkflowState -import io.infinitic.transport.config.Transport +import io.infinitic.transport.config.InMemoryTransportConfig +import io.infinitic.transport.config.PulsarTransportConfig import io.infinitic.utils.Listener import io.infinitic.workers.InfiniticWorker -import io.infinitic.workers.config.WorkerConfig +import io.infinitic.workers.config.InfiniticWorkerConfig import io.kotest.core.config.AbstractProjectConfig import io.kotest.core.listeners.AfterProjectListener import io.kotest.core.listeners.BeforeProjectListener @@ -43,33 +43,45 @@ import kotlin.time.Duration.Companion.milliseconds * If Docker is available, the tests are done on Pulsar, if not they are done in memory * Docker is available on GitHub. */ + +fun main() { + Test.start() +} + internal object Test { + private val pulsarServer = DockerOnly().pulsarServer - private val workerConfig = WorkerConfig.fromResource("/pulsar.yml", "/register.yml").let { - when (pulsarServer) { - null -> it.copy(transport = Transport.inMemory) - else -> it.copy( - transport = Transport.pulsar, - pulsar = it.pulsar!!.copy( - brokerServiceUrl = pulsarServer.pulsarBrokerUrl, - webServiceUrl = pulsarServer.httpServiceUrl, - policies = it.pulsar!!.policies.copy(delayedDeliveryTickTimeMillis = 1), // useful for tests - ), - ) - } + private val workerConfig by lazy { + InfiniticWorkerConfig + .fromYamlResource("/pulsar.yml", "/register.yml").let { + when (pulsarServer) { + null -> it.copy(transport = InMemoryTransportConfig()) + else -> it.copy( + transport = PulsarTransportConfig( + pulsar = (it.transport as PulsarTransportConfig).pulsar.copy( + brokerServiceUrl = pulsarServer.pulsarBrokerUrl, + webServiceUrl = pulsarServer.httpServiceUrl, + policies = (it.transport as PulsarTransportConfig).pulsar.policies.copy( + delayedDeliveryTickTimeMillis = 1, + ), + ), + ), + ) + } + } } - val worker = InfiniticWorker.fromConfig(workerConfig) - val client = worker.client + val worker by lazy { InfiniticWorker(workerConfig) } + val client by lazy { worker.client } fun start() { + pulsarServer?.start() worker.startAsync() } fun stop() { worker.close() - client.close() pulsarServer?.stop() } } @@ -107,7 +119,7 @@ internal suspend fun InfiniticWorker.getWorkflowState( // that's why we are waiting here a bit before getting the state delay(200) - return registry.workflowStateEngines[WorkflowName(name)]!!.storage.getState(WorkflowId(id)) + return getWorkflowStateEngineConfig(name)?.workflowStateStorage?.getState(WorkflowId(id)) } object ProjectConfig : AbstractProjectConfig() { diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/scaffolds/engine.kt b/infinitic-tests/src/test/kotlin/io/infinitic/scaffolds/engine.kt index 014a05d75..3f3b48e5e 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/scaffolds/engine.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/scaffolds/engine.kt @@ -37,7 +37,7 @@ import java.util.concurrent.Executors * - that uncaught exception will terminate the app * - that there is no deadlock when back messages are not synchronous */ -fun main() { +private fun main() { val threadPool = Executors.newCachedThreadPool() val scope = CoroutineScope(threadPool.asCoroutineDispatcher()) @@ -63,7 +63,7 @@ private fun CoroutineScope.startEngine( } } -suspend fun wait() = delay((Math.random() * 10L).toLong()) +private suspend fun wait() = delay((Math.random() * 10L).toLong()) fun CoroutineScope.startEngine() { val msgNb = 1000 diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/tests/branches/BranchesWorkflowTests.kt b/infinitic-tests/src/test/kotlin/io/infinitic/tests/branches/BranchesWorkflowTests.kt index a10aae2b8..a15e45a62 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/tests/branches/BranchesWorkflowTests.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/tests/branches/BranchesWorkflowTests.kt @@ -41,11 +41,7 @@ internal class BranchesWorkflowTests : StringSpec( val utilWorkflow = client.newWorkflow(UtilWorkflow::class.java) // the first test has a large timeout to deal with Pulsar initialization - "Initialization".config(timeout = 2.minutes) { - branchesWorkflow.seq3() shouldBe "23ba" - } - - "Sequential Workflow with an async branch" { + "Sequential Workflow with an async branch".config(timeout = 2.minutes) { branchesWorkflow.seq3() shouldBe "23ba" worker.getWorkflowState() shouldBe null diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/tests/channels/ChannelWorkflowTests.kt b/infinitic-tests/src/test/kotlin/io/infinitic/tests/channels/ChannelWorkflowTests.kt index 135d9aa28..6f053aaed 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/tests/channels/ChannelWorkflowTests.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/tests/channels/ChannelWorkflowTests.kt @@ -99,14 +99,12 @@ internal class ChannelWorkflowTests : deferred.await() shouldBe "Instant" } - "Sending event before waiting for it prevents catching" { + "Event is discarded if sent before waiting for it" { val deferred = client.dispatch(channelsWorkflow::channel3) - later { - client.getWorkflowById( - ChannelsWorkflow::class.java, - deferred.id, - ).channelStrA.send("test") + later(10) { + client.getWorkflowById(ChannelsWorkflow::class.java, deferred.id) + .channelStrA.send("test") } deferred.await() shouldBe "Instant" diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflow.kt b/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflow.kt index 5c1a971bf..87b2d2bd7 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflow.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflow.kt @@ -91,7 +91,7 @@ interface ITimeoutWorkflow : WithTimeout { // the workflow method 'withMethodTimeout' has a 100ms timeout fun withTimeoutOnMethod(duration: Long): Long - override fun getTimeoutInSeconds(): Double? = 0.4 + override fun getTimeoutSeconds(): Double? = 0.4 } class ITimeoutsWorkflowImpl : Workflow(), ITimeoutWorkflow { diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflowTests.kt b/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflowTests.kt index 575c868ab..03a6df48b 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflowTests.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/tests/timeouts/TimeoutsWorkflowTests.kt @@ -33,6 +33,7 @@ import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe import io.kotest.matchers.types.shouldBeInstanceOf +import kotlin.time.Duration.Companion.seconds internal class TimeoutsWorkflowTests : StringSpec( @@ -42,15 +43,9 @@ internal class TimeoutsWorkflowTests : val timeoutsWorkflow = client.newWorkflow(TimeoutsWorkflow::class.java) val iTimeoutsWorkflow = client.newWorkflow(ITimeoutWorkflow::class.java) - "Synchronous call of a workflow running for more than its timeout should throw" { - shouldThrow { timeoutsWorkflow.withTimeoutOnMethod(2000) } - } - - "Synchronous call of a workflow running for less than its timeout should NOT throw" { - shouldNotThrowAny { timeoutsWorkflow.withTimeoutOnMethod(10) shouldBe 10 } - } - - "Synchronous call of a child-workflow running for more than its timeout should throw" { + "Synchronous call of a child-workflow running for more than its timeout should throw".config( + timeout = 30.seconds, + ) { val e = shouldThrow { timeoutsWorkflow.withTimeoutOnChild(2000) } e.deferredException.shouldBeInstanceOf() @@ -59,6 +54,14 @@ internal class TimeoutsWorkflowTests : cause.workflowMethodName shouldBe "withTimeoutOnMethod" } + "Synchronous call of a workflow running for more than its timeout should throw" { + shouldThrow { timeoutsWorkflow.withTimeoutOnMethod(2000) } + } + + "Synchronous call of a workflow running for less than its timeout should NOT throw" { + shouldNotThrowAny { timeoutsWorkflow.withTimeoutOnMethod(1) shouldBe 1 } + } + "Synchronous call of a child-workflow running for less than its timeout should NOT throw" { shouldNotThrowAny { timeoutsWorkflow.withTimeoutOnChild(10) shouldBe 10 } } diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/utils/UtilService.kt b/infinitic-tests/src/test/kotlin/io/infinitic/utils/UtilService.kt index 66270f6fe..fa8b2af73 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/utils/UtilService.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/utils/UtilService.kt @@ -109,7 +109,7 @@ class UtilServiceImpl : UtilService { @Retry(NoRetry::class) override fun successAtRetry() = when (Task.retrySequence) { - 0 -> throw ExpectedException("expected exception") + 0 -> throw ExpectedException() else -> "ok" } @@ -121,7 +121,7 @@ class UtilServiceImpl : UtilService { override fun getRetry(): Double? = Task.withRetry?.getSecondsBeforeRetry(0, RuntimeException()) - override fun getTimeout(): Double? = Task.withTimeout?.getTimeoutInSeconds() + override fun getTimeout(): Double? = Task.withTimeout?.getTimeoutSeconds() override fun withTimeout(wait: Long): Long { Thread.sleep(wait) diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/utils/retries.kt b/infinitic-tests/src/test/kotlin/io/infinitic/utils/retries.kt index e47881371..c639bbd07 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/utils/retries.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/utils/retries.kt @@ -25,11 +25,11 @@ package io.infinitic.utils import io.infinitic.tasks.WithRetry -class NoRetry : WithRetry { +internal class NoRetry : WithRetry { override fun getSecondsBeforeRetry(retry: Int, e: Exception) = null } -class Only1Retry : WithRetry { +internal class Only1Retry : WithRetry { override fun getSecondsBeforeRetry(retry: Int, e: Exception) = when (retry) { 0 -> 1.0 else -> null diff --git a/infinitic-tests/src/test/kotlin/io/infinitic/utils/timeouts.kt b/infinitic-tests/src/test/kotlin/io/infinitic/utils/timeouts.kt index c12ea1dc3..4f96ac304 100644 --- a/infinitic-tests/src/test/kotlin/io/infinitic/utils/timeouts.kt +++ b/infinitic-tests/src/test/kotlin/io/infinitic/utils/timeouts.kt @@ -26,9 +26,9 @@ package io.infinitic.utils import io.infinitic.tasks.WithTimeout class After100MilliSeconds : WithTimeout { - override fun getTimeoutInSeconds() = 0.1 + override fun getTimeoutSeconds() = 0.1 } class After1Second : WithTimeout { - override fun getTimeoutInSeconds() = 1.0 + override fun getTimeoutSeconds() = 1.0 } diff --git a/infinitic-tests/src/test/resources/pulsar.yml b/infinitic-tests/src/test/resources/pulsar.yml index a687cc425..709083adb 100644 --- a/infinitic-tests/src/test/resources/pulsar.yml +++ b/infinitic-tests/src/test/resources/pulsar.yml @@ -21,16 +21,17 @@ # # Licensor: infinitic.io -# Comment the line below to perform tests on a local Pulsar cluster -transport: inMemory +storage: + inMemory: -pulsar: - brokerServiceUrl: "pulsar://localhost:6650/" - webServiceUrl: "http://localhost:8080" - tenant: infinitic - namespace: dev - consumer: - negativeAckRedeliveryDelaySeconds: 0.3 +transport: + pulsar: + brokerServiceUrl: "pulsar://localhost:6650/" + webServiceUrl: "http://localhost:8080" + tenant: infinitic + namespace: dev + consumer: + negativeAckRedeliveryDelaySeconds: 0.3 # authentication: # issuerUrl: diff --git a/infinitic-tests/src/test/resources/register.yml b/infinitic-tests/src/test/resources/register.yml index 60ba40878..1ab0c0f6e 100644 --- a/infinitic-tests/src/test/resources/register.yml +++ b/infinitic-tests/src/test/resources/register.yml @@ -25,76 +25,138 @@ services: - name: annotatedService - class: io.infinitic.utils.AnnotatedServiceImpl - concurrency: 5 + tagEngine: + executor: + class: io.infinitic.utils.AnnotatedServiceImpl + concurrency: 5 - name: io.infinitic.utils.UtilService - class: io.infinitic.utils.UtilServiceImpl - concurrency: 5 - timeoutInSeconds: 100 - retry: - maximumRetries: 1 - randomFactor: 0 + tagEngine: + executor: + class: io.infinitic.utils.UtilServiceImpl + concurrency: 5 + timeoutSeconds: 100 + retry: + maximumRetries: 1 + randomFactor: 0 - name: io.infinitic.utils.TestService - class: io.infinitic.utils.TestServiceImpl - concurrency: 5 + tagEngine: + executor: + class: io.infinitic.utils.TestServiceImpl + concurrency: 5 - name: io.infinitic.tests.jsonView.JsonViewService - class: io.infinitic.tests.jsonView.JsonViewServiceImpl - concurrency: 5 + tagEngine: + executor: + class: io.infinitic.tests.jsonView.JsonViewServiceImpl + concurrency: 5 workflows: - name: annotatedWorkflow - class: io.infinitic.utils.AnnotatedWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.utils.AnnotatedWorkflowImpl + concurrency: 2 - name: io.infinitic.utils.UtilWorkflow - class: io.infinitic.utils.UtilWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.utils.UtilWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.branches.BranchesWorkflow - class: io.infinitic.tests.branches.BranchesWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.branches.BranchesWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.channels.ChannelsWorkflow - class: io.infinitic.tests.channels.ChannelsWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.channels.ChannelsWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.children.ChildrenWorkflow - class: io.infinitic.tests.children.ChildrenWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.children.ChildrenWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.context.ContextWorkflow - class: io.infinitic.tests.context.ContextWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.context.ContextWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.deferred.DeferredWorkflow - class: io.infinitic.tests.deferred.DeferredWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.deferred.DeferredWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.errors.ErrorsWorkflow - class: io.infinitic.tests.errors.ErrorsWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.errors.ErrorsWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.inline.InlineWorkflow - class: io.infinitic.tests.inline.InlineWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.inline.InlineWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.properties.PropertiesWorkflow - class: io.infinitic.tests.properties.PropertiesWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.properties.PropertiesWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.syntax.SyntaxWorkflow - class: io.infinitic.tests.syntax.SyntaxWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.syntax.SyntaxWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.tags.TagWorkflow - class: io.infinitic.tests.tags.TagWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.tags.TagWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.timers.TimerWorkflow - class: io.infinitic.tests.timers.TimerWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.timers.TimerWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.timeouts.TimeoutsWorkflow - class: io.infinitic.tests.timeouts.TimeoutsWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.timeouts.TimeoutsWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.timeouts.ITimeoutWorkflow - class: io.infinitic.tests.timeouts.ITimeoutsWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.timeouts.ITimeoutsWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.versioning.VersionedWorkflow - classes: - - io.infinitic.tests.versioning.VersionedWorkflowImpl - - io.infinitic.tests.versioning.VersionedWorkflowImpl_1 - concurrency: 2 + tagEngine: + stateEngine: + executor: + classes: + - io.infinitic.tests.versioning.VersionedWorkflowImpl + - io.infinitic.tests.versioning.VersionedWorkflowImpl_1 + concurrency: 2 - name: Delegation - class: io.infinitic.tests.delegation.DelegationWorkflowImpl - concurrency: 2 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.delegation.DelegationWorkflowImpl + concurrency: 2 - name: io.infinitic.tests.jsonView.JsonViewWorkflow - class: io.infinitic.tests.jsonView.JsonViewWorkflowImpl - concurrency: 5 + tagEngine: + stateEngine: + executor: + class: io.infinitic.tests.jsonView.JsonViewWorkflowImpl + concurrency: 5 diff --git a/infinitic-tests/src/test/resources/simplelogger.properties b/infinitic-tests/src/test/resources/simplelogger.properties index 44f6b6a24..8858a8cc1 100644 --- a/infinitic-tests/src/test/resources/simplelogger.properties +++ b/infinitic-tests/src/test/resources/simplelogger.properties @@ -20,6 +20,8 @@ org.slf4j.simpleLogger.log.io.infinitic.tasks.tag.TaskTagEngine=info org.slf4j.simpleLogger.log.io.infinitic.tasks.executor.TaskExecutor=info org.slf4j.simpleLogger.log.io.infinitic.tasks.executor.TaskEventHandler=info org.slf4j.simpleLogger.log.io.infinitic.tasks.executor.TaskRetryHandler=info +org.slf4j.simpleLogger.log.io.infinitic.pulsar.consumers.Consumer=warn +org.slf4j.simpleLogger.log.io.infinitic.pulsar.producers.Producer=warn # Set to true if you want the current date and time to be included in output messages. # Default is false, and will output the number of milliseconds elapsed since startup. org.slf4j.simpleLogger.showDateTime=true diff --git a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryChannels.kt b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryChannels.kt index fe9726d80..53a43801a 100644 --- a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryChannels.kt +++ b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryChannels.kt @@ -51,58 +51,58 @@ import java.util.concurrent.ConcurrentHashMap class InMemoryChannels { - private val clientChannels = - ConcurrentHashMap>() - private val serviceTagChannels = - ConcurrentHashMap>() - private val workflowTagChannels = - ConcurrentHashMap>() - private val serviceExecutorChannels = - ConcurrentHashMap>() - private val serviceEventsChannels = - ConcurrentHashMap>() - private val delayedServiceExecutorChannels = - ConcurrentHashMap>>() - private val workflowCmdChannels = - ConcurrentHashMap>() - private val workflowEngineChannels = - ConcurrentHashMap>() - private val delayedWorkflowEngineChannels = - ConcurrentHashMap>>() - private val workflowEventsChannels = - ConcurrentHashMap>() - private val workflowTaskExecutorChannels = - ConcurrentHashMap>() - private val workflowTaskEventsChannels = - ConcurrentHashMap>() - private val delayedWorkflowTaskExecutorChannels = - ConcurrentHashMap>>() + internal val clientChannels = + ConcurrentHashMap>(100) + internal val serviceTagEngineChannels = + ConcurrentHashMap>(100) + internal val workflowTagEngineChannels = + ConcurrentHashMap>(100) + internal val serviceExecutorChannels = + ConcurrentHashMap>(100) + internal val serviceExecutorEventsChannels = + ConcurrentHashMap>(100) + internal val retryServiceExecutorChannels = + ConcurrentHashMap>>(100) + internal val workflowStateCmdChannels = + ConcurrentHashMap>(100) + internal val workflowStateEngineChannels = + ConcurrentHashMap>(100) + internal val workflowStateTimerChannels = + ConcurrentHashMap>>(100) + internal val workflowStateEventChannels = + ConcurrentHashMap>(100) + internal val workflowExecutorChannels = + ConcurrentHashMap>(100) + internal val workflowExecutorEventChannels = + ConcurrentHashMap>(100) + internal val retryWorkflowExecutorChannels = + ConcurrentHashMap>>(100) @Suppress("UNCHECKED_CAST") fun Topic.channel(entity: String): Channel = when (this) { ClientTopic -> clientChannels.getOrPut(entity, newChannel()) - ServiceTagEngineTopic -> serviceTagChannels.getOrPut(entity, newChannel()) - WorkflowTagEngineTopic -> workflowTagChannels.getOrPut(entity, newChannel()) + ServiceTagEngineTopic -> serviceTagEngineChannels.getOrPut(entity, newChannel()) + WorkflowTagEngineTopic -> workflowTagEngineChannels.getOrPut(entity, newChannel()) ServiceExecutorTopic -> serviceExecutorChannels.getOrPut(entity, newChannel()) - ServiceExecutorEventTopic -> serviceEventsChannels.getOrPut(entity, newChannel()) - WorkflowStateCmdTopic -> workflowCmdChannels.getOrPut(entity, newChannel()) - WorkflowStateEngineTopic -> workflowEngineChannels.getOrPut(entity, newChannel()) - WorkflowStateEventTopic -> workflowEventsChannels.getOrPut(entity, newChannel()) - WorkflowExecutorTopic -> workflowTaskExecutorChannels.getOrPut(entity, newChannel()) - WorkflowExecutorEventTopic -> workflowTaskEventsChannels.getOrPut(entity, newChannel()) + ServiceExecutorEventTopic -> serviceExecutorEventsChannels.getOrPut(entity, newChannel()) + WorkflowStateCmdTopic -> workflowStateCmdChannels.getOrPut(entity, newChannel()) + WorkflowStateEngineTopic -> workflowStateEngineChannels.getOrPut(entity, newChannel()) + WorkflowStateEventTopic -> workflowStateEventChannels.getOrPut(entity, newChannel()) + WorkflowExecutorTopic -> workflowExecutorChannels.getOrPut(entity, newChannel()) + WorkflowExecutorEventTopic -> workflowExecutorEventChannels.getOrPut(entity, newChannel()) else -> thisShouldNotHappen() } as Channel @Suppress("UNCHECKED_CAST") fun Topic.channelForDelayed(entity: String): Channel> { return when (this) { - RetryServiceExecutorTopic -> delayedServiceExecutorChannels.getOrPut(entity, newChannel()) - WorkflowStateTimerTopic -> delayedWorkflowEngineChannels.getOrPut( + RetryServiceExecutorTopic -> retryServiceExecutorChannels.getOrPut(entity, newChannel()) + WorkflowStateTimerTopic -> workflowStateTimerChannels.getOrPut( entity, newChannel(), ) - RetryWorkflowExecutorTopic -> delayedWorkflowTaskExecutorChannels.getOrPut( + RetryWorkflowExecutorTopic -> retryWorkflowExecutorChannels.getOrPut( entity, newChannel(), ) @@ -111,7 +111,7 @@ class InMemoryChannels { } as Channel> } - private fun newChannel(): () -> Channel = { Channel(Channel.UNLIMITED) } + private fun newChannel(): () -> Channel = { Channel(10000) } } internal val Channel<*>.id diff --git a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticConsumer.kt b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticConsumer.kt index 289ea6270..a73426346 100644 --- a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticConsumer.kt +++ b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticConsumer.kt @@ -22,7 +22,6 @@ */ package io.infinitic.inMemory -import io.github.oshai.kotlinlogging.KLogger import io.github.oshai.kotlinlogging.KotlinLogging import io.infinitic.common.data.MillisInstant import io.infinitic.common.messages.Message @@ -32,43 +31,16 @@ import io.infinitic.common.transport.MainSubscription import io.infinitic.common.transport.Subscription import io.infinitic.common.transport.acceptDelayed import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay -import kotlinx.coroutines.job import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking class InMemoryInfiniticConsumer( private val mainChannels: InMemoryChannels, private val eventListenerChannels: InMemoryChannels, ) : InfiniticConsumer { - - override lateinit var workerLogger: KLogger - - // Coroutine scope used to receive messages - private val consumingScope = CoroutineScope(Dispatchers.IO) - - override fun join() { - runBlocking { - consumingScope.coroutineContext.job.children.forEach { - try { - it.join() - } catch (e: CancellationException) { - // do nothing - } - } - } - } - - override fun close() { - consumingScope.cancel() - join() - } - + override suspend fun start( subscription: Subscription, entity: String, @@ -104,7 +76,7 @@ class InMemoryInfiniticConsumer( concurrency: Int ) = coroutineScope { repeat(concurrency) { - launch(consumingScope.coroutineContext) { + launch { try { for (message in channel) { try { @@ -132,7 +104,7 @@ class InMemoryInfiniticConsumer( concurrency: Int ) = coroutineScope { repeat(concurrency) { - launch(consumingScope.coroutineContext) { + launch { try { for (delayedMessage in channel) { try { diff --git a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticProducer.kt b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticProducer.kt index 81d8dd7ad..471bca74d 100644 --- a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticProducer.kt +++ b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticProducer.kt @@ -35,8 +35,6 @@ class InMemoryInfiniticProducer( private val eventListenerChannels: InMemoryChannels ) : InfiniticProducer { - private val logger = KotlinLogging.logger {} - override var name = DEFAULT_NAME private fun Topic.channelsForMessage(message: S): List> { @@ -83,6 +81,8 @@ class InMemoryInfiniticProducer( companion object { private const val DEFAULT_NAME = "inMemory" + + private val logger = KotlinLogging.logger {} } } diff --git a/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticResources.kt b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticResources.kt new file mode 100644 index 000000000..769a02507 --- /dev/null +++ b/infinitic-transport-inMemory/src/main/kotlin/io/infinitic/inMemory/InMemoryInfiniticResources.kt @@ -0,0 +1,44 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.inMemory + +import io.infinitic.common.transport.InfiniticResources + +class InMemoryInfiniticResources( + private val mainChannels: InMemoryChannels, +) : InfiniticResources { + + override suspend fun getServices() = + mainChannels.serviceExecutorChannels.keys().toList().toSet() + .union(mainChannels.serviceTagEngineChannels.keys().toList().toSet()) + + override suspend fun getWorkflows() = + mainChannels.workflowExecutorChannels.keys().toList().toSet() + .union(mainChannels.workflowTagEngineChannels.keys().toList().toSet()) + .union(mainChannels.workflowStateEngineChannels.keys().toList().toSet()) + + override suspend fun deleteTopicForClient(clientName: String) = + Result.success(mainChannels.clientChannels.remove(clientName)?.let { clientName }) +} + + diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticConsumer.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticConsumer.kt index 7210ed5b9..84cac08ff 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticConsumer.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticConsumer.kt @@ -22,11 +22,8 @@ */ package io.infinitic.pulsar -import io.github.oshai.kotlinlogging.KLogger -import io.infinitic.autoclose.autoClose import io.infinitic.common.data.MillisInstant import io.infinitic.common.messages.Message -import io.infinitic.common.transport.ClientTopic import io.infinitic.common.transport.EventListenerSubscription import io.infinitic.common.transport.InfiniticConsumer import io.infinitic.common.transport.MainSubscription @@ -40,55 +37,14 @@ import io.infinitic.pulsar.resources.defaultName import io.infinitic.pulsar.resources.defaultNameDLQ import io.infinitic.pulsar.resources.schema import io.infinitic.pulsar.resources.type -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout import org.apache.pulsar.client.api.SubscriptionInitialPosition -import java.util.concurrent.atomic.AtomicBoolean class PulsarInfiniticConsumer( private val consumer: Consumer, private val pulsarResources: PulsarResources, - val shutdownGracePeriodInSeconds: Double + val shutdownGracePeriodSeconds: Double ) : InfiniticConsumer { - override lateinit var workerLogger: KLogger - - override fun join() = consumer.join() - - private var isClosed: AtomicBoolean = AtomicBoolean(false) - - private lateinit var clientName: String - - override fun close() { - // we test if consumingScope is active, just in case - // the user tries to manually close an already closed resource - if (isClosed.compareAndSet(false, true)) { - runBlocking { - try { - withTimeout((shutdownGracePeriodInSeconds * 1000L).toLong()) { - // By cancelling the consumer coroutine, we interrupt the main loop of consumption - workerLogger.info { "Processing ongoing messages..." } - consumer.cancel() - consumer.join() - workerLogger.info { "All ongoing messages have been processed." } - // delete client topic after all in-memory messages have been processed - deleteClientTopics() - // then close other resources (typically pulsar client & admin) - autoClose() - } - } catch (e: TimeoutCancellationException) { - workerLogger.warn { - "The grace period (${shutdownGracePeriodInSeconds}s) allotted to close was insufficient. " + - "Some ongoing messages may not have been processed properly." - } - } - } - } - } - override suspend fun start( subscription: Subscription, entity: String, @@ -100,8 +56,7 @@ class PulsarInfiniticConsumer( // we do nothing here, as WorkflowTaskExecutorTopic and ServiceExecutorTopic // do not need a distinct topic to handle delayed messages in Pulsar RetryWorkflowExecutorTopic, RetryServiceExecutorTopic -> return - // record client name to be able to delete topic at closing - ClientTopic -> clientName = entity + else -> Unit } @@ -148,24 +103,5 @@ class PulsarInfiniticConsumer( is EventListenerSubscription -> name?.let { SubscriptionInitialPosition.Earliest } ?: defaultInitialPosition } - - private suspend fun deleteClientTopics() { - if (::clientName.isInitialized) coroutineScope { - launch { - val clientTopic = with(pulsarResources) { ClientTopic.fullName(clientName) } - workerLogger.debug { "Deleting client topic '$clientTopic'." } - pulsarResources.deleteTopic(clientTopic) - .onFailure { workerLogger.warn(it) { "Unable to delete client topic '$clientTopic'." } } - .onSuccess { workerLogger.info { "Client topic '$clientTopic' deleted." } } - } - launch { - val clientDLQTopic = with(pulsarResources) { ClientTopic.fullNameDLQ(clientName) } - workerLogger.debug { "Deleting client DLQ topic '$clientDLQTopic'." } - pulsarResources.deleteTopic(clientDLQTopic) - .onFailure { workerLogger.warn(it) { "Unable to delete client DLQ topic '$clientDLQTopic'." } } - .onSuccess { workerLogger.info { "Client DLQ topic '$clientDLQTopic' deleted." } } - } - } - } } diff --git a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceExecutor.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticResources.kt similarity index 65% rename from infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceExecutor.kt rename to infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticResources.kt index f00db2b76..14c05300b 100644 --- a/infinitic-common/src/main/kotlin/io/infinitic/common/workers/registry/RegisteredServiceExecutor.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/PulsarInfiniticResources.kt @@ -20,16 +20,20 @@ * * Licensor: infinitic.io */ -package io.infinitic.common.workers.registry +package io.infinitic.pulsar -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout +import io.infinitic.common.transport.InfiniticResources +import io.infinitic.pulsar.resources.PulsarResources -typealias ServiceFactory = () -> Any +class PulsarInfiniticResources( + private val pulsarResources: PulsarResources +) : InfiniticResources { + override suspend fun getServices(): Set = + pulsarResources.getServiceNames() -data class RegisteredServiceExecutor( - val concurrency: Int, - val factory: ServiceFactory, - val withTimeout: WithTimeout?, - val withRetry: WithRetry? -) + override suspend fun getWorkflows(): Set = + pulsarResources.getWorkflowNames() + + override suspend fun deleteTopicForClient(clientName: String) = + pulsarResources.deleteTopicForClient(clientName) +} diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdmin.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdmin.kt index 1a2a4ba4c..4e9942939 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdmin.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdmin.kt @@ -26,7 +26,6 @@ import io.github.oshai.kotlinlogging.KotlinLogging import io.infinitic.pulsar.config.policies.PoliciesConfig import kotlinx.coroutines.delay import kotlinx.coroutines.future.await -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.apache.pulsar.client.admin.PulsarAdmin @@ -55,6 +54,8 @@ class PulsarInfiniticAdmin( private val namespaces = pulsarAdmin.namespaces() private val topicsMutex = Mutex() + private val namespacesMutex = Mutex() + private val tenantsMutex = Mutex() /** * Get set of clusters' name @@ -121,26 +122,41 @@ class PulsarInfiniticAdmin( * - Result.success(TenantInfo) if tenant exists or has been created * - Result.failure(e) in case of error **/ - suspend fun initTenantOnce( + suspend fun syncInitTenantOnce( tenant: String, allowedClusters: Set?, adminRoles: Set? - ): Result = initializedTenants.computeIfAbsent(tenant) { - runBlocking { - try { - // get tenant info or create it - val tenantInfo = getTenantInfo(tenant).getOrThrow() - ?: createTenant(tenant, allowedClusters, adminRoles).getOrThrow() - - checkTenantInfo(tenant, tenantInfo, allowedClusters, adminRoles) - Result.success(tenantInfo) - } catch (e: Exception) { - logger.info(e) { "Unable to check/create tenant '$tenant'" } - Result.failure(e) - } + ): Result { + // First check if the tenant is already initialized, to avoid locking unnecessarily + initializedTenants[tenant]?.let { return it } + + // Locking the section where we might do an expensive initialization + return tenantsMutex.withLock { + // Double-checked locking to ensure the tenant hasn't been initialized by another coroutine + initializedTenants[tenant]?.let { return it } + + // Initialize the topic if it has not been initialized yet + initTenantOnce(tenant, allowedClusters, adminRoles) + .also { initializedTenants[tenant] = it } } } + private suspend fun initTenantOnce( + tenant: String, + allowedClusters: Set?, + adminRoles: Set? + ): Result = try { + // get tenant info or create it + val tenantInfo = getTenantInfo(tenant).getOrThrow() + ?: createTenant(tenant, allowedClusters, adminRoles).getOrThrow() + + checkTenantInfo(tenant, tenantInfo, allowedClusters, adminRoles) + Result.success(tenantInfo) + } catch (e: Exception) { + logger.info(e) { "Unable to check/create tenant '$tenant'" } + Result.failure(e) + } + /** * Ensure once that namespace exists. @@ -152,25 +168,38 @@ class PulsarInfiniticAdmin( * - Result.success(Policies) if tenant exists or has been created * - Result.failure(e) in case of error **/ - suspend fun initNamespaceOnce( + suspend fun syncInitNamespaceOnce( fullNamespace: String, config: PoliciesConfig - ): Result = initializedNamespaces.computeIfAbsent(fullNamespace) { - runBlocking { - try { - // get Namespace policies or create it - val policies = getNamespacePolicies(fullNamespace).getOrThrow() - ?: createNamespace(fullNamespace, config).getOrThrow() - - checkNamespacePolicies(policies, config.getPulsarPolicies()) - Result.success(policies) - } catch (e: Exception) { - logger.info(e) { "Unable to check/create namespace '$fullNamespace'" } - Result.failure(e) - } + ): Result { + // First check if the namespace is already initialized, to avoid locking unnecessarily + initializedNamespaces[fullNamespace]?.let { return it } + + // Locking the section where we might do an expensive initialization + return namespacesMutex.withLock { + // Double-checked locking to ensure the namespace hasn't been initialized by another coroutine + initializedNamespaces[fullNamespace]?.let { return it } + + // Initialize the topic if it has not been initialized yet + initNamespaceOnce(fullNamespace, config) + .also { initializedNamespaces[fullNamespace] = it } } } + suspend fun initNamespaceOnce( + fullNamespace: String, + config: PoliciesConfig + ): Result = try { + // get Namespace policies or create it + val policies = getNamespacePolicies(fullNamespace).getOrThrow() + ?: createNamespace(fullNamespace, config).getOrThrow() + + checkNamespacePolicies(policies, config.getPulsarPolicies()) + Result.success(policies) + } catch (e: Exception) { + logger.info(e) { "Unable to check/create namespace '$fullNamespace'" } + Result.failure(e) + } /** * Ensure once that topic exists. @@ -184,34 +213,48 @@ class PulsarInfiniticAdmin( * - Result.success(Unit) if topic exists or has been created * - Result.failure(e) in case of error **/ - suspend fun initTopicOnce( + suspend fun syncInitTopicOnce( topic: String, isPartitioned: Boolean, messageTTLPolicy: Int, - retry: Int = 0 - ): Result = initializedTopics[topic] - ?: try { - // get topic info or creates it - val topicInfo = getTopicInfo(topic).getOrThrow() - ?: createTopic(topic, isPartitioned, messageTTLPolicy).getOrThrow() - checkTopicInfo(topic, topicInfo, TopicInfo(isPartitioned, messageTTLPolicy)) - Result.success(topicInfo) - } catch (e: PulsarAdminException.ConflictException) { - when { - retry >= 3 -> { - logger.warn(e) { "Unable to check/create topic '$topic'" } - Result.failure(e) - } + ): Result { + // First check if the topic is already initialized, to avoid locking unnecessarily + initializedTopics[topic]?.let { return it } + + // Locking the section where we might do an expensive initialization + return topicsMutex.withLock { + // Double-checked locking to ensure the topic hasn't been initialized by another coroutine + initializedTopics[topic]?.let { return it } + + // Initialize the topic if it has not been initialized yet + initTopicOnce(topic, isPartitioned, messageTTLPolicy) + .also { initializedTopics[topic] = it } + } + } - else -> { - delay(Random.nextLong(100)) - initTopicOnce(topic, isPartitioned, messageTTLPolicy, retry + 1) - } - } - } catch (e: Exception) { - logger.warn(e) { "Unable to check/create topic '$topic'" } + private suspend fun initTopicOnce( + topic: String, + isPartitioned: Boolean, + messageTTLPolicy: Int, + retry: Int = 0 + ): Result = try { + // get topic info or create it if it doesn't exist + val topicInfo = getTopicInfo(topic).getOrThrow() + ?: createTopic(topic, isPartitioned, messageTTLPolicy).getOrThrow() + checkTopicInfo(topic, topicInfo, TopicInfo(isPartitioned, messageTTLPolicy)) + Result.success(topicInfo) + } catch (e: PulsarAdminException.ConflictException) { + if (retry >= 3) { + logger.warn(e) { "Unable to check/create topic '$topic' after $retry retries." } Result.failure(e) - }.also { topicsMutex.withLock { initializedTopics[topic] = it } } + } else { + delay(Random.nextLong(100)) + initTopicOnce(topic, isPartitioned, messageTTLPolicy, retry + 1) + } + } catch (e: Exception) { + logger.warn(e) { "Unable to check/create topic '$topic'" } + Result.failure(e) + } /** * Delete topic. @@ -220,14 +263,14 @@ class PulsarInfiniticAdmin( * - Result.success(Unit) in case of success or already deleted topic * - Result.failure(e) in case of error */ - suspend fun deleteTopic(topic: String): Result = try { + suspend fun deleteTopic(topic: String): Result = try { logger.debug { "Deleting topic $topic." } topics.deleteAsync(topic, true).await() logger.info { "Topic '$topic' deleted." } Result.success(Unit) } catch (e: PulsarAdminException.NotFoundException) { logger.debug { "Unable to delete topic '$topic' that does not exist." } - Result.success(Unit) + Result.success(null) } catch (e: PulsarAdminException) { logger.warn(e) { "Unable to delete topic '$topic'." } Result.failure(e) @@ -370,11 +413,11 @@ class PulsarInfiniticAdmin( Result.failure(e) } - fun setTopicTTL(topic: String, messageTTLInSecond: Int): Result = try { - logger.debug { "Topic '$topic': setting messageTTLInSecond=$messageTTLInSecond." } - topicPolicies.setMessageTTL(topic, messageTTLInSecond) - logger.info { "Topic '$topic': messageTTLInSecond=$messageTTLInSecond set." } - Result.success(messageTTLInSecond) + fun setTopicTTL(topic: String, messageTTLSeconds: Int): Result = try { + logger.debug { "Topic '$topic': setting messageTTLInSecond=$messageTTLSeconds." } + topicPolicies.setMessageTTL(topic, messageTTLSeconds) + logger.info { "Topic '$topic': messageTTLInSecond=$messageTTLSeconds set." } + Result.success(messageTTLSeconds) } catch (e: PulsarAdminException) { logger.warn(e) { "Topic '$topic': Unable to set message TTL for topic." } Result.failure(e) @@ -509,8 +552,8 @@ class PulsarInfiniticAdmin( } private fun PoliciesConfig.getPulsarPolicies() = PulsarPolicies().also { - it.retention_policies = RetentionPolicies(retentionTimeInMinutes, retentionSizeInMB) - it.message_ttl_in_seconds = messageTTLInSeconds + it.retention_policies = RetentionPolicies(retentionTimeMinutes, retentionSizeMB) + it.message_ttl_in_seconds = messageTTLSeconds it.delayed_delivery_policies = DelayedDeliveryPoliciesImpl(delayedDeliveryTickTimeMillis, true) it.schema_compatibility_strategy = schemaCompatibilityStrategy @@ -553,13 +596,13 @@ class PulsarInfiniticAdmin( companion object { private const val DEFAULT_NUM_PARTITIONS = 3 - // thread-safe set of initialized tenants - private val initializedTenants = ConcurrentHashMap>() + // set of initialized tenants + private val initializedTenants = mutableMapOf>() - // thread-safe set of initialized namespaces - private val initializedNamespaces = ConcurrentHashMap>() + // set of initialized namespaces + private val initializedNamespaces = mutableMapOf>() - // thread-safe set of initialized topics (topic name includes tenant and namespace) + // set of initialized topics (topic name includes tenant and namespace) private val initializedTopics = mutableMapOf>() // thread-safe set of (topic, subscription name) that have already been checked for consumers diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/PulsarConfig.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/PulsarConfig.kt index 1d3fce0da..b807fee08 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/PulsarConfig.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/PulsarConfig.kt @@ -24,6 +24,9 @@ package io.infinitic.pulsar.config import com.sksamuel.hoplite.Secret import io.github.oshai.kotlinlogging.KotlinLogging +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString import io.infinitic.pulsar.config.auth.AuthenticationAthenzConfig import io.infinitic.pulsar.config.auth.AuthenticationOAuth2Config import io.infinitic.pulsar.config.auth.AuthenticationSaslConfig @@ -40,6 +43,27 @@ import org.apache.pulsar.client.impl.auth.oauth2.AuthenticationFactoryOAuth2 import org.apache.pulsar.client.impl.auth.AuthenticationAthenz as PulsarAuthenticationAthenz import org.apache.pulsar.client.impl.auth.AuthenticationSasl as PulsarAuthenticationSasl +/** + * Configuration class for setting up Apache Pulsar connection properties. + * + * @property brokerServiceUrl URL for the Pulsar broker service. + * @property webServiceUrl URL for the Pulsar web service. + * @property tenant Pulsar tenant name. + * @property namespace Pulsar namespace name. + * @property allowedClusters Set of allowed clusters (optional). + * @property adminRoles Set of admin roles (optional). + * @property tlsAllowInsecureConnection Whether TLS allows insecure connections (optional). + * @property tlsEnableHostnameVerification Whether TLS enables hostname verification (optional). + * @property tlsTrustCertsFilePath Path to the TLS trust certificates file (optional). + * @property useKeyStoreTls Whether to use KeyStore for TLS (optional). + * @property tlsTrustStoreType Type of the TLS trust store (e.g., JKS, PKCS12) (optional). + * @property tlsTrustStorePath Path to the TLS trust store (optional). + * @property tlsTrustStorePassword Password for the TLS trust store (optional). + * @property authentication Client authentication configuration (optional). + * @property policies Pulsar policies configuration (default value provided). + * @property producer Pulsar producer configuration (default value provided). + * @property consumer Pulsar consumer configuration (default value provided). + */ @Suppress("unused") data class PulsarConfig( val brokerServiceUrl: String, // "pulsar://localhost:6650/", @@ -48,11 +72,11 @@ data class PulsarConfig( val namespace: String, val allowedClusters: Set? = null, val adminRoles: Set? = null, - val tlsAllowInsecureConnection: Boolean = false, - val tlsEnableHostnameVerification: Boolean = false, + val tlsAllowInsecureConnection: Boolean? = null, + val tlsEnableHostnameVerification: Boolean? = null, val tlsTrustCertsFilePath: String? = null, - val useKeyStoreTls: Boolean = false, - val tlsTrustStoreType: TlsTrustStoreType = TlsTrustStoreType.JKS, + val useKeyStoreTls: Boolean? = null, + val tlsTrustStoreType: TlsTrustStoreType? = null, val tlsTrustStorePath: String? = null, val tlsTrustStorePassword: Secret? = null, val authentication: ClientAuthenticationConfig? = null, @@ -62,14 +86,33 @@ data class PulsarConfig( ) { companion object { - @JvmStatic - fun builder() = PulsarConfigBuilder() - private val logger = KotlinLogging.logger {} private const val PULSAR_PROTOCOL = "pulsar://" private const val PULSAR_PROTOCOL_SSL = "pulsar+ssl://" private const val HTTP_PROTOCOL = "http://" private const val HTTP_PROTOCOL_SSL = "https://" + private const val HIDDEN = "******" + + /** + * Create PulsarConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): PulsarConfig = + loadFromYamlFile(*files) + + /** + * Create PulsarConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): PulsarConfig = + loadFromYamlResource(*resources) + + /** + * Create PulsarConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): PulsarConfig = + loadFromYamlString(*yamls) } init { @@ -97,52 +140,59 @@ data class PulsarConfig( } } - require(tenant.isNotEmpty()) { "tenant can NOT be empty" } + require(tenant.isNotBlank()) { "tenant can NOT be blank" } - require(namespace.isNotEmpty()) { "namespace can NOT be empty" } + require(namespace.isNotBlank()) { "namespace can NOT be blank" } - if (useKeyStoreTls) { + if (useKeyStoreTls == true) { require(tlsTrustStorePath != null) { - "tlsTrustStorePath MUST be defined if useKeyStoreTls is true" + "Configuration Error: 'tlsTrustStorePath' is required when 'useKeyStoreTls' is set to true. Please specify a valid path to the trust store." } require(tlsTrustStorePassword != null) { - "tlsTrustStorePassword MUST be defined if useKeyStoreTls is true" + "Configuration Error: 'tlsTrustStorePassword' is required when 'useKeyStoreTls' is set to true. Please provide the trust store password." + } + require(tlsTrustStoreType != null) { + "Configuration Error: 'tlsTrustStoreType' is required when 'useKeyStoreTls' is set to true. Please specify the type of trust store (e.g., JKS, PKCS12)." } } } val admin: PulsarAdmin by lazy { - val log = mutableMapOf() + val log = mutableMapOf() PulsarAdmin.builder().apply { serviceHttpUrl(webServiceUrl) log["serviceHttpUrl"] = webServiceUrl - allowTlsInsecureConnection(tlsAllowInsecureConnection) - log["allowTlsInsecureConnection"] = tlsAllowInsecureConnection + tlsAllowInsecureConnection?.let { + allowTlsInsecureConnection(it) + log["allowTlsInsecureConnection"] = it + } - enableTlsHostnameVerification(tlsEnableHostnameVerification) - log["enableTlsHostnameVerification"] = tlsEnableHostnameVerification + tlsEnableHostnameVerification?.let { + enableTlsHostnameVerification(it) + log["enableTlsHostnameVerification"] = it + } - if (tlsTrustCertsFilePath != null) { - tlsTrustCertsFilePath(tlsTrustCertsFilePath) - log["tlsTrustCertsFilePath"] = tlsTrustCertsFilePath + tlsTrustCertsFilePath?.let { + tlsTrustCertsFilePath(it) + log["tlsTrustCertsFilePath"] = it } - if (useKeyStoreTls) { + if (useKeyStoreTls == true) { useKeyStoreTls(true) tlsTrustStoreType(tlsTrustStoreType.toString()) - tlsTrustStorePath(tlsTrustStorePath!!) - tlsTrustStorePassword(tlsTrustStorePassword!!.value) + tlsTrustStorePath(tlsTrustStorePath) + tlsTrustStorePassword(tlsTrustStorePassword?.value) log["useKeyStoreTls"] = true log["tlsTrustStoreType"] = tlsTrustStoreType log["tlsTrustStorePath"] = tlsTrustStorePath - log["tlsTrustStorePassword"] = tlsTrustStorePassword + log["tlsTrustStorePassword"] = HIDDEN } when (authentication) { is AuthenticationTokenConfig -> { authentication(AuthenticationFactory.token(authentication.token.value)) - log["AuthenticationFactory.token"] = authentication.token + log["AuthenticationFactory.token"] = HIDDEN } is AuthenticationAthenzConfig -> { @@ -152,7 +202,7 @@ data class PulsarConfig( mapper.writeValueAsString(authentication), ), ) - log["AuthenticationFactory.AuthenticationAthenz"] = "****************" + log["AuthenticationFactory.AuthenticationAthenz"] = HIDDEN } is AuthenticationSaslConfig -> { @@ -162,7 +212,7 @@ data class PulsarConfig( mapper.writeValueAsString(authentication), ), ) - log["AuthenticationFactory.AuthenticationSasl"] = "****************" + log["AuthenticationFactory.AuthenticationSasl"] = HIDDEN } is AuthenticationOAuth2Config -> { @@ -188,37 +238,41 @@ data class PulsarConfig( } val client: PulsarClient by lazy { - val log = mutableMapOf() + val log = mutableMapOf() PulsarClient.builder().apply { serviceUrl(brokerServiceUrl) log["serviceUrl"] = brokerServiceUrl - allowTlsInsecureConnection(tlsAllowInsecureConnection) - log["allowTlsInsecureConnection"] = tlsAllowInsecureConnection + tlsAllowInsecureConnection?.let { + allowTlsInsecureConnection(it) + log["allowTlsInsecureConnection"] = it + } - enableTlsHostnameVerification(tlsEnableHostnameVerification) - log["enableTlsHostnameVerification"] = tlsEnableHostnameVerification + tlsEnableHostnameVerification?.let { + enableTlsHostnameVerification(it) + log["enableTlsHostnameVerification"] = it + } - if (tlsTrustCertsFilePath != null) { - tlsTrustCertsFilePath(tlsTrustCertsFilePath) - log["tlsTrustCertsFilePath"] = tlsTrustCertsFilePath + tlsTrustCertsFilePath?.let { + tlsTrustCertsFilePath(it) + log["tlsTrustCertsFilePath"] = it } - if (useKeyStoreTls) { + if (useKeyStoreTls == true) { useKeyStoreTls(true) tlsTrustStoreType(tlsTrustStoreType.toString()) - tlsTrustStorePath(tlsTrustStorePath!!) - tlsTrustStorePassword(tlsTrustStorePassword!!.value) + tlsTrustStorePath(tlsTrustStorePath) + tlsTrustStorePassword(tlsTrustStorePassword?.value) log["useKeyStoreTls"] = true log["tlsTrustStoreType"] = tlsTrustStoreType log["tlsTrustStorePath"] = tlsTrustStorePath - log["tlsTrustStorePassword"] = tlsTrustStorePassword + log["tlsTrustStorePassword"] = HIDDEN } when (authentication) { is AuthenticationTokenConfig -> { authentication(AuthenticationFactory.token(authentication.token.value)) - log["AuthenticationFactory.token"] = authentication.token + log["AuthenticationFactory.token"] = HIDDEN } is AuthenticationAthenzConfig -> { @@ -228,7 +282,7 @@ data class PulsarConfig( mapper.writeValueAsString(authentication), ), ) - log["AuthenticationFactory.AuthenticationAthenz"] = "****************" + log["AuthenticationFactory.AuthenticationAthenz"] = HIDDEN } is AuthenticationSaslConfig -> { @@ -238,7 +292,7 @@ data class PulsarConfig( mapper.writeValueAsString(authentication), ), ) - log["AuthenticationFactory.AuthenticationSasl"] = "****************" + log["AuthenticationFactory.AuthenticationSasl"] = HIDDEN } is AuthenticationOAuth2Config -> { @@ -262,107 +316,6 @@ data class PulsarConfig( } } } - - /** - * PulsarConfig builder (Useful for Java user) - */ - class PulsarConfigBuilder { - private val default = - PulsarConfig(PULSAR_PROTOCOL, HTTP_PROTOCOL, UNSET, UNSET) - private var tenant = default.tenant - private var namespace = default.namespace - private var brokerServiceUrl = default.brokerServiceUrl - private var webServiceUrl = default.webServiceUrl - private var allowedClusters = default.allowedClusters - private var adminRoles = default.adminRoles - private var tlsAllowInsecureConnection = default.tlsAllowInsecureConnection - private var tlsEnableHostnameVerification = default.tlsEnableHostnameVerification - private var tlsTrustCertsFilePath = default.tlsTrustCertsFilePath - private var useKeyStoreTls = default.useKeyStoreTls - private var tlsTrustStoreType = default.tlsTrustStoreType - private var tlsTrustStorePath = default.tlsTrustStorePath - private var tlsTrustStorePassword = default.tlsTrustStorePassword - private var authentication = default.authentication - private var policies = default.policies - private var producer = default.producer - private var consumer = default.consumer - - fun tenant(tenant: String) = - apply { this.tenant = tenant } - - fun namespace(namespace: String) = - apply { this.namespace = namespace } - - fun brokerServiceUrl(brokerServiceUrl: String) = - apply { this.brokerServiceUrl = brokerServiceUrl } - - fun webServiceUrl(webServiceUrl: String) = - apply { this.webServiceUrl = webServiceUrl } - - fun allowedClusters(allowedClusters: Set) = - apply { this.allowedClusters = allowedClusters } - - fun adminRoles(adminRoles: Set) = - apply { this.adminRoles = adminRoles } - - fun tlsAllowInsecureConnection(tlsAllowInsecureConnection: Boolean) = - apply { this.tlsAllowInsecureConnection = tlsAllowInsecureConnection } - - fun tlsEnableHostnameVerification(tlsEnableHostnameVerification: Boolean) = - apply { this.tlsEnableHostnameVerification = tlsEnableHostnameVerification } - - fun tlsTrustCertsFilePath(tlsTrustCertsFilePath: String) = - apply { this.tlsTrustCertsFilePath = tlsTrustCertsFilePath } - - fun useKeyStoreTls(useKeyStoreTls: Boolean) = - apply { this.useKeyStoreTls = useKeyStoreTls } - - fun tlsTrustStoreType(tlsTrustStoreType: TlsTrustStoreType) = - apply { this.tlsTrustStoreType = tlsTrustStoreType } - - fun tlsTrustStorePath(tlsTrustStorePath: String) = - apply { this.tlsTrustStorePath = tlsTrustStorePath } - - fun tlsTrustStorePassword(tlsTrustStorePassword: Secret) = - apply { this.tlsTrustStorePassword = tlsTrustStorePassword } - - fun authentication(authentication: ClientAuthenticationConfig) = - apply { this.authentication = authentication } - - fun policies(policiesConfig: PoliciesConfig) = - apply { this.policies = policiesConfig } - - fun producer(producerConfig: ProducerConfig) = - apply { this.producer = producerConfig } - - fun consumer(consumerConfig: ConsumerConfig) = - apply { this.consumer = consumerConfig } - - fun build() = PulsarConfig( - brokerServiceUrl.removeUnset(PULSAR_PROTOCOL), - webServiceUrl.removeUnset(HTTP_PROTOCOL), - tenant.removeUnset(), - namespace.removeUnset(), - allowedClusters, - adminRoles, - tlsAllowInsecureConnection, - tlsEnableHostnameVerification, - tlsTrustCertsFilePath, - useKeyStoreTls, - tlsTrustStoreType, - tlsTrustStorePath, - tlsTrustStorePassword, - authentication, - policies, - producer, - consumer, - ) - } } -private const val UNSET = "INFINITIC_UNSET_STRING" -private fun String.removeUnset(unset: String = UNSET): String = when (this) { - unset -> "" - else -> this -} diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationOAuth2Config.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationOAuth2Config.kt index fd32885d6..f08e0d040 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationOAuth2Config.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationOAuth2Config.kt @@ -24,12 +24,13 @@ package io.infinitic.pulsar.config.auth import java.net.URL +@Suppress("unused") data class AuthenticationOAuth2Config( val issuerUrl: URL, val privateKey: URL, val audience: String ) : ClientAuthenticationConfig() { - + /** * AuthenticationOAuth2Config builder (Useful for Java user) */ @@ -38,9 +39,9 @@ data class AuthenticationOAuth2Config( private var privateKey: URL? = null private var audience: String? = null - fun issuerUrl(issuerUrl: URL) = apply { this.issuerUrl = issuerUrl } - fun privateKey(privateKey: URL) = apply { this.privateKey = privateKey } - fun audience(audience: String) = apply { this.audience = audience } + fun setIssuerUrl(issuerUrl: URL) = apply { this.issuerUrl = issuerUrl } + fun setPrivateKey(privateKey: URL) = apply { this.privateKey = privateKey } + fun setAudience(audience: String) = apply { this.audience = audience } fun build() = AuthenticationOAuth2Config( issuerUrl ?: throw IllegalArgumentException("issuerUrl must be set"), diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationSaslConfig.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationSaslConfig.kt index 31cce255b..75c207757 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationSaslConfig.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/auth/AuthenticationSaslConfig.kt @@ -24,6 +24,7 @@ package io.infinitic.pulsar.config.auth import com.sksamuel.hoplite.Secret +@Suppress("unused") data class AuthenticationSaslConfig( val tenantDomain: String, val tenantService: String, @@ -42,11 +43,11 @@ data class AuthenticationSaslConfig( private var privateKey: String? = null private var keyId: String? = null - fun tenantDomain(tenantDomain: String) = apply { this.tenantDomain = tenantDomain } - fun tenantService(tenantService: String) = apply { this.tenantService = tenantService } - fun providerDomain(providerDomain: String) = apply { this.providerDomain = providerDomain } - fun privateKey(privateKey: String) = apply { this.privateKey = privateKey } - fun keyId(keyId: String) = apply { this.keyId = keyId } + fun setTenantDomain(tenantDomain: String) = apply { this.tenantDomain = tenantDomain } + fun setTenantService(tenantService: String) = apply { this.tenantService = tenantService } + fun setProviderDomain(providerDomain: String) = apply { this.providerDomain = providerDomain } + fun setPrivateKey(privateKey: String) = apply { this.privateKey = privateKey } + fun setKeyId(keyId: String) = apply { this.keyId = keyId } fun build() = AuthenticationSaslConfig( tenantDomain ?: throw IllegalArgumentException("tenantDomain must be set"), diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/policies/PoliciesConfig.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/policies/PoliciesConfig.kt index 27ad658f1..d8f7a3d41 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/policies/PoliciesConfig.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/config/policies/PoliciesConfig.kt @@ -27,13 +27,13 @@ import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy @Suppress("unused") data class PoliciesConfig( // Retain messages for 7 days - val retentionTimeInMinutes: Int = 60 * 24 * 7, + val retentionTimeMinutes: Int = 60 * 24 * 7, // Retain messages up to 1GB - val retentionSizeInMB: Long = 1024, + val retentionSizeMB: Long = 1024, // Expire messages after 14 days - val messageTTLInSeconds: Int = 3600 * 24 * 14, + val messageTTLSeconds: Int = 3600 * 24 * 14, // Expire delayed messages after 1 year - val timerTTLInSeconds: Int = 3600 * 24 * 366, + val timerTTLSeconds: Int = 3600 * 24 * 366, // Delayed delivery tick time = 1 second val delayedDeliveryTickTimeMillis: Long = 1000, // Changes allowed: add optional fields, delete fields @@ -58,10 +58,10 @@ data class PoliciesConfig( */ class PoliciesConfigBuilder { private val default = PoliciesConfig() - private var retentionTimeInMinutes = default.retentionTimeInMinutes - private var retentionSizeInMB = default.retentionSizeInMB - private var messageTTLInSeconds = default.messageTTLInSeconds - private var timerTTLInSeconds = default.timerTTLInSeconds + private var retentionTimeMinutes = default.retentionTimeMinutes + private var retentionSizeMB = default.retentionSizeMB + private var messageTTLSeconds = default.messageTTLSeconds + private var timerTTLSeconds = default.timerTTLSeconds private var delayedDeliveryTickTimeMillis = default.delayedDeliveryTickTimeMillis private var schemaCompatibilityStrategy = default.schemaCompatibilityStrategy private var allowAutoTopicCreation = default.allowAutoTopicCreation @@ -69,17 +69,17 @@ data class PoliciesConfig( private var isAllowAutoUpdateSchema = default.isAllowAutoUpdateSchema private var deduplicationEnabled = default.deduplicationEnabled - fun setRetentionTimeInMinutes(retentionTimeInMinutes: Int) = - apply { this.retentionTimeInMinutes = retentionTimeInMinutes } + fun setRetentionTimeMinutes(retentionTimeMinutes: Int) = + apply { this.retentionTimeMinutes = retentionTimeMinutes } - fun setRetentionSizeInMB(retentionSizeInMB: Long) = - apply { this.retentionSizeInMB = retentionSizeInMB } + fun setRetentionSizeMB(retentionSizeMB: Long) = + apply { this.retentionSizeMB = retentionSizeMB } - fun setMessageTTLInSeconds(messageTTLInSeconds: Int) = - apply { this.messageTTLInSeconds = messageTTLInSeconds } + fun setMessageTTLSeconds(messageTTLSeconds: Int) = + apply { this.messageTTLSeconds = messageTTLSeconds } - fun setTimerTTLInSeconds(timerTTLInSeconds: Int) = - apply { this.timerTTLInSeconds = timerTTLInSeconds } + fun setTimerTTLSeconds(timerTTLSeconds: Int) = + apply { this.timerTTLSeconds = timerTTLSeconds } fun setDelayedDeliveryTickTimeMillis(delayedDeliveryTickTimeMillis: Long) = apply { this.delayedDeliveryTickTimeMillis = delayedDeliveryTickTimeMillis } @@ -100,10 +100,10 @@ data class PoliciesConfig( apply { this.deduplicationEnabled = deduplicationEnabled } fun build() = PoliciesConfig( - retentionTimeInMinutes, - retentionSizeInMB, - messageTTLInSeconds, - timerTTLInSeconds, + retentionTimeMinutes, + retentionSizeMB, + messageTTLSeconds, + timerTTLSeconds, delayedDeliveryTickTimeMillis, schemaCompatibilityStrategy, allowAutoTopicCreation, diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/Consumer.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/Consumer.kt index 8d0e35852..4d762399e 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/Consumer.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/Consumer.kt @@ -27,55 +27,29 @@ import io.infinitic.common.data.MillisInstant import io.infinitic.common.messages.Envelope import io.infinitic.common.messages.Message import io.infinitic.pulsar.client.PulsarInfiniticClient -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.NonCancellable -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.future.await import kotlinx.coroutines.isActive -import kotlinx.coroutines.job import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.apache.pulsar.client.api.Consumer import org.apache.pulsar.client.api.MessageId import org.apache.pulsar.client.api.Schema import org.apache.pulsar.client.api.SubscriptionInitialPosition import org.apache.pulsar.client.api.SubscriptionType -import java.util.concurrent.CancellationException import java.util.concurrent.CompletionException -import java.util.concurrent.Executors +import kotlin.coroutines.cancellation.CancellationException import org.apache.pulsar.client.api.Message as PulsarMessage class Consumer( val client: PulsarInfiniticClient, private val consumerConfig: ConsumerConfig ) { - private val logger = KotlinLogging.logger {} - private val consumingScope = - CoroutineScope(Executors.newCachedThreadPool().asCoroutineDispatcher()) - - private val isActive get() = consumingScope.isActive - - fun cancel() { - if (isActive) consumingScope.cancel() - } - - fun join() = runBlocking { - consumingScope.coroutineContext.job.children.forEach { - try { - it.join() - } catch (e: CancellationException) { - // do nothing - } - } - } - internal suspend fun > startListening( handler: suspend (S, MillisInstant) -> Unit, beforeDlq: (suspend (S?, Exception) -> Unit)?, @@ -91,7 +65,7 @@ class Consumer( ) { when (subscriptionType) { SubscriptionType.Key_Shared -> { - logger.debug { "Starting $concurrency $subscriptionType consumers on topic $topic with subscription $subscriptionName" } + logger.info { "Starting $concurrency $subscriptionType consumers on topic $topic with subscription $subscriptionName" } val consumers = List(concurrency) { getConsumer( @@ -110,7 +84,7 @@ class Consumer( } else -> { - logger.debug { "Starting a $subscriptionType consumer on topic $topic with subscription $subscriptionName" } + logger.info { "Starting a $subscriptionType consumer on topic $topic with subscription $subscriptionName" } val consumer = getConsumer( schema = schema, @@ -139,42 +113,47 @@ class Consumer( val channel = Channel>() // start executor coroutines - launch(consumingScope.coroutineContext) { - val jobs = List(concurrency) { - launch { - try { - for (pulsarMessage: PulsarMessage in channel) { - // this ensures that ongoing messages are processed - // even after scope is cancelled following an interruption or an Error - withContext(NonCancellable) { - processPulsarMessage(consumer, handler, beforeDlq, consumer.topic, pulsarMessage) - } + val jobs = List(concurrency) { + launch { + try { + for (pulsarMessage: PulsarMessage in channel) { + // this ensures that ongoing messages are processed + // even after scope is cancelled following an interruption or an Error + withContext(NonCancellable) { + processPulsarMessage(consumer, handler, beforeDlq, consumer.topic, pulsarMessage) } - } catch (e: CancellationException) { - logDebug(consumer.topic) { "Processor #$it closed in ${consumer.consumerName} after cancellation" } } - } - } - // start message receiver - while (isActive) { - try { - // await() is a suspendable and should be used instead of get() - val pulsarMessage = consumer.receiveAsync().await() - logDebug(consumer.topic, pulsarMessage.messageId) { "Received pulsar message" } - channel.send(pulsarMessage) } catch (e: CancellationException) { - logDebug(consumer.topic) { "Exiting receiving loop in ${consumer.consumerName}" } - // if current scope is canceled, we just exit the while loop - break + logInfo(consumer.topic) { + "Processor #$it closed in ${consumer.consumerName} after cancellation." + } } catch (e: Throwable) { - e.rethrowError(consumer.topic, where = "in ${consumer.consumerName}") - continue + logInfo(consumer.topic) { + "Processor #$it closed in ${consumer.consumerName} after error $e." + + "Waiting for completion of other ongoing messages" + } + throw e } } - logDebug(consumer.topic) { "Waiting completion of ongoing messages in ${consumer.consumerName}" } - withContext(NonCancellable) { jobs.joinAll() } - closeConsumer(consumer) } + // start message receiver + while (isActive) { + try { + // await() is a suspendable and should be used instead of get() + val pulsarMessage = consumer.receiveAsync().await() + logDebug(consumer.topic, pulsarMessage.messageId) { "Received pulsar message" } + channel.send(pulsarMessage) + } catch (e: CancellationException) { + logInfo(consumer.topic) { "Exiting receiving loop in ${consumer.consumerName}" } + // if current scope is canceled, we just exit the while loop + break + } catch (e: Throwable) { + e.rethrowError(consumer.topic, where = "in ${consumer.consumerName}") + continue + } + } + withContext(NonCancellable) { jobs.joinAll() } + closeConsumer(consumer) } private suspend fun > startListeningWithKey( @@ -185,7 +164,7 @@ class Consumer( // For Key_Shared subscription, we must create a new consumer for each executor coroutine consumers.forEach { consumer -> - launch(consumingScope.coroutineContext) { + launch { while (isActive) { try { // await() is a suspendable and should be used instead of get() @@ -210,10 +189,10 @@ class Consumer( } private fun closeConsumer(consumer: Consumer<*>) { - logger.debug { "Closing consumer ${consumer.consumerName} after cancellation" } + logger.debug { "Closing consumer ${consumer.consumerName}" } client.closeConsumer(consumer) - .onSuccess { logger.info { "Consumer ${consumer.consumerName} closed after cancellation" } } - .onFailure { logger.warn(it) { "Unable to close consumer ${consumer.consumerName} after cancellation" } } + .onSuccess { logger.info { "Consumer ${consumer.consumerName} closed" } } + .onFailure { logger.warn(it) { "Unable to close consumer ${consumer.consumerName}" } } } private suspend fun > processPulsarMessage( diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/ConsumerConfig.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/ConsumerConfig.kt index 3f2329af2..a2131533a 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/ConsumerConfig.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/consumers/ConsumerConfig.kt @@ -91,77 +91,77 @@ data class ConsumerConfig( private var startPaused = default.startPaused private var maxRedeliverCount = default.maxRedeliverCount - fun loadConf(loadConf: Map) = + fun setLoadConf(loadConf: Map) = apply { this.loadConf = loadConf } - fun subscriptionProperties(subscriptionProperties: Map) = + fun setSubscriptionProperties(subscriptionProperties: Map) = apply { this.subscriptionProperties = subscriptionProperties } - fun ackTimeoutSeconds(ackTimeoutSeconds: Double) = + fun setAckTimeoutSeconds(ackTimeoutSeconds: Double) = apply { this.ackTimeoutSeconds = ackTimeoutSeconds } - fun isAckReceiptEnabled(isAckReceiptEnabled: Boolean) = + fun setIsAckReceiptEnabled(isAckReceiptEnabled: Boolean) = apply { this.isAckReceiptEnabled = isAckReceiptEnabled } - fun ackTimeoutTickTimeSeconds(ackTimeoutTickTimeSeconds: Double) = + fun setAckTimeoutTickTimeSeconds(ackTimeoutTickTimeSeconds: Double) = apply { this.ackTimeoutTickTimeSeconds = ackTimeoutTickTimeSeconds } - fun negativeAckRedeliveryDelaySeconds(negativeAckRedeliveryDelaySeconds: Double) = + fun setNegativeAckRedeliveryDelaySeconds(negativeAckRedeliveryDelaySeconds: Double) = apply { this.negativeAckRedeliveryDelaySeconds = negativeAckRedeliveryDelaySeconds } - fun defaultCryptoKeyReader(defaultCryptoKeyReader: String) = + fun setDefaultCryptoKeyReader(defaultCryptoKeyReader: String) = apply { this.defaultCryptoKeyReader = defaultCryptoKeyReader } - fun cryptoFailureAction(cryptoFailureAction: ConsumerCryptoFailureAction) = + fun setCryptoFailureAction(cryptoFailureAction: ConsumerCryptoFailureAction) = apply { this.cryptoFailureAction = cryptoFailureAction } - fun receiverQueueSize(receiverQueueSize: Int) = + fun setReceiverQueueSize(receiverQueueSize: Int) = apply { this.receiverQueueSize = receiverQueueSize } - fun acknowledgmentGroupTimeSeconds(acknowledgmentGroupTimeSeconds: Double) = + fun setAcknowledgmentGroupTimeSeconds(acknowledgmentGroupTimeSeconds: Double) = apply { this.acknowledgmentGroupTimeSeconds = acknowledgmentGroupTimeSeconds } - fun replicateSubscriptionState(replicateSubscriptionState: Boolean) = + fun setReplicateSubscriptionState(replicateSubscriptionState: Boolean) = apply { this.replicateSubscriptionState = replicateSubscriptionState } - fun maxTotalReceiverQueueSizeAcrossPartitions(maxTotalReceiverQueueSizeAcrossPartitions: Int) = + fun setMaxTotalReceiverQueueSizeAcrossPartitions(maxTotalReceiverQueueSizeAcrossPartitions: Int) = apply { this.maxTotalReceiverQueueSizeAcrossPartitions = maxTotalReceiverQueueSizeAcrossPartitions } - fun priorityLevel(priorityLevel: Int) = + fun setPriorityLevel(priorityLevel: Int) = apply { this.priorityLevel = priorityLevel } - fun properties(properties: Map) = + fun setProperties(properties: Map) = apply { this.properties = properties } - fun autoUpdatePartitions(autoUpdatePartitions: Boolean) = + fun setAutoUpdatePartitions(autoUpdatePartitions: Boolean) = apply { this.autoUpdatePartitions = autoUpdatePartitions } - fun autoUpdatePartitionsIntervalSeconds(autoUpdatePartitionsIntervalSeconds: Double) = + fun setAutoUpdatePartitionsIntervalSeconds(autoUpdatePartitionsIntervalSeconds: Double) = apply { this.autoUpdatePartitionsIntervalSeconds = autoUpdatePartitionsIntervalSeconds } - fun enableBatchIndexAcknowledgment(enableBatchIndexAcknowledgment: Boolean) = + fun setEnableBatchIndexAcknowledgment(enableBatchIndexAcknowledgment: Boolean) = apply { this.enableBatchIndexAcknowledgment = enableBatchIndexAcknowledgment } - fun maxPendingChunkedMessage(maxPendingChunkedMessage: Int) = + fun setMaxPendingChunkedMessage(maxPendingChunkedMessage: Int) = apply { this.maxPendingChunkedMessage = maxPendingChunkedMessage } - fun autoAckOldestChunkedMessageOnQueueFull(autoAckOldestChunkedMessageOnQueueFull: Boolean) = + fun setAutoAckOldestChunkedMessageOnQueueFull(autoAckOldestChunkedMessageOnQueueFull: Boolean) = apply { this.autoAckOldestChunkedMessageOnQueueFull = autoAckOldestChunkedMessageOnQueueFull } - fun expireTimeOfIncompleteChunkedMessageSeconds(expireTimeOfIncompleteChunkedMessageSeconds: Double) = + fun setExpireTimeOfIncompleteChunkedMessageSeconds(expireTimeOfIncompleteChunkedMessageSeconds: Double) = apply { this.expireTimeOfIncompleteChunkedMessageSeconds = expireTimeOfIncompleteChunkedMessageSeconds } - fun startPaused(startPaused: Boolean) = + fun setStartPaused(startPaused: Boolean) = apply { this.startPaused = startPaused } - fun maxRedeliverCount(maxRedeliverCount: Int) = + fun setMaxRedeliverCount(maxRedeliverCount: Int) = apply { this.maxRedeliverCount = maxRedeliverCount } fun build() = ConsumerConfig( diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarResources.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarResources.kt index 3052baeb6..23b4f1683 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarResources.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarResources.kt @@ -24,6 +24,7 @@ package io.infinitic.pulsar.resources import io.infinitic.common.messages.Message +import io.infinitic.common.transport.ClientTopic import io.infinitic.common.transport.MainSubscription import io.infinitic.common.transport.Topic import io.infinitic.common.transport.isTimer @@ -61,12 +62,12 @@ class PulsarResources( suspend fun getTopicsFullName(): Set = admin.getTopicsSet(namespaceFullName).getOrThrow() /** Set of service's names for current tenant and namespace */ - suspend fun getServicesName(): Set = getTopicsFullName().mapNotNull { + suspend fun getServiceNames(): Set = getTopicsFullName().mapNotNull { getServiceNameFromTopicName(it.removePrefix(topicFullName(""))) }.toSet() /** Set of workflow's names for current tenant and namespace */ - suspend fun getWorkflowsName(): Set = getTopicsFullName().mapNotNull { + suspend fun getWorkflowNames(): Set = getTopicsFullName().mapNotNull { getWorkflowNameFromTopicName(it.removePrefix(topicFullName(""))) }.toSet() @@ -122,7 +123,7 @@ class PulsarResources( /** * Delete a topic by name */ - suspend fun deleteTopic(topic: String): Result = admin.deleteTopic(topic) + suspend fun deleteTopic(topic: String): Result = admin.deleteTopic(topic) /** * Check if a topic exists, and create it if not @@ -134,16 +135,16 @@ class PulsarResources( isTimed: Boolean, ): Result { // initialize tenant once (do nothing on error) - admin.initTenantOnce(tenant, allowedClusters, adminRoles) + admin.syncInitTenantOnce(tenant, allowedClusters, adminRoles) // initialize namespace once (do nothing on error) - admin.initNamespaceOnce(namespaceFullName, policies) + admin.syncInitNamespaceOnce(namespaceFullName, policies) // initialize topic once (do nothing on error) val ttl = when (isTimed) { - true -> policies.timerTTLInSeconds - false -> policies.messageTTLInSeconds + true -> policies.timerTTLSeconds + false -> policies.messageTTLSeconds } - return admin.initTopicOnce(topic, isPartitioned, ttl).map { } + return admin.syncInitTopicOnce(topic, isPartitioned, ttl).map { } } /** @@ -157,6 +158,18 @@ class PulsarResources( ): Result = topic?.let { initTopicOnce(it, isPartitioned, isDelayed) } ?: Result.success(null) + /** + * Deletes the topic that corresponds to the specified client name. + * Returns: + * - Result.success(null) if the topic does not exist + * - Result.success(topicName) if the topic has been deleted + * - Result.failure(e) if the deletion failed + */ + suspend fun deleteTopicForClient(clientName: String): Result { + val topicName = ClientTopic.fullName(clientName) + return deleteTopic(topicName).map { it?.let { topicName } } + } + companion object { /** Create TopicManager from a Pulsar configuration instance */ fun from(pulsarConfig: PulsarConfig) = PulsarResources( diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarSubscription.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarSubscription.kt index 9c8db6422..4f97b6a3e 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarSubscription.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/PulsarSubscription.kt @@ -28,13 +28,13 @@ import io.infinitic.common.transport.Subscription import org.apache.pulsar.client.api.SubscriptionInitialPosition import org.apache.pulsar.client.api.SubscriptionType -val Subscription<*>.type +internal val Subscription<*>.type get() = when (withKey) { true -> SubscriptionType.Key_Shared false -> SubscriptionType.Shared } -val Subscription<*>.defaultName +internal val Subscription<*>.defaultName get() = when (this) { // IMPORTANT: subscription name MUST stay UNCHANGED through all Infinitic versions // as Pulsar identify subscriptions through their name. @@ -44,9 +44,9 @@ val Subscription<*>.defaultName is EventListenerSubscription -> "listener-subscription" } -val Subscription<*>.defaultNameDLQ get() = "$defaultName-dlq" +internal val Subscription<*>.defaultNameDLQ get() = "$defaultName-dlq" -val Subscription<*>.defaultInitialPosition +internal val Subscription<*>.defaultInitialPosition get() = when (this) { is MainSubscription -> SubscriptionInitialPosition.Earliest is EventListenerSubscription -> SubscriptionInitialPosition.Latest diff --git a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/Topics.kt b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/Topics.kt index 55b2e4221..800e2a2dc 100644 --- a/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/Topics.kt +++ b/infinitic-transport-pulsar/src/main/kotlin/io/infinitic/pulsar/resources/Topics.kt @@ -175,7 +175,7 @@ internal val Topic.envelopeClass: KClass> } as KClass> @Suppress("UNCHECKED_CAST") -fun Topic.envelope(message: S) = when (this) { +internal fun Topic.envelope(message: S) = when (this) { NamingTopic -> thisShouldNotHappen() ClientTopic -> ClientEnvelope.from(message as ClientMessage) WorkflowTagEngineTopic -> WorkflowTagEnvelope.from(message as WorkflowTagEngineMessage) diff --git a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdminTests.kt b/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdminTests.kt index 846d25cff..577dc3290 100644 --- a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdminTests.kt +++ b/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/admin/PulsarInfiniticAdminTests.kt @@ -52,8 +52,8 @@ class PulsarInfiniticAdminTests : // This test ensures that this case is handled correctly. shouldNotThrow { CoroutineScope(Dispatchers.IO).async { - launch { admin.initTenantOnce("test-tenant", null, null).getOrThrow() } - launch { admin.initTenantOnce("test-tenant", null, null).getOrThrow() } + launch { admin.syncInitTenantOnce("test-tenant", null, null).getOrThrow() } + launch { admin.syncInitTenantOnce("test-tenant", null, null).getOrThrow() } }.await() } } @@ -64,13 +64,15 @@ class PulsarInfiniticAdminTests : // with multiple new tags (there is one topic per tag). // This test ensures that this case is handled correctly. shouldNotThrow { - admin.initTenantOnce("test-tenant", null, null).getOrThrow() + admin.syncInitTenantOnce("test-tenant", null, null).getOrThrow() CoroutineScope(Dispatchers.IO).async { launch { - admin.initNamespaceOnce("test-tenant/test-namespace", PoliciesConfig()).getOrThrow() + admin.syncInitNamespaceOnce("test-tenant/test-namespace", PoliciesConfig()) + .getOrThrow() } launch { - admin.initNamespaceOnce("test-tenant/test-namespace", PoliciesConfig()).getOrThrow() + admin.syncInitNamespaceOnce("test-tenant/test-namespace", PoliciesConfig()) + .getOrThrow() } }.await() } @@ -83,8 +85,8 @@ class PulsarInfiniticAdminTests : // This test ensures that this case is handled correctly. shouldNotThrow { coroutineScope { - launch { admin.initTopicOnce("test-topic", true, 60).getOrThrow() } - launch { admin.initTopicOnce("test-topic", true, 60).getOrThrow() } + launch { admin.syncInitTopicOnce("test-topic", true, 60).getOrThrow() } + launch { admin.syncInitTopicOnce("test-topic", true, 60).getOrThrow() } } } } diff --git a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/consumers/ConsumerTests.kt b/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/consumers/ConsumerTests.kt index de28d27ac..5bf2bc181 100644 --- a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/consumers/ConsumerTests.kt +++ b/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/consumers/ConsumerTests.kt @@ -42,7 +42,12 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.comparables.shouldBeLessThan import io.kotest.matchers.doubles.shouldBeLessThan import io.kotest.matchers.ints.shouldBeExactly +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.cancel import kotlinx.coroutines.delay +import kotlinx.coroutines.job +import kotlinx.coroutines.launch import net.bytebuddy.utility.RandomString import org.apache.pulsar.client.api.PulsarClient import org.apache.pulsar.client.api.SubscriptionInitialPosition @@ -51,6 +56,7 @@ import java.time.Duration import java.time.Instant import java.util.concurrent.CompletableFuture import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger @EnabledIf(DockerOnly::class) @@ -59,7 +65,10 @@ class ConsumerTests : StringSpec( val pulsarServer = DockerOnly().pulsarServer!! val client = PulsarInfiniticClient( - PulsarClient.builder().serviceUrl(pulsarServer.pulsarBrokerUrl).build(), + PulsarClient + .builder() + .serviceUrl(pulsarServer.pulsarBrokerUrl) + .build(), ) val producer = Producer(client, ProducerConfig()) val zero = MillisDuration(0) @@ -87,29 +96,33 @@ class ConsumerTests : StringSpec( } } - suspend fun startAsync( + fun getScope() = CoroutineScope(Executors.newCachedThreadPool().asCoroutineDispatcher()) + + fun CoroutineScope.startAsync( consumer: Consumer, handler: suspend (ServiceExecutorMessage, MillisInstant) -> Unit, topic: String, concurrency: Int, withKey: Boolean = false - ) = consumer.startListening( - handler = handler, - beforeDlq = { _, _ -> }, - schema = ServiceExecutorTopic.schema, - topic = topic, - topicDlq = null, - subscriptionName = topic + "Consumer", - subscriptionNameDlq = "", - subscriptionType = if (withKey) SubscriptionType.Key_Shared else SubscriptionType.Shared, - subscriptionInitialPosition = SubscriptionInitialPosition.Earliest, - consumerName = "consumerTest", - concurrency = concurrency, - ) + ) = launch { + consumer.startListening( + handler = handler, + beforeDlq = { _, _ -> }, + schema = ServiceExecutorTopic.schema, + topic = topic, + topicDlq = null, + subscriptionName = topic + "Consumer", + subscriptionNameDlq = "", + subscriptionType = if (withKey) SubscriptionType.Key_Shared else SubscriptionType.Shared, + subscriptionInitialPosition = SubscriptionInitialPosition.Earliest, + consumerName = "consumerTest", + concurrency = concurrency, + ) + } "producing 10000 message in bulk should take less than 1 ms in average" { val topic = RandomString(10).nextString() - sendMessage(topic, 10000).shouldBeLessThan(1.0) + sendMessage(topic, 10000) shouldBeLessThan 1.0 } "consuming 1000 messages (1ms) without concurrency should take less than 5 ms in average" { @@ -120,6 +133,8 @@ class ConsumerTests : StringSpec( val counter = AtomicInteger(0) lateinit var start: Instant + val scope = getScope() + val handler: suspend (ServiceExecutorMessage, MillisInstant) -> Unit = { _, _ -> if (counter.get() == 0) start = Instant.now() // emulate a 1ms task @@ -129,18 +144,18 @@ class ConsumerTests : StringSpec( if (it == total) { averageMillisToConsume = (start.fromNow() / total) println("Average time to consume a message: $averageMillisToConsume ms") - consumer.cancel() + scope.cancel() } } } // start consumers - startAsync(consumer, handler, topic, 1) + scope.startAsync(consumer, handler, topic, 1) // send messages sendMessage(topic, total) // wait for scope cancellation - consumer.join() + scope.coroutineContext.job.join() - averageMillisToConsume.shouldBeLessThan(5.0) + averageMillisToConsume shouldBeLessThan 5.0 } "consuming 1000 messages (100ms) with 100 concurrency (shared) should take less than 5 ms in average" { @@ -151,6 +166,8 @@ class ConsumerTests : StringSpec( val counter = AtomicInteger(0) lateinit var start: Instant + val scope = getScope() + val handler: ((ServiceExecutorMessage, MillisInstant) -> Unit) = { _, _ -> if (counter.get() == 0) start = Instant.now() // emulate a 100ms task @@ -160,21 +177,21 @@ class ConsumerTests : StringSpec( if (it == total) { averageMillisToConsume = (start.fromNow() / total) println("Average time to consume a message: $averageMillisToConsume ms") - consumer.cancel() + scope.cancel() } } } // start consumers - startAsync(consumer, handler, topic, 100) + scope.startAsync(consumer, handler, topic, 100) // send messages sendMessage(topic, total) // wait for scope cancellation - consumer.join() + scope.coroutineContext.job.join() - averageMillisToConsume.shouldBeLessThan(5.0) + averageMillisToConsume shouldBeLessThan 5.0 } - "consuming 1000 messages (100ms) with 100 concurrency (key-shared) should take less than 5 ms in average" { + "consuming 1000 messages (100ms) with 100 concurrency (key-shared) should take less than 10 ms in average" { val consumer = Consumer(client, ConsumerConfig()) val topic = RandomString(10).nextString() var averageMillisToConsume = 100.0 @@ -182,6 +199,8 @@ class ConsumerTests : StringSpec( val counter = AtomicInteger(0) lateinit var start: Instant + val scope = getScope() + val handler: ((ServiceExecutorMessage, MillisInstant) -> Unit) = { _, _ -> if (counter.get() == 0) start = Instant.now() // emulate a 100ms task @@ -191,18 +210,18 @@ class ConsumerTests : StringSpec( if (it == total) { averageMillisToConsume = (start.fromNow() / total) println("Average time to consume a message: $averageMillisToConsume ms") - consumer.cancel() + scope.cancel() } } } // start consumers - startAsync(consumer, handler, topic, 100, true) + scope.startAsync(consumer, handler, topic, 100, true) // send messages sendMessage(topic, total, true) // wait for scope cancellation - consumer.join() + scope.coroutineContext.job.join() - averageMillisToConsume.shouldBeLessThan(5.0) + averageMillisToConsume.shouldBeLessThan(10.0) } "graceful shutdown with Shared" { @@ -213,6 +232,8 @@ class ConsumerTests : StringSpec( val messageClosed = CopyOnWriteArrayList() val total = 1000 + val scope = getScope() + val handler: ((ServiceExecutorMessage, MillisInstant) -> Unit) = { _, _ -> counter.incrementAndGet().let { // begin of task @@ -224,18 +245,17 @@ class ConsumerTests : StringSpec( } } // start consumers - startAsync(consumer, handler, topic, 100) + scope.startAsync(consumer, handler, topic, 100) // send messages sendMessage(topic, total) // cancel after 0.4s - later(400) { consumer.cancel() } + later(400) { scope.cancel() } // wait for scope cancellation - consumer.join() + scope.coroutineContext.job.join() // for the test to be meaningful, all messages should not have been processed messageOpen.count().shouldBeLessThan(total) messageClosed.count().shouldBeLessThan(total) - messageOpen.count().shouldBeExactly(messageClosed.count()) } @@ -247,6 +267,8 @@ class ConsumerTests : StringSpec( val messageClosed = CopyOnWriteArrayList() val total = 1000 + val scope = getScope() + val handler: ((ServiceExecutorMessage, MillisInstant) -> Unit) = { _, _ -> counter.incrementAndGet().let { // begin of task @@ -258,18 +280,17 @@ class ConsumerTests : StringSpec( } } // start consumers - startAsync(consumer, handler, topic, 100, true) + scope.startAsync(consumer, handler, topic, 100, true) // send messages sendMessage(topic, total, true) // cancel after 1s - later(1000) { consumer.cancel() } + later(1000) { scope.cancel() } // wait for scope cancellation - consumer.join() + scope.coroutineContext.job.join() // for the test to be meaningful, all messages should not have been processed messageOpen.count().shouldBeLessThan(total) messageClosed.count().shouldBeLessThan(total) - messageOpen.count().shouldBeExactly(messageClosed.count()) } }, diff --git a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/resources/PulsarResourcesTest.kt b/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/resources/PulsarResourcesTest.kt index 5f2ad3ad5..17c8c7bd3 100644 --- a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/resources/PulsarResourcesTest.kt +++ b/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/resources/PulsarResourcesTest.kt @@ -85,15 +85,15 @@ class PulsarResourcesTest : StringSpec( "should be able to init delayed topic even if I can not check tenant and namespace" { coEvery { - pulsarInfiniticAdmin.initTenantOnce(any(), any(), any()) + pulsarInfiniticAdmin.syncInitTenantOnce(any(), any(), any()) } returns Result.failure(mockk()) coEvery { - pulsarInfiniticAdmin.initNamespaceOnce(any(), any()) + pulsarInfiniticAdmin.syncInitNamespaceOnce(any(), any()) } returns Result.failure(mockk()) coEvery { - pulsarInfiniticAdmin.initTopicOnce(any(), any(), any(), any()) + pulsarInfiniticAdmin.syncInitTopicOnce(any(), any(), any()) } returns Result.success(mockk()) val topic = TestFactory.random() @@ -105,9 +105,9 @@ class PulsarResourcesTest : StringSpec( ).isSuccess shouldBe true coVerifyAll { - pulsarInfiniticAdmin.initTenantOnce("tenantTest", allowedClusters, adminRoles) - pulsarInfiniticAdmin.initNamespaceOnce("tenantTest/namespaceTest", policiesConfig) - pulsarInfiniticAdmin.initTopicOnce(topic, true, policiesConfig.timerTTLInSeconds) + pulsarInfiniticAdmin.syncInitTenantOnce("tenantTest", allowedClusters, adminRoles) + pulsarInfiniticAdmin.syncInitNamespaceOnce("tenantTest/namespaceTest", policiesConfig) + pulsarInfiniticAdmin.syncInitTopicOnce(topic, true, policiesConfig.timerTTLSeconds) } } @@ -120,7 +120,7 @@ class PulsarResourcesTest : StringSpec( coEvery { getTopicsFullName() } returns setOf(topic) } - mockResources.getWorkflowsName() shouldBe setOf(workflowName) + mockResources.getWorkflowNames() shouldBe setOf(workflowName) } } @@ -133,7 +133,7 @@ class PulsarResourcesTest : StringSpec( coEvery { getTopicsFullName() } returns setOf(topic) } - mockResources.getServicesName() shouldBe setOf(serviceName) + mockResources.getServiceNames() shouldBe setOf(serviceName) } } diff --git a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/InMemoryTransportConfig.kt b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/InMemoryTransportConfig.kt new file mode 100644 index 000000000..c58d62f85 --- /dev/null +++ b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/InMemoryTransportConfig.kt @@ -0,0 +1,51 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.transport.config + +import io.infinitic.inMemory.InMemoryChannels +import io.infinitic.inMemory.InMemoryInfiniticConsumer +import io.infinitic.inMemory.InMemoryInfiniticProducer +import io.infinitic.inMemory.InMemoryInfiniticResources + +data class InMemoryTransportConfig( + override val shutdownGracePeriodSeconds: Double = 5.0 +) : TransportConfig() { + + init { + require(shutdownGracePeriodSeconds > 0) { "shutdownGracePeriodSeconds must be > 0" } + } + + override val cloudEventSource: String = "inMemory" + + private val mainChannels = InMemoryChannels() + private val eventListenerChannels = InMemoryChannels() + + override val resources = InMemoryInfiniticResources(mainChannels) + override val consumer = InMemoryInfiniticConsumer(mainChannels, eventListenerChannels) + override val producer = InMemoryInfiniticProducer(mainChannels, eventListenerChannels) + + override fun close() { + // Do nothing + } +} + diff --git a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/PulsarTransportConfig.kt b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/PulsarTransportConfig.kt new file mode 100644 index 000000000..f62c55b4c --- /dev/null +++ b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/PulsarTransportConfig.kt @@ -0,0 +1,212 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.transport.config + +import com.sksamuel.hoplite.Secret +import io.infinitic.properties.isLazyInitialized +import io.infinitic.pulsar.PulsarInfiniticConsumer +import io.infinitic.pulsar.PulsarInfiniticProducer +import io.infinitic.pulsar.PulsarInfiniticResources +import io.infinitic.pulsar.client.PulsarInfiniticClient +import io.infinitic.pulsar.config.PulsarConfig +import io.infinitic.pulsar.config.TlsTrustStoreType +import io.infinitic.pulsar.config.auth.ClientAuthenticationConfig +import io.infinitic.pulsar.config.policies.PoliciesConfig +import io.infinitic.pulsar.consumers.Consumer +import io.infinitic.pulsar.consumers.ConsumerConfig +import io.infinitic.pulsar.producers.Producer +import io.infinitic.pulsar.producers.ProducerConfig +import io.infinitic.pulsar.resources.PulsarResources +import java.net.URLEncoder + +@Suppress("unused") +data class PulsarTransportConfig( + val pulsar: PulsarConfig, + override val shutdownGracePeriodSeconds: Double = 30.0 +) : TransportConfig() { + + init { + require(shutdownGracePeriodSeconds > 0) { "shutdownGracePeriodSeconds must be > 0" } + } + + override fun close() { + if (pulsar::admin.isLazyInitialized) pulsar.admin.close() + if (pulsar::client.isLazyInitialized) pulsar.client.close() + } + + /** This is used as source prefix for CloudEvents */ + override val cloudEventSource: String = pulsar.brokerServiceUrl.removeSuffix("/") + "/" + + URLEncoder.encode(pulsar.tenant, Charsets.UTF_8) + "/" + + URLEncoder.encode(pulsar.namespace, Charsets.UTF_8) + + + private val pulsarInfiniticClient by lazy { PulsarInfiniticClient(pulsar.client) } + private val pulsarResources by lazy { PulsarResources.from(pulsar) } + + /** Infinitic Resources */ + override val resources by lazy { + PulsarInfiniticResources(pulsarResources) + } + + /** Infinitic Consumer */ + override val consumer by lazy { + PulsarInfiniticConsumer( + Consumer(pulsarInfiniticClient, pulsar.consumer), + pulsarResources, + shutdownGracePeriodSeconds, + ) + } + + /** Infinitic Producer */ + override val producer by lazy { + PulsarInfiniticProducer( + Producer(pulsarInfiniticClient, pulsar.producer), + pulsarResources, + ) + } + + companion object { + @JvmStatic + fun builder() = PulsarTransportConfigBuilder() + } + + /** + * PulsarConfig builder + */ + class PulsarTransportConfigBuilder : TransportConfigBuilder { + private var shutdownGracePeriodSeconds: Double = 30.0 + private var brokerServiceUrl: String? = null + private var webServiceUrl: String? = null + private var tenant: String? = null + private var namespace: String? = null + private var allowedClusters: Set? = null + private var adminRoles: Set? = null + private var tlsAllowInsecureConnection: Boolean? = null + private var tlsEnableHostnameVerification: Boolean? = null + private var tlsTrustCertsFilePath: String? = null + private var useKeyStoreTls: Boolean? = null + private var tlsTrustStoreType: TlsTrustStoreType? = null + private var tlsTrustStorePath: String? = null + private var tlsTrustStorePassword: Secret? = null + private var authentication: ClientAuthenticationConfig? = null + private var policies: PoliciesConfig = PoliciesConfig() + private var producer: ProducerConfig = ProducerConfig() + private var consumer: ConsumerConfig = ConsumerConfig() + + fun setShutdownGracePeriodSeconds(shutdownGracePeriodSeconds: Double) = + apply { this.shutdownGracePeriodSeconds = shutdownGracePeriodSeconds } + + fun setTenant(tenant: String) = + apply { this.tenant = tenant } + + fun setNamespace(namespace: String) = + apply { this.namespace = namespace } + + fun setBrokerServiceUrl(brokerServiceUrl: String) = + apply { this.brokerServiceUrl = brokerServiceUrl } + + fun setWebServiceUrl(webServiceUrl: String) = + apply { this.webServiceUrl = webServiceUrl } + + fun setAllowedClusters(allowedClusters: Set) = + apply { this.allowedClusters = allowedClusters } + + fun setAdminRoles(adminRoles: Set) = + apply { this.adminRoles = adminRoles } + + fun setTlsAllowInsecureConnection(tlsAllowInsecureConnection: Boolean) = + apply { this.tlsAllowInsecureConnection = tlsAllowInsecureConnection } + + fun setTlsEnableHostnameVerification(tlsEnableHostnameVerification: Boolean) = + apply { this.tlsEnableHostnameVerification = tlsEnableHostnameVerification } + + fun setTlsTrustCertsFilePath(tlsTrustCertsFilePath: String) = + apply { this.tlsTrustCertsFilePath = tlsTrustCertsFilePath } + + fun setUseKeyStoreTls(useKeyStoreTls: Boolean) = + apply { this.useKeyStoreTls = useKeyStoreTls } + + fun setTlsTrustStoreType(tlsTrustStoreType: TlsTrustStoreType) = + apply { this.tlsTrustStoreType = tlsTrustStoreType } + + fun setTlsTrustStorePath(tlsTrustStorePath: String) = + apply { this.tlsTrustStorePath = tlsTrustStorePath } + + fun setTlsTrustStorePassword(tlsTrustStorePassword: Secret) = + apply { this.tlsTrustStorePassword = tlsTrustStorePassword } + + fun setAuthentication(authentication: ClientAuthenticationConfig) = + apply { this.authentication = authentication } + + fun setPolicies(policiesConfig: PoliciesConfig) = + apply { this.policies = policiesConfig } + + fun setPolicies(policiesConfig: PoliciesConfig.PoliciesConfigBuilder) = + setPolicies(policiesConfig.build()) + + fun setProducer(producerConfig: ProducerConfig) = + apply { this.producer = producerConfig } + + fun setProducer(producerConfig: ProducerConfig.ProducerConfigBuilder) = + setProducer(producerConfig.build()) + + fun setConsumer(consumerConfig: ConsumerConfig) = + apply { this.consumer = consumerConfig } + + fun setConsumer(consumerConfig: ConsumerConfig.ConsumerConfigBuilder) = + setConsumer(consumerConfig.build()) + + override fun build(): PulsarTransportConfig { + require(brokerServiceUrl != null) { "${PulsarConfig::brokerServiceUrl.name} must not be null" } + require(webServiceUrl != null) { "${PulsarConfig::webServiceUrl.name} must not be null" } + require(tenant != null) { "${PulsarConfig::tenant.name} must not be null" } + require(namespace != null) { "${PulsarConfig::namespace.name} must not be null" } + + val pulsarConfig = PulsarConfig( + brokerServiceUrl!!, + webServiceUrl!!, + tenant!!, + namespace!!, + allowedClusters, + adminRoles, + tlsAllowInsecureConnection, + tlsEnableHostnameVerification, + tlsTrustCertsFilePath, + useKeyStoreTls, + tlsTrustStoreType, + tlsTrustStorePath, + tlsTrustStorePassword, + authentication, + policies, + producer, + consumer, + ) + + return PulsarTransportConfig( + pulsar = pulsarConfig, + shutdownGracePeriodSeconds = shutdownGracePeriodSeconds, + ) + } + } +} + diff --git a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfig.kt b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfig.kt index f4c3367c5..4d4cfbc2a 100644 --- a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfig.kt +++ b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfig.kt @@ -22,86 +22,38 @@ */ package io.infinitic.transport.config -import io.infinitic.autoclose.addAutoCloseResource import io.infinitic.common.transport.InfiniticConsumer import io.infinitic.common.transport.InfiniticProducer -import io.infinitic.inMemory.InMemoryChannels -import io.infinitic.inMemory.InMemoryInfiniticConsumer -import io.infinitic.inMemory.InMemoryInfiniticProducer -import io.infinitic.pulsar.PulsarInfiniticConsumer -import io.infinitic.pulsar.PulsarInfiniticProducer -import io.infinitic.pulsar.client.PulsarInfiniticClient -import io.infinitic.pulsar.config.PulsarConfig -import io.infinitic.pulsar.consumers.Consumer -import io.infinitic.pulsar.producers.Producer -import io.infinitic.pulsar.resources.PulsarResources -import java.net.URLEncoder - -data class TransportConfig( - /** Transport configuration */ - override val transport: Transport, - - /** Pulsar configuration */ - override val pulsar: PulsarConfig?, - - /** Shutdown Grace Period */ - override val shutdownGracePeriodInSeconds: Double -) : TransportConfigInterface { - - init { - if (transport == Transport.pulsar) { - require(pulsar != null) { "Missing Pulsar configuration" } - } - - require(shutdownGracePeriodInSeconds >= 0) { "shutdownGracePeriodInSeconds must be >= 0" } - } - - /** This is used as source prefix for CloudEvents */ - val source: String = when (transport) { - Transport.pulsar -> pulsar!!.brokerServiceUrl.removeSuffix("/") + "/" + - URLEncoder.encode(pulsar.tenant, Charsets.UTF_8) + "/" + - URLEncoder.encode(pulsar.namespace, Charsets.UTF_8) - - Transport.inMemory -> "inmemory" +import io.infinitic.common.transport.InfiniticResources + +sealed class TransportConfig: AutoCloseable { + /** + * Specifies the duration, in seconds, allowed for the system to gracefully shut down. + * During this period, the system will attempt to complete handle ongoing messages + */ + abstract val shutdownGracePeriodSeconds: Double + + /** + * This property denotes the origin of the CloudEvents being generated or consumed. + */ + abstract val cloudEventSource: String + + /** + * This property provides methods to fetch available services and workflows, + */ + abstract val resources: InfiniticResources + + /** + * This consumer is responsible for subscribing to and handling messages + */ + abstract val consumer: InfiniticConsumer + + /** + * This producer is responsible for sending messages asynchronously to specified topics + */ + abstract val producer: InfiniticProducer + + interface TransportConfigBuilder { + fun build(): TransportConfig } - - // we provide consumer and producer together, - // as they must share the same configuration - private val cp: Pair = - when (transport) { - Transport.pulsar -> with(PulsarResources.from(pulsar!!)) { - val client = PulsarInfiniticClient(pulsar.client) - - val consumer = PulsarInfiniticConsumer( - Consumer(client, pulsar.consumer), - this, - shutdownGracePeriodInSeconds, - ) - // Pulsar client and admin will be closed with consumer - consumer.addAutoCloseResource(pulsar.client) - consumer.addAutoCloseResource(pulsar.admin) - - val producer = PulsarInfiniticProducer( - Producer(client, pulsar.producer), - this, - ) - - Pair(consumer, producer) - } - - Transport.inMemory -> { - val mainChannels = InMemoryChannels() - val eventListenerChannels = InMemoryChannels() - val consumer = InMemoryInfiniticConsumer(mainChannels, eventListenerChannels) - val producer = InMemoryInfiniticProducer(mainChannels, eventListenerChannels) - Pair(consumer, producer) - } - } - - /** Infinitic Consumer */ - val consumer = cp.first - - /** Infinitic Producer */ - val producer = cp.second } - diff --git a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfigInterface.kt b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfigInterface.kt index 5f25eb8b3..8803c27fd 100644 --- a/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfigInterface.kt +++ b/infinitic-transport/src/main/kotlin/io/infinitic/transport/config/TransportConfigInterface.kt @@ -22,15 +22,7 @@ */ package io.infinitic.transport.config -import io.infinitic.pulsar.config.PulsarConfig - interface TransportConfigInterface { /** Transport configuration */ - val transport: Transport - - /** Pulsar configuration */ - val pulsar: PulsarConfig? - - /** Shutdown Grace Period */ - val shutdownGracePeriodInSeconds: Double + val transport: TransportConfig } diff --git a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/config/PulsarConfigTests.kt b/infinitic-transport/src/test/kotlin/io/infinitic/transport/config/PulsarConfigTests.kt similarity index 67% rename from infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/config/PulsarConfigTests.kt rename to infinitic-transport/src/test/kotlin/io/infinitic/transport/config/PulsarConfigTests.kt index 756a8d312..5ed01d8f2 100644 --- a/infinitic-transport-pulsar/src/test/kotlin/io/infinitic/pulsar/config/PulsarConfigTests.kt +++ b/infinitic-transport/src/test/kotlin/io/infinitic/transport/config/PulsarConfigTests.kt @@ -21,9 +21,10 @@ * Licensor: infinitic.io */ -package io.infinitic.pulsar.config +package io.infinitic.transport.config import io.infinitic.common.fixtures.DockerOnly +import io.infinitic.pulsar.config.PulsarConfig import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.annotation.EnabledIf import io.kotest.core.spec.style.StringSpec @@ -39,43 +40,43 @@ class PulsarConfigTests : StringSpec( val dev = "dev" "Can create PulsarConfig through builder" { - val config = PulsarConfig.builder() - .brokerServiceUrl(brokerServiceUrl) - .webServiceUrl(webServiceUrl) - .tenant(tenant) - .namespace(dev) + val config = PulsarTransportConfig.builder() + .setBrokerServiceUrl(brokerServiceUrl) + .setWebServiceUrl(webServiceUrl) + .setTenant(tenant) + .setNamespace(dev) .build() - config shouldBe PulsarConfig(brokerServiceUrl, webServiceUrl, tenant, dev) + config.pulsar shouldBe PulsarConfig(brokerServiceUrl, webServiceUrl, tenant, dev) } "Create PulsarConfig without brokerServiceUrl should throw" { val e = shouldThrow { - PulsarConfig.builder() - .webServiceUrl(webServiceUrl) - .tenant(tenant) - .namespace(dev) + PulsarTransportConfig.builder() + .setWebServiceUrl(webServiceUrl) + .setTenant(tenant) + .setNamespace(dev) .build() } - e.message shouldContain "pulsar://localhost:6650" + e.message shouldContain "brokerServiceUrl" } "Create PulsarConfig without webServiceUrl should throw" { val e = shouldThrow { - PulsarConfig.builder() - .brokerServiceUrl(brokerServiceUrl) - .tenant(tenant) - .namespace(dev) + PulsarTransportConfig.builder() + .setBrokerServiceUrl(brokerServiceUrl) + .setTenant(tenant) + .setNamespace(dev) .build() } - e.message shouldContain "http://localhost:8080" + e.message shouldContain "webServiceUrl" } "Create PulsarConfig without tenant should throw" { val e = shouldThrow { - PulsarConfig.builder() - .brokerServiceUrl(brokerServiceUrl) - .webServiceUrl(webServiceUrl) - .namespace(dev) + PulsarTransportConfig.builder() + .setBrokerServiceUrl(brokerServiceUrl) + .setWebServiceUrl(webServiceUrl) + .setNamespace(dev) .build() } e.message shouldContain "tenant" @@ -83,10 +84,10 @@ class PulsarConfigTests : StringSpec( "Create PulsarConfig without namespace should throw" { val e = shouldThrow { - PulsarConfig.builder() - .brokerServiceUrl(brokerServiceUrl) - .webServiceUrl(webServiceUrl) - .tenant(tenant) + PulsarTransportConfig.builder() + .setBrokerServiceUrl(brokerServiceUrl) + .setWebServiceUrl(webServiceUrl) + .setTenant(tenant) .build() } e.message shouldContain "namespace" diff --git a/infinitic-utils/build.gradle.kts b/infinitic-utils/build.gradle.kts index 41b8b7d30..19a92f98a 100644 --- a/infinitic-utils/build.gradle.kts +++ b/infinitic-utils/build.gradle.kts @@ -21,4 +21,9 @@ * Licensor: infinitic.io */ +dependencies { + implementation(Libs.Hoplite.core) + implementation(Libs.Hoplite.yaml) +} + apply("../publish.gradle.kts") diff --git a/infinitic-utils/src/main/kotlin/io/infinitic/config/loaders.kt b/infinitic-utils/src/main/kotlin/io/infinitic/config/loaders.kt new file mode 100644 index 000000000..a3d32101d --- /dev/null +++ b/infinitic-utils/src/main/kotlin/io/infinitic/config/loaders.kt @@ -0,0 +1,76 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.config + +import com.sksamuel.hoplite.ConfigLoaderBuilder +import com.sksamuel.hoplite.PropertySource +import com.sksamuel.hoplite.yaml.YamlPropertySource +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.File + +val logger = KotlinLogging.logger("io.infinitic.common.config.loaders") + +/** + * Loads a configuration object of type [T] from the given list of resource names. + * + * @param resources The list of resource names to be loaded. + * @return The loaded configuration object. + */ +inline fun loadFromYamlResource(vararg resources: String): T { + val builder = ConfigLoaderBuilder.default() + resources.map { builder.addSource(PropertySource.resource(it, false)) } + val config = builder.build().loadConfigOrThrow() + logger.info { "Config loaded from resource: $config" } + + return config +} + +/** + * Loads a configuration of type [T] from the specified list of files. + * + * @param files The list of file paths to load the configuration from. + * @return The loaded configuration of type [T]. + */ +inline fun loadFromYamlFile(vararg files: String): T { + val builder = ConfigLoaderBuilder.default() + files.map { builder.addSource(PropertySource.file(File(it), false)) } + val config = builder.build().loadConfigOrThrow() + logger.info { "Config loaded from file: $config" } + + return config +} + +/** + * Loads a configuration of type [T] from a YAML string. + * + * @param yamls The YAML strings to load the configuration from. + * @return The loaded configuration object of type T. + */ +inline fun loadFromYamlString(vararg yamls: String): T { + val builder = ConfigLoaderBuilder.default() + yamls.map { builder.addSource(YamlPropertySource(it)) } + val config = builder.build().loadConfigOrThrow() + logger.info { "Config loaded from yaml: $config" } + + return config +} diff --git a/infinitic-task-tag/src/main/kotlin/io/infinitic/tasks/tag/config/ServiceTagEngineConfig.kt b/infinitic-utils/src/main/kotlin/io/infinitic/properties/properties.kt similarity index 74% rename from infinitic-task-tag/src/main/kotlin/io/infinitic/tasks/tag/config/ServiceTagEngineConfig.kt rename to infinitic-utils/src/main/kotlin/io/infinitic/properties/properties.kt index 86f5115c8..cd43bf447 100644 --- a/infinitic-task-tag/src/main/kotlin/io/infinitic/tasks/tag/config/ServiceTagEngineConfig.kt +++ b/infinitic-utils/src/main/kotlin/io/infinitic/properties/properties.kt @@ -20,19 +20,15 @@ * * Licensor: infinitic.io */ -package io.infinitic.tasks.tag.config -import io.infinitic.storage.config.StorageConfig +package io.infinitic.properties -data class ServiceTagEngineConfig( - var concurrency: Int? = null, - var storage: StorageConfig? = null, -) { - var isDefault: Boolean = false +import kotlin.reflect.KProperty0 +import kotlin.reflect.jvm.isAccessible - init { - concurrency?.let { - require(it >= 0) { "concurrency must be positive" } - } - } -} +/** + * This returns `true` if property is lazy and has been initialized, + * otherwise it returns `false`. + */ +val KProperty0<*>.isLazyInitialized + get() = (apply { isAccessible = true }.getDelegate() as? Lazy<*>)?.isInitialized() ?: false diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorker.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorker.kt index 199f543fb..3e297833b 100644 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorker.kt +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorker.kt @@ -25,26 +25,19 @@ package io.infinitic.workers import io.cloudevents.CloudEvent import io.github.oshai.kotlinlogging.KLogger import io.github.oshai.kotlinlogging.KotlinLogging -import io.infinitic.autoclose.addAutoCloseResource -import io.infinitic.autoclose.autoClose import io.infinitic.clients.InfiniticClient +import io.infinitic.cloudEvents.logs.CLOUD_EVENTS_SERVICE_EXECUTOR +import io.infinitic.cloudEvents.logs.CLOUD_EVENTS_SERVICE_TAG_ENGINE +import io.infinitic.cloudEvents.logs.CLOUD_EVENTS_WORKFLOW_EXECUTOR +import io.infinitic.cloudEvents.logs.CLOUD_EVENTS_WORKFLOW_STATE_ENGINE +import io.infinitic.cloudEvents.logs.CLOUD_EVENTS_WORKFLOW_TAG_ENGINE import io.infinitic.common.data.MillisInstant -import io.infinitic.common.logs.SERVICE_EXECUTOR_CLOUD_EVENTS -import io.infinitic.common.logs.SERVICE_TAG_ENGINE_CLOUD_EVENTS -import io.infinitic.common.logs.WORKFLOW_EXECUTOR_CLOUD_EVENTS -import io.infinitic.common.logs.WORKFLOW_STATE_ENGINE_CLOUD_EVENTS -import io.infinitic.common.logs.WORKFLOW_TAG_ENGINE_CLOUD_EVENTS import io.infinitic.common.messages.Message import io.infinitic.common.tasks.data.ServiceName import io.infinitic.common.tasks.events.messages.ServiceExecutorEventMessage import io.infinitic.common.tasks.executors.messages.ExecuteTask import io.infinitic.common.tasks.executors.messages.ServiceExecutorMessage import io.infinitic.common.tasks.tags.messages.ServiceTagMessage -import io.infinitic.common.tasks.tags.storage.TaskTagStorage -import io.infinitic.common.transport.InfiniticConsumer -import io.infinitic.common.transport.InfiniticProducer -import io.infinitic.common.transport.LoggedInfiniticConsumer -import io.infinitic.common.transport.LoggedInfiniticProducer import io.infinitic.common.transport.MainSubscription import io.infinitic.common.transport.RetryServiceExecutorTopic import io.infinitic.common.transport.RetryWorkflowExecutorTopic @@ -60,29 +53,38 @@ import io.infinitic.common.transport.WorkflowStateEventTopic import io.infinitic.common.transport.WorkflowStateTimerTopic import io.infinitic.common.transport.WorkflowTagEngineTopic import io.infinitic.common.transport.create +import io.infinitic.common.transport.logged.LoggedInfiniticConsumer +import io.infinitic.common.transport.logged.LoggedInfiniticProducer import io.infinitic.common.workflows.data.workflows.WorkflowName +import io.infinitic.common.workflows.emptyWorkflowContext import io.infinitic.common.workflows.engine.messages.WorkflowCmdMessage import io.infinitic.common.workflows.engine.messages.WorkflowEventMessage import io.infinitic.common.workflows.engine.messages.WorkflowStateEngineMessage -import io.infinitic.common.workflows.engine.storage.WorkflowStateStorage import io.infinitic.common.workflows.tags.messages.WorkflowTagEngineMessage -import io.infinitic.common.workflows.tags.storage.WorkflowTagStorage import io.infinitic.events.toJsonString import io.infinitic.events.toServiceCloudEvent import io.infinitic.events.toWorkflowCloudEvent import io.infinitic.logger.ignoreNull -import io.infinitic.pulsar.PulsarInfiniticConsumer import io.infinitic.tasks.Task +import io.infinitic.tasks.WithTimeout import io.infinitic.tasks.executor.TaskEventHandler import io.infinitic.tasks.executor.TaskExecutor import io.infinitic.tasks.executor.TaskRetryHandler import io.infinitic.tasks.tag.TaskTagEngine import io.infinitic.tasks.tag.storage.LoggedTaskTagStorage -import io.infinitic.transport.config.TransportConfig -import io.infinitic.workers.config.WorkerConfig -import io.infinitic.workers.config.WorkerConfigInterface -import io.infinitic.workers.register.InfiniticRegister -import io.infinitic.workers.register.InfiniticRegisterImpl +import io.infinitic.workers.config.ConfigGetterInterface +import io.infinitic.workers.config.EventListenerConfig +import io.infinitic.workers.config.InfiniticWorkerConfig +import io.infinitic.workers.config.InfiniticWorkerConfigInterface +import io.infinitic.workers.config.ServiceConfig +import io.infinitic.workers.config.ServiceExecutorConfig +import io.infinitic.workers.config.ServiceTagEngineConfig +import io.infinitic.workers.config.WorkflowConfig +import io.infinitic.workers.config.WorkflowExecutorConfig +import io.infinitic.workers.config.WorkflowStateEngineConfig +import io.infinitic.workers.config.WorkflowTagEngineConfig +import io.infinitic.workers.registry.ExecutorRegistry +import io.infinitic.workflows.Workflow import io.infinitic.workflows.engine.WorkflowStateCmdHandler import io.infinitic.workflows.engine.WorkflowStateEngine import io.infinitic.workflows.engine.WorkflowStateEventHandler @@ -92,104 +94,159 @@ import io.infinitic.workflows.tag.WorkflowTagEngine import io.infinitic.workflows.tag.storage.LoggedWorkflowTagStorage import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.future.future +import kotlinx.coroutines.job import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout import java.util.concurrent.CompletableFuture +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.cancellation.CancellationException import kotlin.system.exitProcess @Suppress("unused") -class InfiniticWorker private constructor( - val register: InfiniticRegister, - val consumer: InfiniticConsumer, - val producer: InfiniticProducer, - private val source: String -) : AutoCloseable, InfiniticRegister by register { +class InfiniticWorker( + val config: InfiniticWorkerConfigInterface, +) : AutoCloseable, ConfigGetterInterface { - /** Infinitic Client */ - val client = InfiniticClient(consumer, producer) + private val registry = ExecutorRegistry(config.services, config.workflows) - init { - consumer.workerLogger = logger - - Runtime.getRuntime().addShutdownHook( - Thread { - logger.info { "Closing worker..." } - close() - logger.info { "Worker closed." } - }, - ) - } + private fun getService(serviceName: String): ServiceConfig? = + config.services.firstOrNull { it.name == serviceName } - companion object { - private val logger = KotlinLogging.logger {} + private fun getWorkflow(workflowName: String): WorkflowConfig? = + config.workflows.firstOrNull { it.name == workflowName } - /** Create [InfiniticWorker] from config */ - @JvmStatic - fun fromConfig(workerConfig: WorkerConfigInterface): InfiniticWorker = with(workerConfig) { + override fun getEventListenerConfig() = + config.eventListener - val transportConfig = TransportConfig(transport, pulsar, shutdownGracePeriodInSeconds) + override fun getServiceExecutorConfigs() = + config.services.mapNotNull { it.executor } - /** Infinitic Consumer */ - val consumer = transportConfig.consumer + override fun getServiceTagEngineConfigs() = + config.services.mapNotNull { it.tagEngine } - /** Infinitic Producer */ - val producer = transportConfig.producer + override fun getWorkflowExecutorConfigs() = + config.workflows.mapNotNull { it.executor } - // set name, if it exists in the worker configuration - name?.let { producer.name = it } + override fun getWorkflowTagEngineConfigs() = + config.workflows.mapNotNull { it.tagEngine } - /** Infinitic Register */ - // if an exception is thrown, we ensure to close the previously created resource - val register = try { - InfiniticRegisterImpl.fromConfig(this) - } catch (e: Exception) { - consumer.close() - throw e - } + override fun getWorkflowStateEngineConfigs() = + config.workflows.mapNotNull { it.stateEngine } + + override fun getServiceExecutorConfig(serviceName: String) = + getService(serviceName)?.executor + + override fun getServiceTagEngineConfig(serviceName: String) = + getService(serviceName)?.tagEngine + + override fun getWorkflowExecutorConfig(workflowName: String) = + getWorkflow(workflowName)?.executor + + override fun getWorkflowTagEngineConfig(workflowName: String) = + getWorkflow(workflowName)?.tagEngine + + override fun getWorkflowStateEngineConfig(workflowName: String) = + getWorkflow(workflowName)?.stateEngine + + private val resources by lazy { + config.transport.resources + } + private val consumer by lazy { + config.transport.consumer + } + private val producer by lazy { + config.transport.producer.apply { config.name?.let { name = it } } + } + + private val shutdownGracePeriodSeconds = config.transport.shutdownGracePeriodSeconds + private val source = config.transport.cloudEventSource + private val beautifyLogs = config.logs.beautify + + /** + * Indicates whether the InfiniticWorker instance is started. + * + * This variable is used to manage the state of the worker, ensuring that it can + * be safely closed and that no operations occur once the worker is marked as closed. + * When set to `false`, the worker is closed or not yet started; + * when set to `true`, the worker is operational. + */ + private var isStarted: AtomicBoolean = AtomicBoolean(false) + + private lateinit var completableStart: CompletableFuture + + /** Coroutine scope used to launch consumers and await their termination */ + private var scope = CoroutineScope(Dispatchers.IO) + + /** Infinitic Client */ + val client = InfiniticClient(config) - /** Infinitic Worker */ - InfiniticWorker(register, consumer, producer, transportConfig.source).also { - // close consumer with the worker - it.addAutoCloseResource(consumer) - // close storages with the worker - it.addAutoCloseResource(register) + init { + Runtime.getRuntime().addShutdownHook(Thread { close() }) + } + + override fun close() { + if (isStarted.compareAndSet(true, false)) runBlocking { + logger.info { "Closing worker..." } + try { + scope.cancel() + logger.info { "Processing ongoing messages..." } + withTimeout((shutdownGracePeriodSeconds * 1000).toLong()) { + scope.coroutineContext.job.join() + logger.info { "All ongoing messages have been processed." } + } + } catch (e: TimeoutCancellationException) { + logger.warn { + "The grace period (${shutdownGracePeriodSeconds}s) allotted when closing the worker was insufficient. " + + "Some ongoing messages may not have been processed properly." + } + } finally { + client.close() } + logger.info { "Worker closed." } } + } + + + companion object { + private val logger = KotlinLogging.logger {} + private const val NONE = "none" + + @JvmStatic + fun builder() = InfiniticWorkerBuilder() - /** Create [InfiniticWorker] from config in resources */ + /** Create [InfiniticWorker] from yaml resources */ @JvmStatic - fun fromConfigResource(vararg resources: String): InfiniticWorker = - fromConfig(WorkerConfig.fromResource(*resources)) + fun fromYamlResource(vararg resources: String) = + InfiniticWorker(InfiniticWorkerConfig.fromYamlResource(*resources)) - /** Create [InfiniticWorker] from config in system file */ + /** Create [InfiniticWorker] from yaml files */ @JvmStatic - fun fromConfigFile(vararg files: String): InfiniticWorker = - fromConfig(WorkerConfig.fromFile(*files)) + fun fromYamlFile(vararg files: String): InfiniticWorker = + InfiniticWorker(InfiniticWorkerConfig.fromYamlFile(*files)) /** Create [InfiniticWorker] from yaml strings */ @JvmStatic - fun fromConfigYaml(vararg yamls: String): InfiniticWorker = - fromConfig(WorkerConfig.fromYaml(*yamls)) + fun fromYamlString(vararg yamls: String): InfiniticWorker = + InfiniticWorker(InfiniticWorkerConfig.fromYamlString(*yamls)) } - private val workerRegistry = register.registry - private val sendingMessageToDLQ = { "Unable to process message, sending to Dead Letter Queue" } - override fun close() { - client.close() - autoClose() - } - - /** * Start worker synchronously * (blocks the current thread) */ fun start(): Unit = try { - startAsync().get() + startAsync().join() + } catch (e: CancellationException) { + // do nothing, the worker has been closed } catch (e: Throwable) { - logger.error(e) { "Exiting" } + logger.error(e) { "Error: exiting" } // this will trigger the shutdown hook exitProcess(1) } @@ -198,114 +255,233 @@ class InfiniticWorker private constructor( * Start worker asynchronously */ fun startAsync(): CompletableFuture { - runBlocking(Dispatchers.IO) { - - workerRegistry.workflowTagEngines.forEach { (workflowName, registeredWorkflowTag) -> - // WORKFLOW TAG ENGINE - startWorkflowTagEngine( - workflowName, - registeredWorkflowTag.concurrency, - registeredWorkflowTag.storage, - ) - } + if (isStarted.compareAndSet(false, true)) { + scope = CoroutineScope(Dispatchers.IO) - workerRegistry.workflowStateEngines.forEach { (workflowName, registeredWorkflowEngine) -> - // WORKFLOW STATE ENGINE - startWorkflowStateEngine( - workflowName, - registeredWorkflowEngine.concurrency, - registeredWorkflowEngine.storage, - ) - } + completableStart = scope.future { - workerRegistry.workflowExecutors.forEach { (workflowName, registeredWorkflowExecutor) -> - // WORKFLOW EXECUTOR - startWorkflowExecutor( - workflowName, - registeredWorkflowExecutor.concurrency, - ) - } - - workerRegistry.serviceTagEngines.forEach { (serviceName, registeredServiceTag) -> - // SERVICE TAG ENGINE - startTaskTagEngine( - serviceName, - registeredServiceTag.concurrency, - registeredServiceTag.storage, - ) - } + getServiceTagEngineConfigs().forEach { startServiceTagEngine(it) } - workerRegistry.serviceExecutors.forEach { (serviceName, registeredEventListener) -> - // SERVICE EXECUTOR - startServiceExecutor( - serviceName, - registeredEventListener.concurrency, - ) - } - - workerRegistry.workflowEventListeners.forEach { (workflowName, registeredEventListener) -> - // WORKFLOW TASK EVENT LISTENER - startWorkflowExecutorEventListener( - workflowName, - registeredEventListener.concurrency, - registeredEventListener.subscriptionName, - SubscriptionType.EVENT_LISTENER, - ) { message, publishedAt -> - message.toServiceCloudEvent(publishedAt, source)?.let { - registeredEventListener.eventListener.onEvent(it) - } ?: Unit + config.services.forEach { serviceConfig -> + logger.info { "Service ${serviceConfig.name}:" } + // Start SERVICE TAG ENGINE + serviceConfig.tagEngine?.let { startServiceTagEngine(it) } + // Start SERVICE EXECUTOR + serviceConfig.executor?.let { startServiceExecutor(it) } } - // WORKFLOW EVENT LISTENER - startWorkflowStateEventListener( - workflowName, - registeredEventListener.concurrency, - registeredEventListener.subscriptionName, - SubscriptionType.EVENT_LISTENER, - ) { message, publishedAt -> - message.toWorkflowCloudEvent(publishedAt, source)?.let { - registeredEventListener.eventListener.onEvent(it) - } ?: Unit + config.workflows.forEach { workflowConfig -> + logger.info { "Workflow ${workflowConfig.name}:" } + // Start WORKFLOW TAG ENGINE + workflowConfig.tagEngine?.let { startWorkflowTagEngine(it) } + // Start WORKFLOW STATE ENGINE + workflowConfig.stateEngine?.let { startWorkflowStateEngine(it) } + // Start WORKFLOW EXECUTOR + workflowConfig.executor?.let { startWorkflowExecutor(it) } } - } - workerRegistry.serviceEventListeners.forEach { (serviceName, registeredEventListener) -> - // SERVICE EVENT LISTENER - startServiceEventListener( - serviceName, - registeredEventListener.concurrency, - registeredEventListener.subscriptionName, - SubscriptionType.EVENT_LISTENER, - ) { message: Message, publishedAt: MillisInstant -> - message.toServiceCloudEvent(publishedAt, source)?.let { - registeredEventListener.eventListener.onEvent(it) - } ?: Unit + config.eventListener?.let { startEventListener(it) } + + logger.info { + "Worker \"${producer.name}\" ready (shutdownGracePeriodSeconds=${shutdownGracePeriodSeconds}s)" } } } + return completableStart + } + + private fun WithTimeout?.toLog() = + this?.getTimeoutSeconds()?.let { String.format("%.2fs", it) } ?: NONE + + private fun logEventListenerStart(config: EventListenerConfig) { + logger.info { + "* Service Event Listener".padEnd(25) + ": (" + + "concurrency: ${config.concurrency}, " + + "class: ${config.listener::class.java.name}" + + (config.subscriptionName?.let { ", subscription: $it" } ?: "") + + ")" + } + } + + private fun logServiceExecutorStart(config: ServiceExecutorConfig) { + logger.info { + "* Service Executor".padEnd(25) + ": (" + + "concurrency: ${config.concurrency}, " + + "class: ${config.factory()::class.java.name}, " + + "timeout: ${config.withTimeout?.toLog()}, " + + "withRetry: ${config.withRetry ?: NONE})" + } + } + + private fun logServiceTagEngineStart(config: ServiceTagEngineConfig) { + logger.info { + "* Service Tag Engine".padEnd(25) + ": (" + + "concurrency: ${config.concurrency}, " + + "storage: ${config.storage?.type}, " + + "cache: ${config.storage?.cache?.type ?: NONE}, " + + "compression: ${config.storage?.compression ?: NONE})" + } + } + private fun logWorkflowExecutorStart(config: WorkflowExecutorConfig) { + Workflow.setContext(emptyWorkflowContext) logger.info { - "Worker \"${producer.name}\" ready" + when (consumer is PulsarInfiniticConsumer) { - true -> " (shutdownGracePeriodInSeconds=${consumer.shutdownGracePeriodInSeconds}s)" - false -> "" + "* Workflow Executor".padEnd(25) + ": (" + + "concurrency: ${config.concurrency}, " + + "classes: ${ + config.factories.map { it.invoke()::class.java }.joinToString { it.name } + }, " + + "timeout: ${config.withTimeout?.toLog()}, " + + "withRetry: ${config.withRetry ?: NONE}" + + (config.checkMode?.let { ", checkMode: $it" } ?: "") + + ")" + } + } + + private fun logWorkflowTagEngineStart(config: WorkflowTagEngineConfig) { + logger.info { + "* Workflow Tag Engine".padEnd(25) + ": (" + + "concurrency: ${config.concurrency}, " + + "storage: ${config.storage?.type}, " + + "cache: ${config.storage?.cache?.type ?: NONE}, " + + "compression: ${config.storage?.compression ?: NONE})" + } + } + + private fun logWorkflowStateEngineStart(config: WorkflowStateEngineConfig) { + logger.info { + "* Workflow State Engine".padEnd(25) + ": (" + + "concurrency: ${config.concurrency}, " + + "storage: ${config.storage?.type}, " + + "cache: ${config.storage?.cache?.type ?: NONE}, " + + "compression: ${config.storage?.compression ?: NONE})" + } + } + + private fun CoroutineScope.startServiceTagEngine(config: ServiceTagEngineConfig) { + // Log Service Tag Engine configuration + logServiceTagEngineStart(config) + + val logsEventLogger = KotlinLogging.logger( + "$CLOUD_EVENTS_SERVICE_TAG_ENGINE.${config.serviceName}", + ).ignoreNull() + + // TASK-TAG + launch { + val logger = TaskTagEngine.logger + val loggedStorage = LoggedTaskTagStorage(logger, config.serviceTagStorage) + val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) + val loggedProducer = LoggedInfiniticProducer(logger, producer) + + val taskTagEngine = TaskTagEngine(loggedStorage, loggedProducer) + + val handler: suspend (ServiceTagMessage, MillisInstant) -> Unit = + { message, publishedAt -> + logsEventLogger.logServiceCloudEvent(message, publishedAt, source) + taskTagEngine.handle(message, publishedAt) + } + + loggedConsumer.start( + subscription = MainSubscription(ServiceTagEngineTopic), + entity = config.serviceName, + handler = handler, + beforeDlq = null, + concurrency = config.concurrency, + ) + } + } + + private fun CoroutineScope.startServiceExecutor(config: ServiceExecutorConfig) { + // Log Service Executor configuration + logServiceExecutorStart(config) + + val logsEventLogger = KotlinLogging.logger( + "$CLOUD_EVENTS_SERVICE_EXECUTOR.${config.serviceName}", + ).ignoreNull() + + // TASK-EXECUTOR + launch { + val logger = TaskExecutor.logger + val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) + val loggedProducer = LoggedInfiniticProducer(logger, producer) + val taskExecutor = TaskExecutor(registry, loggedProducer, client) + + val handler: suspend (ServiceExecutorMessage, MillisInstant) -> Unit = + { message, publishedAt -> + logsEventLogger.logServiceCloudEvent(message, publishedAt, source) + taskExecutor.handle(message, publishedAt) + } + + val beforeDlq: suspend (ServiceExecutorMessage?, Exception) -> Unit = { message, cause -> + when (message) { + null -> Unit + is ExecuteTask -> with(taskExecutor) { + message.sendTaskFailed(cause, Task.meta, sendingMessageToDLQ) + } + } } + + loggedConsumer.start( + subscription = MainSubscription(ServiceExecutorTopic), + entity = config.serviceName, + handler = handler, + beforeDlq = beforeDlq, + concurrency = config.concurrency, + ) + } + + // TASK-EXECUTOR-DELAY + launch { + val logger = TaskRetryHandler.logger + val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) + val loggedProducer = LoggedInfiniticProducer(logger, producer) + + val taskRetryHandler = TaskRetryHandler(loggedProducer) + + loggedConsumer.start( + subscription = MainSubscription(RetryServiceExecutorTopic), + entity = config.serviceName, + handler = taskRetryHandler::handle, + beforeDlq = null, + concurrency = config.concurrency, + ) } - return CompletableFuture.supplyAsync { consumer.join() } + // TASK-EVENTS + launch { + val logger = TaskEventHandler.logger + val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) + val loggedProducer = LoggedInfiniticProducer(logger, producer) + val taskEventHandler = TaskEventHandler(loggedProducer) + + val handler: suspend (ServiceExecutorEventMessage, MillisInstant) -> Unit = + { message, publishedAt -> + logsEventLogger.logServiceCloudEvent(message, publishedAt, source) + taskEventHandler.handle(message, publishedAt) + } + + loggedConsumer.start( + subscription = MainSubscription(ServiceExecutorEventTopic), + entity = config.serviceName, + handler = handler, + beforeDlq = null, + concurrency = config.concurrency, + ) + } } - private fun CoroutineScope.startWorkflowTagEngine( - workflowName: WorkflowName, - concurrency: Int, - storage: WorkflowTagStorage - ) { - val eventLogger = KotlinLogging.logger("$WORKFLOW_TAG_ENGINE_CLOUD_EVENTS.$workflowName") - .ignoreNull() + private fun CoroutineScope.startWorkflowTagEngine(config: WorkflowTagEngineConfig) { + // Log Workflow State Engine configuration + logWorkflowTagEngineStart(config) + + val logsEventLogger = KotlinLogging.logger( + "$CLOUD_EVENTS_WORKFLOW_TAG_ENGINE.${config.workflowName}", + ).ignoreNull() // WORKFLOW-TAG launch { val logger = WorkflowTagEngine.logger - val loggedStorage = LoggedWorkflowTagStorage(logger, storage) + val loggedStorage = LoggedWorkflowTagStorage(logger, config.workflowTagStorage) val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) val loggedProducer = LoggedInfiniticProducer(logger, producer) @@ -313,27 +489,28 @@ class InfiniticWorker private constructor( val handler: suspend (WorkflowTagEngineMessage, MillisInstant) -> Unit = { message, publishedAt -> - eventLogger.logWorkflowCloudEvent(message, publishedAt, source) + logsEventLogger.logWorkflowCloudEvent(message, publishedAt, source) workflowTagEngine.handle(message, publishedAt) } loggedConsumer.start( subscription = MainSubscription(WorkflowTagEngineTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = handler, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } } - private fun CoroutineScope.startWorkflowStateEngine( - workflowName: WorkflowName, - concurrency: Int, - storage: WorkflowStateStorage - ) { - val eventLogger = KotlinLogging.logger("$WORKFLOW_STATE_ENGINE_CLOUD_EVENTS.$workflowName") - .ignoreNull() + private fun CoroutineScope.startWorkflowStateEngine(config: WorkflowStateEngineConfig) { + // Log Workflow State Engine configuration + logWorkflowStateEngineStart(config) + + val logsEventLogger = KotlinLogging.logger( + "$CLOUD_EVENTS_WORKFLOW_STATE_ENGINE.${config.workflowName}", + ).ignoreNull() + // WORKFLOW-CMD launch { val logger = WorkflowStateCmdHandler.logger @@ -344,23 +521,23 @@ class InfiniticWorker private constructor( val handler: suspend (WorkflowStateEngineMessage, MillisInstant) -> Unit = { message, publishedAt -> - eventLogger.logWorkflowCloudEvent(message, publishedAt, source) + logsEventLogger.logWorkflowCloudEvent(message, publishedAt, source) workflowStateCmdHandler.handle(message, publishedAt) } loggedConsumer.start( subscription = MainSubscription(WorkflowStateCmdTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = handler, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } // WORKFLOW-STATE-ENGINE launch { val logger = WorkflowStateEngine.logger - val loggedStorage = LoggedWorkflowStateStorage(logger, storage) + val loggedStorage = LoggedWorkflowStateStorage(logger, config.workflowStateStorage) val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) val loggedProducer = LoggedInfiniticProducer(logger, producer) @@ -369,17 +546,17 @@ class InfiniticWorker private constructor( val handler: suspend (WorkflowStateEngineMessage, MillisInstant) -> Unit = { message, publishedAt -> if (message !is WorkflowCmdMessage) { - eventLogger.logWorkflowCloudEvent(message, publishedAt, source) + logsEventLogger.logWorkflowCloudEvent(message, publishedAt, source) } workflowStateEngine.handle(message, publishedAt) } loggedConsumer.start( subscription = MainSubscription(WorkflowStateEngineTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = handler, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } @@ -394,10 +571,10 @@ class InfiniticWorker private constructor( // we do not use loggedConsumer to avoid logging twice the messages coming from delayed topics loggedConsumer.start( subscription = MainSubscription(WorkflowStateTimerTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = workflowStateTimerHandler::handle, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } @@ -411,26 +588,27 @@ class InfiniticWorker private constructor( val handler: suspend (WorkflowEventMessage, MillisInstant) -> Unit = { message, publishedAt -> - eventLogger.logWorkflowCloudEvent(message, publishedAt, source) + logsEventLogger.logWorkflowCloudEvent(message, publishedAt, source) workflowStateEventHandler.handle(message, publishedAt) } loggedConsumer.start( subscription = MainSubscription(WorkflowStateEventTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = handler, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } } - private fun CoroutineScope.startWorkflowExecutor( - workflowName: WorkflowName, - concurrency: Int - ) { - val eventLogger = KotlinLogging.logger("$WORKFLOW_EXECUTOR_CLOUD_EVENTS.$workflowName") - .ignoreNull() + private fun CoroutineScope.startWorkflowExecutor(config: WorkflowExecutorConfig) { + // Log Workflow Executor configuration + logWorkflowExecutorStart(config) + + val logsEventLogger = KotlinLogging.logger( + "$CLOUD_EVENTS_WORKFLOW_EXECUTOR.${config.workflowName}", + ).ignoreNull() // WORKFLOW-TASK_EXECUTOR launch { @@ -438,11 +616,11 @@ class InfiniticWorker private constructor( val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) val loggedProducer = LoggedInfiniticProducer(logger, producer) - val workflowTaskExecutor = TaskExecutor(workerRegistry, loggedProducer, client) + val workflowTaskExecutor = TaskExecutor(registry, loggedProducer, client) val handler: suspend (ServiceExecutorMessage, MillisInstant) -> Unit = { message, publishedAt -> - eventLogger.logServiceCloudEvent(message, publishedAt, source) + logsEventLogger.logServiceCloudEvent(message, publishedAt, source) workflowTaskExecutor.handle(message, publishedAt) } @@ -457,10 +635,10 @@ class InfiniticWorker private constructor( loggedConsumer.start( subscription = MainSubscription(WorkflowExecutorTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = handler, beforeDlq = beforeDlq, - concurrency = concurrency, + concurrency = config.concurrency, ) } @@ -474,10 +652,10 @@ class InfiniticWorker private constructor( // we do not use loggedConsumer to avoid logging twice the messages coming from delayed topics loggedConsumer.start( subscription = MainSubscription(RetryWorkflowExecutorTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = taskRetryHandler::handle, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } @@ -491,128 +669,112 @@ class InfiniticWorker private constructor( val handler: suspend (ServiceExecutorEventMessage, MillisInstant) -> Unit = { message, publishedAt -> - eventLogger.logServiceCloudEvent(message, publishedAt, source) + logsEventLogger.logServiceCloudEvent(message, publishedAt, source) workflowTaskEventHandler.handle(message, publishedAt) } loggedConsumer.start( subscription = MainSubscription(WorkflowExecutorEventTopic), - entity = workflowName.toString(), + entity = config.workflowName, handler = handler, beforeDlq = null, - concurrency = concurrency, + concurrency = config.concurrency, ) } } - private fun CoroutineScope.startTaskTagEngine( - serviceName: ServiceName, - concurrency: Int, - storage: TaskTagStorage - ) { - val eventLogger = KotlinLogging.logger("$SERVICE_TAG_ENGINE_CLOUD_EVENTS.$serviceName") - .ignoreNull() + private fun CoroutineScope.startEventListener(config: EventListenerConfig) { + logEventListenerStart(config) - // TASK-TAG launch { - val logger = TaskTagEngine.logger - val loggedStorage = LoggedTaskTagStorage(logger, storage) - val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) - val loggedProducer = LoggedInfiniticProducer(logger, producer) - - val taskTagEngine = TaskTagEngine(loggedStorage, loggedProducer) + checkNewServices(config) { serviceName -> + logger.info { "EventListener starting listening Service $serviceName" } - val handler: suspend (ServiceTagMessage, MillisInstant) -> Unit = - { message, publishedAt -> - eventLogger.logServiceCloudEvent(message, publishedAt, source) - taskTagEngine.handle(message, publishedAt) + startServiceEventListener( + ServiceName(serviceName), + config.concurrency, + config.subscriptionName, + SubscriptionType.EVENT_LISTENER, + ) { message: Message, publishedAt: MillisInstant -> + message.toServiceCloudEvent(publishedAt, source)?.let { cloudEvent -> + config.listener.onEvent(cloudEvent) } - - loggedConsumer.start( - subscription = MainSubscription(ServiceTagEngineTopic), - entity = serviceName.toString(), - handler = handler, - beforeDlq = null, - concurrency = concurrency, - ) + } + } } - } - private fun CoroutineScope.startServiceExecutor( - serviceName: ServiceName, - concurrency: Int - ) { - val eventLogger = KotlinLogging.logger("$SERVICE_EXECUTOR_CLOUD_EVENTS.$serviceName") - .ignoreNull() - - // TASK-EXECUTOR launch { - val logger = TaskExecutor.logger - val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) - val loggedProducer = LoggedInfiniticProducer(logger, producer) - val taskExecutor = TaskExecutor(workerRegistry, loggedProducer, client) - - val handler: suspend (ServiceExecutorMessage, MillisInstant) -> Unit = - { message, publishedAt -> - eventLogger.logServiceCloudEvent(message, publishedAt, source) - taskExecutor.handle(message, publishedAt) + checkNewWorkflows(config) { workflowName -> + logger.info { "EventListener starting listening Workflow $workflowName" } + startWorkflowExecutorEventListener( + WorkflowName(workflowName), + config.concurrency, + config.subscriptionName, + SubscriptionType.EVENT_LISTENER, + ) { message, publishedAt -> + message.toServiceCloudEvent(publishedAt, source)?.let { cloudEvent -> + config.listener.onEvent(cloudEvent) } - - val beforeDlq: suspend (ServiceExecutorMessage?, Exception) -> Unit = { message, cause -> - when (message) { - null -> Unit - is ExecuteTask -> with(taskExecutor) { - message.sendTaskFailed(cause, Task.meta, sendingMessageToDLQ) + } + startWorkflowStateEventListener( + WorkflowName(workflowName), + config.concurrency, + config.subscriptionName, + SubscriptionType.EVENT_LISTENER, + ) { message, publishedAt -> + message.toWorkflowCloudEvent(publishedAt, source)?.let { cloudEvent -> + config.listener.onEvent(cloudEvent) } } } - - loggedConsumer.start( - subscription = MainSubscription(ServiceExecutorTopic), - entity = serviceName.toString(), - handler = handler, - beforeDlq = beforeDlq, - concurrency = concurrency, - ) } + } - // TASK-EXECUTOR-DELAY - launch { - val logger = TaskRetryHandler.logger - val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) - val loggedProducer = LoggedInfiniticProducer(logger, producer) + private fun CoroutineScope.checkNewServices( + config: EventListenerConfig, + starter: CoroutineScope.(String) -> Unit + ) = launch { + val processedServices = mutableSetOf() - val taskRetryHandler = TaskRetryHandler(loggedProducer) + while (true) { + // Retrieve the list of services + val currentServices = resources.getServices().filter { config.includeService(it) } - loggedConsumer.start( - subscription = MainSubscription(RetryServiceExecutorTopic), - entity = serviceName.toString(), - handler = taskRetryHandler::handle, - beforeDlq = null, - concurrency = concurrency, - ) + // Determine new services that haven't been processed + val newServices = currentServices.filterNot { it in processedServices } + + // Launch starter for each new service + for (service in newServices) { + starter(service) + // Add the service to the set of processed services + processedServices.add(service) + } + + delay((config.refreshDelaySeconds * 1000).toLong()) } + } - // TASK-EVENTS - launch { - val logger = TaskEventHandler.logger - val loggedConsumer = LoggedInfiniticConsumer(logger, consumer) - val loggedProducer = LoggedInfiniticProducer(logger, producer) - val taskEventHandler = TaskEventHandler(loggedProducer) + private fun CoroutineScope.checkNewWorkflows( + config: EventListenerConfig, + starter: CoroutineScope.(String) -> Unit + ) = launch { + val processedWorkflows = mutableSetOf() - val handler: suspend (ServiceExecutorEventMessage, MillisInstant) -> Unit = - { message, publishedAt -> - eventLogger.logServiceCloudEvent(message, publishedAt, source) - taskEventHandler.handle(message, publishedAt) - } + while (true) { + // Retrieve the list of workflows + val currentServices = resources.getWorkflows().filter { config.includeWorkflow(it) } - loggedConsumer.start( - subscription = MainSubscription(ServiceExecutorEventTopic), - entity = serviceName.toString(), - handler = handler, - beforeDlq = null, - concurrency = concurrency, - ) + // Determine new workflows that haven't been processed + val newWorkflows = currentServices.filterNot { it in processedWorkflows } + + // Launch starter for each new workflow + for (workflow in newWorkflows) { + starter(workflow) + // Add the workflow to the set of processed workflows + processedWorkflows.add(workflow) + } + + delay((config.refreshDelaySeconds * 1000).toLong()) } } @@ -668,7 +830,7 @@ class InfiniticWorker private constructor( ) { // WORKFLOW-TASK-EXECUTOR topic launch { - this@InfiniticWorker.consumer.start( + consumer.start( subscription = subscriptionType.create(WorkflowExecutorTopic, subscriptionName), entity = workflowName.toString(), handler = handler, @@ -678,7 +840,7 @@ class InfiniticWorker private constructor( } // WORKFLOW-TASK-EXECUTOR-DELAY topic launch { - this@InfiniticWorker.consumer.start( + consumer.start( subscription = subscriptionType.create(RetryWorkflowExecutorTopic, subscriptionName), entity = workflowName.toString(), handler = handler, @@ -688,7 +850,7 @@ class InfiniticWorker private constructor( } // WORKFLOW-TASK-EVENTS topic launch { - this@InfiniticWorker.consumer.start( + consumer.start( subscription = subscriptionType.create(WorkflowExecutorEventTopic, subscriptionName), entity = workflowName.toString(), handler = handler, @@ -707,7 +869,7 @@ class InfiniticWorker private constructor( ) { // WORKFLOW-CMD topic launch { - this@InfiniticWorker.consumer.start( + consumer.start( subscription = subscriptionType.create(WorkflowStateCmdTopic, subscriptionName), entity = workflowName.toString(), handler = handler, @@ -717,7 +879,7 @@ class InfiniticWorker private constructor( } // WORKFLOW-STATE-ENGINE topic launch { - this@InfiniticWorker.consumer.start( + consumer.start( subscription = subscriptionType.create(WorkflowStateEngineTopic, subscriptionName), entity = workflowName.toString(), handler = { message: Message, publishedAt: MillisInstant -> @@ -731,7 +893,7 @@ class InfiniticWorker private constructor( } // WORKFLOW-EVENTS topic launch { - this@InfiniticWorker.consumer.start( + consumer.start( subscription = subscriptionType.create(WorkflowStateEventTopic, subscriptionName), entity = workflowName.toString(), handler = handler, @@ -749,7 +911,7 @@ class InfiniticWorker private constructor( ) { try { info { - message.eventProducer(publishedAt, prefix)?.toJsonString(register.logsConfig.beautify) + message.eventProducer(publishedAt, prefix)?.toJsonString(beautifyLogs) } } catch (e: Exception) { // Failure to log shouldn't break the application diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorkerBuilder.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorkerBuilder.kt new file mode 100644 index 000000000..fbbfff96e --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/InfiniticWorkerBuilder.kt @@ -0,0 +1,132 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +@file:Suppress("unused") + +package io.infinitic.workers + +import io.infinitic.storage.config.StorageConfig +import io.infinitic.transport.config.TransportConfig +import io.infinitic.workers.config.EventListenerConfig +import io.infinitic.workers.config.InfiniticWorkerConfig +import io.infinitic.workers.config.LogsConfig +import io.infinitic.workers.config.ServiceConfig +import io.infinitic.workers.config.ServiceExecutorConfig +import io.infinitic.workers.config.ServiceTagEngineConfig +import io.infinitic.workers.config.WorkflowConfig +import io.infinitic.workers.config.WorkflowExecutorConfig +import io.infinitic.workers.config.WorkflowStateEngineConfig +import io.infinitic.workers.config.WorkflowTagEngineConfig + +/** + * InfiniticWorker builder + */ +class InfiniticWorkerBuilder { + private var name: String? = null + private var transport: TransportConfig? = null + private var storage: StorageConfig? = null + private var logs: LogsConfig = LogsConfig() + private var workflows: MutableList = mutableListOf() + private var services: MutableList = mutableListOf() + private var eventListener: EventListenerConfig? = null + + private fun getOrCreateServiceConfig(serviceName: String) = + services.firstOrNull { it.name == serviceName } + ?: ServiceConfig(serviceName).also { services.add(it) } + + private fun getOrCreateWorkflowConfig(workflowName: String) = + workflows.firstOrNull { it.name == workflowName } + ?: WorkflowConfig(workflowName).also { workflows.add(it) } + + fun setName(name: String) = + apply { this.name = name } + + fun setTransport(transport: TransportConfig) = + apply { this.transport = transport } + + fun setTransport(transport: TransportConfig.TransportConfigBuilder) = + setTransport(transport.build()) + + fun setStorage(storage: StorageConfig) = + apply { this.storage = storage } + + fun setStorage(storage: StorageConfig.StorageConfigBuilder) = + setStorage(storage.build()) + + fun setLogs(logs: LogsConfig) = + apply { this.logs = logs } + + fun setLogs(logs: LogsConfig.LogConfigBuilder) = + setLogs(logs.build()) + + fun addServiceExecutor(config: ServiceExecutorConfig) = + apply { getOrCreateServiceConfig(config.serviceName).executor = config } + + fun addServiceExecutor(config: ServiceExecutorConfig.ServiceExecutorConfigBuilder) = + addServiceExecutor(config.build()) + + fun addServiceTagEngine(config: ServiceTagEngineConfig) = + apply { getOrCreateServiceConfig(config.serviceName).tagEngine = config } + + fun addServiceTagEngine(config: ServiceTagEngineConfig.ServiceTagEngineConfigBuilder) = + addServiceTagEngine(config.build()) + + fun addWorkflowExecutor(config: WorkflowExecutorConfig) = + apply { getOrCreateWorkflowConfig(config.workflowName).executor = config } + + fun addWorkflowExecutor(config: WorkflowExecutorConfig.WorkflowExecutorConfigBuilder) = + addWorkflowExecutor(config.build()) + + fun addWorkflowStateEngine(config: WorkflowStateEngineConfig) = + apply { getOrCreateWorkflowConfig(config.workflowName).stateEngine = config } + + fun addWorkflowStateEngine(config: WorkflowStateEngineConfig.WorkflowStateEngineConfigBuilder) = + addWorkflowStateEngine(config.build()) + + fun addWorkflowTagEngine(config: WorkflowTagEngineConfig) = + apply { getOrCreateWorkflowConfig(config.workflowName).tagEngine = config } + + fun addWorkflowTagEngine(config: WorkflowTagEngineConfig.WorkflowTagEngineConfigBuilder) = + addWorkflowTagEngine(config.build()) + + fun setEventListener(eventListener: EventListenerConfig?) = + apply { this.eventListener = eventListener } + + fun setEventListener(eventListener: EventListenerConfig.EventListenerConfigBuilder) = + setEventListener(eventListener.build()) + + fun build(): InfiniticWorker { + require(transport != null) { "transport must not be null" } + + val config = InfiniticWorkerConfig( + name, + transport!!, + storage, + logs, + workflows, + services, + eventListener, + ) + + return InfiniticWorker(config) + } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ConfigGetterInterface.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ConfigGetterInterface.kt new file mode 100644 index 000000000..0a78392ae --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ConfigGetterInterface.kt @@ -0,0 +1,47 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +interface ConfigGetterInterface { + fun getEventListenerConfig(): EventListenerConfig? + + fun getServiceExecutorConfigs(): List + + fun getWorkflowExecutorConfigs(): List + + fun getWorkflowTagEngineConfigs(): List + + fun getWorkflowStateEngineConfigs(): List + + fun getServiceTagEngineConfigs(): List + + fun getServiceExecutorConfig(serviceName: String): ServiceExecutorConfig? + + fun getServiceTagEngineConfig(serviceName: String): ServiceTagEngineConfig? + + fun getWorkflowExecutorConfig(workflowName: String): WorkflowExecutorConfig? + + fun getWorkflowTagEngineConfig(workflowName: String): WorkflowTagEngineConfig? + + fun getWorkflowStateEngineConfig(workflowName: String): WorkflowStateEngineConfig? +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/EventListenerConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/EventListenerConfig.kt new file mode 100644 index 000000000..399a3edc2 --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/EventListenerConfig.kt @@ -0,0 +1,203 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.cloudEvents.CloudEventListener +import io.infinitic.cloudEvents.SelectionConfig +import io.infinitic.common.utils.annotatedName +import io.infinitic.common.utils.getInstance +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString + +sealed class EventListenerConfig { + abstract val listener: CloudEventListener + abstract val concurrency: Int + abstract val subscriptionName: String? + abstract val allowedServices: List? + abstract val disallowedServices: List + abstract val allowedWorkflows: List? + abstract val disallowedWorkflows: List + abstract val refreshDelaySeconds: Double + + fun includeService(service: String): Boolean { + return !disallowedServices.contains(service) && (allowedServices?.contains(service) != false) + } + + fun includeWorkflow(workflow: String): Boolean { + return !disallowedWorkflows.contains(workflow) && (allowedWorkflows?.contains(workflow) != false) + } + + companion object { + @JvmStatic + fun builder() = EventListenerConfigBuilder() + + /** + * Create EventListenerConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): EventListenerConfig = + loadFromYamlFile(*files) + + /** + * Create EventListenerConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): EventListenerConfig = + loadFromYamlResource(*resources) + + /** + * Create EventListenerConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): EventListenerConfig = + loadFromYamlString(*yamls) + } + + /** + * EventListenerConfig builder + */ + class EventListenerConfigBuilder { + private var listener: CloudEventListener? = null + private var concurrency: Int = 1 + private var subscriptionName: String? = null + private var allowedServices: MutableList? = null + private val disallowedServices: MutableList = mutableListOf() + private var allowedWorkflows: MutableList? = null + private val disallowedWorkflows: MutableList = mutableListOf() + private var refreshDelaySeconds: Double = 60.0 + + fun setListener(cloudEventListener: CloudEventListener) = + apply { this.listener = cloudEventListener } + + fun setConcurrency(concurrency: Int) = + apply { this.concurrency = concurrency } + + fun setSubscriptionName(subscriptionName: String) = + apply { this.subscriptionName = subscriptionName } + + fun disallowServices(vararg services: String) = + apply { services.forEach { disallowedServices.add(it) } } + + fun allowServices(vararg services: String) = + apply { + services.forEach { service -> + allowedServices = (allowedServices ?: mutableListOf()).apply { add(service) } + } + } + + fun disallowWorkflows(vararg workflows: String) = + apply { workflows.forEach { disallowedWorkflows.add(it) } } + + fun allowWorkflows(vararg workflows: String) = + apply { + workflows.forEach { workflow -> + allowedWorkflows = (allowedWorkflows ?: mutableListOf()).apply { add(workflow) } + } + } + + fun disallowServices(vararg services: Class<*>) = + apply { disallowServices(*(services.map { it.annotatedName }.toTypedArray())) } + + fun allowServices(vararg services: Class<*>) = + apply { allowServices(*(services.map { it.annotatedName }.toTypedArray())) } + + fun disallowWorkflows(vararg workflows: Class<*>) = + apply { disallowWorkflows(*(workflows.map { it.annotatedName }.toTypedArray())) } + + fun allowWorkflows(vararg workflows: Class<*>) = + apply { allowWorkflows(*(workflows.map { it.annotatedName }.toTypedArray())) } + + fun setRefreshDelaySeconds(refreshDelaySeconds: Double) = + apply { this.refreshDelaySeconds = refreshDelaySeconds } + + fun build(): EventListenerConfig { + require(listener != null) { "${EventListenerConfig::listener.name} must not be null" } + + return BuiltEventListenerConfig( + listener!!, + concurrency, + subscriptionName, + refreshDelaySeconds, + allowedServices, + disallowedServices, + allowedWorkflows, + disallowedWorkflows, + ) + } + } +} + +/** + * EventListenerConfig built from builder + */ +data class BuiltEventListenerConfig( + override val listener: CloudEventListener, + override val concurrency: Int, + override val subscriptionName: String?, + override val refreshDelaySeconds: Double, + override val allowedServices: MutableList?, + override val disallowedServices: MutableList, + override val allowedWorkflows: MutableList?, + override val disallowedWorkflows: MutableList, +) : EventListenerConfig() + +/** + * EventListenerConfig loaded from YAML + */ +data class LoadedEventListenerConfig( + val `class`: String, + override val concurrency: Int = 1, + override val subscriptionName: String? = null, + override val refreshDelaySeconds: Double = 60.0, + val services: SelectionConfig = SelectionConfig(), + val workflows: SelectionConfig = SelectionConfig() +) : EventListenerConfig() { + + override val listener: CloudEventListener + override var allowedServices = services.allow + override val disallowedServices = services.disallow + override var allowedWorkflows = workflows.allow + override val disallowedWorkflows = workflows.disallow + + init { + with(`class`) { + require(isNotEmpty()) { error("'class' must not be empty") } + val obj = getInstance().getOrThrow() + require(obj is CloudEventListener) { + error("Class '$this' must implement '${CloudEventListener::class.java.name}'") + } + listener = obj + } + + require(concurrency > 0) { error("'${::concurrency.name}' must be > 0, but was $concurrency") } + + require(refreshDelaySeconds >= 0) { error("'${::refreshDelaySeconds.name}' must be >= 0, but was $refreshDelaySeconds") } + + subscriptionName?.let { + require(it.isNotEmpty()) { error("'when provided, ${::subscriptionName.name}' must not be empty") } + } + } + + private fun error(txt: String) = "eventListener: $txt" +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfig.kt new file mode 100644 index 000000000..5ccc826ec --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfig.kt @@ -0,0 +1,108 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ + +package io.infinitic.workers.config + +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString +import io.infinitic.storage.config.StorageConfig +import io.infinitic.transport.config.TransportConfig + +@Suppress("unused") +data class InfiniticWorkerConfig( + /** Worker name */ + override val name: String? = null, + + /** Transport configuration */ + override val transport: TransportConfig, + + /** Default storage */ + override val storage: StorageConfig? = null, + + /** Logs configuration */ + override val logs: LogsConfig = LogsConfig(), + + /** Workflows configuration */ + override val workflows: List = listOf(), + + /** Services configuration */ + override val services: List = listOf(), + + /** Default event listener configuration */ + override val eventListener: EventListenerConfig? = null, + + ) : InfiniticWorkerConfigInterface { + + init { + workflows.forEach { workflowConfig -> + workflowConfig.stateEngine?.let { + it.setStorage(storage) + ?: throw IllegalArgumentException("Storage undefined for Workflow State Engine of '${workflowConfig.name}") + } + workflowConfig.tagEngine?.let { + it.setStorage(storage) + ?: throw IllegalArgumentException("Storage undefined for Workflow Tag Engine of '${workflowConfig.name}") + } + } + services.forEach { serviceConfig -> + serviceConfig.tagEngine?.let { + it.setStorage(storage) + ?: throw IllegalArgumentException("Storage undefined for Service Tag Engine of '${serviceConfig.name}") + } + } + } + + companion object { + /** + * Create InfiniticWorkerConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): InfiniticWorkerConfig = + loadFromYamlFile(*files) + + /** + * Create InfiniticWorkerConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): InfiniticWorkerConfig = + loadFromYamlResource(*resources) + + /** + * Create InfiniticWorkerConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): InfiniticWorkerConfig = + loadFromYamlString(*yamls) + } +} + +internal interface WithMutableStorage { + var storage: StorageConfig? +} + +private fun WithMutableStorage.setStorage(storage: StorageConfig?): StorageConfig? { + this.storage = this.storage ?: storage + + return this.storage +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/RegisterConfigInterface.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfigInterface.kt similarity index 81% rename from infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/RegisterConfigInterface.kt rename to infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfigInterface.kt index 438a69795..7b677681f 100644 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/RegisterConfigInterface.kt +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/InfiniticWorkerConfigInterface.kt @@ -20,11 +20,17 @@ * * Licensor: infinitic.io */ -package io.infinitic.workers.register.config -import io.infinitic.events.config.EventListenerConfig +package io.infinitic.workers.config + +import io.infinitic.clients.config.InfiniticClientConfigInterface +import io.infinitic.storage.config.StorageConfig + +@Suppress("unused") +interface InfiniticWorkerConfigInterface: InfiniticClientConfigInterface { + /** Default storage */ + val storage: StorageConfig? -interface RegisterConfigInterface { /** Logs configuration */ val logs: LogsConfig @@ -34,12 +40,6 @@ interface RegisterConfigInterface { /** Services configuration */ val services: List - /** Default service configuration */ - val serviceDefault: ServiceConfigDefault? - - /** Default workflow configuration */ - val workflowDefault: WorkflowConfigDefault? - /** Default event listener configuration */ val eventListener: EventListenerConfig? } diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/LogsConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/LogsConfig.kt similarity index 94% rename from infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/LogsConfig.kt rename to infinitic-worker/src/main/kotlin/io/infinitic/workers/config/LogsConfig.kt index c4b3bfe22..a30ca5a47 100644 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/LogsConfig.kt +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/LogsConfig.kt @@ -20,7 +20,7 @@ * * Licensor: infinitic.io */ -package io.infinitic.workers.register.config +package io.infinitic.workers.config @Suppress("unused") data class LogsConfig( @@ -39,7 +39,7 @@ data class LogsConfig( private val default = LogsConfig() private var beautify = default.beautify - fun beautify(beautify: Boolean) = + fun setBeautify(beautify: Boolean) = apply { this.beautify = beautify } fun build() = LogsConfig( diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceConfig.kt new file mode 100644 index 000000000..f04b44971 --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceConfig.kt @@ -0,0 +1,84 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.utils.isImplementationOf +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString + +@Suppress("unused") +data class ServiceConfig( + val name: String, + var executor: ServiceExecutorConfig? = null, + var tagEngine: ServiceTagEngineConfig? = null, +) { + init { + require(name.isNotBlank()) { "'${::name.name}' can not be blank" } + + executor?.let { + if (it is LoadedServiceExecutorConfig) it.setServiceName(name) + val instance = it.factory() + require(instance::class.java.isImplementationOf(name)) { + error("Class '${instance::class.java.name}' must be an implementation of Service '$name', but is not.") + } + } + + tagEngine?.setServiceName(name) + } + + companion object { + /** + * Create ServiceConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): ServiceConfig = + loadFromYamlFile(*files) + + /** + * Create ServiceConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): ServiceConfig = + loadFromYamlResource(*resources) + + /** + * Create ServiceExecutorConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): ServiceConfig = + loadFromYamlString(*yamls) + } +} + +internal interface WithMutableServiceName { + var serviceName: String +} + +private fun WithMutableServiceName.setServiceName(name: String) { + if (serviceName == name) return + if (serviceName.isNotBlank()) { + throw IllegalStateException("${::serviceName.name} is already set to '$serviceName'") + } + serviceName = name +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceExecutorConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceExecutorConfig.kt new file mode 100644 index 000000000..33fe72701 --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceExecutorConfig.kt @@ -0,0 +1,202 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.utils.getInstance +import io.infinitic.common.workers.config.RetryPolicy +import io.infinitic.common.workers.config.UNSET_RETRY_POLICY +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout + +private typealias ServiceFactory = () -> Any + +internal const val UNSET_TIMEOUT = Double.MAX_VALUE + +@Suppress("unused") +sealed class ServiceExecutorConfig { + + abstract val serviceName: String + abstract val factory: ServiceFactory + abstract val concurrency: Int + + /** + * WithRetry instance for this executor + * Set to WithRetry.UNSET if not defined + * (it can still be defined by annotation or interface within the TaskExecutor) + */ + abstract val withRetry: WithRetry? + + /** + * WithTimout instance for this executor + * Set to WithTimeout.UNSET if not defined + * (it can still be defined by annotation or interface within the TaskExecutor) + */ + abstract val withTimeout: WithTimeout? + + companion object { + @JvmStatic + fun builder() = ServiceExecutorConfigBuilder() + + /** + * Create ServiceExecutorConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): ServiceExecutorConfig = + loadFromYamlFile(*files) + + /** + * Create ServiceExecutorConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): ServiceExecutorConfig = + loadFromYamlResource(*resources) + + /** + * Create ServiceExecutorConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): ServiceExecutorConfig = + loadFromYamlString(*yamls) + } + + /** + * ServiceConfig builder + */ + class ServiceExecutorConfigBuilder { + private var serviceName: String? = null + private var factory: ServiceFactory? = null + private var concurrency: Int = 1 + private var timeoutSeconds: Double? = UNSET_TIMEOUT + private var withRetry: WithRetry? = WithRetry.UNSET + + fun setServiceName(name: String) = + apply { this.serviceName = name } + + fun setFactory(factory: () -> Any) = + apply { this.factory = factory } + + fun setConcurrency(concurrency: Int) = + apply { this.concurrency = concurrency } + + fun setTimeoutSeconds(timeoutSeconds: Double) = + apply { this.timeoutSeconds = timeoutSeconds } + + fun withRetry(retry: WithRetry) = + apply { this.withRetry = retry } + + fun build(): ServiceExecutorConfig { + serviceName.checkServiceName() + require(factory != null) { "${::factory.name} must not be null" } + concurrency.checkConcurrency() + timeoutSeconds?.checkTimeout() + + return BuiltServiceExecutorConfig( + serviceName!!, + factory!!, + concurrency, + withRetry, + timeoutSeconds.withTimeout, + ) + } + } +} + +/** + * ServiceExecutorConfig built from builder + */ +data class BuiltServiceExecutorConfig( + override val serviceName: String, + override val factory: ServiceFactory, + override val concurrency: Int, + override val withRetry: WithRetry?, + override val withTimeout: WithTimeout?, +) : ServiceExecutorConfig() + +/** + * ServiceExecutorConfig loaded from YAML + */ +data class LoadedServiceExecutorConfig( + override var serviceName: String = "", + val `class`: String, + override val concurrency: Int = 1, + val timeoutSeconds: Double? = UNSET_TIMEOUT, + val retry: RetryPolicy? = UNSET_RETRY_POLICY, +) : ServiceExecutorConfig(), WithMutableServiceName { + + override val factory: () -> Any = { `class`.getInstance().getOrThrow() } + + override val withTimeout = timeoutSeconds.withTimeout + + override val withRetry: WithRetry? = retry.withRetry + + init { + `class`.checkClass() + concurrency.checkConcurrency() + timeoutSeconds?.checkTimeout() + retry?.check() + } + + @JvmName("replaceServiceName") + internal fun setServiceName(name: String) { + if (serviceName == name) return + if (serviceName.isNotBlank()) { + throw IllegalStateException("serviceName is already set to '$serviceName'") + } + serviceName = name + } +} + +internal val Double?.withTimeout + get() = when (this) { + UNSET_TIMEOUT -> WithTimeout.UNSET + else -> this?.let { WithTimeout { it } } + } + +internal val WithRetry?.withRetry + get() = when (this) { + is UNSET_RETRY_POLICY -> WithRetry.UNSET + else -> this + } + +internal fun String.checkClass() { + require(isNotEmpty()) { "class can not be empty" } + + getInstance().getOrThrow() +} + +internal fun String?.checkServiceName() { + require(this != null) { "serviceName must not be null" } + require(this.isNotBlank()) { "serviceName must not be blank" } +} + +internal fun Int?.checkConcurrency() { + require(this != null) { "concurrency must not be null" } + require(this > 0) { "concurrency must be > 0, but was $this" } +} + +internal fun Double.checkTimeout() { + require(this > 0) { "timeoutSeconds must be > 0, but was $this" } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceTagEngineConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceTagEngineConfig.kt new file mode 100644 index 000000000..143aa86f5 --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/ServiceTagEngineConfig.kt @@ -0,0 +1,102 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.exceptions.thisShouldNotHappen +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString +import io.infinitic.storage.config.StorageConfig +import io.infinitic.tasks.tag.storage.BinaryTaskTagStorage + +data class ServiceTagEngineConfig( + override var serviceName: String = "", + val concurrency: Int = 1, + override var storage: StorageConfig? = null, +) : WithMutableServiceName, WithMutableStorage { + init { + require(concurrency > 0) { "${::concurrency.name} must be > 0" } + } + + val serviceTagStorage by lazy { + (storage ?: thisShouldNotHappen()).let { + BinaryTaskTagStorage(it.keyValue, it.keySet) + } + } + + companion object { + @JvmStatic + fun builder() = ServiceTagEngineConfigBuilder() + + /** + * Create ServiceTagEngineConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): ServiceTagEngineConfig = + loadFromYamlFile(*files) + + /** + * Create ServiceTagEngineConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): ServiceTagEngineConfig = + loadFromYamlResource(*resources) + + /** + * Create ServiceTagEngineConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): ServiceTagEngineConfig = + loadFromYamlString(*yamls) + + } + + /** + * ServiceTagEngineConfig builder + */ + class ServiceTagEngineConfigBuilder { + private val default = ServiceTagEngineConfig() + private var serviceName = default.serviceName + private var concurrency = default.concurrency + private var storage = default.storage + + fun setServiceName(serviceName: String) = + apply { this.serviceName = serviceName } + + fun setConcurrency(concurrency: Int) = + apply { this.concurrency = concurrency } + + fun setStorage(storage: StorageConfig) = + apply { this.storage = storage } + + fun build(): ServiceTagEngineConfig { + serviceName.checkServiceName() + + return ServiceTagEngineConfig( + serviceName, + concurrency, + storage, + ) + } + } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfig.kt deleted file mode 100644 index 41079853b..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkerConfig.kt +++ /dev/null @@ -1,168 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ - -package io.infinitic.workers.config - -import io.infinitic.common.config.loadConfigFromFile -import io.infinitic.common.config.loadConfigFromResource -import io.infinitic.common.config.loadConfigFromYaml -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.pulsar.config.PulsarConfig -import io.infinitic.storage.config.StorageConfig -import io.infinitic.transport.config.Transport -import io.infinitic.workers.register.config.LogsConfig -import io.infinitic.workers.register.config.ServiceConfig -import io.infinitic.workers.register.config.ServiceConfigDefault -import io.infinitic.workers.register.config.WorkflowConfig -import io.infinitic.workers.register.config.WorkflowConfigDefault - -@Suppress("unused") -data class WorkerConfig( - /** Worker name */ - override val name: String? = null, - - /** Worker name */ - override val shutdownGracePeriodInSeconds: Double = 30.0, - - /** Transport configuration */ - override val transport: Transport = Transport.pulsar, - - /** Pulsar configuration */ - override val pulsar: PulsarConfig? = null, - - /** Default storage */ - override val storage: StorageConfig? = null, - - /** Logs configuration */ - override val logs: LogsConfig = LogsConfig(), - - /** Workflows configuration */ - override val workflows: List = listOf(), - - /** Services configuration */ - override val services: List = listOf(), - - /** Default service configuration */ - override val serviceDefault: ServiceConfigDefault? = null, - - /** Default workflow configuration */ - override val workflowDefault: WorkflowConfigDefault? = null, - - /** Default event listener configuration */ - override val eventListener: EventListenerConfig? = null, - - ) : WorkerConfigInterface { - - init { -// check retry values - serviceDefault?.retry?.check() - services.forEach { it.retry?.check() } - workflowDefault?.retry?.check() - workflows.forEach { it.retry?.check() } - } - - companion object { - /** Create ClientConfig from files in file system */ - @JvmStatic - fun fromFile(vararg files: String): WorkerConfig = - loadConfigFromFile(*files) - - /** Create ClientConfig from files in resources directory */ - @JvmStatic - fun fromResource(vararg resources: String): WorkerConfig = - loadConfigFromResource(*resources) - - /** Create ClientConfig from yaml strings */ - @JvmStatic - fun fromYaml(vararg yamls: String): WorkerConfig = - loadConfigFromYaml(*yamls) - - @JvmStatic - fun builder() = WorkerConfigBuilder() - } - - /** - * WorkerConfig builder (Useful for Java user) - */ - class WorkerConfigBuilder { - private val default = WorkerConfig() - private var name = default.name - private var shutdownGracePeriodInSeconds = default.shutdownGracePeriodInSeconds - private var transport = default.transport - private var pulsar = default.pulsar - private var storage = default.storage - private var logs = default.logs - private var workflows = default.workflows - private var services = default.services - private var serviceDefault = default.serviceDefault - private var workflowDefault = default.workflowDefault - private var eventListener = default.eventListener - - fun name(name: String) = - apply { this.name = name } - - fun shutdownGracePeriodInSeconds(shutdownGracePeriodInSeconds: Double) = - apply { this.shutdownGracePeriodInSeconds = shutdownGracePeriodInSeconds } - - fun transport(transport: Transport) = - apply { this.transport = transport } - - fun pulsar(pulsar: PulsarConfig?) = - apply { this.pulsar = pulsar } - - fun storage(storage: StorageConfig?) = - apply { this.storage = storage } - - fun logs(logs: LogsConfig) = - apply { this.logs = logs } - - fun workflows(workflows: List) = - apply { this.workflows = workflows } - - fun services(services: List) = - apply { this.services = services } - - fun serviceDefault(serviceDefault: ServiceConfigDefault?) = - apply { this.serviceDefault = serviceDefault } - - fun workflowDefault(workflowDefault: WorkflowConfigDefault?) = - apply { this.workflowDefault = workflowDefault } - - fun eventListener(eventListener: EventListenerConfig?) = - apply { this.eventListener = eventListener } - - fun build() = WorkerConfig( - name, - shutdownGracePeriodInSeconds, - transport, - pulsar, - storage, - logs, - workflows, - services, - serviceDefault, - workflowDefault, - eventListener, - ) - } -} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowConfig.kt new file mode 100644 index 000000000..1dc28d6b7 --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowConfig.kt @@ -0,0 +1,86 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.utils.isImplementationOf +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString + +data class WorkflowConfig( + val name: String, + var executor: WorkflowExecutorConfig? = null, + var tagEngine: WorkflowTagEngineConfig? = null, + var stateEngine: WorkflowStateEngineConfig? = null, +) { + init { + require(name.isNotEmpty()) { "'${::name.name}' can not be empty" } + + executor?.let { + if (it is LoadedWorkflowExecutorConfig) it.setWorkflowName(name) + val instances = it.factories.map { it() } + instances.forEach { instance -> + require(instance::class.java.isImplementationOf(name)) { + error("Class '${instance::class.java.name}' must be an implementation of Workflow '$name'") + } + } + } + tagEngine?.setWorkflowName(name) + stateEngine?.setWorkflowName(name) + } + + companion object { + /** + * Create WorkflowConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): WorkflowConfig = + loadFromYamlFile(*files) + + /** + * Create WorkflowConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): WorkflowConfig = + loadFromYamlResource(*resources) + + /** + * Create WorkflowConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): WorkflowConfig = + loadFromYamlString(*yamls) + } +} + +interface WithMutableWorkflowName { + var workflowName: String +} + +private fun WithMutableWorkflowName.setWorkflowName(name: String) { + if (workflowName == name) return + if (workflowName.isNotBlank()) { + throw IllegalStateException("${::workflowName.name} is already set to '$workflowName'") + } + workflowName = name +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowExecutorConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowExecutorConfig.kt new file mode 100644 index 000000000..a55bcf2ad --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowExecutorConfig.kt @@ -0,0 +1,256 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.utils.getInstance +import io.infinitic.common.workers.config.RetryPolicy +import io.infinitic.common.workers.config.UNSET_RETRY_POLICY +import io.infinitic.common.workers.config.WorkflowVersion +import io.infinitic.common.workflows.emptyWorkflowContext +import io.infinitic.common.workflows.executors.getProperties +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout +import io.infinitic.workflows.Workflow +import io.infinitic.workflows.WorkflowCheckMode + +internal typealias WorkflowFactory = () -> Workflow +internal typealias WorkflowFactories = List + +sealed class WorkflowExecutorConfig { + + abstract val workflowName: String + abstract val factories: WorkflowFactories + abstract val concurrency: Int + abstract val withRetry: WithRetry? + abstract val withTimeout: WithTimeout? + abstract val checkMode: WorkflowCheckMode? + + companion object { + @JvmStatic + fun builder() = WorkflowExecutorConfigBuilder() + + /** + * Create WorkflowExecutorConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): WorkflowExecutorConfig = + loadFromYamlFile(*files) + + /** + * Create WorkflowExecutorConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): WorkflowExecutorConfig = + loadFromYamlResource(*resources) + + /** + * Create WorkflowExecutorConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): WorkflowExecutorConfig = + loadFromYamlString(*yamls) + } + + /** + * ServiceConfig builder + */ + class WorkflowExecutorConfigBuilder { + private var workflowName: String? = null + private var factories: MutableList<() -> Workflow> = mutableListOf() + private var concurrency: Int = 1 + private var timeoutSeconds: Double? = UNSET_TIMEOUT + private var withRetry: WithRetry? = WithRetry.UNSET + private var checkMode: WorkflowCheckMode? = null + + fun setWorkflowName(workflowName: String) = + apply { this.workflowName = workflowName } + + fun addFactory(factory: () -> Workflow) = + apply { this.factories.add(factory) } + + fun setConcurrency(concurrency: Int) = + apply { this.concurrency = concurrency } + + fun setTimeoutSeconds(timeoutSeconds: Double) = + apply { this.timeoutSeconds = timeoutSeconds } + + fun withRetry(retry: WithRetry) = + apply { this.withRetry = retry } + + fun setCheckMode(checkMode: WorkflowCheckMode) = + apply { this.checkMode = checkMode } + + fun build(): WorkflowExecutorConfig { + workflowName.checkWorkflowName() + + // Needed if the workflow context is referenced within the properties of the workflow + Workflow.setContext(emptyWorkflowContext) + + require(factories.isNotEmpty()) { "At least one factory must be defined" } + factories.checkVersionUniqueness() + factories.checkInstanceUniqueness() + + concurrency.checkConcurrency() + timeoutSeconds?.checkTimeout() + + return BuiltWorkflowExecutorConfig( + workflowName!!, + factories, + concurrency, + timeoutSeconds.withTimeout, + withRetry, + checkMode, + ) + } + } +} + +/** + * WorkflowExecutorConfig built from builders + */ +data class BuiltWorkflowExecutorConfig( + override val workflowName: String, + override val factories: WorkflowFactories, + override val concurrency: Int, + override var withTimeout: WithTimeout? = null, + override var withRetry: WithRetry? = null, + override var checkMode: WorkflowCheckMode? = null, +) : WorkflowExecutorConfig() + +/** + * WorkflowExecutorConfig loaded from YAML + */ +data class LoadedWorkflowExecutorConfig( + override var workflowName: String = "", + val `class`: String? = null, + val classes: List? = null, + override val concurrency: Int = 1, + val timeoutSeconds: Double? = UNSET_TIMEOUT, + var retry: RetryPolicy? = UNSET_RETRY_POLICY, + override var checkMode: WorkflowCheckMode? = null, +) : WorkflowExecutorConfig(), WithMutableWorkflowName { + private val allInstances = mutableListOf() + + override val withTimeout = timeoutSeconds.withTimeout + + override val withRetry: WithRetry? = retry.withRetry + + override val factories: WorkflowFactories by lazy { + allInstances.map { { it::class.java.getInstance().getOrThrow() } } + } + + init { + // Needed if the workflow context is referenced within the properties of the workflow + Workflow.setContext(emptyWorkflowContext) + + require((`class` != null) || (classes != null)) { + "'${::`class`.name}' and '${::classes.name}' can not be both null" + } + + `class`?.let { + require(`class`.isNotEmpty()) { "'${::`class`.name}' can not be empty" } + allInstances.add(getInstance(it)) + } + + classes?.forEachIndexed { index, s: String -> + require(s.isNotEmpty()) { "'${::classes.name}[$index]' can not be empty" } + allInstances.add(getInstance(s)) + } + factories.checkVersionUniqueness() + factories.checkInstanceUniqueness() + + concurrency.checkConcurrency() + timeoutSeconds?.checkTimeout() + retry?.check() + } + + private fun getInstance(className: String): Workflow { + val instance = className.getInstance().getOrThrow() + + require(instance is Workflow) { + "Class '$className' must extend '${Workflow::class.java.name}'" + } + + return instance + } +} + +internal fun String?.checkWorkflowName() { + require(this != null) { "workflowName must not be null" } + require(this.isNotBlank()) { "workflowName must not be blank" } +} + +internal fun WorkflowFactories.checkInstanceUniqueness() { + // check that each factory returns a different instance, if this instance has properties + forEachIndexed { index, function -> + val instance1 = function.invoke() + val instance2 = function.invoke() + + require( + (instance1 !== instance2) || + (instance1.getProperties().isEmpty() && instance2.getProperties().isEmpty()), + ) { + "The workflow factory $index returned the same object instance twice. " + + "Because your workflow contains properties, this will create threading issues. " + + "Please ensure a new workflow instance is returned each time." + } + } +} + +@JvmName("checkFactories") +internal fun WorkflowFactories.checkVersionUniqueness() { + // check that each factory is associated with a different version + val instances = mapIndexed { index, factory -> + try { + factory() + } catch (e: Exception) { + throw IllegalArgumentException("Error when running factory #$index", e) + } + } + instances.checkVersionUniqueness() +} + +@JvmName("checkWorkflows") +internal fun List.checkVersionUniqueness() { + // check that each class is associated with a different version + val versions = map { WorkflowVersion.from(it::class.java) } + + val duplicatedVersions = getDuplicatedVersionsWithIndices(versions) + + duplicatedVersions.forEach { (version, indices) -> + throw IllegalArgumentException("Version $version is duplicated at factory: $indices") + } +} + +private fun getDuplicatedVersionsWithIndices(versions: List): Map> { + val indexMap = mutableMapOf>() + + versions.forEachIndexed { index, version -> + indexMap.computeIfAbsent(version) { mutableListOf() }.add(index) + } + + return indexMap.filter { it.value.size > 1 } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfig.kt new file mode 100644 index 000000000..ea9854476 --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfig.kt @@ -0,0 +1,100 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.exceptions.thisShouldNotHappen +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString +import io.infinitic.storage.config.StorageConfig +import io.infinitic.workflows.engine.storage.BinaryWorkflowStateStorage + +data class WorkflowStateEngineConfig( + override var workflowName: String = "", + val concurrency: Int = 1, + override var storage: StorageConfig? = null, +) : WithMutableWorkflowName, WithMutableStorage { + init { + require(concurrency >= 0) { "concurrency must be positive" } + } + + val workflowStateStorage by lazy { + BinaryWorkflowStateStorage((storage ?: thisShouldNotHappen()).keyValue) + } + + companion object { + @JvmStatic + fun builder() = WorkflowStateEngineConfigBuilder() + + /** + * Create WorkflowStateEngineConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): WorkflowStateEngineConfig = + loadFromYamlFile(*files) + + /** + * Create WorkflowStateEngineConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): WorkflowStateEngineConfig = + loadFromYamlResource(*resources) + + /** + * Create WorkflowStateEngineConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): WorkflowStateEngineConfig = + loadFromYamlString(*yamls) + } + + /** + * WorkflowStateEngineConfig builder (Useful for Java user) + */ + class WorkflowStateEngineConfigBuilder { + private val default = WorkflowStateEngineConfig() + private var workflowName = default.workflowName + private var concurrency = default.concurrency + private var storage = default.storage + + fun setWorkflowName(workflowName: String) = + apply { this.workflowName = workflowName } + + fun setConcurrency(concurrency: Int) = + apply { this.concurrency = concurrency } + + fun setStorage(storage: StorageConfig) = + apply { this.storage = storage } + + fun build(): WorkflowStateEngineConfig { + workflowName.checkWorkflowName() + concurrency.checkConcurrency() + + return WorkflowStateEngineConfig( + workflowName, + concurrency, + storage, + ) + } + } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfig.kt new file mode 100644 index 000000000..f984be8bf --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfig.kt @@ -0,0 +1,103 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.common.exceptions.thisShouldNotHappen +import io.infinitic.config.loadFromYamlFile +import io.infinitic.config.loadFromYamlResource +import io.infinitic.config.loadFromYamlString +import io.infinitic.storage.config.StorageConfig +import io.infinitic.workflows.tag.storage.BinaryWorkflowTagStorage + +data class WorkflowTagEngineConfig( + override var workflowName: String = "", + var concurrency: Int = 1, + override var storage: StorageConfig? = null, +) : WithMutableWorkflowName, WithMutableStorage { + + init { + require(concurrency >= 0) { "concurrency must be positive" } + } + + val workflowTagStorage by lazy { + (storage ?: thisShouldNotHappen()).let { + BinaryWorkflowTagStorage(it.keyValue, it.keySet) + } + } + + companion object { + @JvmStatic + fun builder() = WorkflowTagEngineConfigBuilder() + + /** + * Create WorkflowTagEngineConfig from files in file system + */ + @JvmStatic + fun fromYamlFile(vararg files: String): WorkflowTagEngineConfig = + loadFromYamlFile(*files) + + /** + * Create WorkflowTagEngineConfig from files in resources directory + */ + @JvmStatic + fun fromYamlResource(vararg resources: String): WorkflowTagEngineConfig = + loadFromYamlResource(*resources) + + /** + * Create WorkflowTagEngineConfig from yaml strings + */ + @JvmStatic + fun fromYamlString(vararg yamls: String): WorkflowTagEngineConfig = + loadFromYamlString(*yamls) + } + + /** + * WorkflowTagEngineConfig builder (Useful for Java user) + */ + class WorkflowTagEngineConfigBuilder { + private val default = WorkflowTagEngineConfig() + private var workflowName = default.workflowName + private var concurrency = default.concurrency + private var storage = default.storage + + fun setWorkflowName(workflowName: String) = + apply { this.workflowName = workflowName } + + fun setConcurrency(concurrency: Int) = + apply { this.concurrency = concurrency } + + fun setStorage(storage: StorageConfig) = + apply { this.storage = storage } + + fun build(): WorkflowTagEngineConfig { + workflowName.checkWorkflowName() + concurrency.checkConcurrency() + + return WorkflowTagEngineConfig( + workflowName, + concurrency, + storage, + ) + } + } +} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegister.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegister.kt deleted file mode 100644 index 625042880..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegister.kt +++ /dev/null @@ -1,172 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register - -import io.infinitic.cloudEvents.CloudEventListener -import io.infinitic.common.workers.registry.ServiceFactory -import io.infinitic.common.workers.registry.WorkerRegistry -import io.infinitic.common.workers.registry.WorkflowFactories -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.storage.config.StorageConfig -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout -import io.infinitic.workers.register.config.LogsConfig -import io.infinitic.workers.register.config.ServiceConfigDefault -import io.infinitic.workers.register.config.UNDEFINED_WITH_RETRY -import io.infinitic.workers.register.config.UNDEFINED_WITH_TIMEOUT -import io.infinitic.workers.register.config.WorkflowConfigDefault -import io.infinitic.workflows.Workflow -import io.infinitic.workflows.WorkflowCheckMode - -@Suppress("unused") -interface InfiniticRegister : AutoCloseable { - - val registry: WorkerRegistry - - /** - * Logs configuration - */ - var logsConfig: LogsConfig - - /** - * Default value of Storage - */ - var defaultStorage: StorageConfig - - /** - * Default value of EventListener - */ - var defaultEventListener: EventListenerConfig? - - /** - * Service default values - */ - var serviceDefault: ServiceConfigDefault - - /** - * Workflow default values - */ - var workflowDefault: WorkflowConfigDefault - - /** - * - * Register Services - * - */ - - /** Register service tag engine */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerServiceTagEngine( - serviceName: String, - concurrency: Int, - storageConfig: StorageConfig? = null - ) - - /** Register service */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerServiceExecutor( - serviceName: String, - serviceFactory: ServiceFactory, - concurrency: Int, - withTimeout: WithTimeout? = UNDEFINED_WITH_TIMEOUT, - withRetry: WithRetry? = UNDEFINED_WITH_RETRY - ) - - - /** Register service event listener */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerServiceEventListener( - serviceName: String, - concurrency: Int, - eventListener: CloudEventListener, - subscriptionName: String? = null, - ) - - /** - * - * Register workflows - * - */ - - /** Register workflow tag engine */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerWorkflowTagEngine( - workflowName: String, - concurrency: Int, - storageConfig: StorageConfig? = null - ) - - /** Register workflow executor */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerWorkflowExecutor( - workflowName: String, - factories: WorkflowFactories, - concurrency: Int, - withTimeout: WithTimeout?, - withRetry: WithRetry?, - checkMode: WorkflowCheckMode? = null, - ) - - /** Register workflow executor */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerWorkflowExecutor( - workflowName: String, - factory: () -> Workflow, - concurrency: Int, - withTimeout: WithTimeout? = UNDEFINED_WITH_TIMEOUT, - withRetry: WithRetry? = UNDEFINED_WITH_RETRY, - checkMode: WorkflowCheckMode? = null, - ) = registerWorkflowExecutor( - workflowName, - listOf(factory), - concurrency, - withTimeout, - withRetry, - checkMode, - ) - - /** Register workflow state engine */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerWorkflowStateEngine( - workflowName: String, - concurrency: Int, - storageConfig: StorageConfig? = null - ) - - /** Register workflow event listener */ - @Suppress("OVERLOADS_INTERFACE") - @JvmOverloads - fun registerWorkflowEventListener( - workflowName: String, - concurrency: Int, - eventListener: CloudEventListener, - subscriptionName: String? = null, - ) -} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegisterImpl.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegisterImpl.kt deleted file mode 100644 index ae8c20f5e..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/InfiniticRegisterImpl.kt +++ /dev/null @@ -1,403 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register - -import io.github.oshai.kotlinlogging.KotlinLogging -import io.infinitic.cloudEvents.CloudEventListener -import io.infinitic.common.tasks.data.ServiceName -import io.infinitic.common.utils.merge -import io.infinitic.common.workers.registry.RegisteredEventListener -import io.infinitic.common.workers.registry.RegisteredServiceExecutor -import io.infinitic.common.workers.registry.RegisteredServiceTagEngine -import io.infinitic.common.workers.registry.RegisteredWorkflowExecutor -import io.infinitic.common.workers.registry.RegisteredWorkflowStateEngine -import io.infinitic.common.workers.registry.RegisteredWorkflowTagEngine -import io.infinitic.common.workers.registry.ServiceFactory -import io.infinitic.common.workers.registry.WorkerRegistry -import io.infinitic.common.workers.registry.WorkflowFactories -import io.infinitic.common.workflows.data.workflows.WorkflowName -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.storage.config.StorageConfig -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout -import io.infinitic.tasks.tag.storage.BinaryTaskTagStorage -import io.infinitic.workers.InfiniticWorker -import io.infinitic.workers.config.WorkerConfigInterface -import io.infinitic.workers.register.config.DEFAULT_CONCURRENCY -import io.infinitic.workers.register.config.LogsConfig -import io.infinitic.workers.register.config.ServiceConfigDefault -import io.infinitic.workers.register.config.UNDEFINED_EVENT_LISTENER -import io.infinitic.workers.register.config.UNDEFINED_WITH_RETRY -import io.infinitic.workers.register.config.UNDEFINED_WITH_TIMEOUT -import io.infinitic.workers.register.config.WorkflowConfigDefault -import io.infinitic.workflows.WorkflowCheckMode -import io.infinitic.workflows.engine.storage.BinaryWorkflowStateStorage -import io.infinitic.workflows.tag.storage.BinaryWorkflowTagStorage -import java.security.InvalidParameterException -import java.util.concurrent.ConcurrentHashMap - -class InfiniticRegisterImpl(override var logsConfig: LogsConfig) : InfiniticRegister { - - // thread-safe set of all storage instances used - private val storages = ConcurrentHashMap.newKeySet() - - override val registry = WorkerRegistry() - - override var defaultStorage: StorageConfig = StorageConfig() - override var defaultEventListener: EventListenerConfig? = null - override var serviceDefault: ServiceConfigDefault = ServiceConfigDefault() - override var workflowDefault: WorkflowConfigDefault = WorkflowConfigDefault() - - override fun close() { - storages.forEach { - try { - logger.info { "Closing KeyValueStorage $it" } - it.keyValue.close() - } catch (e: Exception) { - logger.warn(e) { "Unable to close KeyValueStorage $it" } - } - try { - logger.info { "Closing KeySetStorage $it" } - it.keySet.close() - } catch (e: Exception) { - logger.warn(e) { "Unable to close KeySetStorage $it" } - } - } - } - - /** Register Service Executor */ - override fun registerServiceExecutor( - serviceName: String, - serviceFactory: ServiceFactory, - concurrency: Int, - withTimeout: WithTimeout?, - withRetry: WithRetry?, - ) { - val service = ServiceName(serviceName) - - val withT = when (withTimeout) { - null -> null - UNDEFINED_WITH_TIMEOUT -> serviceDefault.withTimeout - else -> withTimeout - } - - val withR = when (withRetry) { - null -> null - UNDEFINED_WITH_RETRY -> serviceDefault.retry - else -> withRetry - } - - registry.serviceExecutors[service] = RegisteredServiceExecutor( - concurrency, - serviceFactory, - withT, - withR, - ).also { - logger.info { - "* Service Executor".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "class: ${it.factory()::class.java.name}, " + - "timeout: ${ - it.withTimeout?.getTimeoutInSeconds()?.let { String.format("%.2fs", it) } ?: NONE - }, " + - "withRetry: ${it.withRetry ?: NONE})" - } - } - } - - /** Register Service Tag Engine */ - override fun registerServiceTagEngine( - serviceName: String, - concurrency: Int, - storageConfig: StorageConfig? - ) { - val service = ServiceName(serviceName) - val storage = storageConfig ?: serviceDefault.tagEngine?.storage ?: defaultStorage - storages.add(storage) - - registry.serviceTagEngines[service] = RegisteredServiceTagEngine( - concurrency, - BinaryTaskTagStorage(storage.keyValue, storage.keySet), - ).also { - logger.info { - "* Service Tag Engine".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "storage: ${storage.type}, " + - "cache: ${storage.cache?.type ?: NONE}, " + - "compression: ${storage.compression ?: NONE})" - } - } - } - - /** Register Service Event Listener */ - override fun registerServiceEventListener( - serviceName: String, - concurrency: Int, - eventListener: CloudEventListener, - subscriptionName: String?, - ) { - val subName = subscriptionName - ?: serviceDefault.eventListener?.subscriptionName - ?: defaultEventListener?.subscriptionName - - registry.serviceEventListeners[ServiceName(serviceName)] = RegisteredEventListener( - eventListener, - concurrency, - subName, - ).also { - logger.info { - "* Service Event Listener".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "class: ${it.eventListener::class.java.name}" + - (it.subscriptionName?.let { ", subscription: $it" } ?: "") + - ")" - } - } - } - - /** Register Workflow Executor */ - override fun registerWorkflowExecutor( - workflowName: String, - factories: WorkflowFactories, - concurrency: Int, - withTimeout: WithTimeout?, - withRetry: WithRetry?, - checkMode: WorkflowCheckMode?, - ) { - val workflow = WorkflowName(workflowName) - - val withT = when (withTimeout) { - null -> null - UNDEFINED_WITH_TIMEOUT -> workflowDefault.withTimeout - else -> withTimeout - } - - val withR = when (withRetry) { - null -> null - UNDEFINED_WITH_RETRY -> workflowDefault.retry - else -> withRetry - } - - registry.workflowExecutors[workflow] = RegisteredWorkflowExecutor( - workflow, - factories, - concurrency, - withT, - withR, - checkMode ?: workflowDefault.checkMode, - ).also { - logger.info { - "* Workflow Executor".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "classes: ${it.classes.joinToString { it.simpleName }}, " + - "timeout: ${ - it.withTimeout?.getTimeoutInSeconds()?.let { String.format("%.2fs", it) } ?: NONE - }, " + - "withRetry: ${it.withRetry ?: NONE}" + - (it.checkMode?.let { ", checkMode: $it" } ?: "") + - ")" - } - } - } - - /** Register Workflow State Engine */ - override fun registerWorkflowStateEngine( - workflowName: String, - concurrency: Int, - storageConfig: StorageConfig?, - ) { - val workflow = WorkflowName(workflowName) - - val storage = storageConfig ?: workflowDefault.stateEngine?.storage ?: defaultStorage - storages.add(storage) - - registry.workflowStateEngines[workflow] = RegisteredWorkflowStateEngine( - concurrency, - BinaryWorkflowStateStorage(storage.keyValue), - ).also { - logger.info { - "* Workflow State Engine".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "storage: ${storage.type}, " + - "cache: ${storage.cache?.type ?: NONE}, " + - "compression: ${storage.compression ?: NONE})" - } - } - } - - /** Register Workflow Tag Engine */ - override fun registerWorkflowTagEngine( - workflowName: String, - concurrency: Int, - storageConfig: StorageConfig?, - ) { - val storage = storageConfig ?: workflowDefault.tagEngine?.storage ?: defaultStorage - storages.add(storage) - - registry.workflowTagEngines[WorkflowName(workflowName)] = RegisteredWorkflowTagEngine( - concurrency, - BinaryWorkflowTagStorage(storage.keyValue, storage.keySet), - ).also { - logger.info { - "* Workflow Tag Engine".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "storage: ${storage.type}, " + - "cache: ${storage.cache?.type ?: NONE}, " + - "compression: ${storage.compression ?: NONE})" - } - } - } - - /** Register Workflow Event Listener */ - override fun registerWorkflowEventListener( - workflowName: String, - concurrency: Int, - eventListener: CloudEventListener, - subscriptionName: String?, - ) { - val workflow = WorkflowName(workflowName) - val subName = subscriptionName - ?: workflowDefault.eventListener?.subscriptionName - ?: defaultEventListener?.subscriptionName - - registry.workflowEventListeners[workflow] = RegisteredEventListener( - eventListener, - concurrency, - subName, - ).also { - logger.info { - "* Workflow Event Listener".padEnd(25) + ": (" + - "concurrency: ${it.concurrency}, " + - "class: ${it.eventListener::class.java.name}" + - (it.subscriptionName?.let { ", subscription: $it" } ?: "") + - ")" - } - } - } - - private fun getDefaultServiceConcurrency(name: String) = - registry.serviceExecutors[ServiceName(name)]?.concurrency - ?: serviceDefault.concurrency - ?: DEFAULT_CONCURRENCY - - private fun getDefaultWorkflowConcurrency(name: String) = - registry.workflowExecutors[WorkflowName(name)]?.concurrency - ?: workflowDefault.concurrency - ?: DEFAULT_CONCURRENCY - - companion object { - private val logger = KotlinLogging.logger(InfiniticWorker::class.java.name) - - private const val NONE = "none" - - /** Create [InfiniticRegisterImpl] from config */ - @JvmStatic - fun fromConfig(workerConfig: WorkerConfigInterface): InfiniticRegisterImpl = - InfiniticRegisterImpl(workerConfig.logs).apply { - - workerConfig.storage?.let { defaultStorage = it } - workerConfig.serviceDefault?.let { serviceDefault = it } - workerConfig.workflowDefault?.let { workflowDefault = it } - workerConfig.eventListener?.let { defaultEventListener = it } - - for (workflowConfig in workerConfig.workflows) with(workflowConfig) { - logger.info { "Workflow $name:" } - - // Workflow Executors are registered first, as it defines some default values for the others - if (allClasses.isNotEmpty()) { - registerWorkflowExecutor( - name, - allClasses.map { { it.getDeclaredConstructor().newInstance() } }, - concurrency ?: getDefaultWorkflowConcurrency(name), - withTimeout, - withRetry, - checkMode, - ) - } - // Workflow Tag Engine - tagEngine?.merge(workflowDefault.tagEngine)?.let { - registerWorkflowTagEngine( - name, - it.concurrency ?: getDefaultWorkflowConcurrency(name), - it.storage, - ) - } - // Workflow State Engine - stateEngine?.merge(workflowDefault.stateEngine)?.let { - registerWorkflowStateEngine( - name, - it.concurrency ?: getDefaultWorkflowConcurrency(name), - it.storage, - ) - } - // Workflow Event Listener - eventListener?.merge(workflowDefault.eventListener)?.merge(defaultEventListener)?.let { - if (it != UNDEFINED_EVENT_LISTENER) when (val instance = it.instance) { - null -> throw InvalidParameterException("Missing declaration of " + CloudEventListener::class.java.name) - else -> registerWorkflowEventListener( - name, - it.concurrency ?: getDefaultWorkflowConcurrency(name), - instance, - it.subscriptionName, - ) - } - } - } - - for (service in workerConfig.services) with(service) { - logger.info { "Service $name:" } - - // Service Executors are registered first, as it defines some default values for the others - `class`?.let { - registerServiceExecutor( - name, - { getInstance() }, - concurrency ?: getDefaultServiceConcurrency(name), - withTimeout, - withRetry, - ) - } - // Service Tag Engine - tagEngine?.merge(serviceDefault.tagEngine)?.let { - registerServiceTagEngine( - name, - it.concurrency ?: getDefaultServiceConcurrency(name), - it.storage, - ) - } - // Service Event Listener - eventListener?.merge(serviceDefault.eventListener)?.merge(defaultEventListener)?.let { - if (it != UNDEFINED_EVENT_LISTENER) when (val instance = it.instance) { - null -> throw InvalidParameterException("Missing declaration of " + CloudEventListener::class.java.name) - else -> registerServiceEventListener( - name, - it.concurrency ?: getDefaultServiceConcurrency(name), - instance, - it.subscriptionName, - ) - } - } - } - } - } -} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfig.kt deleted file mode 100644 index f1b07f564..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfig.kt +++ /dev/null @@ -1,137 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register.config - -import io.infinitic.common.utils.getInstance -import io.infinitic.common.utils.isImplementationOf -import io.infinitic.common.workers.config.RetryPolicy -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout -import io.infinitic.tasks.tag.config.ServiceTagEngineConfig - -@Suppress("unused") -data class ServiceConfig( - val name: String, - val `class`: String? = null, - var concurrency: Int? = null, - var timeoutInSeconds: Double? = UNDEFINED_TIMEOUT, - var retry: RetryPolicy? = UNDEFINED_RETRY, - var tagEngine: ServiceTagEngineConfig? = DEFAULT_SERVICE_TAG_ENGINE, - var eventListener: EventListenerConfig? = UNDEFINED_EVENT_LISTENER, -) { - fun getInstance(): Any = `class`!!.getInstance().getOrThrow() - - val withTimeout: WithTimeout? = when (timeoutInSeconds) { - null -> null - UNDEFINED_TIMEOUT -> UNDEFINED_WITH_TIMEOUT - else -> WithTimeout { timeoutInSeconds } - } - - val withRetry: WithRetry? = when (retry) { - null -> null - UNDEFINED_RETRY -> UNDEFINED_WITH_RETRY - else -> retry - } - - init { - require(name.isNotEmpty()) { "'${::name.name}' can not be empty" } - - `class`?.let { klass -> - require(klass.isNotEmpty()) { error("'class' can not be empty") } - - val instance = getInstance() - - require(instance::class.java.isImplementationOf(name)) { - error("Class '${instance::class.java.name}' is not an implementation of this service - check your configuration") - } - - concurrency?.let { - require(it >= 0) { error("'${::concurrency.name}' must be an integer >= 0") } - } - - timeoutInSeconds?.let { timeout -> - require(timeout > 0 || timeout == UNDEFINED_TIMEOUT) { error("'${::timeoutInSeconds.name}' must be an integer > 0") } - } - } - } - - companion object { - @JvmStatic - fun builder() = ServiceConfigBuilder() - } - - /** - * ServiceConfig builder (Useful for Java user) - */ - class ServiceConfigBuilder { - private val default = ServiceConfig(UNSET) - private var name = default.name - private var `class` = default.`class` - private var concurrency = default.concurrency - private var timeoutInSeconds = default.timeoutInSeconds - private var retry = default.retry - private var tagEngine = default.tagEngine - private var eventListener = default.eventListener - - fun name(name: String) = - apply { this.name = name } - - fun `class`(`class`: String) = - apply { this.`class` = `class` } - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun timeoutInSeconds(timeoutInSeconds: Double) = - apply { this.timeoutInSeconds = timeoutInSeconds } - - fun retry(retry: RetryPolicy) = - apply { this.retry = retry } - - fun tagEngine(tagEngine: ServiceTagEngineConfig) = - apply { this.tagEngine = tagEngine } - - fun eventListener(eventListener: EventListenerConfig) = - apply { this.eventListener = eventListener } - - fun build() = ServiceConfig( - name.noUnset, - `class`, - concurrency, - timeoutInSeconds, - retry, - tagEngine, - eventListener, - ) - } - - private fun error(txt: String) = "Service $name: $txt" -} - -private const val UNSET = "INFINITIC_UNSET_STRING" -private val String.noUnset: String - get() = when (this) { - UNSET -> "" - else -> this - } diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfigDefault.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfigDefault.kt deleted file mode 100644 index 5093acd0a..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/ServiceConfigDefault.kt +++ /dev/null @@ -1,94 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register.config - -import io.infinitic.common.workers.config.RetryPolicy -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.tasks.WithTimeout -import io.infinitic.tasks.tag.config.ServiceTagEngineConfig - -@Suppress("unused") -data class ServiceConfigDefault( - val concurrency: Int? = null, - val timeoutInSeconds: Double? = null, - val retry: RetryPolicy? = null, - val tagEngine: ServiceTagEngineConfig? = null, - val eventListener: EventListenerConfig? = null, -) { - init { - concurrency?.let { - require(it >= 0) { error("'${::concurrency.name}' must be positive") } - } - - timeoutInSeconds?.let { - require(it > 0) { error("'${::timeoutInSeconds.name}' must be strictly positive") } - } - } - - val withTimeout: WithTimeout? = when (timeoutInSeconds) { - null -> null - else -> WithTimeout { timeoutInSeconds } - } - - companion object { - @JvmStatic - fun builder() = ServiceConfigDefaultBuilder() - } - - /** - * ServiceConfigDefault builder (Useful for Java user) - */ - class ServiceConfigDefaultBuilder { - private val default = ServiceConfigDefault() - private var concurrency = default.concurrency - private var timeoutInSeconds = default.timeoutInSeconds - private var retry = default.retry - private var tagEngine = default.tagEngine - private var eventListener = default.eventListener - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun timeoutInSeconds(timeoutInSeconds: Double) = - apply { this.timeoutInSeconds = timeoutInSeconds } - - fun retry(retry: RetryPolicy) = - apply { this.retry = retry } - - fun tagEngine(tagEngine: ServiceTagEngineConfig) = - apply { this.tagEngine = tagEngine } - - fun eventListener(eventListener: EventListenerConfig) = - apply { this.eventListener = eventListener } - - fun build() = ServiceConfigDefault( - concurrency, - timeoutInSeconds, - retry, - tagEngine, - eventListener, - ) - } - - private fun error(msg: String) = "default service: $msg" -} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfig.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfig.kt deleted file mode 100644 index b8ea1d5cf..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfig.kt +++ /dev/null @@ -1,186 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register.config - -import io.infinitic.common.utils.getInstance -import io.infinitic.common.utils.isImplementationOf -import io.infinitic.common.workers.config.RetryPolicy -import io.infinitic.common.workflows.emptyWorkflowContext -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout -import io.infinitic.workflows.Workflow -import io.infinitic.workflows.WorkflowCheckMode -import io.infinitic.workflows.engine.config.WorkflowStateEngineConfig -import io.infinitic.workflows.tag.config.WorkflowTagEngineConfig -import io.infinitic.workflows.Workflow as WorkflowBase - -@Suppress("unused") -data class WorkflowConfig( - val name: String, - val `class`: String? = null, - val classes: List? = null, - var concurrency: Int? = null, - var timeoutInSeconds: Double? = UNDEFINED_TIMEOUT, - var retry: RetryPolicy? = UNDEFINED_RETRY, - var checkMode: WorkflowCheckMode? = null, - var tagEngine: WorkflowTagEngineConfig? = DEFAULT_WORKFLOW_TAG_ENGINE, - var stateEngine: WorkflowStateEngineConfig? = DEFAULT_WORKFLOW_STATE_ENGINE, - var eventListener: EventListenerConfig? = UNDEFINED_EVENT_LISTENER, -) { - val allClasses = mutableListOf>() - - val withTimeout: WithTimeout? = when (timeoutInSeconds) { - null -> null - UNDEFINED_TIMEOUT -> UNDEFINED_WITH_TIMEOUT - else -> WithTimeout { timeoutInSeconds } - } - - val withRetry: WithRetry? = when (retry) { - null -> null - UNDEFINED_RETRY -> UNDEFINED_WITH_RETRY - else -> retry - } - - init { - require(name.isNotEmpty()) { "name can not be empty" } - - when { - (`class` == null) && (classes == null) -> require(tagEngine != null || stateEngine != null || eventListener != null) { - error("'${::`class`.name}', '${::classes.name}', '${::tagEngine.name}', '${::stateEngine.name}' and '${::eventListener.name}' can not be all null") - } - - else -> { - `class`?.let { - require(`class`.isNotEmpty()) { error("'${::`class`.name}' can not be empty") } - allClasses.add(getWorkflowClass(it)) - } - classes?.forEachIndexed { index, s: String -> - require(s.isNotEmpty()) { error("'${::classes.name}[$index]' can not be empty") } - allClasses.add(getWorkflowClass(s)) - } - - concurrency?.let { - require(it >= 0) { error("'${::concurrency.name}' must be an integer >= 0") } - } - - timeoutInSeconds?.let { timeout -> - require(timeout > 0 || timeout == UNDEFINED_TIMEOUT) { error("'${::timeoutInSeconds.name}' must be an integer > 0") } - } - } - } - } - - companion object { - @JvmStatic - fun builder() = WorkflowConfigBuilder() - } - - /** - * WorkflowConfig builder (Useful for Java user) - */ - class WorkflowConfigBuilder { - private val default = WorkflowConfig(UNSET) - private var name = default.name - private var `class` = default.`class` - private var classes = default.classes - private var concurrency = default.concurrency - private var timeoutInSeconds = default.timeoutInSeconds - private var retry = default.retry - private var checkMode = default.checkMode - private var tagEngine = default.tagEngine - private var stateEngine = default.stateEngine - private var eventListener = default.eventListener - - fun name(name: String) = - apply { this.name = name } - - fun `class`(`class`: String) = - apply { this.`class` = `class` } - - fun classes(classes: MutableList) = - apply { this.classes = classes } - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun timeoutInSeconds(timeoutInSeconds: Double) = - apply { this.timeoutInSeconds = timeoutInSeconds } - - fun retry(retry: RetryPolicy) = - apply { this.retry = retry } - - fun checkMode(checkMode: WorkflowCheckMode) = - apply { this.checkMode = checkMode } - - fun tagEngine(tagEngine: WorkflowTagEngineConfig) = - apply { this.tagEngine = tagEngine } - - fun stateEngine(stateEngine: WorkflowStateEngineConfig) = - apply { this.stateEngine = stateEngine } - - fun eventListener(eventListener: EventListenerConfig) = - apply { this.eventListener = eventListener } - - fun build() = WorkflowConfig( - name.noUnset, - `class`, - classes, - concurrency, - timeoutInSeconds, - retry, - checkMode, - tagEngine, - stateEngine, - eventListener, - ) - } - - private fun getWorkflowClass(className: String): Class { - // make sure to have a context to be able to create the workflow instance - Workflow.setContext(emptyWorkflowContext) - - val instance = className.getInstance().getOrThrow() - val klass = instance::class.java - - require(klass.isImplementationOf(name)) { - error("Class '${klass.name}' is not an implementation of '$name' - check your configuration") - } - - require(instance is WorkflowBase) { - error("Class '${klass.name}' must extend '${WorkflowBase::class.java.name}'") - } - - @Suppress("UNCHECKED_CAST") - return klass as Class - } - - private fun error(txt: String) = "Workflow $name: $txt" -} - -private const val UNSET = "INFINITIC_UNSET_STRING" -private val String.noUnset: String - get() = when (this) { - UNSET -> "" - else -> this - } diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfigDefault.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfigDefault.kt deleted file mode 100644 index 18274c041..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/WorkflowConfigDefault.kt +++ /dev/null @@ -1,108 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register.config - -import io.infinitic.common.workers.config.RetryPolicy -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.tasks.WithTimeout -import io.infinitic.workflows.WorkflowCheckMode -import io.infinitic.workflows.engine.config.WorkflowStateEngineConfig -import io.infinitic.workflows.tag.config.WorkflowTagEngineConfig - -@Suppress("unused") -data class WorkflowConfigDefault( - val concurrency: Int? = null, - val timeoutInSeconds: Double? = null, - val retry: RetryPolicy? = null, - val tagEngine: WorkflowTagEngineConfig? = null, - var stateEngine: WorkflowStateEngineConfig? = null, - val checkMode: WorkflowCheckMode? = null, - val eventListener: EventListenerConfig? = null, -) { - init { - concurrency?.let { - require(it >= 0) { error("'${::concurrency.name}' must be positive") } - } - - timeoutInSeconds?.let { - require(it > 0) { error("'${::timeoutInSeconds.name}' must be strictly positive") } - } - } - - val withTimeout: WithTimeout? = when (timeoutInSeconds) { - null -> null - else -> WithTimeout { timeoutInSeconds } - } - - companion object { - @JvmStatic - fun builder() = WorkflowConfigDefaultBuilder() - } - - /** - * WorkflowConfigDefault builder (Useful for Java user) - */ - class WorkflowConfigDefaultBuilder { - val default = WorkflowConfigDefault() - private var concurrency = default.concurrency - private var timeoutInSeconds = default.timeoutInSeconds - private var retry = default.retry - private var tagEngine = default.tagEngine - private var stateEngine = default.stateEngine - private var checkMode = default.checkMode - private var eventListener = default.eventListener - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun timeoutInSeconds(timeoutInSeconds: Double) = - apply { this.timeoutInSeconds = timeoutInSeconds } - - fun retry(retry: RetryPolicy) = - apply { this.retry = retry } - - fun tagEngine(tagEngine: WorkflowTagEngineConfig) = - apply { this.tagEngine = tagEngine } - - fun stateEngine(stateEngine: WorkflowStateEngineConfig) = - apply { this.stateEngine = stateEngine } - - fun checkMode(checkMode: WorkflowCheckMode) = - apply { this.checkMode = checkMode } - - fun eventListener(eventListener: EventListenerConfig) = - apply { this.eventListener = eventListener } - - fun build() = WorkflowConfigDefault( - concurrency, - timeoutInSeconds, - retry, - tagEngine, - stateEngine, - checkMode, - eventListener, - ) - } - - private fun error(msg: String) = "default workflow: $msg" -} diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/default.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/default.kt deleted file mode 100644 index ff352ef79..000000000 --- a/infinitic-worker/src/main/kotlin/io/infinitic/workers/register/config/default.kt +++ /dev/null @@ -1,54 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register.config - -import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy -import io.infinitic.events.config.EventListenerConfig -import io.infinitic.tasks.WithRetry -import io.infinitic.tasks.WithTimeout -import io.infinitic.tasks.tag.config.ServiceTagEngineConfig -import io.infinitic.workflows.engine.config.WorkflowStateEngineConfig -import io.infinitic.workflows.tag.config.WorkflowTagEngineConfig - -/** - * Note: Final default values for withRetry, withTimeout and workflow check mode - * are in TaskExecutors as they can be defined through annotations as well - */ -internal const val DEFAULT_CONCURRENCY = 1 - -internal const val UNDEFINED_TIMEOUT = -Double.MAX_VALUE - -internal val UNDEFINED_WITH_TIMEOUT = WithTimeout { UNDEFINED_TIMEOUT } - -internal val UNDEFINED_RETRY = ExponentialBackoffRetryPolicy().apply { isDefined = false } - -internal val UNDEFINED_WITH_RETRY = WithRetry { _: Int, _: Exception -> null } - -internal val UNDEFINED_EVENT_LISTENER = EventListenerConfig().apply { isDefined = false } - -internal val DEFAULT_SERVICE_TAG_ENGINE = ServiceTagEngineConfig().apply { isDefault = true } - -internal val DEFAULT_WORKFLOW_STATE_ENGINE = WorkflowStateEngineConfig().apply { isDefault = true } - -internal val DEFAULT_WORKFLOW_TAG_ENGINE = WorkflowTagEngineConfig().apply { isDefault = true } - diff --git a/infinitic-worker/src/main/kotlin/io/infinitic/workers/registry/ExecutorRegistry.kt b/infinitic-worker/src/main/kotlin/io/infinitic/workers/registry/ExecutorRegistry.kt new file mode 100644 index 000000000..8d584f68b --- /dev/null +++ b/infinitic-worker/src/main/kotlin/io/infinitic/workers/registry/ExecutorRegistry.kt @@ -0,0 +1,135 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.registry + +import io.infinitic.common.exceptions.thisShouldNotHappen +import io.infinitic.common.registry.ExecutorRegistryInterface +import io.infinitic.common.tasks.data.ServiceName +import io.infinitic.common.workers.config.WorkflowVersion +import io.infinitic.common.workflows.WorkflowContext +import io.infinitic.common.workflows.data.workflowTasks.WorkflowTaskParameters +import io.infinitic.common.workflows.data.workflows.WorkflowName +import io.infinitic.common.workflows.emptyWorkflowContext +import io.infinitic.exceptions.workflows.UnknownWorkflowVersionException +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout +import io.infinitic.workers.config.ServiceConfig +import io.infinitic.workers.config.ServiceExecutorConfig +import io.infinitic.workers.config.WorkflowConfig +import io.infinitic.workers.config.WorkflowExecutorConfig +import io.infinitic.workers.config.WorkflowFactory +import io.infinitic.workflows.Workflow +import io.infinitic.workflows.WorkflowCheckMode + +class ExecutorRegistry( + private val services: List, + private val workflows: List, +) : ExecutorRegistryInterface { + + override fun getServiceExecutorInstance(serviceName: ServiceName): Any = + getServiceExecutor(serviceName).factory.invoke() + + override fun getServiceExecutorWithTimeout(serviceName: ServiceName): WithTimeout? = + getServiceExecutor(serviceName).withTimeout + + override fun getServiceExecutorWithRetry(serviceName: ServiceName): WithRetry? = + getServiceExecutor(serviceName).withRetry + + override fun getWorkflowExecutorInstance(workflowTaskParameters: WorkflowTaskParameters): Workflow = + with(workflowTaskParameters) { + // set WorkflowContext before Workflow instance creation + Workflow.setContext( + WorkflowContext( + workflowName = workflowName.toString(), + workflowId = workflowId.toString(), + methodName = workflowMethod.methodName.toString(), + methodId = workflowMethod.workflowMethodId.toString(), + meta = workflowMeta.map, + tags = workflowTags.map { it.tag }.toSet(), + ), + ) + + getInstanceByVersion(workflowName, workflowVersion) + } + + override fun getWorkflowExecutorWithTimeout(workflowName: WorkflowName): WithTimeout? = + getWorkflowExecutor(workflowName).withTimeout + + override fun getWorkflowExecutorWithRetry(workflowName: WorkflowName): WithRetry? = + getWorkflowExecutor(workflowName).withRetry + + override fun getWorkflowExecutorCheckMode(workflowName: WorkflowName): WorkflowCheckMode? = + getWorkflowExecutor(workflowName).checkMode + + private fun getService(serviceName: ServiceName): ServiceConfig? = + services.firstOrNull { it.name == serviceName.name } + + private fun getServiceExecutor(serviceName: ServiceName): ServiceExecutorConfig = + getService(serviceName)?.executor ?: thisShouldNotHappen() + + private fun getWorkflow(workflowName: WorkflowName): WorkflowConfig? = + workflows.firstOrNull { it.name == workflowName.name } + + private fun getWorkflowExecutor(workflowName: WorkflowName): WorkflowExecutorConfig = + getWorkflow(workflowName)?.executor ?: thisShouldNotHappen() + + private fun getInstanceByVersion( + workflowName: WorkflowName, + workflowVersion: WorkflowVersion? + ): Workflow = getFactory( + workflowName, + workflowVersion ?: getLastVersion(workflowName), + ).invoke() + + private val factoryByVersionByWorkflowName = workflows + .associateBy { WorkflowName(it.name) } + .mapValues { it.value.executor?.instanceByVersion } + + private fun getFactory( + workflowName: WorkflowName, + workflowVersion: WorkflowVersion + ): WorkflowFactory = + (factoryByVersionByWorkflowName[workflowName] ?: thisShouldNotHappen())[workflowVersion] + ?: throw UnknownWorkflowVersionException(workflowName, workflowVersion) + + private fun getLastVersion(workflowName: WorkflowName) = + factoryByVersionByWorkflowName[workflowName]?.keys?.maxOrNull() + ?: thisShouldNotHappen() + + private val WorkflowExecutorConfig.instanceByVersion: Map + get() { + // this is needed in case the workflow properties use the workflow context + Workflow.setContext(emptyWorkflowContext) + + // list of versions + val versions = factories.mapIndexed { index, factory -> + try { + factory() + } catch (e: Exception) { + throw IllegalArgumentException("Error when running factory #$index", e) + } + }.map { WorkflowVersion.from(it::class.java) } + + return versions.zip(factories).toMap() + } +} diff --git a/infinitic-worker/src/test/java/io/infinitic/workers/InfiniticWorkerTests.java b/infinitic-worker/src/test/java/io/infinitic/workers/InfiniticWorkerTests.java deleted file mode 100644 index 82d313ba3..000000000 --- a/infinitic-worker/src/test/java/io/infinitic/workers/InfiniticWorkerTests.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - *

- * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - *

- * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - *

- * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - *

- * Software: Infinitic - *

- * License: MIT License (https://opensource.org/licenses/MIT) - *

- * Licensor: infinitic.io - */ -package io.infinitic.workers; - -import io.infinitic.common.transport.InfiniticConsumer; -import io.infinitic.common.transport.InfiniticProducer; -import io.infinitic.pulsar.PulsarInfiniticConsumer; -import io.infinitic.pulsar.PulsarInfiniticProducer; -import io.infinitic.pulsar.config.PulsarConfig; -import io.infinitic.storage.config.PostgresConfig; -import io.infinitic.storage.config.StorageConfig; -import io.infinitic.workers.config.WorkerConfig; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertInstanceOf; - - -class InfiniticWorkerTests { - - @Test - public void canCreateWorkerProgrammaticallyWithPulsar() { - PulsarConfig pulsar = PulsarConfig.builder() - .brokerServiceUrl("pulsar://localhost:6650") - .webServiceUrl("http://localhost:8080") - .tenant("infinitic") - .namespace("dev") - .build(); - - StorageConfig storage = StorageConfig.builder() - .postgres(PostgresConfig - .builder() - .build() - ).build(); - - - WorkerConfig workerConfig = WorkerConfig.builder() - .pulsar(pulsar) - .storage(storage) - .build(); - - try (InfiniticWorker worker = InfiniticWorker.fromConfig(workerConfig)) { - InfiniticConsumer c = worker.getConsumer(); - InfiniticProducer p = worker.getProducer(); - assertInstanceOf(PulsarInfiniticConsumer.class, c); - assertInstanceOf(PulsarInfiniticProducer.class, p); - } - } -} diff --git a/infinitic-worker/src/test/java/io/infinitic/workers/JavaInfiniticWorkerTests.java b/infinitic-worker/src/test/java/io/infinitic/workers/JavaInfiniticWorkerTests.java new file mode 100644 index 000000000..5393426e3 --- /dev/null +++ b/infinitic-worker/src/test/java/io/infinitic/workers/JavaInfiniticWorkerTests.java @@ -0,0 +1,123 @@ +/** + * "Commons Clause" License Condition v1.0 + *

+ * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + *

+ * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + *

+ * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + *

+ * Software: Infinitic + *

+ * License: MIT License (https://opensource.org/licenses/MIT) + *

+ * Licensor: infinitic.io + */ +package io.infinitic.workers; + +import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy; +import io.infinitic.storage.config.PostgresStorageConfig; +import io.infinitic.storage.config.StorageConfig; +import io.infinitic.transport.config.PulsarTransportConfig; +import io.infinitic.workers.config.*; +import io.infinitic.workflows.Workflow; +import io.infinitic.workflows.WorkflowCheckMode; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + + +class JavaInfiniticWorkerTests { + + @Test + public void canCreateWorkerThroughBuilders() { + assertDoesNotThrow(() -> { + PulsarTransportConfig transport = PulsarTransportConfig.builder() + .setBrokerServiceUrl("pulsar://localhost:6650") + .setWebServiceUrl("http://localhost:8080") + .setTenant("infinitic") + .setNamespace("dev") + .build(); + + StorageConfig storage = PostgresStorageConfig.builder() + .setHost("localhost") + .setPort(5432) + .setUserName("postgres") + .setPassword("password") + .build(); + + + EventListenerConfig eventListener = EventListenerConfig.builder() + .setListener(new TestEventListener()) + .allowServices("service1") + .allowServices(ServiceA.class) + .disallowServices("service2") + .disallowServices(ServiceA.class) + .allowWorkflows("workflow1") + .allowWorkflows(ServiceA.class) + .disallowWorkflows("workflow2") + .disallowWorkflows(ServiceA.class) + .setConcurrency(7) + .build(); + + ServiceExecutorConfig serviceExecutor = ServiceExecutorConfig.builder() + .setServiceName("service1") + .setConcurrency(3) + .setFactory(ServiceA::new) + .withRetry(new ExponentialBackoffRetryPolicy()) + .setTimeoutSeconds(4.0) + .build(); + + ServiceTagEngineConfig serviceTagEngine = ServiceTagEngineConfig.builder() + .setConcurrency(6) + .setServiceName("service1") + .build(); + + WorkflowExecutorConfig workflowExecutor = WorkflowExecutorConfig.builder() + .setConcurrency(5) + .setTimeoutSeconds(3.0) + .addFactory(WorkflowA::new) + .setWorkflowName("workflow1") + .setCheckMode(WorkflowCheckMode.simple) + .build(); + + WorkflowTagEngineConfig workflowTagEngine = WorkflowTagEngineConfig.builder() + .setConcurrency(4) + .setWorkflowName("workflow1") + .setStorage(storage) + .build(); + + WorkflowStateEngineConfig workflowStateEngine = WorkflowStateEngineConfig.builder() + .setConcurrency(6) + .setWorkflowName("workflow1") + .build(); + + try (InfiniticWorker worker = InfiniticWorker.builder() + .setTransport(transport) + .setStorage(storage) + .setEventListener(eventListener) + .addServiceExecutor(serviceExecutor) + .addServiceTagEngine(serviceTagEngine) + .addWorkflowExecutor(workflowExecutor) + .addWorkflowTagEngine(workflowTagEngine) + .addWorkflowStateEngine(workflowStateEngine) + .build() + ) { + // ok + } + }); + } +} + +class ServiceA { +} + +class WorkflowA extends Workflow { +} diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/InfiniticWorkerTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/InfiniticWorkerTests.kt index 330d0ebff..abf31322b 100644 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/InfiniticWorkerTests.kt +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/InfiniticWorkerTests.kt @@ -22,4 +22,421 @@ */ package io.infinitic.workers +import io.cloudevents.CloudEvent +import io.infinitic.cloudEvents.CloudEventListener +import io.infinitic.common.fixtures.later +import io.infinitic.storage.config.InMemoryConfig +import io.infinitic.storage.config.InMemoryStorageConfig +import io.infinitic.storage.config.MySQLConfig +import io.infinitic.transport.config.InMemoryTransportConfig +import io.infinitic.workers.config.EventListenerConfig +import io.infinitic.workers.config.InfiniticWorkerConfig +import io.infinitic.workers.config.ServiceExecutorConfig +import io.infinitic.workers.config.ServiceTagEngineConfig +import io.infinitic.workers.config.WorkflowExecutorConfig +import io.infinitic.workers.config.WorkflowStateEngineConfig +import io.infinitic.workers.config.WorkflowTagEngineConfig +import io.infinitic.workers.samples.ServiceA +import io.infinitic.workers.samples.ServiceAImpl +import io.infinitic.workers.samples.WorkflowA +import io.infinitic.workers.samples.WorkflowAImpl +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.shouldBeInstanceOf +internal class InfiniticWorkerTests : StringSpec( + { + val transport = InMemoryTransportConfig() + + class TestEventListener : CloudEventListener { + override fun onEvent(event: CloudEvent) {} + } + + val eventListener = EventListenerConfig.builder() + .setListener(TestEventListener()) + .build() + + val storage = InMemoryStorageConfig(InMemoryConfig()) + + val serviceName = ServiceA::class.java.name + val serviceClass = ServiceAImpl::class.java.name + + val serviceTagEngine = ServiceTagEngineConfig.builder() + .setServiceName(serviceName) + .setStorage(storage) + .build() + + val serviceExecutor = ServiceExecutorConfig.builder() + .setServiceName(serviceName) + .setFactory { ServiceAImpl() } + .build() + + val workflowName = WorkflowA::class.java.name + val workflowClass = WorkflowAImpl::class.java.name + + val workflowTagEngine = WorkflowTagEngineConfig.builder() + .setWorkflowName(workflowName) + .setStorage(storage) + .build() + + val workflowExecutor = WorkflowExecutorConfig.builder() + .setWorkflowName(workflowName) + .addFactory { WorkflowAImpl() } + .build() + + val workflowStateEngine = WorkflowStateEngineConfig.builder() + .setWorkflowName(workflowName) + .setStorage(storage) + .build() + + "Can create Infinitic Worker as Event Listener through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .setEventListener(eventListener) + .build() + } + worker.getEventListenerConfig() shouldBe eventListener + } + + "Can create Infinitic Worker as Event Listener through Yaml" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +eventListener: + class: ${TestEventListener::class.java.name} + """, + ) + } + with(worker.getEventListenerConfig()) { + this?.listener?.shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Service Executor through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .addServiceExecutor(serviceExecutor) + .build() + } + worker.getServiceExecutorConfig(serviceName) shouldBe serviceExecutor + } + + "Can create Infinitic Worker as Service Executor through Yaml" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +services: +- name: $serviceName + executor: + class: $serviceClass + """, + ) + } + with(worker.getServiceExecutorConfig(serviceName)) { + this?.factory?.invoke().shouldBeInstanceOf() + this?.serviceName shouldBe serviceName + } + } + + "Can create Infinitic Worker as Service Tag Engine through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .addServiceTagEngine(serviceTagEngine) + .build() + } + worker.getServiceTagEngineConfig(serviceName) shouldBe serviceTagEngine + } + + "Can create Infinitic Worker as Service Tag Engine through Yaml" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +services: +- name: $serviceName + tagEngine: + storage: + inMemory: + """, + ) + } + with(worker.getServiceTagEngineConfig(serviceName)) { + this?.serviceName shouldBe serviceName + this?.concurrency shouldBe 1 + this?.storage.shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Service Tag Engine through Yaml with default storage" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +storage: + inMemory: +services: +- name: $serviceName + tagEngine: + """, + ) + } + with(worker.getServiceTagEngineConfig(serviceName)) { + this?.serviceName shouldBe serviceName + this?.concurrency shouldBe 1 + this?.storage.shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Workflow Executor through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .addWorkflowExecutor(workflowExecutor) + .build() + } + worker.getWorkflowExecutorConfig(workflowName) shouldBe workflowExecutor + } + + "Can create Infinitic Worker as Workflow Executor through Yaml" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +workflows: +- name: $workflowName + executor: + class: $workflowClass + """, + ) + } + with(worker.getWorkflowExecutorConfig(workflowName)) { + this?.workflowName shouldBe workflowName + this?.factories?.get(0)?.invoke().shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Workflow Tag Engine through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .addWorkflowTagEngine(workflowTagEngine) + .build() + } + worker.getWorkflowTagEngineConfig(workflowName) shouldBe workflowTagEngine + } + + "Can create Infinitic Worker as Workflow Tag Engine through Yaml" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +workflows: +- name: $workflowName + tagEngine: + storage: + inMemory: + """, + ) + } + with(worker.getWorkflowTagEngineConfig(workflowName)) { + this?.workflowName shouldBe workflowName + this?.concurrency shouldBe 1 + this?.storage.shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Workflow Tag Engine through Yaml with default storage" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +storage: + inMemory: +workflows: +- name: $workflowName + tagEngine: + """, + ) + } + with(worker.getWorkflowTagEngineConfig(workflowName)) { + this?.workflowName shouldBe workflowName + this?.concurrency shouldBe 1 + this?.storage.shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Workflow State Engine through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .addWorkflowStateEngine(workflowStateEngine) + .build() + } + worker.getWorkflowStateEngineConfig(workflowName) shouldBe workflowStateEngine + } + + "Can create Infinitic Worker as Workflow State Engine through Yaml" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +workflows: +- name: $workflowName + stateEngine: + storage: + inMemory: + """, + ) + } + with(worker.getWorkflowStateEngineConfig(workflowName)) { + this?.workflowName shouldBe workflowName + this?.concurrency shouldBe 1 + this?.storage.shouldBeInstanceOf() + } + } + + "Can create Infinitic Worker as Workflow State Engine through Yaml with default storage" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +storage: + inMemory: +workflows: +- name: $workflowName + stateEngine: + """, + ) + } + with(worker.getWorkflowStateEngineConfig(workflowName)) { + this?.workflowName shouldBe workflowName + this?.concurrency shouldBe 1 + this?.storage.shouldBeInstanceOf() + } + } + + "Can create complete Infinitic Worker through builder" { + val worker = shouldNotThrowAny { + InfiniticWorker.builder() + .setTransport(transport) + .setEventListener(eventListener) + .addServiceExecutor(serviceExecutor) + .addServiceTagEngine(serviceTagEngine) + .addWorkflowTagEngine(workflowTagEngine) + .addWorkflowExecutor(workflowExecutor) + .addWorkflowStateEngine(workflowStateEngine) + .build() + } + worker.getEventListenerConfig() shouldBe eventListener + worker.getServiceExecutorConfig(serviceName) shouldBe serviceExecutor + worker.getServiceTagEngineConfig(serviceName) shouldBe serviceTagEngine + worker.getWorkflowTagEngineConfig(workflowName) shouldBe workflowTagEngine + worker.getWorkflowExecutorConfig(workflowName) shouldBe workflowExecutor + worker.getWorkflowStateEngineConfig(workflowName) shouldBe workflowStateEngine + } + + "Can create complete Infinitic Worker through Yaml with default storage" { + val worker = shouldNotThrowAny { + InfiniticWorker.fromYamlString( + """ +transport: inMemory +eventListener: + class: ${TestEventListener::class.java.name} +storage: + inMemory: +services: +- name: $serviceName + executor: + class: $serviceClass + tagEngine: +workflows: +- name: $workflowName + executor: + class: $workflowClass + stateEngine: + tagEngine: + """, + ) + } + worker.getEventListenerConfig() shouldNotBe null + worker.getServiceExecutorConfig(serviceName) shouldNotBe null + worker.getServiceTagEngineConfig(serviceName) shouldNotBe null + worker.getWorkflowTagEngineConfig(workflowName) shouldNotBe null + worker.getWorkflowExecutorConfig(workflowName) shouldNotBe null + worker.getWorkflowStateEngineConfig(workflowName) shouldNotBe null + } + + "explicit storage should not be replaced by default" { + val db = MySQLConfig(host = "localhost", port = 3306, username = "root", password = "p") + val config = InfiniticWorkerConfig.fromYamlString( + """ +transport: inMemory +storage: + compression: bzip2 + mysql: + host: ${db.host} + port: ${db.port} + username: ${db.username} + password: ${db.password} + +services: + - name: $serviceName + tagEngine: + storage: + inMemory: +workflows: + - name: $workflowName + stateEngine: + storage: + inMemory: + tagEngine: + storage: + inMemory: +""", + ) + with(config.services.first { it.name == serviceName }) { + tagEngine?.storage.shouldBeInstanceOf() + } + with(config.workflows.first { it.name == workflowName }) { + stateEngine?.storage.shouldBeInstanceOf() + tagEngine?.storage.shouldBeInstanceOf() + } + } + + "startAsync() should not block" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addServiceExecutor(serviceExecutor) + .build() + var flag = false + later { + flag = true + worker.close() + } + worker.startAsync() + flag shouldBe false + } + + "start() should block, and be released when closed" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addServiceExecutor(serviceExecutor) + .build() + + var flag = false + later(2000) { + flag = true + worker.close() + } + worker.start() + flag shouldBe true + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ConfigGetterInterfaceTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ConfigGetterInterfaceTests.kt new file mode 100644 index 000000000..cb79494f7 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ConfigGetterInterfaceTests.kt @@ -0,0 +1,135 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.cloudevents.CloudEvent +import io.infinitic.cloudEvents.CloudEventListener +import io.infinitic.storage.config.InMemoryConfig +import io.infinitic.storage.config.InMemoryStorageConfig +import io.infinitic.transport.config.InMemoryTransportConfig +import io.infinitic.workers.InfiniticWorker +import io.infinitic.workers.samples.ServiceA +import io.infinitic.workers.samples.ServiceAImpl +import io.infinitic.workers.samples.WorkflowA +import io.infinitic.workers.samples.WorkflowAImpl +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe + +internal class ConfigGetterInterfaceTests : StringSpec( + { + class TestEventListener : CloudEventListener { + override fun onEvent(event: CloudEvent) {} + } + + val transport = InMemoryTransportConfig() + + val eventListener = EventListenerConfig.builder() + .setListener(TestEventListener()) + .build() + + val storage = InMemoryStorageConfig(InMemoryConfig()) + + val serviceName = ServiceA::class.java.name + + val serviceTagEngine = ServiceTagEngineConfig.builder() + .setServiceName(serviceName) + .setStorage(storage) + .build() + + val serviceExecutor = ServiceExecutorConfig.builder() + .setServiceName(serviceName) + .setFactory { ServiceAImpl() } + .build() + + val workflowName = WorkflowA::class.java.name + + val workflowTagEngine = WorkflowTagEngineConfig.builder() + .setWorkflowName(workflowName) + .setStorage(storage) + .build() + + val workflowExecutor = WorkflowExecutorConfig.builder() + .setWorkflowName(workflowName) + .addFactory { WorkflowAImpl() } + .build() + + val workflowStateEngine = WorkflowStateEngineConfig.builder() + .setWorkflowName(workflowName) + .setStorage(storage) + .build() + + "Can access EventListenerConfig" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .setEventListener(eventListener) + .build() + + worker.getEventListenerConfig() shouldBe eventListener + } + + "Can access ServiceExecutorConfig" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addServiceExecutor(serviceExecutor) + .build() + + worker.getServiceExecutorConfig(serviceName) shouldBe serviceExecutor + } + + "Can access ServiceTagEngineConfig" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addServiceTagEngine(serviceTagEngine) + .build() + + worker.getServiceTagEngineConfig(serviceName) shouldBe serviceTagEngine + } + + "Can access WorkflowExecutorConfig" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addWorkflowExecutor(workflowExecutor) + .build() + + worker.getWorkflowExecutorConfig(workflowName) shouldBe workflowExecutor + } + + "Can access WorkflowStateEngineConfig" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addWorkflowStateEngine(workflowStateEngine) + .build() + + worker.getWorkflowStateEngineConfig(workflowName) shouldBe workflowStateEngine + } + + "Can access WorkflowTagEngineConfig" { + val worker = InfiniticWorker.builder() + .setTransport(transport) + .addWorkflowTagEngine(workflowTagEngine) + .build() + + worker.getWorkflowTagEngineConfig(workflowName) shouldBe workflowTagEngine + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/EventListenerConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/EventListenerConfigTests.kt new file mode 100644 index 000000000..75dab9d58 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/EventListenerConfigTests.kt @@ -0,0 +1,199 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.cloudevents.CloudEvent +import io.infinitic.cloudEvents.CloudEventListener +import io.infinitic.common.utils.annotatedName +import io.infinitic.workers.samples.ServiceA +import io.infinitic.workers.samples.ServiceAImpl +import io.infinitic.workers.samples.WorkflowA +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrowAny +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf + +internal class TestEventListener : CloudEventListener { + override fun onEvent(event: CloudEvent) {} +} + +internal class EventListenerConfigTests : StringSpec( + { + val listener = TestEventListener() + + "Can create EventListenerConfig through builder with default parameters" { + val config = shouldNotThrowAny { + EventListenerConfig.builder() + .setListener(listener) + .build() + } + + config.shouldBeInstanceOf() + config.listener shouldBe listener + config.concurrency shouldBe 1 + config.subscriptionName shouldBe null + config.refreshDelaySeconds shouldBe 60.0 + config.allowedServices shouldBe null + config.allowedWorkflows shouldBe null + config.disallowedServices.size shouldBe 0 + config.disallowedWorkflows.size shouldBe 0 + } + + "Can create EventListenerConfig through Yaml with default parameters" { + val config = shouldNotThrowAny { + EventListenerConfig.fromYamlString( + """ +class: ${TestEventListener::class.java.name} + """, + ) + } + + config.shouldBeInstanceOf() + config.listener::class shouldBe TestEventListener::class + config.concurrency shouldBe 1 + config.subscriptionName shouldBe null + config.refreshDelaySeconds shouldBe 60.0 + config.allowedServices shouldBe null + config.allowedWorkflows shouldBe null + config.disallowedServices.size shouldBe 0 + config.disallowedWorkflows.size shouldBe 0 + } + + "Can create EventListenerConfig through builder with all parameters" { + val config = shouldNotThrowAny { + EventListenerConfig.builder() + .setListener(listener) + .setConcurrency(10) + .setSubscriptionName("subscriptionName") + .setRefreshDelaySeconds(10.0) + .allowServices("service1", "service2") + .allowServices("service3") + .allowServices(ServiceA::class.java) + .allowWorkflows("workflow1", "workflow2") + .allowWorkflows("workflow3") + .allowWorkflows(WorkflowA::class.java) + .disallowServices("service4", "service5") + .disallowServices("service6") + .disallowServices(ServiceA::class.java) + .disallowWorkflows("workflow4", "workflow5") + .disallowWorkflows("workflow6") + .disallowWorkflows(WorkflowA::class.java) + .build() + } + + config.shouldBeInstanceOf() + config.listener shouldBe listener + config.concurrency shouldBe 10 + config.subscriptionName shouldBe "subscriptionName" + config.refreshDelaySeconds shouldBe 10.0 + config.allowedServices shouldBe listOf( + "service1", + "service2", + "service3", + ServiceA::class.java.annotatedName, + ) + config.allowedWorkflows shouldBe listOf( + "workflow1", + "workflow2", + "workflow3", + WorkflowA::class.java.annotatedName, + ) + config.disallowedServices shouldBe listOf( + "service4", + "service5", + "service6", + ServiceA::class.java.annotatedName, + ) + config.disallowedWorkflows shouldBe listOf( + "workflow4", + "workflow5", + "workflow6", + WorkflowA::class.java.annotatedName, + ) + } + + "Can create EventListenerConfig through YMAL with all parameters" { + val config = shouldNotThrowAny { + EventListenerConfig.fromYamlString( + """ +class: ${TestEventListener::class.java.name} +concurrency: 10 +subscriptionName: subscriptionName +refreshDelaySeconds: 10 +services: + allow: + - service1 + - service2 + - service3 + disallow: + - service4 + - service5 + - service6 +workflows: + allow: + - workflow1 + - workflow2 + - workflow3 + disallow: + - workflow4 + - workflow5 + - workflow6 + """, + ) + } + config.shouldBeInstanceOf() + config.listener::class shouldBe TestEventListener::class + config.concurrency shouldBe 10 + config.subscriptionName shouldBe "subscriptionName" + config.refreshDelaySeconds shouldBe 10.0 + config.allowedServices shouldBe listOf("service1", "service2", "service3") + config.allowedWorkflows shouldBe listOf("workflow1", "workflow2", "workflow3") + config.disallowedServices shouldBe listOf("service4", "service5", "service6") + config.disallowedWorkflows shouldBe listOf("workflow4", "workflow5", "workflow6") + } + + "Listener not implementing CloudEventListener should throw exception" { + val e = shouldThrowAny { + EventListenerConfig.fromYamlString( + """ +class: ${ServiceAImpl::class.java.name} + """, + ) + } + e.message shouldContain "CloudEventListener" + } + + "Listener not found should throw exception" { + val e = shouldThrowAny { + EventListenerConfig.fromYamlString( + """ +class: UnknownClass + """, + ) + } + e.message shouldContain "Class 'UnknownClass' not found" + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/RetryPolicyTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/RetryPolicyTests.kt deleted file mode 100644 index b031d800d..000000000 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/RetryPolicyTests.kt +++ /dev/null @@ -1,79 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.config - -import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.string.shouldContain - -internal class RetryPolicyTests : - StringSpec( - { - "initialDelayInSeconds must be > 0" { - val e = shouldThrow { - ExponentialBackoffRetryPolicy(minimumSeconds = 0.0).check() - } - e.message!! shouldContain ExponentialBackoffRetryPolicy::minimumSeconds.name - - val f = shouldThrow { - ExponentialBackoffRetryPolicy(minimumSeconds = -1.0).check() - } - f.message!! shouldContain ExponentialBackoffRetryPolicy::minimumSeconds.name - } - - "backoffCoefficient can not be > 0" { - val e = shouldThrow { - ExponentialBackoffRetryPolicy(backoffCoefficient = 0.0).check() - } - e.message!! shouldContain ExponentialBackoffRetryPolicy::backoffCoefficient.name - - val f = - shouldThrow { - ExponentialBackoffRetryPolicy(backoffCoefficient = -1.0).check() - } - f.message!! shouldContain ExponentialBackoffRetryPolicy::backoffCoefficient.name - } - - "maximumSeconds can not be > 0" { - val e = shouldThrow { - ExponentialBackoffRetryPolicy(maximumSeconds = 0.0).check() - } - e.message!! shouldContain ExponentialBackoffRetryPolicy::maximumSeconds.name - - val f = - shouldThrow { - ExponentialBackoffRetryPolicy(maximumSeconds = -1.0).check() - } - f.message!! shouldContain ExponentialBackoffRetryPolicy::maximumSeconds.name - } - - "maximumRetries can not be >= 0" { - val f = shouldThrow { - ExponentialBackoffRetryPolicy(maximumRetries = -1).check() - } - f.message!! shouldContain ExponentialBackoffRetryPolicy::maximumRetries.name - } - }, - ) - diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceConfigTests.kt index 82a3e02ea..121eeb116 100644 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceConfigTests.kt +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceConfigTests.kt @@ -22,21 +22,84 @@ */ package io.infinitic.workers.config -import io.infinitic.workers.register.config.ServiceConfig +import com.sksamuel.hoplite.ConfigException +import io.infinitic.workers.samples.ServiceA +import io.infinitic.workers.samples.ServiceAImpl +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf -internal class ServiceConfigTests : - StringSpec( - { - val name = "testName" +internal class ServiceConfigTests : StringSpec( + { + val serviceName = ServiceA::class.java.name + val serviceClass = ServiceAImpl::class.java.name - "Can create ServiceConfig through builder" { - val serviceConfig = ServiceConfig.builder() - .name(name) - .build() + "Can create ServiceConfig through YAML with an executor" { + val config = shouldNotThrowAny { + ServiceConfig.fromYamlString( + """ +name: $serviceName +executor: + class: $serviceClass + concurrency: 10 + """, + ) + } + + config.name shouldBe serviceName + config.executor.shouldBeInstanceOf() + config.tagEngine shouldBe null + } + + "Can create ServiceConfig through YAML with a Tag Engine" { + val config = shouldNotThrowAny { + ServiceConfig.fromYamlString( + """ +name: $serviceName +tagEngine: + concurrency: 10 + """, + ) + } + + config.name shouldBe serviceName + config.executor shouldBe null + config.tagEngine.shouldBeInstanceOf() + } + + "Can create ServiceConfig through YAML with both an Executor and a Tag Engine" { + val config = shouldNotThrowAny { + ServiceConfig.fromYamlString( + """ +name: $serviceName +executor: + class: $serviceClass + concurrency: 10 +tagEngine: + concurrency: 10 + """, + ) + } + + config.name shouldBe serviceName + config.executor.shouldBeInstanceOf() + config.tagEngine.shouldBeInstanceOf() + } - serviceConfig shouldBe ServiceConfig(name) + "class must implements the Service" { + val e = shouldThrow { + ServiceConfig.fromYamlString( + """ +name: UnknownService +executor: + class: $serviceClass + """, + ) } - }, - ) + e.message shouldContain "'$serviceClass' must be an implementation of Service 'UnknownService'" + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceExecutorConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceExecutorConfigTests.kt new file mode 100644 index 000000000..045635f54 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceExecutorConfigTests.kt @@ -0,0 +1,231 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import com.sksamuel.hoplite.ConfigException +import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout +import io.infinitic.workers.samples.ServiceA +import io.infinitic.workers.samples.ServiceAImpl +import io.infinitic.workers.samples.ServiceWithExceptionInInitializerError +import io.infinitic.workers.samples.ServiceWithInvocationTargetException +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf + +internal class ServiceExecutorConfigTests : StringSpec( + { + val serviceName = ServiceA::class.java.name + + "Can create ServiceExecutorConfig through builder with default parameters" { + val config = shouldNotThrowAny { + ServiceExecutorConfig.builder() + .setServiceName(serviceName) + .setFactory { ServiceAImpl() } + .build() + } + + config.shouldBeInstanceOf() + config.serviceName shouldBe serviceName + config.factory.invoke().shouldBeInstanceOf() + config.concurrency shouldBe 1 + config.withRetry shouldBe WithRetry.UNSET + config.withTimeout shouldBe WithTimeout.UNSET + } + + "Can create ServiceExecutorConfig through builder with all parameters" { + val withRetry = ExponentialBackoffRetryPolicy() + val config = shouldNotThrowAny { + ServiceExecutorConfig.builder() + .setServiceName(serviceName) + .setFactory { ServiceAImpl() } + .setConcurrency(10) + .setTimeoutSeconds(3.0) + .withRetry(withRetry) + .build() + } + + config.shouldBeInstanceOf() + config.factory.invoke().shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.withRetry shouldBe withRetry + config.withTimeout?.getTimeoutSeconds() shouldBe 3.0 + } + + "Concurrency must be positive when building ServiceExecutorConfig" { + val e = shouldThrow { + ServiceExecutorConfig.builder() + .setServiceName(serviceName) + .setFactory { ServiceAImpl() } + .setConcurrency(0) + .build() + } + e.message shouldContain "concurrency" + } + + "serviceName is mandatory when building ServiceExecutorConfig" { + val e = shouldThrow { + ServiceExecutorConfig.builder() + .setFactory { ServiceAImpl() } + .setConcurrency(1) + .build() + } + e.message shouldContain "serviceName" + } + + "Factory is mandatory when building ServiceExecutorConfig" { + val e = shouldThrow { + ServiceExecutorConfig.builder() + .setServiceName(serviceName) + .setConcurrency(1) + .build() + } + e.message shouldContain "factory" + } + + "Can create ServiceExecutorConfig through YAML with default parameters" { + val config = shouldNotThrowAny { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceAImpl::class.java.name} + """, + ) + } + + config.factory.invoke().shouldBeInstanceOf() + config.concurrency shouldBe 1 + config.withRetry shouldBe WithRetry.UNSET + config.withTimeout shouldBe WithTimeout.UNSET + } + + "Can create ServiceExecutorConfig through YAML with all parameters" { + val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = 4.0) + val config = shouldNotThrowAny { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceAImpl::class.java.name} +concurrency: 10 +timeoutSeconds: 3.0 +retry: + minimumSeconds: 4 + """, + ) + } + + config.factory.invoke().shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.withTimeout?.getTimeoutSeconds() shouldBe 3.0 + config.withRetry shouldBe withRetry + } + + "class is mandatory when building ServiceExecutorConfig from YAML" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +concurrency: 10 + """, + ) + } + + e.message shouldContain "class" + } + + "Unknown class in ignoredExceptions should throw" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceAImpl::class.java.name} +retry: + ignoredExceptions: + - foobar +""", + ) + } + e.message shouldContain "Class 'foobar' not found" + } + + "Class not an Exception in ignoredExceptions should throw" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceAImpl::class.java.name} +retry: + ignoredExceptions: + - ${ServiceA::class.java.name} +""", + ) + } + e.message shouldContain "must be an Exception" + } + + "timeout must be > 0 when building ServiceExecutorConfig from YAML" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceAImpl::class.java.name} +timeoutSeconds: 0 +""", + ) + } + e.message shouldContain "timeoutSeconds must be > 0" + } + + "task with InvocationTargetException should throw cause" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceWithInvocationTargetException::class.java.name} + """, + ) + } + e.message shouldContain + "Error during class '${ServiceWithInvocationTargetException::class.java.name}' instantiation" + } + + "task with ServiceWithExceptionInInitializerError should throw cause" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +class: ${ServiceWithExceptionInInitializerError::class.java.name} + """, + ) + } + e.message shouldContain "ExceptionInInitializerError" + } + + "service Unknown" { + val e = shouldThrow { + ServiceExecutorConfig.fromYamlString( + """ +class: io.infinitic.workers.samples.UnknownService + """, + ) + } + e.message shouldContain "Class 'io.infinitic.workers.samples.UnknownService' not found" + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceTagEngineConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceTagEngineConfigTests.kt new file mode 100644 index 000000000..7a46dcc25 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/ServiceTagEngineConfigTests.kt @@ -0,0 +1,113 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.storage.config.InMemoryStorageConfig +import io.infinitic.workers.samples.ServiceA +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf + +internal class ServiceTagEngineConfigTests : StringSpec( + { + val serviceName = ServiceA::class.java.name + val storage = InMemoryStorageConfig.builder().build() + + "Can create ServiceTagEngineConfig through builder with default parameters" { + val config = shouldNotThrowAny { + ServiceTagEngineConfig.builder() + .setServiceName(serviceName) + .build() + } + + config.serviceName shouldBe serviceName + config.shouldBeInstanceOf() + config.storage shouldBe null + config.concurrency shouldBe 1 + } + + "Can create ServiceTagEngineConfig through builder with all parameters" { + val config = shouldNotThrowAny { + ServiceTagEngineConfig.builder() + .setServiceName(serviceName) + .setConcurrency(10) + .setStorage(storage) + .build() + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.storage shouldBe storage + } + + "ServiceName is mandatory when building ServiceTagEngineConfig through builder" { + val e = shouldThrow { + ServiceTagEngineConfig.builder() + .build() + } + e.message shouldContain "serviceName" + } + + "Concurrency must be positive when building ServiceTagEngineConfig" { + val e = shouldThrow { + ServiceTagEngineConfig.builder() + .setServiceName(serviceName) + .setConcurrency(0) + .build() + } + e.message shouldContain "concurrency" + } + + "Can create ServiceTagEngineConfig through YAML without serviceName" { + val config = shouldNotThrowAny { + ServiceTagEngineConfig.fromYamlString( + """ +concurrency: 10 + """, + ) + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + } + + "Can create ServiceTagEngineConfig through YAML with all parameters" { + val config = shouldNotThrowAny { + ServiceTagEngineConfig.fromYamlString( + """ +concurrency: 10 +storage: + inMemory: + """, + ) + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.storage shouldBe storage + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkerConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkerConfigTests.kt deleted file mode 100644 index ad903c84b..000000000 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkerConfigTests.kt +++ /dev/null @@ -1,265 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.config - -import com.sksamuel.hoplite.ConfigException -import io.infinitic.cloudEvents.CloudEventListener -import io.infinitic.common.config.loadConfigFromYaml -import io.infinitic.common.tasks.data.ServiceName -import io.infinitic.workers.register.config.UNDEFINED_RETRY -import io.infinitic.workers.register.config.UNDEFINED_TIMEOUT -import io.infinitic.workers.samples.ServiceA -import io.infinitic.workers.samples.ServiceAImpl -import io.infinitic.workers.samples.WorkflowAImpl -import io.infinitic.workers.samples.WorkflowAImpl2 -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.shouldBe -import io.kotest.matchers.shouldNotBe -import io.kotest.matchers.string.shouldContain - -internal class WorkerConfigTests : - StringSpec( - { - - val serviceName = ServiceName(ServiceA::class.java.name) - val serviceImplName = ServiceAImpl::class.java.name - - "Can create WorkerConfig through builder" { - val workerName = "nameTest" - val workerConfig = WorkerConfig - .builder() - .name(workerName) - .build() - - workerConfig shouldBe WorkerConfig(workerName) - } - - "Unknown class in ignoredExceptions should throw" { - val e = shouldThrow { - loadConfigFromYaml( - """ -transport: inMemory -serviceDefault: - retry: - ignoredExceptions: - - foobar -""", - ) - } - e.message!! shouldContain "Class 'foobar' not found" - } - - "Unknown class in ignoredExceptions in task should throw" { - val e = shouldThrow { - loadConfigFromYaml( - """ -transport: inMemory -services: - - name: $serviceName - class: $serviceImplName - retry: - ignoredExceptions: - - foobar -""", - ) - } - e.message!! shouldContain "Class 'foobar' not found" - } - - "No Exception class in ignoredExceptions should throw" { - val e = shouldThrow { - loadConfigFromYaml( - """ -transport: inMemory -serviceDefault: - retry: - ignoredExceptions: - - io.infinitic.workers.InfiniticWorker -""", - ) - } - e.message!! shouldContain - "'io.infinitic.workers.InfiniticWorker' in ignoredExceptions must be an Exception" - } - - "No Exception class in ignoredExceptions in task should throw" { - val e = shouldThrow { - loadConfigFromYaml( - """ -transport: inMemory -services: - - name: $serviceName - class: $serviceImplName - retry: - ignoredExceptions: - - io.infinitic.workers.InfiniticWorker -""", - ) - } - e.message!! shouldContain - "'io.infinitic.workers.InfiniticWorker' in ignoredExceptions must be an Exception" - } - - "timeout in task should be positive" { - val e = shouldThrow { - loadConfigFromYaml( - """ -transport: inMemory -services: - - name: $serviceName - class: $serviceImplName - timeoutInSeconds: 0 -""", - ) - } - e.message!! shouldContain "timeoutInSeconds" - } - - "task instance should not be reused" { - val config = WorkerConfig.fromResource("/config/services/instance.yml") - - val task = config.services.first { it.name == ServiceA::class.java.name } - task.getInstance() shouldNotBe task.getInstance() - } - - "task with InvocationTargetException should throw cause" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/services/invocationTargetException.yml") - } - e.message!! shouldContain - "Error during class 'io.infinitic.workers.samples.ServiceWithInvocationTargetException' instantiation" - } - - "workflow with InvocationTargetException should throw cause" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/workflows/invocationTargetException.yml") - } - e.message!! shouldContain - "Error during class 'io.infinitic.workers.samples.WorkflowWithInvocationTargetException' instantiation" - } - - "task with ExceptionInInitializerError should throw cause" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/services/exceptionInInitializerError.yml") - } - e.message!! shouldContain "Underlying error was java.lang.ExceptionInInitializerError" - } - - "workflow with ExceptionInInitializerError should throw cause" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/workflows/exceptionInInitializerError.yml") - } - e.message!! shouldContain "Underlying error was java.lang.ExceptionInInitializerError" - } - - "service Unknown" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/services/unknown.yml") - } - e.message!! shouldContain "Class 'io.infinitic.workers.samples.UnknownService' not found" - } - - "Service Event Listener Unknown" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/services/unknownListener.yml") - } - e.message!! shouldContain "Class 'io.infinitic.workers.samples.UnknownListener' not found" - } - - "not a Service Event Listener" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/services/incompatibleListener.yml") - } - e.message!! shouldContain - "Class 'io.infinitic.workers.samples.ServiceAImpl' must implement '${CloudEventListener::class.java.name}'" - } - - "Service Event Listener should inherit concurrency from Service" { - val config = - WorkerConfig.fromResource("/config/services/instanceWithListener.yml") - - config.services - } - - "workflow Unknown" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/workflows/unknown.yml") - } - e.message!! shouldContain "Class 'io.infinitic.workers.samples.UnknownWorkflow' not found" - } - - "not a workflow" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/workflows/notAWorkflow.yml") - } - e.message!! shouldContain - "Class 'io.infinitic.workers.samples.NotAWorkflow' must extend 'io.infinitic.workflows.Workflow'" - } - - "checking default service config" { - val config = WorkerConfig.fromResource("/config/services/instance.yml") - - config.serviceDefault shouldBe null - config.services.size shouldBe 1 - config.services[0].retry shouldBe UNDEFINED_RETRY - config.services[0].timeoutInSeconds shouldBe UNDEFINED_TIMEOUT - config.services[0].concurrency shouldBe null - } - - "checking default workflow config" { - val config = WorkerConfig.fromResource("/config/workflows/instance.yml") - - config.workflowDefault shouldBe null - config.workflows.size shouldBe 1 - config.workflows[0].retry shouldBe UNDEFINED_RETRY - config.workflows[0].timeoutInSeconds shouldBe UNDEFINED_TIMEOUT - config.workflows[0].checkMode shouldBe null - config.workflows[0].concurrency shouldBe null - } - - "checking the compatibility between name and class in services" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/services/incompatibleServiceName.yml") - } - e.message!! shouldContain - "Class '${ServiceAImpl::class.java.name}' is not an implementation of this service" - } - - "checking the compatibility between name and class in workflows" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/workflows/incompatibleWorkflowName.yml") - } - e.message!! shouldContain - "Class '${WorkflowAImpl::class.java.name}' is not an implementation of 'UnknownWorkflow'" - } - - "checking the compatibility between name and classes in workflows" { - val e = shouldThrow { - WorkerConfig.fromResource("/config/workflows/incompatibleWorkflowsName.yml") - } - e.message!! shouldContain - "Class '${WorkflowAImpl2::class.java.name}' is not an implementation of 'io.infinitic.workers.samples.WorkflowA'" - } - }, - ) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowConfigTests.kt index 907ca04be..b04bb1439 100644 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowConfigTests.kt +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowConfigTests.kt @@ -22,21 +22,115 @@ */ package io.infinitic.workers.config -import io.infinitic.workers.register.config.WorkflowConfig +import com.sksamuel.hoplite.ConfigException +import io.infinitic.workers.samples.WorkflowA +import io.infinitic.workers.samples.WorkflowAImpl +import io.infinitic.workers.samples.WorkflowAImpl_1 +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf internal class WorkflowConfigTests : StringSpec( { - val name = "testName" + val workflowName = WorkflowA::class.java.name + val workflowClass = WorkflowAImpl::class.java.name - "Can create WorkflowConfig through builder" { - val workflowConfig = WorkflowConfig.builder() - .name(name) - .build() + "Can create WorkflowConfig through YAML with an executor" { + val config = shouldNotThrowAny { + WorkflowConfig.fromYamlString( + """ +name: $workflowName +executor: + class: $workflowClass + """, + ) + } - workflowConfig shouldBe WorkflowConfig(name) + config.name shouldBe workflowName + config.executor.shouldBeInstanceOf() + config.tagEngine shouldBe null + config.stateEngine shouldBe null + } + + "Can create WorkflowConfig through YAML with an executor with multiple versions" { + shouldNotThrowAny { + WorkflowConfig.fromYamlString( + """ +name: $workflowName +executor: + classes: + - $workflowClass + - ${WorkflowAImpl_1::class.java.name} + """, + ) + } + } + + "Can create WorkflowConfig through YAML with a Tag Engine" { + val config = shouldNotThrowAny { + WorkflowConfig.fromYamlString( + """ +name: $workflowName +tagEngine: + """, + ) + } + + config.name shouldBe workflowName + config.executor shouldBe null + config.tagEngine.shouldBeInstanceOf() + config.stateEngine shouldBe null + } + + "Can create WorkflowConfig through YAML with a State Engine" { + val config = shouldNotThrowAny { + WorkflowConfig.fromYamlString( + """ +name: $workflowName +stateEngine: + """, + ) + } + + config.name shouldBe workflowName + config.executor shouldBe null + config.tagEngine shouldBe null + config.stateEngine.shouldBeInstanceOf() + } + + "Can create WorkflowConfig through YAML with executor, tag engine and state engine" { + val config = shouldNotThrowAny { + WorkflowConfig.fromYamlString( + """ +name: $workflowName +executor: + class: $workflowClass +tagEngine: +stateEngine: + """, + ) + } + config.name shouldBe workflowName + config.executor.shouldBeInstanceOf() + config.tagEngine.shouldBeInstanceOf() + config.stateEngine.shouldBeInstanceOf() + } + + "class must implements the Workflow" { + val e = shouldThrow { + WorkflowConfig.fromYamlString( + """ +name: UnknownWorkflow +executor: + class: $workflowClass + """, + ) + } + e.message shouldContain "'$workflowClass' must be an implementation of Workflow 'UnknownWorkflow'" } }, ) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowExecutorConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowExecutorConfigTests.kt new file mode 100644 index 000000000..04918efdc --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowExecutorConfigTests.kt @@ -0,0 +1,426 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import com.sksamuel.hoplite.ConfigException +import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy +import io.infinitic.common.workflows.emptyWorkflowContext +import io.infinitic.tasks.WithRetry +import io.infinitic.tasks.WithTimeout +import io.infinitic.workers.samples.NotAWorkflow +import io.infinitic.workers.samples.ServiceA +import io.infinitic.workers.samples.WorkflowAImpl +import io.infinitic.workers.samples.WorkflowWithExceptionInInitializerError +import io.infinitic.workers.samples.WorkflowWithInvocationTargetException +import io.infinitic.workflows.Workflow +import io.infinitic.workflows.WorkflowCheckMode +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf + +internal class WorkflowExecutorConfigTests : StringSpec( + { + val workflowName = WorkflowAImpl::class.java.name + + "Can create WorkflowExecutorConfig through builder with default parameters" { + val config = shouldNotThrowAny { + WorkflowExecutorConfig.builder() + .setWorkflowName(workflowName) + .addFactory { WorkflowAImpl() } + .build() + } + + config.workflowName shouldBe workflowName + config.shouldBeInstanceOf() + config.factories.size shouldBe 1 + config.factories[0].invoke().shouldBeInstanceOf() + config.concurrency shouldBe 1 + config.withRetry shouldBe WithRetry.UNSET + config.withTimeout shouldBe WithTimeout.UNSET + } + + "Can create WorkflowExecutorConfig through builder with all parameters" { + val withRetry = ExponentialBackoffRetryPolicy() + val config = shouldNotThrowAny { + WorkflowExecutorConfig.builder() + .setWorkflowName(workflowName) + .addFactory { WorkflowAImpl() } + .setConcurrency(10) + .setTimeoutSeconds(3.0) + .withRetry(withRetry) + .setCheckMode(WorkflowCheckMode.strict) + .build() + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.withRetry shouldBe withRetry + config.withTimeout?.getTimeoutSeconds() shouldBe 3.0 + config.checkMode shouldBe WorkflowCheckMode.strict + } + + "workflowName is mandatory when building WorkflowExecutorConfig through builder" { + val e = shouldThrow { + WorkflowExecutorConfig.builder() + .addFactory { WorkflowAImpl() } + .build() + } + e.message shouldContain "workflowName" + } + + "Concurrency must be positive when building WorkflowExecutorConfig" { + val e = shouldThrow { + WorkflowExecutorConfig.builder() + .setWorkflowName(workflowName) + .addFactory { WorkflowAImpl() } + .setConcurrency(0) + .build() + } + e.message shouldContain "concurrency" + } + + "Factory is mandatory when building WorkflowExecutorConfig" { + val e = shouldThrow { + WorkflowExecutorConfig.builder() + .setWorkflowName(workflowName) + .setConcurrency(1) + .build() + } + e.message shouldContain "factory" + } + + "Can create WorkflowExecutorConfig through YAML without workflowName" { + val config = shouldNotThrowAny { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowAImpl::class.java.name} + """, + ) + } + + config.workflowName.isBlank() shouldBe true + config.factories.size shouldBe 1 + config.factories[0].invoke().shouldBeInstanceOf() + config.concurrency shouldBe 1 + config.withRetry shouldBe WithRetry.UNSET + config.withTimeout shouldBe WithTimeout.UNSET + } + + "Can create WorkflowExecutorConfig through YAML with all parameters" { + val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = 4.0) + val config = shouldNotThrowAny { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowAImpl::class.java.name} +concurrency: 10 +timeoutSeconds: 3.0 +checkMode: strict +retry: + minimumSeconds: 4 + """, + ) + } + + config.concurrency shouldBe 10 + config.withTimeout?.getTimeoutSeconds() shouldBe 3.0 + config.withRetry shouldBe withRetry + config.checkMode shouldBe WorkflowCheckMode.strict + } + + "class is mandatory when building WorkflowExecutorConfig from YAML" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +concurrency: 10 + """, + ) + } + + e.message shouldContain "class" + } + + "Unknown class in ignoredExceptions should throw" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowAImpl::class.java.name} +retry: + ignoredExceptions: + - foobar +""", + ) + } + e.message shouldContain "Class 'foobar' not found" + } + + "Class not an Exception in ignoredExceptions should throw" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowAImpl::class.java.name} +retry: + ignoredExceptions: + - ${ServiceA::class.java.name} +""", + ) + } + e.message shouldContain "must be an Exception" + } + + "timeout must be > 0 when building ServiceExecutorConfig from YAML" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowAImpl::class.java.name} +timeoutSeconds: 0 +""", + ) + } + e.message shouldContain "timeoutSeconds must be > 0" + } + + "InvocationTargetException should throw cause" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowWithInvocationTargetException::class.java.name} + """, + ) + } + e.message shouldContain + "Error during class '${WorkflowWithInvocationTargetException::class.java.name}' instantiation" + } + + "ExceptionInInitializer should throw cause" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${WorkflowWithExceptionInInitializerError::class.java.name} + """, + ) + } + e.message shouldContain "ExceptionInInitializerError" + } + + "workflow Unknown" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: io.infinitic.workers.samples.UnknownWorkflow + """, + ) + } + e.message shouldContain "Class 'io.infinitic.workers.samples.UnknownWorkflow' not found" + } + + "Not a workflow" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +class: ${NotAWorkflow::class.java.name} + """, + ) + } + e.message shouldContain "Class '${NotAWorkflow::class.java.name}' must extend '${Workflow::class.java.name}'" + } + + class MyWorkflow : Workflow() + class MyWorkflow_0 : Workflow() + class MyWorkflow_1 : Workflow() + class MyWorkflow_2 : Workflow() + class MyWorkflow_WithContext : Workflow() { + val id = workflowId + } + + "Can create WorkflowExecutorConfig with multiple versions" { + val config = shouldNotThrowAny { + WorkflowExecutorConfig.builder() + .setWorkflowName(MyWorkflow::class.java.name) + .addFactory { MyWorkflow() } + .addFactory { MyWorkflow_1() } + .addFactory { MyWorkflow_2() } + .build() + } + + config.shouldBeInstanceOf() + config.factories.size shouldBe 3 + config.factories[0].invoke().shouldBeInstanceOf() + config.factories[1].invoke().shouldBeInstanceOf() + config.factories[2].invoke().shouldBeInstanceOf() + } + + "Can create WorkflowExecutorConfig through YAML with multiple versions" { + val config = shouldNotThrowAny { + WorkflowExecutorConfig.fromYamlString( + """ +classes: + - ${MyWorkflow::class.java.name} + - ${MyWorkflow_1::class.java.name} + - ${MyWorkflow_2::class.java.name} + """, + ) + } + config.shouldBeInstanceOf() + config.factories.size shouldBe 3 + config.factories[0].invoke().shouldBeInstanceOf() + config.factories[1].invoke().shouldBeInstanceOf() + config.factories[2].invoke().shouldBeInstanceOf() + } + + "Exception when workflows have same version (through builder)" { + val e = shouldThrow { + WorkflowExecutorConfig.builder() + .setWorkflowName(MyWorkflow::class.java.name) + .addFactory { MyWorkflow() } + .addFactory { MyWorkflow_0() } + .addFactory { MyWorkflow_1() } + .addFactory { MyWorkflow_2() } + .build() + } + e.message shouldContain "duplicated" + } + + "Exception when workflows have same version (through YAML)" { + val e = shouldThrow { + WorkflowExecutorConfig.fromYamlString( + """ +classes: + - ${MyWorkflow::class.java.name} + - ${MyWorkflow_0::class.java.name} + - ${MyWorkflow_1::class.java.name} + - ${MyWorkflow_2::class.java.name} + """, + ) + } + e.message shouldContain "duplicated" + } + + "should NOT throw when factory returns the same instance but has no properties" { + val w = MyWorkflow() + shouldNotThrowAny { + WorkflowExecutorConfig.builder() + .setWorkflowName(MyWorkflow::class.java.name) + .addFactory { w } + .build() + } + } + + "should throw when factory returns the same instance but has properties" { + Workflow.setContext(emptyWorkflowContext) + val w = MyWorkflow_WithContext() + val e = shouldThrow { + WorkflowExecutorConfig.builder() + .setWorkflowName(MyWorkflow::class.java.name) + .addFactory { w } + .build() + } + e.message shouldContain "same object instance twice" + } + +// "Exception when requesting unknown version" { +// val e = shouldThrow { +// val w = MyWorkflow() +// val r = RegisteredWorkflowExecutor( +// WorkflowName("foo"), +// listOf { w }, +// 42, +// null, +// null, +// null, +// ) +// r.getInstanceByVersion(WorkflowVersion(1)) +// } +// e.message shouldContain "Unknown version '1' for Workflow 'foo'" +// } +// +// +// +// "Get instance with single class" { +// val rw = RegisteredWorkflowExecutor( +// WorkflowName("foo"), listOf { MyWorkflow() }, 42, null, null, null, +// ) +// // get explicit version 0 +// rw.getInstanceByVersion(WorkflowVersion(0))::class.java shouldBe MyWorkflow::class.java +// // get default version +// rw.getInstanceByVersion(null)::class.java shouldBe MyWorkflow::class.java +// // get unknown version +// val e = shouldThrow { +// rw.getInstanceByVersion(WorkflowVersion(1)) +// } +// e.message shouldContain "Unknown version '1'" +// } +// +// "Get instance with single class with version" { +// val rw = +// RegisteredWorkflowExecutor( +// WorkflowName("foo"), listOf { MyWorkflow_2() }, 42, null, null, null, +// ) +// // get explicit version 0 +// rw.getInstanceByVersion(WorkflowVersion(2))::class.java shouldBe MyWorkflow_2::class.java +// // get default version +// rw.getInstanceByVersion(null)::class.java shouldBe MyWorkflow_2::class.java +// // get unknown version +// val e = shouldThrow { +// rw.getInstanceByVersion(WorkflowVersion(1)) +// } +// e.message shouldContain "Unknown version '1'" +// } +// +// "Get instance with multiple classes" { +// val rw = RegisteredWorkflowExecutor( +// WorkflowName("foo"), +// listOf({ MyWorkflow() }, { MyWorkflow_2() }), +// 42, +// null, +// null, +// null, +// ) +// // get explicit version 0 +// rw.getInstanceByVersion(WorkflowVersion(0))::class.java shouldBe MyWorkflow::class.java +// // get explicit version 2 +// rw.getInstanceByVersion(WorkflowVersion(2))::class.java shouldBe MyWorkflow_2::class.java +// // get default version +// rw.getInstanceByVersion(null)::class.java shouldBe MyWorkflow_2::class.java +// // get unknown version +// val e = shouldThrow { +// rw.getInstanceByVersion( +// WorkflowVersion( +// 1, +// ), +// ) +// } +// e.message shouldContain "Unknown version '1'" +// } +// +// "Get instance with single class using context" { +// shouldNotThrowAny { +// RegisteredWorkflowExecutor( +// WorkflowName("foo"), listOf { MyWorkflow_WithContext() }, 42, null, null, null, +// ) +// } +// } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfigTests.kt new file mode 100644 index 000000000..0a5424d85 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowStateEngineConfigTests.kt @@ -0,0 +1,115 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.storage.config.InMemoryStorageConfig +import io.infinitic.workers.samples.WorkflowA +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf + +internal class WorkflowStateEngineConfigTests : StringSpec( + { + val workflowName = WorkflowA::class.java.name + val storage = InMemoryStorageConfig.builder().build() + + "Can create WorkflowStateEngineConfig through builder with default parameters" { + val config = shouldNotThrowAny { + WorkflowStateEngineConfig.builder() + .setWorkflowName(workflowName) + .build() + } + + config.workflowName shouldBe workflowName + config.shouldBeInstanceOf() + config.storage shouldBe null + config.concurrency shouldBe 1 + } + + "Can create WorkflowStateEngineConfig through builder with all parameters" { + val config = shouldNotThrowAny { + WorkflowStateEngineConfig.builder() + .setWorkflowName(workflowName) + .setConcurrency(10) + .setStorage(storage) + .build() + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.storage shouldBe storage + } + + "WorkflowName is mandatory when building WorkflowStateEngineConfig through builder" { + val e = shouldThrow { + WorkflowStateEngineConfig.builder() + .build() + } + e.message shouldContain "workflowName" + } + + "Concurrency must be positive when building WorkflowStateEngineConfig" { + val e = shouldThrow { + WorkflowStateEngineConfig.builder() + .setWorkflowName(workflowName) + .setConcurrency(0) + .build() + } + e.message shouldContain "concurrency" + } + + "Can create WorkflowStateEngineConfig through YAML without workflowName" { + val config = shouldNotThrowAny { + WorkflowStateEngineConfig.fromYamlString( + """ +concurrency: 10 + """, + ) + } + + config.shouldBeInstanceOf() + config.workflowName.isBlank() shouldBe true + config.concurrency shouldBe 10 + config.storage shouldBe null + } + + "Can create WorkflowStateEngineConfig through YAML with all parameters" { + val config = shouldNotThrowAny { + WorkflowStateEngineConfig.fromYamlString( + """ +concurrency: 10 +storage: + inMemory: + """, + ) + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.storage shouldBe storage + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfigTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfigTests.kt new file mode 100644 index 000000000..dd4135a17 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/config/WorkflowTagEngineConfigTests.kt @@ -0,0 +1,114 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.config + +import io.infinitic.storage.config.InMemoryStorageConfig +import io.infinitic.workers.samples.WorkflowA +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.types.shouldBeInstanceOf + +internal class WorkflowTagEngineConfigTests : StringSpec( + { + val workflowName = WorkflowA::class.java.name + val storage = InMemoryStorageConfig.builder().build() + + "Can create WorkflowTagEngineConfig through builder with default parameters" { + val config = shouldNotThrowAny { + WorkflowTagEngineConfig.builder() + .setWorkflowName(workflowName) + .build() + } + + config.shouldBeInstanceOf() + config.workflowName shouldBe workflowName + config.storage shouldBe null + config.concurrency shouldBe 1 + } + + "Can create WorkflowTagEngineConfig through builder with all parameters" { + val config = shouldNotThrowAny { + WorkflowTagEngineConfig.builder() + .setWorkflowName(workflowName) + .setConcurrency(10) + .setStorage(storage) + .build() + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.storage shouldBe storage + } + + "workflowName is mandatory when building WorkflowTagEngineConfig through builder" { + val e = shouldThrow { + WorkflowTagEngineConfig.builder().build() + } + e.message shouldContain "workflowName" + } + + "Concurrency must be positive when building WorkflowTagEngineConfig" { + val e = shouldThrow { + WorkflowTagEngineConfig.builder() + .setWorkflowName(workflowName) + .setConcurrency(0) + .build() + } + e.message shouldContain "concurrency" + } + + "Can create WorkflowTagEngineConfig through YAML without workflowName" { + val config = shouldNotThrowAny { + WorkflowTagEngineConfig.fromYamlString( + """ +concurrency: 10 + """, + ) + } + + config.shouldBeInstanceOf() + config.workflowName.isBlank() shouldBe true + config.concurrency shouldBe 10 + config.storage shouldBe null + } + + "Can create WorkflowTagEngineConfig through YAML with all parameters" { + val config = shouldNotThrowAny { + WorkflowTagEngineConfig.fromYamlString( + """ +concurrency: 10 +storage: + inMemory: + """, + ) + } + + config.shouldBeInstanceOf() + config.concurrency shouldBe 10 + config.storage shouldBe storage + } + }, +) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/register/InfiniticRegisterTests.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/register/InfiniticRegisterTests.kt deleted file mode 100644 index f1e1f1051..000000000 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/register/InfiniticRegisterTests.kt +++ /dev/null @@ -1,887 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workers.register - -import io.infinitic.common.config.loadConfigFromYaml -import io.infinitic.common.tasks.data.ServiceName -import io.infinitic.common.workers.config.ExponentialBackoffRetryPolicy -import io.infinitic.common.workers.registry.RegisteredServiceTagEngine -import io.infinitic.common.workflows.data.workflows.WorkflowName -import io.infinitic.workers.config.WorkerConfig -import io.infinitic.workers.samples.EventListenerFake -import io.infinitic.workers.samples.EventListenerImpl -import io.infinitic.workers.samples.ServiceA -import io.infinitic.workers.samples.ServiceAImpl -import io.infinitic.workers.samples.WorkflowA -import io.infinitic.workers.samples.WorkflowAImpl -import io.kotest.assertions.throwables.shouldThrow -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.shouldBe -import io.kotest.matchers.shouldNotBe -import io.kotest.matchers.string.shouldContain -import io.kotest.matchers.types.shouldBeInstanceOf -import java.security.InvalidParameterException -import java.util.* -import kotlin.random.Random - -private const val yaml = """ -transport: inMemory -storage: inMemory -""" - -private open class TestException : Exception() - -private class ChildTestException : TestException() - -internal class InfiniticRegisterTests : - StringSpec( - { - val serviceName = ServiceName(ServiceA::class.java.name) - val serviceImplName = ServiceAImpl::class.java.name - val workflowName = WorkflowName(WorkflowA::class.java.name) - val workflowImplName = WorkflowAImpl::class.java.name - val eventListenerImplName = EventListenerImpl::class.java.name - - "checking default service settings" { - val config = WorkerConfig.fromYaml( - yaml, - """ -services: - - name: $serviceName - class: $serviceImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - withTimeout shouldBe null - withRetry shouldBe null - concurrency shouldBe 1 - } - with(register.registry.serviceTagEngines[serviceName]!!) { - shouldBeInstanceOf() - concurrency shouldBe 1 - } - register.registry.serviceEventListeners[serviceName] shouldBe null - } - - "checking default service settings with explicit concurrency" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val config = WorkerConfig.fromYaml( - yaml, - """ -services: - - name: $serviceName - class: $serviceImplName - concurrency: $concurrency -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.serviceTagEngines[serviceName]!!) { - this.concurrency shouldBe concurrency - } - - } - - "checking explicit service settings" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val timeoutInSeconds = Random.nextDouble() - val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = Random.nextDouble()) - val config = WorkerConfig.fromYaml( - yaml, - """ -services: - - name: $serviceName - class: $serviceImplName - concurrency: $concurrency - timeoutInSeconds: $timeoutInSeconds - retry: - minimumSeconds: ${withRetry.minimumSeconds} - tagEngine: - concurrency: 42 - eventListener: - class: $eventListenerImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - withTimeout!!.getTimeoutInSeconds() shouldBe timeoutInSeconds - this.withRetry shouldBe withRetry - this.concurrency shouldBe concurrency - } - with(register.registry.serviceTagEngines[serviceName]!!) { - this.concurrency shouldBe 42 - } - with(register.registry.serviceEventListeners[serviceName]) { - this shouldNotBe null - this!!.concurrency shouldBe concurrency - } - } - - "Get service settings from serviceDefault" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val timeoutInSeconds = Random.nextDouble() - val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = Random.nextDouble()) - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - timeoutInSeconds: $timeoutInSeconds - retry: - minimumSeconds: ${withRetry.minimumSeconds} - eventListener: - class: $eventListenerImplName -services: - - name: $serviceName - class: $serviceImplName - concurrency: $concurrency -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - withTimeout?.getTimeoutInSeconds() shouldBe timeoutInSeconds - this.withRetry shouldBe withRetry - this.concurrency shouldBe concurrency - } - with(register.registry.serviceEventListeners[serviceName]) { - this shouldNotBe null - this!!.concurrency shouldBe concurrency - } - } - - "Explicit service concurrency setting should not be overridden by serviceDefault" { - val concurrency = Random.nextInt(from = 2, until = Int.MAX_VALUE) - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - concurrency: 1 -services: - - name: $serviceName - class: $serviceImplName - concurrency: $concurrency -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - this.concurrency shouldBe concurrency - } - } - - "Explicit service timeout setting should not be overridden by serviceDefault" { - val timeoutInSeconds = Random.nextDouble() - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - timeoutInSeconds: 1. -services: - - name: $serviceName - class: $serviceImplName - timeoutInSeconds: $timeoutInSeconds -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - withTimeout?.getTimeoutInSeconds() shouldBe timeoutInSeconds - } - } - - "Explicit null service timeout setting should not be overridden by serviceDefault" { - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - timeoutInSeconds: 1. -services: - - name: $serviceName - class: $serviceImplName - timeoutInSeconds: null -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - withTimeout?.getTimeoutInSeconds() shouldBe null - } - } - - "Explicit service retry setting should not be overridden by serviceDefault" { - val withRetry = ExponentialBackoffRetryPolicy(maximumRetries = Random.nextInt(100, 1000)) - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - retry: - maximumRetries: 42 -services: - - name: $serviceName - class: $serviceImplName - retry: - maximumRetries: ${withRetry.maximumRetries} -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - this.withRetry shouldBe withRetry - } - } - - "Explicit null service retry setting should not be overridden by serviceDefault" { - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - retry: - maximumRetries: 42 -services: - - name: $serviceName - class: $serviceImplName - retry: null -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceExecutors[serviceName]!!) { - withRetry shouldBe null - } - } - - "Explicit service eventListener setting should not be overridden by serviceDefault" { - val subscriptionName = UUID.randomUUID().toString() - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - eventListener: - class: ${EventListenerFake::class.java.name} - subscriptionName: $subscriptionName -services: - - name: $serviceName - class: $serviceImplName - eventListener: - class: $eventListenerImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceEventListeners[serviceName]) { - this shouldNotBe null - this!!.eventListener::class shouldBe EventListenerImpl::class - this.subscriptionName shouldBe subscriptionName - } - } - - "Explicit null service eventListener setting should not be overridden by serviceDefault" { - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - eventListener: - class: $eventListenerImplName -services: - - name: $serviceName - class: $serviceImplName - eventListener: null -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - register.registry.serviceEventListeners[serviceName] shouldBe null - } - - "if Event Listener class is not defined, it should throw an exception" { - val e = shouldThrow { - val config = WorkerConfig.fromYaml( - yaml, - """ -services: - - name: $serviceName - eventListener: - concurrency: 100 - """, - ) - InfiniticRegisterImpl.fromConfig(config) - } - - e.message.shouldContain("CloudEventListener") - } - - "Get service eventListener settings from eventListener default" { - val config = WorkerConfig.fromYaml( - yaml, - """ -eventListener: - class: $eventListenerImplName -services: - - name: $serviceName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceEventListeners[serviceName]) { - this shouldNotBe null - this!!.eventListener::class shouldBe EventListenerImpl::class - } - } - - "Explicit service eventListener setting should not be overridden by eventListener default" { - val config = WorkerConfig.fromYaml( - yaml, - """ -eventListener: - class: $eventListenerImplName -services: - - name: $serviceName - eventListener: - class: ${EventListenerFake::class.java.name} -""", - ) - println(config) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.serviceEventListeners[serviceName]) { - this shouldNotBe null - this!!.eventListener::class shouldBe EventListenerFake::class - } - } - - "Explicit null service eventListener setting should not be overridden by eventListener default" { - val config = WorkerConfig.fromYaml( - yaml, - """ -eventListener: - class: $eventListenerImplName -services: - - name: $serviceName - eventListener: null -""", - ) - println(config) - val register = InfiniticRegisterImpl.fromConfig(config) - register.registry.serviceEventListeners[serviceName] shouldBe null - } - - "checking default workflow settings" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - class: $workflowImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - withTimeout shouldBe null - withRetry shouldBe null - concurrency shouldBe 1 - checkMode shouldBe null - } - with(register.registry.workflowStateEngines[workflowName]) { - this shouldNotBe null - this!!.concurrency shouldBe 1 - } - with(register.registry.workflowTagEngines[workflowName]) { - this shouldNotBe null - this!!.concurrency shouldBe 1 - } - with(register.registry.workflowEventListeners[workflowName]) { - this shouldBe null - } - } - - "checking default workflow settings with explicit concurrency" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - class: $workflowImplName - concurrency: $concurrency -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.workflowStateEngines[workflowName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.workflowTagEngines[workflowName]!!) { - this.concurrency shouldBe concurrency - } - } - - "checking explicit workflow settings" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val timeoutInSeconds = Random.nextDouble() - val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = Random.nextDouble()) - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - class: $workflowImplName - concurrency: $concurrency - timeoutInSeconds: $timeoutInSeconds - retry: - minimumSeconds: ${withRetry.minimumSeconds} - eventListener: - class: $eventListenerImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - withTimeout!!.getTimeoutInSeconds() shouldBe timeoutInSeconds - this.withRetry shouldBe withRetry - this.concurrency shouldBe concurrency - checkMode shouldBe null - } - with(register.registry.workflowStateEngines[workflowName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.workflowTagEngines[workflowName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.workflowEventListeners[workflowName]!!) { - eventListener::class shouldBe EventListenerImpl::class - } - } - - "Get workflow settings from workflowDefault" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val timeoutInSeconds = Random.nextDouble() - val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = Random.nextDouble()) - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - concurrency: $concurrency - timeoutInSeconds: $timeoutInSeconds - retry: - minimumSeconds: ${withRetry.minimumSeconds} - eventListener: - class: $eventListenerImplName -workflows: - - name: $workflowName - class: $workflowImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - withTimeout!!.getTimeoutInSeconds() shouldBe timeoutInSeconds - this.withRetry shouldBe withRetry - this.concurrency shouldBe concurrency - checkMode shouldBe null - } - with(register.registry.workflowStateEngines[workflowName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.workflowTagEngines[workflowName]!!) { - this.concurrency shouldBe concurrency - } - with(register.registry.workflowEventListeners[workflowName]!!) { - eventListener::class shouldBe EventListenerImpl::class - } - } - - "Explicit workflow concurrency setting should not be overridden by workflowDefault" { - val concurrency = Random.nextInt(from = 1, until = Int.MAX_VALUE) - val config = loadConfigFromYaml( - yaml, - """ -workflowDefault: - concurrency: 42 -workflows: - - name: $workflowName - class: $workflowImplName - concurrency: $concurrency -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - this.concurrency shouldBe concurrency - } - } - - "Explicit workflow timeout setting should not be overridden by workflowDefault" { - val timeoutInSeconds = Random.nextDouble() - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - timeoutInSeconds: 1. -workflows: - - name: $workflowName - class: $workflowImplName - timeoutInSeconds: $timeoutInSeconds -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - withTimeout?.getTimeoutInSeconds() shouldBe timeoutInSeconds - } - } - - "Explicit null workflow timeout setting should not be overridden by workflowDefault" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - timeoutInSeconds: 1. -workflows: - - name: $workflowName - class: $workflowImplName - timeoutInSeconds: null -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - withTimeout shouldBe null - } - } - - "Explicit workflow retry setting should not be overridden by workflowDefault" { - val withRetry = ExponentialBackoffRetryPolicy(minimumSeconds = Random.nextDouble()) - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - retry: - maximumRetries: 42 -workflows: - - name: $workflowName - class: $workflowImplName - retry: - minimumSeconds: ${withRetry.minimumSeconds} -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowExecutors[workflowName]!!) { - this.withRetry shouldBe withRetry - } - } - - "Explicit null workflow retry setting should not be overridden by workflowDefault" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - retry: - maximumRetries: 42 -workflows: - - name: $workflowName - class: $workflowImplName - retry: null -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - register.registry.workflowExecutors[workflowName]!!.withRetry shouldBe null - } - - "Explicit workflow eventListener setting should not be overridden by workflowDefault" { - val subscriptionName = UUID.randomUUID().toString() - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - eventListener: - class: ${EventListenerFake::class.java.name} - subscriptionName: $subscriptionName -workflows: - - name: $workflowName - class: $workflowImplName - eventListener: - class: $eventListenerImplName -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - with(register.registry.workflowEventListeners[workflowName]) { - this shouldNotBe null - this!!.eventListener::class shouldBe EventListenerImpl::class - this.subscriptionName shouldBe subscriptionName - } - } - - "Explicit null workflow eventListener setting should not be overridden by workflowDefault" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflowDefault: - eventListener: - class: $eventListenerImplName -workflows: - - name: $workflowName - class: $workflowImplName - eventListener: null -""", - ) - val register = InfiniticRegisterImpl.fromConfig(config) - register.registry.workflowEventListeners[workflowName] shouldBe null - } - - "service executor do not retry if maximumRetries = 0" { - val workerConfig = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - retry: - maximumRetries: 0 -""", - ) - workerConfig.serviceDefault?.retry shouldNotBe null - workerConfig.serviceDefault?.retry!!.getSecondsBeforeRetry(0, Exception()) shouldBe null - } - - "do not retry once reach maximumRetries" { - val workerConfig = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - retry: - maximumRetries: 10 -""", - ) - workerConfig.serviceDefault?.retry shouldNotBe null - workerConfig.serviceDefault?.retry!!.getSecondsBeforeRetry( - 9, - Exception(), - ) shouldNotBe null - workerConfig.serviceDefault?.retry!!.getSecondsBeforeRetry(10, Exception()) shouldBe null - } - - "do not retry for non retryable exception" { - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - retry: - ignoredExceptions: - - ${TestException::class.java.name} -""", - ) - config.serviceDefault?.retry shouldNotBe null - config.serviceDefault?.retry!!.getSecondsBeforeRetry(1, Exception()) shouldNotBe null - config.serviceDefault?.retry!!.getSecondsBeforeRetry( - 1, - TestException(), - ) shouldBe null - config.serviceDefault?.retry!!.getSecondsBeforeRetry( - 1, - ChildTestException(), - ) shouldBe null - } - - "I can deploy a service without tag engine" { - val config = WorkerConfig.fromYaml( - yaml, - """ -services: - - name: $serviceName - class: $serviceImplName - tagEngine: null - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - val tagEngine = register.registry.serviceTagEngines[serviceName] - val executor = register.registry.serviceExecutors[serviceName] - - executor shouldNotBe null - tagEngine shouldBe null - } - - "I can deploy a service tag engin without service" { - val config = WorkerConfig.fromYaml( - yaml, - """ -services: - - name: $serviceName - tagEngine: - concurrency: 5 - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - val tagEngine = register.registry.serviceTagEngines[serviceName] - val executor = register.registry.serviceExecutors[serviceName] - - executor shouldBe null - tagEngine shouldNotBe null - } - - "I can deploy a workflow without tag engine" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - class: $workflowImplName - tagEngine: null - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - val tagEngine = register.registry.workflowTagEngines[workflowName] - val executor = register.registry.workflowExecutors[workflowName] - - executor shouldNotBe null - tagEngine shouldBe null - } - - "I can deploy a workflow tag engine without workflow" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - tagEngine: - concurrency: 5 - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - val tagEngine = register.registry.workflowTagEngines[workflowName] - val executor = register.registry.workflowExecutors[workflowName] - - executor shouldBe null - tagEngine shouldNotBe null - } - - "I can deploy a workflow executor without state engine" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - class: $workflowImplName - concurrency: 5 - stateEngine: null - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - val executor = register.registry.workflowExecutors[workflowName] - executor shouldNotBe null - executor!!.concurrency shouldBe 5 - - val engine = register.registry.workflowStateEngines[workflowName] - engine shouldBe null - } - - "I can deploy a workflow engine without workflow executor" { - val config = WorkerConfig.fromYaml( - yaml, - """ -workflows: - - name: $workflowName - stateEngine: - concurrency: 5 - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - val engine = register.registry.workflowStateEngines[workflowName] - val executor = register.registry.workflowExecutors[workflowName] - - executor shouldBe null - engine shouldNotBe null - engine!!.concurrency shouldBe 5 - } - - "serviceDefault is used if NOT present" { - val concurrency = 10 - val timeoutInSeconds = 400.0 - val maxRetries = 2 - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - concurrency: $concurrency - timeoutInSeconds: $timeoutInSeconds - retry: - maximumRetries: $maxRetries -services: - - name: $serviceName - class: $serviceImplName - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - - val service = register.registry.serviceExecutors[serviceName] - - service shouldNotBe null - service?.concurrency shouldBe concurrency - service?.withTimeout?.getTimeoutInSeconds() shouldBe timeoutInSeconds - service?.withRetry?.getSecondsBeforeRetry(maxRetries, Exception()) shouldBe null - } - - "serviceDefault is used, with a manual registration" { - val concurrency = 10 - val timeoutInSeconds = 400.0 - val withRetry = ExponentialBackoffRetryPolicy(maximumRetries = 2) - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - concurrency: $concurrency - timeoutInSeconds: $timeoutInSeconds - retry: - maximumRetries: ${withRetry.maximumRetries} - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - register.registerServiceExecutor(serviceName.toString(), { }, 7) - - with(register.registry.serviceExecutors[serviceName]) { - this shouldNotBe null - this!!.concurrency shouldBe 7 - this.withTimeout?.getTimeoutInSeconds() shouldBe timeoutInSeconds - this.withRetry shouldBe withRetry - } - } - - "serviceDefault is NOT used, with a manual registration where configuration is present" { - val concurrency = 10 - val timeoutInSeconds = 400.0 - val maxRetries = 2 - val config = WorkerConfig.fromYaml( - yaml, - """ -serviceDefault: - concurrency: $concurrency - timeoutInSeconds: $timeoutInSeconds - retry: - maximumRetries: $maxRetries - """, - ) - val register = InfiniticRegisterImpl.fromConfig(config) - register.registerServiceExecutor( - serviceName = serviceName.toString(), - serviceFactory = { }, - concurrency = 7, - withTimeout = { 100.0 }, - withRetry = { _: Int, _: Exception -> null }, - ) - - val service = register.registry.serviceExecutors[serviceName] - - service shouldNotBe null - service?.concurrency shouldBe 7 - service?.withTimeout?.getTimeoutInSeconds() shouldBe 100.0 - service?.withRetry?.getSecondsBeforeRetry(maxRetries, Exception()) shouldBe null - } - }, - ) diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/ServiceA.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/ServiceA.kt index 13d98d8ee..b6f112ef9 100644 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/ServiceA.kt +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/ServiceA.kt @@ -24,9 +24,9 @@ package io.infinitic.workers.samples -internal interface ServiceA +interface ServiceA -internal class ServiceAImpl : ServiceA +class ServiceAImpl : ServiceA internal class ServiceWithInvocationTargetException : ServiceA { init { diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/WorkflowA.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/WorkflowA.kt index dd80338df..d62d41289 100644 --- a/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/WorkflowA.kt +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/samples/WorkflowA.kt @@ -27,8 +27,10 @@ import io.infinitic.workflows.Workflow internal interface WorkflowA internal class WorkflowAImpl : Workflow(), WorkflowA +internal class WorkflowA2Impl : Workflow(), WorkflowA +internal class WorkflowAImpl_1 : Workflow(), WorkflowA -internal class WorkflowAImpl2 : Workflow() +internal class WorkflowNotAImpl : Workflow() internal class NotAWorkflow : WorkflowA diff --git a/infinitic-worker/src/test/kotlin/io/infinitic/workers/scafold/start.kt b/infinitic-worker/src/test/kotlin/io/infinitic/workers/scafold/start.kt new file mode 100644 index 000000000..71ace8a21 --- /dev/null +++ b/infinitic-worker/src/test/kotlin/io/infinitic/workers/scafold/start.kt @@ -0,0 +1,214 @@ +/** + * "Commons Clause" License Condition v1.0 + * + * The Software is provided to you by the Licensor under the License, as defined below, subject to + * the following condition. + * + * Without limiting other conditions in the License, the grant of rights under the License will not + * include, and the License does not grant to you, the right to Sell the Software. + * + * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you + * under the License to provide to third parties, for a fee or other consideration (including + * without limitation fees for hosting or consulting/ support services related to the Software), a + * product or service whose value derives, entirely or substantially, from the functionality of the + * Software. Any license notice or attribution required by the License must also include this + * Commons Clause License Condition notice. + * + * Software: Infinitic + * + * License: MIT License (https://opensource.org/licenses/MIT) + * + * Licensor: infinitic.io + */ +package io.infinitic.workers.scafold + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.future.future +import kotlinx.coroutines.isActive +import kotlinx.coroutines.job +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.random.Random +import kotlin.system.exitProcess + +private suspend fun processMessage(msg: String) { + println(">> processing $msg") + val d = Random.nextLong(1000) + when { + //d < 50 -> throw RuntimeException("error $msg") + else -> delay(d) + } + println(">>>> processed $msg") +} + +private class ConsumerWithoutKey { + + suspend fun start(concurrency: Int) = coroutineScope { + val channel = Channel() + // start executor coroutines + val jobs = List(concurrency) { + launch { + try { + for (msg: String in channel) { + // this ensures that ongoing messages are processed + // even after scope is cancelled following an interruption or an Error + withContext(NonCancellable) { + processMessage(msg) + } + } + } catch (e: CancellationException) { + println("Processor #$it closed after cancellation") + } catch (e: Exception) { + println("Processor #$it closed after error $e") + println("Waiting processing of ongoing messages") + throw e + } + } + } + // start message receiver + var msg = 0 + while (isActive) { + try { + msg++ + println("receiving $msg") + channel.send(msg.toString()) + } catch (e: CancellationException) { + // if current scope is canceled, we just exit the while loop + break + } + } + withContext(NonCancellable) { jobs.joinAll() } + println("Closing consumer") + } +} + +private class ConsumerWithKey { + + suspend fun start(concurrency: Int) = coroutineScope { + val msg = AtomicInteger(0) + // start executor coroutines + // For Key_Shared subscription, we must create a new consumer for each executor coroutine + List(concurrency) { consumer -> + launch { + while (isActive) { + try { + msg.addAndGet(1) + println("receiving Consumer$consumer: $msg") + // this ensures that ongoing messages are processed + // even after scope is cancelled following an interruption or an Error + withContext(NonCancellable) { + processMessage("Consumer$consumer: $msg") + } + } catch (e: CancellationException) { + println("Exiting receiving loop $consumer") + // if current scope is canceled, we just exit the while loop + throw e + } catch (e: Exception) { + println("Closing consumer $consumer after Exception $e") + throw e + } + } + println("Closing consumer $consumer") + } + } + } +} + +private class Worker { + private val scope = CoroutineScope(Dispatchers.IO) + private val consumerWithoutKey = ConsumerWithoutKey() + private val consumerWithKey = ConsumerWithKey() + private var isClosed: AtomicBoolean = AtomicBoolean(false) + + init { + Runtime.getRuntime().addShutdownHook( + Thread { + close() + }, + ) + } + + fun CoroutineScope.startWithoutKey(concurrency: Int) { + launch { + println("consumerWithoutKey started") + try { + consumerWithoutKey.start(5) + } catch (e: CancellationException) { + // do nothing + } + println("consumerWithoutKey stopped") + } + } + + fun CoroutineScope.startWithKey(concurrency: Int) { + launch { + println("consumerWithKey started") + try { + consumerWithKey.start(5) + } catch (e: CancellationException) { + // do nothing + } + println("consumerWithKey stopped") + } + } + + + fun startAsync() = scope.future { + println("a") + //startWithoutKey(5) + startWithKey(5) + println("b") + } + + fun start() { + try { + startAsync().join() + } catch (e: CancellationException) { + // do nothing + } catch (e: Exception) { + println(e.printStackTrace()) + exitProcess(1) + } + } + + fun close() { + if (isClosed.compareAndSet(false, true)) { + println("Closing worker...") + scope.cancel() + runBlocking { + try { + withTimeout(1000) { + scope.coroutineContext.job.children.forEach { it.join() } + println("all message processed") + } + } catch (e: TimeoutCancellationException) { + println("The grace period allotted to close was insufficient.") + } + } + println("Worker closed.") + } + } +} + +suspend fun main() { + val worker = Worker() +// later(10000) { +// worker.close() +// } + worker.startAsync() + delay(1000) + println("That's all folks") +} diff --git a/infinitic-worker/src/test/resources/config/services/exceptionInInitializerError.yml b/infinitic-worker/src/test/resources/config/services/exceptionInInitializerError.yml index 403de84b2..1e85f56b2 100644 --- a/infinitic-worker/src/test/resources/config/services/exceptionInInitializerError.yml +++ b/infinitic-worker/src/test/resources/config/services/exceptionInInitializerError.yml @@ -21,18 +21,18 @@ # # Licensor: infinitic.io -# Uncomment the line below to perform test on a local Pulsar cluster - -transport: inMemory +# comment the line below to perform test on a local Pulsar cluster storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev services: - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.ServiceWithExceptionInInitializerError + executor: + class: io.infinitic.workers.samples.ServiceWithExceptionInInitializerError diff --git a/infinitic-worker/src/test/resources/config/services/incompatibleListener.yml b/infinitic-worker/src/test/resources/config/services/incompatibleListener.yml index 4f9830f8e..26199e4e8 100644 --- a/infinitic-worker/src/test/resources/config/services/incompatibleListener.yml +++ b/infinitic-worker/src/test/resources/config/services/incompatibleListener.yml @@ -21,19 +21,22 @@ # # Licensor: infinitic.io -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev + +eventListener: + class: io.infinitic.workers.samples.ServiceAImpl services: - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.ServiceAImpl - eventListener: + executor: class: io.infinitic.workers.samples.ServiceAImpl + diff --git a/infinitic-worker/src/test/resources/config/services/incompatibleServiceName.yml b/infinitic-worker/src/test/resources/config/services/incompatibleServiceName.yml index bd8506e62..340b12045 100644 --- a/infinitic-worker/src/test/resources/config/services/incompatibleServiceName.yml +++ b/infinitic-worker/src/test/resources/config/services/incompatibleServiceName.yml @@ -21,16 +21,17 @@ # # Licensor: infinitic.io -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev services: - name: UnknownService - class: io.infinitic.workers.samples.ServiceAImpl + executor: + class: io.infinitic.workers.samples.ServiceAImpl diff --git a/infinitic-worker/src/test/resources/config/services/instance.yml b/infinitic-worker/src/test/resources/config/services/instance.yml index 73e429968..ed3e5d271 100644 --- a/infinitic-worker/src/test/resources/config/services/instance.yml +++ b/infinitic-worker/src/test/resources/config/services/instance.yml @@ -21,17 +21,18 @@ # # Licensor: infinitic.io -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev services: - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.ServiceAImpl + executor: + class: io.infinitic.workers.samples.ServiceAImpl diff --git a/infinitic-worker/src/test/resources/config/services/instanceWithListener.yml b/infinitic-worker/src/test/resources/config/services/instanceWithListener.yml index eaff4cd45..a539f1f57 100644 --- a/infinitic-worker/src/test/resources/config/services/instanceWithListener.yml +++ b/infinitic-worker/src/test/resources/config/services/instanceWithListener.yml @@ -21,19 +21,22 @@ # # Licensor: infinitic.io -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev + +eventListener: + class: io.infinitic.workers.samples.EventListenerImpl services: - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.ServiceAImpl - eventListener: - class: io.infinitic.workers.samples.EventListenerImpl + executor: + class: io.infinitic.workers.samples.ServiceAImpl + diff --git a/infinitic-worker/src/test/resources/config/services/invocationTargetException.yml b/infinitic-worker/src/test/resources/config/services/invocationTargetException.yml deleted file mode 100644 index 3a3970322..000000000 --- a/infinitic-worker/src/test/resources/config/services/invocationTargetException.yml +++ /dev/null @@ -1,38 +0,0 @@ -# "Commons Clause" License Condition v1.0 -# -# The Software is provided to you by the Licensor under the License, as defined -# below, subject to the following condition. -# -# Without limiting other conditions in the License, the grant of rights under the -# License will not include, and the License does not grant to you, the right to -# Sell the Software. -# -# For purposes of the foregoing, “Sell” means practicing any or all of the rights -# granted to you under the License to provide to third parties, for a fee or -# other consideration (including without limitation fees for hosting or -# consulting/ support services related to the Software), a product or service -# whose value derives, entirely or substantially, from the functionality of the -# Software. Any license notice or attribution required by the License must also -# include this Commons Clause License Condition notice. -# -# Software: Infinitic -# -# License: MIT License (https://opensource.org/licenses/MIT) -# -# Licensor: infinitic.io - -# Uncomment the line below to perform test on a local Pulsar cluster - -transport: inMemory -storage: inMemory - -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev - -services: - - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.ServiceWithInvocationTargetException - diff --git a/infinitic-worker/src/test/resources/config/services/unknown.yml b/infinitic-worker/src/test/resources/config/services/unknown.yml index 742dee043..7acf1b06c 100644 --- a/infinitic-worker/src/test/resources/config/services/unknown.yml +++ b/infinitic-worker/src/test/resources/config/services/unknown.yml @@ -21,16 +21,17 @@ # # Licensor: infinitic.io -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev services: - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.UnknownService + executor: + class: io.infinitic.workers.samples.UnknownService diff --git a/infinitic-worker/src/test/resources/config/services/unknownListener.yml b/infinitic-worker/src/test/resources/config/services/unknownListener.yml index 8d5a0957e..b1088cd4b 100644 --- a/infinitic-worker/src/test/resources/config/services/unknownListener.yml +++ b/infinitic-worker/src/test/resources/config/services/unknownListener.yml @@ -21,19 +21,22 @@ # # Licensor: infinitic.io -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev + +eventListener: + class: io.infinitic.workers.samples.UnknownListener services: - name: io.infinitic.workers.samples.ServiceA - class: io.infinitic.workers.samples.ServiceAImpl - eventListener: - class: io.infinitic.workers.samples.UnknownListener + executor: + class: io.infinitic.workers.samples.ServiceAImpl + diff --git a/infinitic-worker/src/test/resources/config/workflows/exceptionInInitializerError.yml b/infinitic-worker/src/test/resources/config/workflows/exceptionInInitializerError.yml index c54b68183..fdc2192c9 100644 --- a/infinitic-worker/src/test/resources/config/workflows/exceptionInInitializerError.yml +++ b/infinitic-worker/src/test/resources/config/workflows/exceptionInInitializerError.yml @@ -23,16 +23,17 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: io.infinitic.workers.samples.WorkflowA - class: io.infinitic.workers.samples.WorkflowWithExceptionInInitializerError + executor: + class: io.infinitic.workers.samples.WorkflowWithExceptionInInitializerError diff --git a/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowName.yml b/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowName.yml index 608ec0c8d..2d199337b 100644 --- a/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowName.yml +++ b/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowName.yml @@ -23,16 +23,17 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: UnknownWorkflow - class: io.infinitic.workers.samples.WorkflowAImpl + executor: + class: io.infinitic.workers.samples.WorkflowAImpl diff --git a/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowsName.yml b/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowsName.yml index 310bc101c..e81577345 100644 --- a/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowsName.yml +++ b/infinitic-worker/src/test/resources/config/workflows/incompatibleWorkflowsName.yml @@ -23,18 +23,19 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: io.infinitic.workers.samples.WorkflowA - classes: - - io.infinitic.workers.samples.WorkflowAImpl - - io.infinitic.workers.samples.WorkflowAImpl2 + executor: + classes: + - io.infinitic.workers.samples.WorkflowAImpl + - io.infinitic.workers.samples.WorkflowAImpl2 diff --git a/infinitic-worker/src/test/resources/config/workflows/instance.yml b/infinitic-worker/src/test/resources/config/workflows/instance.yml index c4c4eb7d5..a89551d65 100644 --- a/infinitic-worker/src/test/resources/config/workflows/instance.yml +++ b/infinitic-worker/src/test/resources/config/workflows/instance.yml @@ -23,16 +23,17 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: io.infinitic.workers.samples.WorkflowA - class: io.infinitic.workers.samples.WorkflowAImpl + executor: + class: io.infinitic.workers.samples.WorkflowAImpl diff --git a/infinitic-worker/src/test/resources/config/workflows/invocationTargetException.yml b/infinitic-worker/src/test/resources/config/workflows/invocationTargetException.yml index 530af3d58..0964d8c6a 100644 --- a/infinitic-worker/src/test/resources/config/workflows/invocationTargetException.yml +++ b/infinitic-worker/src/test/resources/config/workflows/invocationTargetException.yml @@ -23,16 +23,17 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: io.infinitic.workers.samples.WorkflowA - class: io.infinitic.workers.samples.WorkflowWithInvocationTargetException + executor: + class: io.infinitic.workers.samples.WorkflowWithInvocationTargetException diff --git a/infinitic-worker/src/test/resources/config/workflows/notAWorkflow.yml b/infinitic-worker/src/test/resources/config/workflows/notAWorkflow.yml index 6d3662cff..ba20fb4fd 100644 --- a/infinitic-worker/src/test/resources/config/workflows/notAWorkflow.yml +++ b/infinitic-worker/src/test/resources/config/workflows/notAWorkflow.yml @@ -23,16 +23,17 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: io.infinitic.workers.samples.WorkflowA - class: io.infinitic.workers.samples.NotAWorkflow + executor: + class: io.infinitic.workers.samples.NotAWorkflow diff --git a/infinitic-worker/src/test/resources/config/workflows/unknown.yml b/infinitic-worker/src/test/resources/config/workflows/unknown.yml index 81e65e85f..58d2f2900 100644 --- a/infinitic-worker/src/test/resources/config/workflows/unknown.yml +++ b/infinitic-worker/src/test/resources/config/workflows/unknown.yml @@ -23,16 +23,17 @@ # Uncomment the line below to perform test on a local Pulsar cluster -transport: inMemory storage: inMemory -pulsar: - brokerServiceUrl: pulsar://localhost:6650 - webServiceUrl: http://localhost:8080 - tenant: infinitic - namespace: dev +transport: + pulsar: + brokerServiceUrl: pulsar://localhost:6650 + webServiceUrl: http://localhost:8080 + tenant: infinitic + namespace: dev workflows: - name: io.infinitic.workers.samples.WorkflowA - class: io.infinitic.workers.samples.UnknownWorkflow + executor: + class: io.infinitic.workers.samples.UnknownWorkflow diff --git a/infinitic-worker/src/test/resources/simplelogger.properties b/infinitic-worker/src/test/resources/simplelogger.properties new file mode 100644 index 000000000..8858a8cc1 --- /dev/null +++ b/infinitic-worker/src/test/resources/simplelogger.properties @@ -0,0 +1,41 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. +# Log file +#org.slf4j.simpleLogger.logFile=infinitic.log +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=warn +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +org.slf4j.simpleLogger.log.io.infinitic.clients.InfiniticClient=info +org.slf4j.simpleLogger.log.io.infinitic.workers.InfiniticWorker=info +org.slf4j.simpleLogger.log.io.infinitic.workflows.tag.WorkflowTagEngine=info +org.slf4j.simpleLogger.log.io.infinitic.workflows.engine.WorkflowStateEngine=info +org.slf4j.simpleLogger.log.io.infinitic.workflows.engine.WorkflowStateCmdHandler=info +org.slf4j.simpleLogger.log.io.infinitic.workflows.engine.WorkflowStateEventHandler=info +org.slf4j.simpleLogger.log.io.infinitic.workflows.engine.WorkflowStateTimerHandler=info +org.slf4j.simpleLogger.log.io.infinitic.tasks.tag.TaskTagEngine=info +org.slf4j.simpleLogger.log.io.infinitic.tasks.executor.TaskExecutor=info +org.slf4j.simpleLogger.log.io.infinitic.tasks.executor.TaskEventHandler=info +org.slf4j.simpleLogger.log.io.infinitic.tasks.executor.TaskRetryHandler=info +org.slf4j.simpleLogger.log.io.infinitic.pulsar.consumers.Consumer=warn +org.slf4j.simpleLogger.log.io.infinitic.pulsar.producers.Producer=warn +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +org.slf4j.simpleLogger.showDateTime=true +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z +# Set to true if you want to output the current thread name. +# Defaults to true. +org.slf4j.simpleLogger.showThreadName=false +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +# org.slf4j.simpleLogger.showLogName=true +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +org.slf4j.simpleLogger.showShortLogName=true diff --git a/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/WorkflowStateEngine.kt b/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/WorkflowStateEngine.kt index 5b7ddb04f..92937b6eb 100644 --- a/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/WorkflowStateEngine.kt +++ b/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/WorkflowStateEngine.kt @@ -36,7 +36,7 @@ import io.infinitic.common.transport.InfiniticProducer import io.infinitic.common.transport.WorkflowStateEngineTopic import io.infinitic.common.transport.WorkflowStateEventTopic import io.infinitic.common.transport.WorkflowTagEngineTopic -import io.infinitic.common.transport.formatLog +import io.infinitic.common.transport.logged.formatLog import io.infinitic.common.workflows.engine.messages.CancelWorkflow import io.infinitic.common.workflows.engine.messages.CompleteTimers import io.infinitic.common.workflows.engine.messages.CompleteWorkflow diff --git a/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/config/WorkflowStateEngineConfig.kt b/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/config/WorkflowStateEngineConfig.kt deleted file mode 100644 index 861d9a8b1..000000000 --- a/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/config/WorkflowStateEngineConfig.kt +++ /dev/null @@ -1,63 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workflows.engine.config - -import io.infinitic.storage.config.StorageConfig - -data class WorkflowStateEngineConfig( - var concurrency: Int? = null, - var storage: StorageConfig? = null, -) { - var isDefault: Boolean = false - - init { - concurrency?.let { - require(it >= 0) { "concurrency must be positive" } - } - } - - companion object { - @JvmStatic - fun builder() = WorkflowStateEngineConfigBuilder() - } - - /** - * WorkflowStateEngineConfig builder (Useful for Java user) - */ - class WorkflowStateEngineConfigBuilder { - private val default = WorkflowStateEngineConfig() - private var concurrency = default.concurrency - private var storage = default.storage - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun storage(storage: StorageConfig) = - apply { this.storage = storage } - - fun build() = WorkflowStateEngineConfig( - concurrency, - storage, - ).apply { isDefault = false } - } -} diff --git a/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/storage/LoggedWorkflowStateStorage.kt b/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/storage/LoggedWorkflowStateStorage.kt index 78d1bd410..917fa2dab 100644 --- a/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/storage/LoggedWorkflowStateStorage.kt +++ b/infinitic-workflow-engine/src/main/kotlin/io/infinitic/workflows/engine/storage/LoggedWorkflowStateStorage.kt @@ -23,7 +23,7 @@ package io.infinitic.workflows.engine.storage import io.github.oshai.kotlinlogging.KLogger -import io.infinitic.common.transport.formatLog +import io.infinitic.common.transport.logged.formatLog import io.infinitic.common.workflows.data.workflows.WorkflowId import io.infinitic.common.workflows.engine.state.WorkflowState import io.infinitic.common.workflows.engine.storage.WorkflowStateStorage diff --git a/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/WorkflowTagEngine.kt b/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/WorkflowTagEngine.kt index 7c858100f..cf415b251 100644 --- a/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/WorkflowTagEngine.kt +++ b/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/WorkflowTagEngine.kt @@ -326,9 +326,9 @@ class WorkflowTagEngine( val workflowIdsByTag = WorkflowIdsByTag( recipientName = ClientName.from(message.emitterName), - message.workflowName, - message.workflowTag, - workflowIds, + workflowName = message.workflowName, + workflowTag = message.workflowTag, + workflowIds = workflowIds, emitterName = emitterName, ) with(producer) { workflowIdsByTag.sendTo(ClientTopic) } @@ -337,7 +337,7 @@ class WorkflowTagEngine( private fun discardTagWithoutIds(message: WorkflowTagEngineMessage) { logger.info { "discarding as no workflow `${message.workflowName}` found for tag `${message.workflowTag}`" } } - + companion object { val logger = KotlinLogging.logger {} } diff --git a/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/config/WorkflowTagEngineConfig.kt b/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/config/WorkflowTagEngineConfig.kt deleted file mode 100644 index 188352ae8..000000000 --- a/infinitic-workflow-tag/src/main/kotlin/io/infinitic/workflows/tag/config/WorkflowTagEngineConfig.kt +++ /dev/null @@ -1,63 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workflows.tag.config - -import io.infinitic.storage.config.StorageConfig - -data class WorkflowTagEngineConfig( - var concurrency: Int? = null, - var storage: StorageConfig? = null, -) { - var isDefault: Boolean = false - - init { - concurrency?.let { - require(it >= 0) { "concurrency must be positive" } - } - } - - companion object { - @JvmStatic - fun builder() = WorkflowTagEngineConfigBuilder() - } - - /** - * WorkflowTagEngineConfig builder (Useful for Java user) - */ - class WorkflowTagEngineConfigBuilder { - private val default = WorkflowTagEngineConfig() - private var concurrency = default.concurrency - private var storage = default.storage - - fun concurrency(concurrency: Int) = - apply { this.concurrency = concurrency } - - fun storage(storage: StorageConfig) = - apply { this.storage = storage } - - fun build() = WorkflowTagEngineConfig( - concurrency, - storage, - ).apply { isDefault = false } - } -} diff --git a/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/WorkflowTaskImpl.kt b/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/WorkflowTaskImpl.kt index ff3746652..9f26d2dc2 100644 --- a/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/WorkflowTaskImpl.kt +++ b/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/WorkflowTaskImpl.kt @@ -30,6 +30,8 @@ import io.infinitic.common.workflows.data.properties.PropertyName import io.infinitic.common.workflows.data.workflowTasks.WorkflowTask import io.infinitic.common.workflows.data.workflowTasks.WorkflowTaskParameters import io.infinitic.common.workflows.data.workflowTasks.WorkflowTaskReturnValue +import io.infinitic.common.workflows.executors.getProperties +import io.infinitic.common.workflows.executors.setProperties import io.infinitic.workflows.Workflow import io.infinitic.workflows.WorkflowCheckMode import io.infinitic.workflows.setChannelNames diff --git a/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/workflowProperties.kt b/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/workflowProperties.kt deleted file mode 100644 index 9cf5c22a4..000000000 --- a/infinitic-workflow-task/src/main/kotlin/io/infinitic/workflows/workflowTask/workflowProperties.kt +++ /dev/null @@ -1,67 +0,0 @@ -/** - * "Commons Clause" License Condition v1.0 - * - * The Software is provided to you by the Licensor under the License, as defined below, subject to - * the following condition. - * - * Without limiting other conditions in the License, the grant of rights under the License will not - * include, and the License does not grant to you, the right to Sell the Software. - * - * For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you - * under the License to provide to third parties, for a fee or other consideration (including - * without limitation fees for hosting or consulting/ support services related to the Software), a - * product or service whose value derives, entirely or substantially, from the functionality of the - * Software. Any license notice or attribution required by the License must also include this - * Commons Clause License Condition notice. - * - * Software: Infinitic - * - * License: MIT License (https://opensource.org/licenses/MIT) - * - * Licensor: infinitic.io - */ -package io.infinitic.workflows.workflowTask - -import io.github.oshai.kotlinlogging.KLogger -import io.infinitic.annotations.Ignore -import io.infinitic.common.exceptions.thisShouldNotHappen -import io.infinitic.common.workflows.data.properties.PropertyHash -import io.infinitic.common.workflows.data.properties.PropertyName -import io.infinitic.common.workflows.data.properties.PropertyValue -import io.infinitic.common.workflows.executors.getPropertiesFromObject -import io.infinitic.common.workflows.executors.setPropertiesToObject -import io.infinitic.workflows.Channel -import io.infinitic.workflows.Workflow -import org.slf4j.Logger -import java.lang.reflect.Proxy -import kotlin.reflect.full.createType -import kotlin.reflect.full.hasAnnotation -import kotlin.reflect.full.isSubtypeOf -import kotlin.reflect.full.starProjectedType - -internal fun Workflow.setProperties( - propertiesHashValue: Map, - propertiesNameHash: Map -) { - val properties = propertiesNameHash.mapValues { - propertiesHashValue[it.value] - ?: thisShouldNotHappen("unknown hash ${it.value} in $propertiesHashValue") - } - - setPropertiesToObject(this, properties) -} - -// TODO: manage Deferred in properties -internal fun Workflow.getProperties() = - getPropertiesFromObject(this) { - // excludes Channels - !it.first.returnType.isSubtypeOf(Channel::class.starProjectedType) && - // excludes Proxies (tasks and workflows) and null - !(it.second?.let { Proxy.isProxyClass(it::class.java) } ?: true) && - // exclude SLF4J loggers - !it.first.returnType.isSubtypeOf(Logger::class.createType()) && - // exclude KotlinLogging loggers - !it.first.returnType.isSubtypeOf(KLogger::class.createType()) && - // exclude Ignore annotation - !it.first.hasAnnotation() - } diff --git a/infinitic-workflow-task/src/test/kotlin/io/infinitic/workflows/workflowTask/PropertiesTests.kt b/infinitic-workflow-task/src/test/kotlin/io/infinitic/workflows/workflowTask/PropertiesTests.kt index 0c8d0e0f6..2472562b5 100644 --- a/infinitic-workflow-task/src/test/kotlin/io/infinitic/workflows/workflowTask/PropertiesTests.kt +++ b/infinitic-workflow-task/src/test/kotlin/io/infinitic/workflows/workflowTask/PropertiesTests.kt @@ -24,6 +24,7 @@ package io.infinitic.workflows.workflowTask import io.infinitic.common.workflows.data.properties.PropertyName import io.infinitic.common.workflows.data.properties.PropertyValue +import io.infinitic.common.workflows.executors.getProperties import io.infinitic.workflows.workflowTask.workflows.WorkflowAImpl import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.shouldBe diff --git a/publish.gradle.kts b/publish.gradle.kts index 5bf3b5a2b..ec229875a 100644 --- a/publish.gradle.kts +++ b/publish.gradle.kts @@ -59,8 +59,7 @@ fun Project.signing(configure: SigningExtension.() -> Unit): Unit = configure(co fun Project.java(configure: JavaPluginExtension.() -> Unit): Unit = configure(configure) -val publications: PublicationContainer = - (extensions.getByName("publishing") as PublishingExtension).publications +val publications = (extensions.getByName("publishing") as PublishingExtension).publications signing { if (Ci.isRelease) {