From 6d61b826d6889b2ef7408e563d2a4263cffba7d0 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 4 Jul 2024 14:31:19 +0400 Subject: [PATCH 1/5] Correct versions order in readme --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index db81396..c10fdbf 100644 --- a/README.md +++ b/README.md @@ -147,15 +147,16 @@ so we exclude Gradle metadata for these repositories. It's been verified that Sailfish itself is compatible with versions from BOM and therefore safe to use. ## Changelog + +### v2.0.0 ++ books&pages support + ### v0.8.0 #### Changed: * Updated versions of common, BOM and sailfish. -### v2.0.0 -+ books&pages support - ### v0.7.0 #### Changed: From d17b0820dd84cdd7111880264feee69e9f3dc7f7 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 4 Jul 2024 15:57:09 +0400 Subject: [PATCH 2/5] Update dependnecies. Add support for transport protocol --- build.gradle | 85 ++------ gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../th2/codec/json/JsonPipelineCodec.kt | 198 ++++++++++++++---- .../codec/json/JsonPipelineCodecFactory.kt | 51 +++-- .../codec/json/JsonPipelineCodecSettings.kt | 6 +- .../th2/codec/json/ConstantMessageType.kt | 25 ++- 7 files changed, 233 insertions(+), 136 deletions(-) diff --git a/build.gradle b/build.gradle index 601ab73..40c7ebc 100644 --- a/build.gradle +++ b/build.gradle @@ -1,31 +1,17 @@ plugins { - id 'com.palantir.docker' version '0.25.0' apply false id 'org.jetbrains.kotlin.jvm' version "${kotlin_version}" - id "org.owasp.dependencycheck" version "7.2.0" id 'org.jetbrains.kotlin.kapt' version "${kotlin_version}" + id 'com.exactpro.th2.gradle.component' version '0.0.8' } -dependencyCheck { - format='HTML' - failBuildOnCVSS=5 -} - -apply plugin: 'kotlin' -apply plugin: 'application' -apply plugin: 'com.palantir.docker' - ext { - sharedDir = file("${project.rootDir}/shared") - sailfishVersion = '3.3.54' - commonVersion = '5.1.0-dev-version-5+' + sailfishVersion = '3.3.111' + commonVersion = '5.13.1-dev' } group = 'com.exactpro.th2' version = release_version -sourceCompatibility = 11 -targetCompatibility = 11 - ext.excludeSailfish = { rcd -> rcd.excludeModule("com.exactpro.sf", "sailfish-core") rcd.excludeModule("com.exactpro.sf", "sailfish-common") @@ -33,11 +19,11 @@ ext.excludeSailfish = { rcd -> rcd.excludeModule("com.exactpro.sf", "service-http") } +kotlin { + jvmToolchain(11) +} + repositories { - maven { - name 'MavenLocal' - url sharedDir - } maven { name 'Sonatype_snapshots' @@ -85,42 +71,27 @@ repositories { } } -jar { - manifest { - attributes( - 'Created-By': "${System.getProperty('java.version')} (${System.getProperty('java.vendor')})", - 'Specification-Title': '', - 'Specification-Vendor': 'Exactpro Systems LLC', - 'Implementation-Title': project.archivesBaseName, - 'Implementation-Vendor': 'Exactpro Systems LLC', - 'Implementation-Vendor-Id': 'com.exactpro', - 'Implementation-Version': project.version - ) - } -} - dependencies { - api platform('com.exactpro.th2:bom:4.1.0') implementation "com.exactpro.th2:common:${commonVersion}" - implementation 'com.exactpro.th2:codec:5.0.0-dev-version-5+' - implementation 'com.exactpro.th2:sailfish-utils:3.14.0' + implementation 'com.exactpro.th2:codec:5.5.0-dev' + implementation 'com.exactpro.th2:sailfish-utils:4.1.1-dev' implementation "com.exactpro.sf:service-http:${sailfishVersion}" implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: kotlin_version implementation group: 'org.jetbrains.kotlin', name: 'kotlin-reflect', version: kotlin_version - implementation group: 'io.github.microutils', name: 'kotlin-logging', version: '2.0.10' + implementation group: 'io.github.microutils', name: 'kotlin-logging', version: '3.0.5' - testImplementation 'org.junit.jupiter:junit-jupiter:5.9.0' + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.3' testImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit5', version: kotlin_version testImplementation testFixtures("com.exactpro.th2:common:${commonVersion}") - compileOnly 'com.google.auto.service:auto-service:1.0.1' - annotationProcessor 'com.google.auto.service:auto-service:1.0.1' - kapt 'com.google.auto.service:auto-service:1.0.1' + compileOnly 'com.google.auto.service:auto-service:1.1.1' + annotationProcessor 'com.google.auto.service:auto-service:1.1.1' + kapt 'com.google.auto.service:auto-service:1.1.1' } test { @@ -129,30 +100,4 @@ test { application { mainClassName 'com.exactpro.th2.codec.MainKt' -} - -applicationName = 'service' - -distTar { - archiveName "${applicationName}.tar" -} - -dockerPrepare { - dependsOn distTar -} - -docker { - copySpec.from(tarTree("$buildDir/distributions/${applicationName}.tar")) -} - -compileKotlin { - kotlinOptions { - jvmTarget = "11" - } -} - -compileTestKotlin { - kotlinOptions { - jvmTarget = "11" - } -} +} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index ad58860..81d29e0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ kotlin.code.style=official -kotlin_version=1.6.21 +kotlin_version=1.8.22 release_version=2.0.0 vcs_url=https://github.com/th2-net/th2-codec-json \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 10c66bb..2b20ca3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ #Mon May 25 11:22:24 MSK 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodec.kt b/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodec.kt index aa1cb66..a46a272 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodec.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodec.kt @@ -31,6 +31,7 @@ import com.exactpro.sf.services.http.HTTPMessageHelper.REQUEST_URI_ATTRIBUTE import com.exactpro.sf.services.json.JSONDecoder import com.exactpro.sf.services.json.JSONEncoder import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.codec.api.IReportingContext import com.exactpro.th2.codec.json.JsonPipelineCodecFactory.Companion.PROTOCOL import com.exactpro.th2.codec.json.JsonPipelineCodecSettings.MessageTypeDetection.BY_HTTP_METHOD_AND_URI import com.exactpro.th2.codec.json.JsonPipelineCodecSettings.MessageTypeDetection.BY_INNER_FIELD @@ -39,18 +40,26 @@ import com.exactpro.th2.codec.util.toDebugString import com.exactpro.th2.common.grpc.AnyMessage import com.exactpro.th2.common.grpc.Direction.FIRST import com.exactpro.th2.common.grpc.Direction.SECOND -import com.exactpro.th2.common.grpc.MessageGroup -import com.exactpro.th2.common.grpc.RawMessage import com.exactpro.th2.common.message.plusAssign +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageGroup +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage +import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.toByteArray import com.exactpro.th2.sailfish.utils.IMessageToProtoConverter +import com.exactpro.th2.sailfish.utils.MessageWrapper import com.exactpro.th2.sailfish.utils.ProtoToIMessageConverter -import com.exactpro.th2.sailfish.utils.factory.MessageFactoryProxy +import com.exactpro.th2.sailfish.utils.transport.IMessageToTransportConverter +import com.exactpro.th2.sailfish.utils.transport.TransportToIMessageConverter import com.fasterxml.jackson.core.JsonPointer import com.fasterxml.jackson.core.JsonPointer.SEPARATOR import com.fasterxml.jackson.core.JsonPointer.empty import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.google.protobuf.ByteString +import io.netty.buffer.Unpooled import io.netty.channel.embedded.EmbeddedChannel +import com.exactpro.th2.common.grpc.MessageGroup as ProtoMessageGroup +import com.exactpro.th2.common.grpc.RawMessage as ProtoRawMessage typealias MessageName = String typealias MessageType = String @@ -61,6 +70,7 @@ class JsonPipelineCodec( ) : IPipelineCodec { private val messageFactory: IMessageFactory private val protoConverter: ProtoToIMessageConverter + private val transportConverter: TransportToIMessageConverter private val encodeChannel: EmbeddedChannel private val decodeChannel: EmbeddedChannel private val requestInfos: Map @@ -77,7 +87,9 @@ class JsonPipelineCodec( SailfishURI.parse(dictionary.namespace).let { uri -> this.messageFactory = JSONMessageFactory().apply { init(uri, dictionary) } this.protoConverter = - ProtoToIMessageConverter(MessageFactoryProxy(messageFactory, uri, dictionary), dictionary, uri) + ProtoToIMessageConverter(messageFactory, dictionary) + this.transportConverter = + TransportToIMessageConverter(messageFactory, dictionary) } val jsonSettings = this.settings.toJsonSettings() @@ -140,10 +152,10 @@ class JsonPipelineCodec( this.messageNames = messageNames } - override fun encode(messageGroup: MessageGroup): MessageGroup { + override fun encode(messageGroup: ProtoMessageGroup, context: IReportingContext): ProtoMessageGroup { val messages = messageGroup.messagesList - val builder = MessageGroup.newBuilder() + val builder = ProtoMessageGroup.newBuilder() for (message in messages) { if (!message.hasMessage()) { @@ -157,38 +169,10 @@ class JsonPipelineCodec( val parsedMessage = message.message val metadata = parsedMessage.metadata - val messageType = metadata.messageType - val messageStructure = checkNotNull(dictionary.messages[messageType]) { "Unknown message type: $messageType" } - - if (settings.messageTypeDetection == BY_HTTP_METHOD_AND_URI) { - check(messageType in requestInfos || messageType in responseInfos) { "Message type is not a request or response: $messageType" } - } - val sfMessage = protoConverter.fromProtoMessage(parsedMessage, true) + val (rawMessage, additionalMetadataProperties) = encodeMessage(sfMessage, metadata.messageType) - if (settings.messageTypeDetection == BY_INNER_FIELD) { - messagePathProvider.set( - sfMessage, - messageTypePointer, - messageStructure, - messageStructure.requireMessageType(), - replaceIfExist = false - ) - } - - val encodedMessage = encodeChannel.encode(sfMessage) - val rawMessage = checkNotNull(encodedMessage.metaData.rawMessage) { "Encoded messages has no raw message in its metadata: $encodedMessage" } - - val additionalMetadataProperties = when (this.settings.messageTypeDetection) { - BY_HTTP_METHOD_AND_URI -> requestInfos[messageType]?.run { - val paramMessage: IMessage? = sfMessage[REQUEST_URI_MESSAGE] - val paramValues: Map = paramMessage?.run { fieldNames.associateWith(::get) } ?: mapOf() - mapOf(METHOD_METADATA_PROPERTY to method, URI_METADATA_PROPERTY to uri.resolve(paramValues)) - } - else -> null - } - - builder += RawMessage.newBuilder().apply { + builder += ProtoRawMessage.newBuilder().apply { body = ByteString.copyFrom(rawMessage) parentEventId = parsedMessage.parentEventId metadataBuilder.apply { @@ -203,14 +187,87 @@ class JsonPipelineCodec( return builder.build() } - override fun decode(messageGroup: MessageGroup): MessageGroup { + override fun encode(messageGroup: MessageGroup, context: IReportingContext): MessageGroup { + val messages = messageGroup.messages + + val builder = MessageGroup.builder() + + for (message in messages) { + if (message !is ParsedMessage) { + builder.addMessage(message) + continue + } + if (message.run { protocol.isNotEmpty() && protocol != PROTOCOL }) { + builder.addMessage(message) + continue + } + + val sfMessage = transportConverter.fromTransport(message.id.book, message.id.sessionGroup, message, useDictionary = true) + val (rawMessage, additionalMetadataProperties) = encodeMessage(sfMessage, message.type) + + builder.addMessage( + RawMessage.builder() + .setId(message.id) + .setBody(Unpooled.wrappedBuffer(rawMessage)) + .setMetadata(message.metadata) + .setProtocol(PROTOCOL) + .apply { + additionalMetadataProperties?.let { + metadataBuilder().putAll(it) + } + message.eventId?.let { setEventId(it) } + }.build() + ) + } + + return builder.build() + } + + private fun encodeMessage( + sfMessage: MessageWrapper, + messageType: String + ): Pair?> { + val messageStructure = checkNotNull(dictionary.messages[messageType]) { "Unknown message type: $messageType" } + + if (settings.messageTypeDetection == BY_HTTP_METHOD_AND_URI) { + check(messageType in requestInfos || messageType in responseInfos) { "Message type is not a request or response: $messageType" } + } + + + if (settings.messageTypeDetection == BY_INNER_FIELD) { + messagePathProvider.set( + sfMessage, + messageTypePointer, + messageStructure, + messageStructure.requireMessageType(), + replaceIfExist = false + ) + } + + val encodedMessage = encodeChannel.encode(sfMessage) + val rawMessage = + checkNotNull(encodedMessage.metaData.rawMessage) { "Encoded messages has no raw message in its metadata: $encodedMessage" } + + val additionalMetadataProperties = when (this.settings.messageTypeDetection) { + BY_HTTP_METHOD_AND_URI -> requestInfos[messageType]?.run { + val paramMessage: IMessage? = sfMessage[REQUEST_URI_MESSAGE] + val paramValues: Map = paramMessage?.run { fieldNames.associateWith(::get) } ?: mapOf() + mapOf(METHOD_METADATA_PROPERTY to method, URI_METADATA_PROPERTY to uri.resolve(paramValues)) + } + + else -> null + } + return Pair(rawMessage, additionalMetadataProperties) + } + + override fun decode(messageGroup: ProtoMessageGroup, context: IReportingContext): ProtoMessageGroup { val messages = messageGroup.messagesList if (messages.isEmpty() || messages.none(AnyMessage::hasRawMessage)) { return messageGroup } - val builder = MessageGroup.newBuilder() + val builder = ProtoMessageGroup.newBuilder() for (message in messages) { if (!message.hasRawMessage()) { @@ -251,7 +308,7 @@ class JsonPipelineCodec( error("Message ${decodedMessage.name} was rejected due to: ${decodedMessage.metaData.rejectReason}") } - builder += IMESSAGE_CONVERTER.toProtoMessage(decodedMessage).apply { + builder += PROTO_IMESSAGE_CONVERTER.toProtoMessage(decodedMessage).apply { parentEventId = rawMessage.parentEventId metadataBuilder.apply { putAllProperties(metadataProperties) @@ -264,6 +321,66 @@ class JsonPipelineCodec( return builder.build() } + override fun decode(messageGroup: MessageGroup, context: IReportingContext): MessageGroup { + val messages = messageGroup.messages + + if (messages.isEmpty() || messages.none { it is RawMessage }) { + return messageGroup + } + + val builder = MessageGroup.builder() + + for (message in messages) { + if (message !is RawMessage) { + builder.addMessage(message) + continue + } + + val metadataProperties = message.metadata + val body = message.body.toByteArray() + val messageId = message.id + + val messageName = when (settings.messageTypeDetection) { + CONSTANT -> settings.constantMessageType + BY_INNER_FIELD -> { + val json = OBJECT_READER.readTree(body) + json.at(messageTypePointer).takeIf { it.isTextual }?.run { messageNames[textValue()] } + ?: error("No valid type field $messageTypePointer in: $json") + } + BY_HTTP_METHOD_AND_URI -> { + val method = requireNotNull(metadataProperties[METHOD_METADATA_PROPERTY]) { "Message has no '$METHOD_METADATA_PROPERTY' metadata property: $message" } + val uri = requireNotNull(metadataProperties[URI_METADATA_PROPERTY]) { "Message has no '$URI_METADATA_PROPERTY' metadata property: $message" } + + when (messageId.direction) { + Direction.INCOMING -> responseInfos.entries.find { it.value.matches(method, uri) } ?: error("No response for request with '$method' method and URI matching: $uri") + Direction.OUTGOING -> requestInfos.entries.find { it.value.matches(method, uri) } ?: error("No request with '$method' method and URI matching: $uri") + }.key + } + } + + val decodedMessage = decodeChannel.decode(messageFactory.createMessage(messageName).apply { + metaData.rawMessage = body + }) + + if (decodedMessage.metaData.isRejected) { + error("Message ${decodedMessage.name} was rejected due to: ${decodedMessage.metaData.rejectReason}") + } + + builder.addMessage( + TRANSPORT_IMESSAGE_CONVERTER + .toTransportBuilder(decodedMessage) + .apply { + message.eventId?.let { setEventId(it) } + metadataBuilder().putAll(metadataProperties) + setId(messageId) + setProtocol(PROTOCOL) + }.build(), + ) + } + + return builder.build() + } + override fun close() { encodeChannel.runCatching { close().sync() } decodeChannel.runCatching { close().sync() } @@ -281,7 +398,8 @@ class JsonPipelineCodec( private const val URI_METADATA_PROPERTY = "uri" private val CODEC_SETTINGS = HTTPClientSettings() - private val IMESSAGE_CONVERTER = IMessageToProtoConverter() + private val PROTO_IMESSAGE_CONVERTER = IMessageToProtoConverter() + private val TRANSPORT_IMESSAGE_CONVERTER = IMessageToTransportConverter() private val VALID_MESSAGE_ATTRIBUTES = setOf(REQUEST_METHOD_ATTRIBUTE, REQUEST_URI_ATTRIBUTE, REQUEST_RESPONSE_ATTRIBUTE) private val OBJECT_READER = jacksonObjectMapper().reader() diff --git a/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecFactory.kt b/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecFactory.kt index 10f8086..db433ff 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecFactory.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecFactory.kt @@ -20,43 +20,60 @@ import com.exactpro.sf.common.messages.structures.IDictionaryStructure import com.exactpro.sf.common.messages.structures.loaders.JsonYamlDictionaryStructureLoader import com.exactpro.sf.common.messages.structures.loaders.XmlDictionaryStructureLoader import com.exactpro.th2.codec.api.IPipelineCodec +import com.exactpro.th2.codec.api.IPipelineCodecContext import com.exactpro.th2.codec.api.IPipelineCodecFactory import com.exactpro.th2.codec.api.IPipelineCodecSettings import com.google.auto.service.AutoService import mu.KotlinLogging import java.io.InputStream +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock @AutoService(IPipelineCodecFactory::class) @Suppress("unused") class JsonPipelineCodecFactory : IPipelineCodecFactory { + private lateinit var context: IPipelineCodecContext + @Volatile private lateinit var dictionary: IDictionaryStructure + private val dictionaryLock = ReentrantLock() override val settingsClass: Class = JsonPipelineCodecSettings::class.java - @Deprecated("Please migrate to the protocols property") - override val protocol: String = PROTOCOL - - override fun init(dictionary: InputStream) { - check(!this::dictionary.isInitialized) { "factory already initialized" } - this.dictionary = dictionary.readAllBytes().run { - runCatching { - LOGGER.debug { "Loading dictionary as XML" } - XmlDictionaryStructureLoader().load(inputStream()) - }.recoverCatching { - LOGGER.warn(it) { "Cannot load dictionary as XML. Loading as JSON/YAML" } - JsonYamlDictionaryStructureLoader().load(inputStream()) - } - }.getOrThrow() + + override val protocols: Set + get() = PROTOCOLS + + override fun init(pipelineCodecContext: IPipelineCodecContext) { + check(!::context.isInitialized) { "factory already initialized" } + context = pipelineCodecContext } override fun create(settings: IPipelineCodecSettings?): IPipelineCodec { - check(::dictionary.isInitialized) { "'dictionary' was not loaded" } - return JsonPipelineCodec(dictionary, requireNotNull(settings as? JsonPipelineCodecSettings) { + check(::context.isInitialized) { "codec factory is not initialized" } + require(settings is JsonPipelineCodecSettings) { "settings is not an instance of ${JsonPipelineCodecSettings::class.java}: $settings" - }) + } + if (!::dictionary.isInitialized) { + dictionaryLock.withLock { + if (::dictionary.isInitialized) { + return@withLock + } + this.dictionary = context[settings.dictionaryAlias].readAllBytes().run { + runCatching { + LOGGER.debug { "Loading dictionary as XML" } + XmlDictionaryStructureLoader().load(inputStream()) + }.recoverCatching { + LOGGER.warn(it) { "Cannot load dictionary as XML. Loading as JSON/YAML" } + JsonYamlDictionaryStructureLoader().load(inputStream()) + } + }.getOrThrow() + } + } + return JsonPipelineCodec(dictionary, settings) } companion object { const val PROTOCOL = "json" + private val PROTOCOLS = setOf(PROTOCOL) private val LOGGER = KotlinLogging.logger { } } } \ No newline at end of file diff --git a/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecSettings.kt b/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecSettings.kt index 55b20ce..c2726dc 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecSettings.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/json/JsonPipelineCodecSettings.kt @@ -17,6 +17,7 @@ package com.exactpro.th2.codec.json import com.exactpro.sf.services.json.JsonSettings +import com.exactpro.th2.codec.api.DictionaryAlias import com.exactpro.th2.codec.api.IPipelineCodecSettings import com.exactpro.th2.codec.json.JsonPipelineCodecSettings.MessageTypeDetection.BY_HTTP_METHOD_AND_URI import com.exactpro.th2.codec.json.JsonPipelineCodecSettings.MessageTypeDetection.BY_INNER_FIELD @@ -27,7 +28,8 @@ data class JsonPipelineCodecSettings( val messageTypeField: String = "", val rejectUnexpectedFields: Boolean = true, val treatSimpleValuesAsStrings: Boolean = false, - val constantMessageType: String = "" + val constantMessageType: String = "", + val dictionaryAlias: DictionaryAlias = "", ) : IPipelineCodecSettings { init { when (messageTypeDetection) { @@ -35,7 +37,7 @@ data class JsonPipelineCodecSettings( CONSTANT -> check(constantMessageType.isNotBlank()) { "${::messageTypeDetection.name} is $CONSTANT but ${::constantMessageType.name} is blank" } BY_HTTP_METHOD_AND_URI -> {} } - + require(dictionaryAlias.isNotBlank()) { "dictionary alias cannot be blank" } } fun toJsonSettings(): JsonSettings = JsonSettings().apply { diff --git a/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt b/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt index 4cacc22..486374e 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt @@ -16,6 +16,9 @@ package com.exactpro.th2.codec.json +import com.exactpro.th2.codec.api.DictionaryAlias +import com.exactpro.th2.codec.api.IPipelineCodecContext +import com.exactpro.th2.codec.api.impl.ReportingContext import com.exactpro.th2.codec.json.JsonPipelineCodecSettings.MessageTypeDetection.CONSTANT import com.exactpro.th2.common.assertString import com.exactpro.th2.common.grpc.AnyMessage @@ -24,11 +27,14 @@ import com.exactpro.th2.common.grpc.RawMessage import com.exactpro.th2.common.message.addField import com.exactpro.th2.common.message.message import com.exactpro.th2.common.message.messageType +import com.exactpro.th2.common.schema.dictionary.DictionaryType +import com.exactpro.th2.common.schema.grpc.router.GrpcRouter import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode import com.google.protobuf.ByteString import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test +import java.io.InputStream class ConstantMessageType { @@ -37,7 +43,7 @@ class ConstantMessageType { val json = """{"SimpleOne":"SimpleOne Value", "SimpleTwo":"SimpleTwo Value", "SimpleThree":"SimpleThree Value"}""" val rawMessage = RawMessage.newBuilder().setBody(ByteString.copyFrom(json.toByteArray())).build() val group = MessageGroup.newBuilder().addMessages(AnyMessage.newBuilder().setRawMessage(rawMessage)).build() - val decodeResult = codec.decode(group) + val decodeResult = codec.decode(group, ReportingContext()) decodeResult.messagesList[0].message.apply { Assertions.assertEquals("Constant", messageType) assertString("SimpleOne", "SimpleOne Value") @@ -54,7 +60,7 @@ class ConstantMessageType { addField("SimpleThree", "SimpleThree Value") } val group = MessageGroup.newBuilder().addMessages(AnyMessage.newBuilder().setMessage(message)).build() - val decodeResult = codec.encode(group) + val decodeResult = codec.encode(group, ReportingContext()) decodeResult.messagesList[0].rawMessage.apply { val json = mapper.readTree(body.toStringUtf8()) as ObjectNode Assertions.assertEquals("SimpleOne Value", json.get("SimpleOne")?.asText()) @@ -67,7 +73,7 @@ class ConstantMessageType { fun `simple constant messageType test encode with default values`() { val message = message("Constant_Default") val group = MessageGroup.newBuilder().addMessages(AnyMessage.newBuilder().setMessage(message)).build() - val decodeResult = codec.encode(group) + val decodeResult = codec.encode(group, ReportingContext()) decodeResult.messagesList[0].rawMessage.apply { val json = mapper.readTree(body.toStringUtf8()) as ObjectNode Assertions.assertEquals("test1", json.get("SimpleOne")?.asText()) @@ -79,8 +85,17 @@ class ConstantMessageType { companion object { val codec = JsonPipelineCodecFactory().apply { - init(getResourceAsStream("constant_message.xml")) - }.create(JsonPipelineCodecSettings(CONSTANT, constantMessageType = "Constant")) + init(object : IPipelineCodecContext { + override fun get(alias: DictionaryAlias): InputStream = getResourceAsStream(alias) + + override fun get(type: DictionaryType): InputStream = TODO("Not yet implemented") + + override fun getDictionaryAliases(): Set = TODO("Not yet implemented") + + override fun getGrpcRouter(): GrpcRouter = TODO("Not yet implemented") + + }) + }.create(JsonPipelineCodecSettings(CONSTANT, constantMessageType = "Constant", dictionaryAlias = "constant_message.xml")) val mapper = ObjectMapper() } } \ No newline at end of file From 98397afc91694fd08c6c116e140d1c92dfa36ae0 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 4 Jul 2024 16:07:51 +0400 Subject: [PATCH 3/5] Update version and readme --- README.md | 128 +++++++++++++++++++++++++--------------------- gradle.properties | 2 +- 2 files changed, 71 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index c10fdbf..ada1b57 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# JSON Codec v2.0.0 +# JSON Codec v2.1.0 This microservice can encode and decode JSON messages received via HTTP or any other transport @@ -68,74 +68,82 @@ If `messageTypeDetection` is set to `BY_HTTP_METHOD_AND_URI` these messages are Here's an example of `infra-mgr` config required to deploy this service ```yaml -apiVersion: th2.exactpro.com/v1 +apiVersion: th2.exactpro.com/v2 kind: Th2Box metadata: name: codec-json spec: - image-name: ghcr.io/th2-net/th2-codec-json - image-version: 0.7.0 - custom-config: + imageName: ghcr.io/th2-net/th2-codec-json + imageVersion: 0.7.0 + customConfig: + transportLines: + "": + type: TH2_TRANSPORT + useParentEventId: true + general: + type: TH2_TRANSPORT + useParentEventId: false codecSettings: messageTypeDetection: BY_INNER_FIELD messageTypeField: "messageType" rejectUnexpectedFields: true treatSimpleValuesAsStrings: false + dictionaryAlias: ${dictionary_link:dictionary-name} type: th2-codec pins: - # encoder - - name: in_codec_encode - connection-type: mq - attributes: - - encoder_in - - subscribe - - group - - name: out_codec_encode - connection-type: mq - attributes: - - encoder_out - - publish - - group - # decoder - - name: in_codec_decode - connection-type: mq - attributes: - - decoder_in - - subscribe - - group - - name: out_codec_decode - connection-type: mq - attributes: - - decoder_out - - publish - - group - # encoder general (technical) - - name: in_codec_general_encode - connection-type: mq - attributes: - - general_encoder_in - - subscribe - - group - - name: out_codec_general_encode - connection-type: mq - attributes: - - general_encoder_out - - publish - - group - # decoder general (technical) - - name: in_codec_general_decode - connection-type: mq - attributes: - - general_decoder_in - - subscribe - - group - - name: out_codec_general_decode - connection-type: mq - attributes: - - general_decoder_out - - publish - - group - extended-settings: + mq: + subscribers: + # encoder + - name: in_codec_encode + connection-type: mq + attributes: + - encoder_in + - subscribe + - transport-group + - name: in_codec_decode + connection-type: mq + attributes: + - decoder_in + - subscribe + - transport-group + - name: in_codec_general_encode + connection-type: mq + attributes: + - general_encoder_in + - subscribe + - transport-group + - name: in_codec_general_decode + connection-type: mq + attributes: + - general_decoder_in + - subscribe + - transport-group + publishers: + - name: out_codec_encode + connection-type: mq + attributes: + - encoder_out + - publish + - transport-group + - name: out_codec_decode + connection-type: mq + attributes: + - decoder_out + - publish + - transport-group + - name: out_codec_general_encode + connection-type: mq + attributes: + - general_encoder_out + - publish + - transport-group + - name: out_codec_general_decode + connection-type: mq + attributes: + - general_decoder_out + - publish + - transport-group + extendedSettings: service: enabled: false ``` @@ -148,6 +156,10 @@ It's been verified that Sailfish itself is compatible with versions from BOM and ## Changelog +### v2.1.0 + ++ Add support fot th2 transport + ### v2.0.0 + books&pages support diff --git a/gradle.properties b/gradle.properties index 81d29e0..a549ca2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ kotlin.code.style=official kotlin_version=1.8.22 -release_version=2.0.0 +release_version=2.1.0 vcs_url=https://github.com/th2-net/th2-codec-json \ No newline at end of file From b7da650ff15ff69cc44c1580cdce587f33278f5f Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 4 Jul 2024 16:12:04 +0400 Subject: [PATCH 4/5] Update workflows --- .../{docker-publish.yml => build-dev-release.yml} | 15 +++++---------- .github/workflows/build-release.yml | 15 +++++++++++++++ ...{dev-docker-publish.yml => build-snapshot.yml} | 1 + .github/workflows/ci-unwelcome-words.yml | 6 +++--- 4 files changed, 24 insertions(+), 13 deletions(-) rename .github/workflows/{docker-publish.yml => build-dev-release.yml} (59%) create mode 100644 .github/workflows/build-release.yml rename .github/workflows/{dev-docker-publish.yml => build-snapshot.yml} (90%) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/build-dev-release.yml similarity index 59% rename from .github/workflows/docker-publish.yml rename to .github/workflows/build-dev-release.yml index cec50a6..c1b77a6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/build-dev-release.yml @@ -1,20 +1,15 @@ name: Build and publish Docker distributions to Github Container Registry ghcr.io on: - push: - branches: - - master - - version-* - paths: - - gradle.properties -# - package_info.json + workflow_dispatch: jobs: - build-job: + build: uses: th2-net/.github/.github/workflows/compound-java.yml@main with: build-target: 'Docker' - docker-username: ${{ github.actor }} + devRelease: true + createTag: true secrets: docker-password: ${{ secrets.GITHUB_TOKEN }} - \ No newline at end of file + nvd-api-key: ${{ secrets.NVD_APIKEY }} \ No newline at end of file diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 0000000..abb05c4 --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,15 @@ +name: Build and publish Docker distributions to Github Container Registry ghcr.io + +on: + workflow_dispatch: + +jobs: + build: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Docker' + devRelease: false + createTag: true + secrets: + docker-password: ${{ secrets.GITHUB_TOKEN }} + nvd-api-key: ${{ secrets.NVD_APIKEY }} \ No newline at end of file diff --git a/.github/workflows/dev-docker-publish.yml b/.github/workflows/build-snapshot.yml similarity index 90% rename from .github/workflows/dev-docker-publish.yml rename to .github/workflows/build-snapshot.yml index 6f77755..2b806c1 100644 --- a/.github/workflows/dev-docker-publish.yml +++ b/.github/workflows/build-snapshot.yml @@ -17,3 +17,4 @@ jobs: docker-username: ${{ github.actor }} secrets: docker-password: ${{ secrets.GITHUB_TOKEN }} + nvd-api-key: ${{ secrets.NVD_APIKEY }} diff --git a/.github/workflows/ci-unwelcome-words.yml b/.github/workflows/ci-unwelcome-words.yml index cd7adcf..b1e592b 100644 --- a/.github/workflows/ci-unwelcome-words.yml +++ b/.github/workflows/ci-unwelcome-words.yml @@ -5,13 +5,13 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: ${{ github.sha }} - name: Checkout tool - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: exactpro-th2/ci-github-action ref: master From 2ddba7286efd1b67411a9c1cd8899592e32aedb3 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 4 Jul 2024 16:27:07 +0400 Subject: [PATCH 5/5] Update gradle to 8.4 --- gradle/wrapper/gradle-wrapper.properties | 2 +- .../kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2b20ca3..81e64d2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ #Mon May 25 11:22:24 MSK 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt b/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt index 486374e..dcf56f7 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/json/ConstantMessageType.kt @@ -88,6 +88,10 @@ class ConstantMessageType { init(object : IPipelineCodecContext { override fun get(alias: DictionaryAlias): InputStream = getResourceAsStream(alias) + @Deprecated( + "Dictionary types will be removed in future releases of infra", + replaceWith = ReplaceWith("get(alias)") + ) override fun get(type: DictionaryType): InputStream = TODO("Not yet implemented") override fun getDictionaryAliases(): Set = TODO("Not yet implemented")