From 597f5c2df75d40d4205b1e617d6c884b91361ac1 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Mon, 6 Jan 2025 14:14:23 +0100 Subject: [PATCH 1/7] feat: Revert "REVIEWED" to "TRANSLATED" when text changes --- .../TranslationsControllerModificationTest.kt | 3 ++- .../tolgee/model/translation/Translation.kt | 3 +++ .../service/translation/TranslationService.kt | 23 ++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerModificationTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerModificationTest.kt index 92d490c44d..8d03e0fbff 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerModificationTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerModificationTest.kt @@ -329,7 +329,7 @@ class TranslationsControllerModificationTest : ProjectAuthControllerTest("/v2/pr @ProjectJWTAuthTestMethod @Test - fun `updates outdated flag when base updated`() { + fun `updates outdated flag and base state when base updated`() { testData.addTranslationsWithStates() saveTestData() performProjectAuthPut( @@ -342,6 +342,7 @@ class TranslationsControllerModificationTest : ProjectAuthControllerTest("/v2/pr ), ).andAssertThatJson { node("translations.en.outdated").isEqualTo(false) + node("translations.en.state").isEqualTo("TRANSLATED") node("translations.de.outdated").isEqualTo(true) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt b/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt index a8125418fe..e00183b9c5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt +++ b/backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt @@ -86,6 +86,9 @@ class Translation( @field:ColumnDefault("false") var outdated: Boolean = false + val isUntranslated: Boolean + get() = state == TranslationState.UNTRANSLATED + constructor(text: String? = null, key: Key, language: Language) : this(text) { this.key = key this.language = language diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index 41ed46e38c..60e054fc14 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -186,17 +186,24 @@ class TranslationService( translation: Translation, text: String?, ): Translation { - if (translation.text !== text) { + val hasTextChanged = translation.text != text + + if (hasTextChanged) { translation.resetFlags() } + translation.text = text - if (translation.state == TranslationState.UNTRANSLATED && !translation.text.isNullOrEmpty()) { - translation.state = TranslationState.TRANSLATED - } - if (text.isNullOrEmpty()) { - translation.state = TranslationState.UNTRANSLATED - translation.text = null - } + + val hasText = !text.isNullOrEmpty() + + translation.state = + when { + translation.isUntranslated && hasText -> TranslationState.TRANSLATED + hasTextChanged -> TranslationState.TRANSLATED + text.isNullOrEmpty() -> TranslationState.UNTRANSLATED + else -> translation.state + } + return save(translation) } From 251e1ac598348adc3d2e62039a9cd01652d8196e Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Tue, 21 Jan 2025 16:12:45 +0100 Subject: [PATCH 2/7] fix: fix import, test & refactor batch operation tests --- .../batch/BatchChangeTranslationStateTest.kt | 61 +++ .../batch/BatchClearTranslationsTest.kt | 60 +++ .../batch/BatchCopyTranslationsTest.kt | 94 ++++ .../controllers/batch/BatchDeleteKeysTest.kt | 67 +++ .../v2/controllers/batch/BatchJobTestBase.kt | 99 ++++ .../batch/BatchMoveToNamespaceTest.kt | 103 ++++ .../controllers/batch/BatchMtTranslateTest.kt | 70 +++ .../batch/BatchPreTranslateByMtTest.kt | 70 +++ .../v2/controllers/batch/BatchTagKeysTest.kt | 124 +++++ .../batch/StartBatchJobControllerTest.kt | 449 ------------------ .../SingleStepImportControllerTest.kt | 28 ++ .../testDataBuilder/builders/KeyBuilder.kt | 2 + .../testDataBuilder/data/BatchJobsTestData.kt | 34 +- .../data/SingleStepImportTestData.kt | 14 + .../service/dataImport/StoredDataImporter.kt | 3 +- .../service/translation/TranslationService.kt | 11 +- 16 files changed, 826 insertions(+), 463 deletions(-) create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt delete mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt new file mode 100644 index 0000000000..60ee5bba5f --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt @@ -0,0 +1,61 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.enums.TranslationState +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchChangeTranslationStateTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it changes translation state`() { + val keyCount = 100 + val keys = testData.addStateChangeData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(10) + val allLanguageIds = testData.projectBuilder.data.languages.map { it.self.id } + val languagesToChangeStateIds = listOf(testData.germanLanguage.id, testData.englishLanguage.id) + + performProjectAuthPost( + "start-batch-job/set-translation-state", + mapOf( + "keyIds" to keyIds, + "languageIds" to languagesToChangeStateIds, + "state" to "REVIEWED", + ), + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = + translationService.getTranslations( + keys.map { it.id }, + allLanguageIds, + ) + all.count { it.state == TranslationState.REVIEWED }.assert.isEqualTo(keyIds.size * 2) + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt new file mode 100644 index 0000000000..d4aa74ca26 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt @@ -0,0 +1,60 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.enums.TranslationState +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchClearTranslationsTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it clears translations`() { + val keyCount = 1000 + val keys = testData.addStateChangeData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(10) + val allLanguageIds = testData.projectBuilder.data.languages.map { it.self.id } + val languagesToClearIds = listOf(testData.germanLanguage.id, testData.englishLanguage.id) + + performProjectAuthPost( + "start-batch-job/clear-translations", + mapOf( + "keyIds" to keyIds, + "languageIds" to languagesToClearIds, + ), + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = + translationService.getTranslations( + keys.map { it.id }, + allLanguageIds, + ) + all.count { it.state == TranslationState.UNTRANSLATED && it.text == null }.assert.isEqualTo(keyIds.size * 2) + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt new file mode 100644 index 0000000000..93e147b06f --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt @@ -0,0 +1,94 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.enums.TranslationState +import io.tolgee.model.key.Key +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchCopyTranslationsTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it copies translations`() { + val keyCount = 1000 + val keys = testData.addStateChangeData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(10) + val allLanguageIds = testData.projectBuilder.data.languages.map { it.self.id } + val languagesToChangeStateIds = listOf(testData.germanLanguage.id, testData.czechLanguage.id) + + performProjectAuthPost( + "start-batch-job/copy-translations", + mapOf( + "keyIds" to keyIds, + "sourceLanguageId" to testData.englishLanguage.id, + "targetLanguageIds" to languagesToChangeStateIds, + ), + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = + translationService.getTranslations( + keys.map { it.id }, + allLanguageIds, + ) + all.count { it.text?.startsWith("en") == true }.assert.isEqualTo(allKeyIds.size + keyIds.size * 2) + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `it resets the state for key`() { + val key = testData.addKeyWithTranslationsReviewed() + batchJobTestBase.saveAndPrepare(this) + + assertGermanAKeyState(key, TranslationState.REVIEWED) + + val languagesToChangeStateIds = listOf(testData.germanLanguage.id) + + performProjectAuthPost( + "start-batch-job/copy-translations", + mapOf( + "keyIds" to listOf(key.id), + "sourceLanguageId" to testData.englishLanguage.id, + "targetLanguageIds" to languagesToChangeStateIds, + ), + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + assertGermanAKeyState(key, TranslationState.TRANSLATED) + } + } + + private fun assertGermanAKeyState( + key: Key, + translationState: TranslationState, + ) { + translationService.getTranslations(listOf(key.id), listOf(testData.germanLanguage.id)) + .single().state.assert.isEqualTo(translationState) + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt new file mode 100644 index 0000000000..adbeec5c58 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt @@ -0,0 +1,67 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.batch.BatchJob +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchDeleteKeysTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it deletes keys`() { + val keyCount = 100 + val keys = testData.addTranslationOperationData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val keyIds = keys.map { it.id }.toList() + + val result = + performProjectAuthPost( + "start-batch-job/delete-keys", + mapOf( + "keyIds" to keyIds, + ), + ).andIsOk + + batchJobTestBase.waitForJobCompleted(result) + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = keyService.getAll(testData.projectBuilder.self.id) + all.assert.hasSize(1) + } + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + executeInNewTransaction { + val data = + entityManager + .createQuery("""from BatchJob""", BatchJob::class.java) + .resultList + + data.assert.hasSize(1) + data[0].activityRevision.assert.isNotNull + } + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt new file mode 100644 index 0000000000..4c0e5020a9 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt @@ -0,0 +1,99 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.batch.BatchJobChunkExecutionQueue +import io.tolgee.batch.BatchJobService +import io.tolgee.configuration.tolgee.InternalProperties +import io.tolgee.configuration.tolgee.machineTranslation.MachineTranslationProperties +import io.tolgee.development.testDataBuilder.TestDataService +import io.tolgee.development.testDataBuilder.data.BatchJobsTestData +import io.tolgee.fixtures.andAssertThatJson +import io.tolgee.fixtures.waitFor +import io.tolgee.fixtures.waitForNotThrowing +import io.tolgee.model.translation.Translation +import io.tolgee.testing.assert +import jakarta.persistence.EntityManager +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component +import org.springframework.test.web.servlet.ResultActions +import java.util.function.Consumer + +@Component +class BatchJobTestBase { + lateinit var testData: BatchJobsTestData + + @Autowired + lateinit var batchJobOperationQueue: BatchJobChunkExecutionQueue + + @Autowired + lateinit var batchJobService: BatchJobService + + @Autowired + lateinit var machineTranslationProperties: MachineTranslationProperties + + @Autowired + lateinit var entityManager: EntityManager + + var fakeBefore: Boolean = false + + @Autowired + private lateinit var internalProperties: InternalProperties + + @Autowired + private lateinit var testDataService: TestDataService + + fun setup() { + batchJobOperationQueue.clear() + testData = BatchJobsTestData() + fakeBefore = internalProperties.fakeMtProviders + internalProperties.fakeMtProviders = true + machineTranslationProperties.google.apiKey = "mock" + machineTranslationProperties.google.defaultEnabled = true + machineTranslationProperties.google.defaultPrimary = true + machineTranslationProperties.aws.defaultEnabled = false + machineTranslationProperties.aws.accessKey = "mock" + machineTranslationProperties.aws.secretKey = "mock" + } + + fun after() { + internalProperties.fakeMtProviders = fakeBefore + } + + fun saveAndPrepare(testClass: ProjectAuthControllerTest) { + testDataService.saveTestData(testData.root) + testClass.userAccount = testData.user + testClass.projectSupplier = { testData.projectBuilder.self } + } + + fun waitForAllTranslated( + keyIds: List, + keyCount: Int, + expectedCsValue: String = "translated with GOOGLE from en to cs", + ) { + waitForNotThrowing(pollTime = 1000, timeout = 60000) { + @Suppress("UNCHECKED_CAST") + val czechTranslations = + entityManager.createQuery( + """ + from Translation t where t.key.id in :keyIds and t.language.tag = 'cs' + """.trimIndent(), + ).setParameter("keyIds", keyIds).resultList as List + czechTranslations.assert.hasSize(keyCount) + czechTranslations.forEach { + it.text.assert.contains(expectedCsValue) + } + } + } + + fun waitForJobCompleted(resultActions: ResultActions) = + resultActions.andAssertThatJson { + this.node("id").isNumber.satisfies( + Consumer { + waitFor(pollTime = 2000) { + val job = batchJobService.findJobDto(it.toLong()) + job?.status?.completed == true + } + }, + ) + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt new file mode 100644 index 0000000000..294e3127a8 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt @@ -0,0 +1,103 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.batch.BatchJobChunkExecutionQueue +import io.tolgee.batch.BatchJobService +import io.tolgee.fixtures.* +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.testing.ContextRecreatingTest +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +import org.springframework.test.web.servlet.ResultActions +import java.util.function.Consumer + +@AutoConfigureMockMvc +@ContextRecreatingTest +class BatchMoveToNamespaceTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobOperationQueue: BatchJobChunkExecutionQueue + + @Autowired + lateinit var batchJobService: BatchJobService + + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it moves to other namespace`() { + val keys = testData.addNamespaceData() + batchJobTestBase.saveAndPrepare(this) + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(700) + + val result = + performProjectAuthPost( + "start-batch-job/set-keys-namespace", + mapOf( + "keyIds" to keyIds, + "namespace" to "other-namespace", + ), + ).andIsOk + + batchJobTestBase.waitForJobCompleted(result) + val all = keyService.find(keyIds) + all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) + namespaceService.find(testData.projectBuilder.self.id, "namespace1").assert.isNull() + } + + @Test + @ProjectJWTAuthTestMethod + fun `it fails on collision when setting namespaces`() { + testData.addNamespaceData() + val key = testData.projectBuilder.addKey(keyName = "key").self + batchJobTestBase.saveAndPrepare(this) + + val result = + performProjectAuthPost( + "start-batch-job/set-keys-namespace", + mapOf( + "keyIds" to listOf(key.id), + "namespace" to "namespace", + ), + ).andIsOk + + batchJobTestBase.waitForJobCompleted(result) + + val jobId = result.jobId + keyService.get(key.id).namespace.assert.isNull() + batchJobService.findJobDto(jobId)?.status.assert.isEqualTo(BatchJobStatus.FAILED) + } + + val ResultActions.jobId: Long + get() { + var jobId: Long? = null + this.andAssertThatJson { + node("id").isNumber.satisfies( + Consumer { + jobId = it.toLong() + }, + ) + } + return jobId!! + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt new file mode 100644 index 0000000000..0411c47e1d --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt @@ -0,0 +1,70 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.andAssertThatJson +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.isValidId +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchMtTranslateTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it machine translates`() { + val keyCount = 1000 + val keys = testData.addTranslationOperationData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val keyIds = keys.map { it.id }.toList() + + performProjectAuthPost( + "start-batch-job/machine-translate", + mapOf( + "keyIds" to keyIds, + "targetLanguageIds" to + listOf( + testData.projectBuilder.getLanguageByTag("cs")!!.self.id, + testData.projectBuilder.getLanguageByTag("de")!!.self.id, + ), + ), + ) + .andIsOk + .andAssertThatJson { + node("id").isValidId + } + + batchJobTestBase.waitForAllTranslated(keyIds, keyCount) + executeInNewTransaction { + val jobs = + entityManager.createQuery("""from BatchJob""", BatchJob::class.java) + .resultList + jobs.assert.hasSize(1) + val job = jobs[0] + job.status.assert.isEqualTo(BatchJobStatus.SUCCESS) + job.activityRevision.assert.isNotNull + job.activityRevision!!.modifiedEntities.assert.hasSize(2000) + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt new file mode 100644 index 0000000000..07fdfb2af7 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt @@ -0,0 +1,70 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.andAssertThatJson +import io.tolgee.fixtures.andIsOk +import io.tolgee.fixtures.isValidId +import io.tolgee.model.batch.BatchJob +import io.tolgee.model.batch.BatchJobStatus +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchPreTranslateByMtTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it pre-translates by mt`() { + val keyCount = 1000 + val keys = testData.addTranslationOperationData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val keyIds = keys.map { it.id }.toList() + + performProjectAuthPost( + "start-batch-job/pre-translate-by-tm", + mapOf( + "keyIds" to keyIds, + "targetLanguageIds" to + listOf( + testData.projectBuilder.getLanguageByTag("cs")!!.self.id, + testData.projectBuilder.getLanguageByTag("de")!!.self.id, + ), + ), + ) + .andIsOk + .andAssertThatJson { + node("id").isValidId + } + + batchJobTestBase.waitForAllTranslated(keyIds, keyCount, "cs") + executeInNewTransaction { + val jobs = + entityManager.createQuery("""from BatchJob""", BatchJob::class.java) + .resultList + jobs.assert.hasSize(1) + val job = jobs[0] + job.status.assert.isEqualTo(BatchJobStatus.SUCCESS) + job.activityRevision.assert.isNotNull + job.activityRevision!!.modifiedEntities.assert.hasSize(2000) + } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt new file mode 100644 index 0000000000..4c94a6863e --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt @@ -0,0 +1,124 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.ProjectAuthControllerTest +import io.tolgee.fixtures.* +import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod +import io.tolgee.testing.assert +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired + +class BatchTagKeysTest : ProjectAuthControllerTest("/v2/projects/") { + @Autowired + lateinit var batchJobTestBase: BatchJobTestBase + + @BeforeEach + fun setup() { + batchJobTestBase.setup() + } + + @AfterEach + fun after() { + batchJobTestBase.after() + } + + val testData + get() = batchJobTestBase.testData + + @Test + @ProjectJWTAuthTestMethod + fun `it validates tag length`() { + performProjectAuthPost( + "start-batch-job/tag-keys", + mapOf( + "keyIds" to listOf(1), + "tags" to listOf("a".repeat(101)), + ), + ).andIsBadRequest.andPrettyPrint + } + + @Test + @ProjectJWTAuthTestMethod + fun `it tags keys`() { + val keyCount = 1000 + val keys = testData.addTagKeysData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(500) + val newTags = listOf("tag1", "tag3", "a-tag", "b-tag") + + performProjectAuthPost( + "start-batch-job/tag-keys", + mapOf( + "keyIds" to keyIds, + "tags" to newTags, + ), + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = keyService.getKeysWithTagsById(testData.project.id, keyIds) + all.assert.hasSize(keyIds.size) + all.count { + it.keyMeta?.tags?.map { it.name }?.containsAll(newTags) == true + }.assert.isEqualTo(keyIds.size) + } + } + + @Test + @ProjectJWTAuthTestMethod + fun `it untags keys`() { + val keyCount = 1000 + val keys = testData.addTagKeysData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val allKeyIds = keys.map { it.id }.toList() + val keyIds = allKeyIds.take(300) + val tagsToRemove = listOf("tag1", "a-tag", "b-tag") + + performProjectAuthPost( + "start-batch-job/untag-keys", + mapOf( + "keyIds" to keyIds, + "tags" to tagsToRemove, + ), + ).andIsOk + + waitForNotThrowing(pollTime = 1000, timeout = 10000) { + val all = keyService.getKeysWithTagsById(testData.project.id, keyIds) + all.assert.hasSize(keyIds.size) + all.count { + it.keyMeta?.tags?.map { it.name }?.any { tagsToRemove.contains(it) } == false && + it.keyMeta?.tags?.map { it.name }?.contains("tag3") == true + }.assert.isEqualTo(keyIds.size) + } + + val aKeyId = keyService.get(testData.projectBuilder.self.id, "a-key", null) + performProjectAuthPost( + "start-batch-job/untag-keys", + mapOf( + "keyIds" to keyIds, + "tags" to listOf("a-tag"), + ), + ).andIsOk + } + + @Test + @ProjectJWTAuthTestMethod + fun `it deletes tags when not used`() { + val keyCount = 1000 + val keys = testData.addTagKeysData(keyCount) + batchJobTestBase.saveAndPrepare(this) + + val aKeyId = keyService.get(testData.projectBuilder.self.id, "a-key", null).id + performProjectAuthPost( + "start-batch-job/untag-keys", + mapOf( + "keyIds" to listOf(aKeyId), + "tags" to listOf("a-tag"), + ), + ).andIsOk + waitForNotThrowing { tagService.find(testData.projectBuilder.self, "a-tag").assert.isNull() } + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt deleted file mode 100644 index 5f76aa6d79..0000000000 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/StartBatchJobControllerTest.kt +++ /dev/null @@ -1,449 +0,0 @@ -package io.tolgee.api.v2.controllers.batch - -import io.tolgee.ProjectAuthControllerTest -import io.tolgee.batch.BatchJobChunkExecutionQueue -import io.tolgee.batch.BatchJobService -import io.tolgee.development.testDataBuilder.data.BatchJobsTestData -import io.tolgee.fixtures.andAssertThatJson -import io.tolgee.fixtures.andIsBadRequest -import io.tolgee.fixtures.andIsOk -import io.tolgee.fixtures.andPrettyPrint -import io.tolgee.fixtures.isValidId -import io.tolgee.fixtures.waitFor -import io.tolgee.fixtures.waitForNotThrowing -import io.tolgee.model.batch.BatchJob -import io.tolgee.model.batch.BatchJobStatus -import io.tolgee.model.enums.TranslationState -import io.tolgee.model.translation.Translation -import io.tolgee.testing.ContextRecreatingTest -import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod -import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc -import org.springframework.test.web.servlet.ResultActions -import java.util.function.Consumer - -@AutoConfigureMockMvc -@ContextRecreatingTest -class StartBatchJobControllerTest : ProjectAuthControllerTest("/v2/projects/") { - lateinit var testData: BatchJobsTestData - var fakeBefore = false - - @Autowired - lateinit var batchJobOperationQueue: BatchJobChunkExecutionQueue - - @Autowired - lateinit var batchJobService: BatchJobService - - @BeforeEach - fun setup() { - batchJobOperationQueue.clear() - testData = BatchJobsTestData() - fakeBefore = internalProperties.fakeMtProviders - internalProperties.fakeMtProviders = true - machineTranslationProperties.google.apiKey = "mock" - machineTranslationProperties.google.defaultEnabled = true - machineTranslationProperties.google.defaultPrimary = true - machineTranslationProperties.aws.defaultEnabled = false - machineTranslationProperties.aws.accessKey = "mock" - machineTranslationProperties.aws.secretKey = "mock" - } - - @AfterEach - fun after() { - internalProperties.fakeMtProviders = fakeBefore - } - - fun saveAndPrepare() { - testDataService.saveTestData(testData.root) - userAccount = testData.user - this.projectSupplier = { testData.projectBuilder.self } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it pre-translates by mt`() { - val keyCount = 1000 - val keys = testData.addTranslationOperationData(keyCount) - saveAndPrepare() - - val keyIds = keys.map { it.id }.toList() - - performProjectAuthPost( - "start-batch-job/pre-translate-by-tm", - mapOf( - "keyIds" to keyIds, - "targetLanguageIds" to - listOf( - testData.projectBuilder.getLanguageByTag("cs")!!.self.id, - testData.projectBuilder.getLanguageByTag("de")!!.self.id, - ), - ), - ) - .andIsOk - .andAssertThatJson { - node("id").isValidId - } - - waitForAllTranslated(keyIds, keyCount, "cs") - executeInNewTransaction { - val jobs = - entityManager.createQuery("""from BatchJob""", BatchJob::class.java) - .resultList - jobs.assert.hasSize(1) - val job = jobs[0] - job.status.assert.isEqualTo(BatchJobStatus.SUCCESS) - job.activityRevision.assert.isNotNull - job.activityRevision!!.modifiedEntities.assert.hasSize(2000) - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it machine translates`() { - val keyCount = 1000 - val keys = testData.addTranslationOperationData(keyCount) - saveAndPrepare() - - val keyIds = keys.map { it.id }.toList() - - performProjectAuthPost( - "start-batch-job/machine-translate", - mapOf( - "keyIds" to keyIds, - "targetLanguageIds" to - listOf( - testData.projectBuilder.getLanguageByTag("cs")!!.self.id, - testData.projectBuilder.getLanguageByTag("de")!!.self.id, - ), - ), - ) - .andIsOk - .andAssertThatJson { - node("id").isValidId - } - - waitForAllTranslated(keyIds, keyCount) - executeInNewTransaction { - val jobs = - entityManager.createQuery("""from BatchJob""", BatchJob::class.java) - .resultList - jobs.assert.hasSize(1) - val job = jobs[0] - job.status.assert.isEqualTo(BatchJobStatus.SUCCESS) - job.activityRevision.assert.isNotNull - job.activityRevision!!.modifiedEntities.assert.hasSize(2000) - } - } - - private fun waitForAllTranslated( - keyIds: List, - keyCount: Int, - expectedCsValue: String = "translated with GOOGLE from en to cs", - ) { - waitForNotThrowing(pollTime = 1000, timeout = 60000) { - @Suppress("UNCHECKED_CAST") - val czechTranslations = - entityManager.createQuery( - """ - from Translation t where t.key.id in :keyIds and t.language.tag = 'cs' - """.trimIndent(), - ).setParameter("keyIds", keyIds).resultList as List - czechTranslations.assert.hasSize(keyCount) - czechTranslations.forEach { - it.text.assert.contains(expectedCsValue) - } - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it deletes keys`() { - val keyCount = 100 - val keys = testData.addTranslationOperationData(keyCount) - saveAndPrepare() - - val keyIds = keys.map { it.id }.toList() - - performProjectAuthPost( - "start-batch-job/delete-keys", - mapOf( - "keyIds" to keyIds, - ), - ).andIsOk.waitForJobCompleted() - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - val all = keyService.getAll(testData.projectBuilder.self.id) - all.assert.hasSize(1) - } - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - executeInNewTransaction { - val data = - entityManager - .createQuery("""from BatchJob""", BatchJob::class.java) - .resultList - - data.assert.hasSize(1) - data[0].activityRevision.assert.isNotNull - } - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it changes translation state`() { - val keyCount = 100 - val keys = testData.addStateChangeData(keyCount) - saveAndPrepare() - - val allKeyIds = keys.map { it.id }.toList() - val keyIds = allKeyIds.take(10) - val allLanguageIds = testData.projectBuilder.data.languages.map { it.self.id } - val languagesToChangeStateIds = listOf(testData.germanLanguage.id, testData.englishLanguage.id) - - performProjectAuthPost( - "start-batch-job/set-translation-state", - mapOf( - "keyIds" to keyIds, - "languageIds" to languagesToChangeStateIds, - "state" to "REVIEWED", - ), - ).andIsOk - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - val all = - translationService.getTranslations( - keys.map { it.id }, - allLanguageIds, - ) - all.count { it.state == TranslationState.REVIEWED }.assert.isEqualTo(keyIds.size * 2) - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it clears translations`() { - val keyCount = 1000 - val keys = testData.addStateChangeData(keyCount) - saveAndPrepare() - - val allKeyIds = keys.map { it.id }.toList() - val keyIds = allKeyIds.take(10) - val allLanguageIds = testData.projectBuilder.data.languages.map { it.self.id } - val languagesToClearIds = listOf(testData.germanLanguage.id, testData.englishLanguage.id) - - performProjectAuthPost( - "start-batch-job/clear-translations", - mapOf( - "keyIds" to keyIds, - "languageIds" to languagesToClearIds, - ), - ).andIsOk - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - val all = - translationService.getTranslations( - keys.map { it.id }, - allLanguageIds, - ) - all.count { it.state == TranslationState.UNTRANSLATED && it.text == null }.assert.isEqualTo(keyIds.size * 2) - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it copies translations`() { - val keyCount = 1000 - val keys = testData.addStateChangeData(keyCount) - saveAndPrepare() - - val allKeyIds = keys.map { it.id }.toList() - val keyIds = allKeyIds.take(10) - val allLanguageIds = testData.projectBuilder.data.languages.map { it.self.id } - val languagesToChangeStateIds = listOf(testData.germanLanguage.id, testData.czechLanguage.id) - - performProjectAuthPost( - "start-batch-job/copy-translations", - mapOf( - "keyIds" to keyIds, - "sourceLanguageId" to testData.englishLanguage.id, - "targetLanguageIds" to languagesToChangeStateIds, - ), - ).andIsOk - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - val all = - translationService.getTranslations( - keys.map { it.id }, - allLanguageIds, - ) - all.count { it.text?.startsWith("en") == true }.assert.isEqualTo(allKeyIds.size + keyIds.size * 2) - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it validates tag length`() { - performProjectAuthPost( - "start-batch-job/tag-keys", - mapOf( - "keyIds" to listOf(1), - "tags" to listOf("a".repeat(101)), - ), - ).andIsBadRequest.andPrettyPrint - } - - @Test - @ProjectJWTAuthTestMethod - fun `it tags keys`() { - val keyCount = 1000 - val keys = testData.addTagKeysData(keyCount) - saveAndPrepare() - - val allKeyIds = keys.map { it.id }.toList() - val keyIds = allKeyIds.take(500) - val newTags = listOf("tag1", "tag3", "a-tag", "b-tag") - - performProjectAuthPost( - "start-batch-job/tag-keys", - mapOf( - "keyIds" to keyIds, - "tags" to newTags, - ), - ).andIsOk - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - val all = keyService.getKeysWithTagsById(testData.project.id, keyIds) - all.assert.hasSize(keyIds.size) - all.count { - it.keyMeta?.tags?.map { it.name }?.containsAll(newTags) == true - }.assert.isEqualTo(keyIds.size) - } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it untags keys`() { - val keyCount = 1000 - val keys = testData.addTagKeysData(keyCount) - saveAndPrepare() - - val allKeyIds = keys.map { it.id }.toList() - val keyIds = allKeyIds.take(300) - val tagsToRemove = listOf("tag1", "a-tag", "b-tag") - - performProjectAuthPost( - "start-batch-job/untag-keys", - mapOf( - "keyIds" to keyIds, - "tags" to tagsToRemove, - ), - ).andIsOk - - waitForNotThrowing(pollTime = 1000, timeout = 10000) { - val all = keyService.getKeysWithTagsById(testData.project.id, keyIds) - all.assert.hasSize(keyIds.size) - all.count { - it.keyMeta?.tags?.map { it.name }?.any { tagsToRemove.contains(it) } == false && - it.keyMeta?.tags?.map { it.name }?.contains("tag3") == true - }.assert.isEqualTo(keyIds.size) - } - - val aKeyId = keyService.get(testData.projectBuilder.self.id, "a-key", null) - performProjectAuthPost( - "start-batch-job/untag-keys", - mapOf( - "keyIds" to keyIds, - "tags" to listOf("a-tag"), - ), - ).andIsOk - } - - @Test - @ProjectJWTAuthTestMethod - fun `it deletes tags when not used`() { - val keyCount = 1000 - val keys = testData.addTagKeysData(keyCount) - saveAndPrepare() - - val aKeyId = keyService.get(testData.projectBuilder.self.id, "a-key", null).id - performProjectAuthPost( - "start-batch-job/untag-keys", - mapOf( - "keyIds" to listOf(aKeyId), - "tags" to listOf("a-tag"), - ), - ).andIsOk - waitForNotThrowing { tagService.find(testData.projectBuilder.self, "a-tag").assert.isNull() } - } - - @Test - @ProjectJWTAuthTestMethod - fun `it moves to other namespace`() { - val keys = testData.addNamespaceData() - saveAndPrepare() - - val allKeyIds = keys.map { it.id }.toList() - val keyIds = allKeyIds.take(700) - - performProjectAuthPost( - "start-batch-job/set-keys-namespace", - mapOf( - "keyIds" to keyIds, - "namespace" to "other-namespace", - ), - ).andIsOk.waitForJobCompleted() - - val all = keyService.find(keyIds) - all.count { it.namespace?.name == "other-namespace" }.assert.isEqualTo(keyIds.size) - namespaceService.find(testData.projectBuilder.self.id, "namespace1").assert.isNull() - } - - @Test - @ProjectJWTAuthTestMethod - fun `it fails on collision when setting namespaces`() { - testData.addNamespaceData() - val key = testData.projectBuilder.addKey(keyName = "key").self - saveAndPrepare() - - val jobId = - performProjectAuthPost( - "start-batch-job/set-keys-namespace", - mapOf( - "keyIds" to listOf(key.id), - "namespace" to "namespace", - ), - ).andIsOk.waitForJobCompleted().jobId - keyService.get(key.id).namespace.assert.isNull() - batchJobService.findJobDto(jobId)?.status.assert.isEqualTo(BatchJobStatus.FAILED) - } - - fun ResultActions.waitForJobCompleted() = - andAssertThatJson { - node("id").isNumber.satisfies( - Consumer { - waitFor(pollTime = 2000) { - val job = batchJobService.findJobDto(it.toLong()) - job?.status?.completed == true - } - }, - ) - } - - val ResultActions.jobId: Long - get() { - var jobId: Long? = null - this.andAssertThatJson { - node("id").isNumber.satisfies( - Consumer { - jobId = it.toLong() - }, - ) - } - return jobId!! - } -} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/SingleStepImportControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/SingleStepImportControllerTest.kt index 8cffdfa6ec..281e2008cf 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/SingleStepImportControllerTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2ImportController/SingleStepImportControllerTest.kt @@ -10,6 +10,7 @@ import io.tolgee.fixtures.andIsForbidden import io.tolgee.fixtures.andIsOk import io.tolgee.model.batch.BatchJob import io.tolgee.model.enums.Scope +import io.tolgee.model.enums.TranslationState import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert import io.tolgee.util.performSingleStepImport @@ -64,6 +65,33 @@ class SingleStepImportControllerTest : ProjectAuthControllerTest("/v2/projects/" } } + @Test + @ProjectJWTAuthTestMethod + fun `resets translation state`() { + testData.addReviewedTranslation() + saveAndPrepare() + + executeInNewTransaction { + getGermanTestTranslation().state + .assert.isEqualTo(TranslationState.REVIEWED) + } + + performImport( + projectId = testData.project.id, + listOf(Pair("de.json", simpleJson)), + params = mapOf("forceMode" to "OVERRIDE"), + ) + + executeInNewTransaction { + getGermanTestTranslation().state + .assert.isEqualTo(TranslationState.TRANSLATED) + } + } + + private fun getGermanTestTranslation() = + getTestKeyTranslations() + .single { it.language.tag == "de" } + @Test @ProjectJWTAuthTestMethod fun `import po with code references`() { diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/builders/KeyBuilder.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/builders/KeyBuilder.kt index c4604c6119..0d508d9f72 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/builders/KeyBuilder.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/builders/KeyBuilder.kt @@ -32,6 +32,8 @@ class KeyBuilder( return addOperation(projectBuilder.data.translations, builder, ft) } + val translations get() = projectBuilder.data.translations.filter { it.self.key === self } + fun addMeta(ft: FT): KeyMetaBuilder { data.meta = KeyMetaBuilder(keyBuilder = this).apply { ft(this.self) } return data.meta!! diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt index 12d33ba050..e1532cfe53 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BatchJobsTestData.kt @@ -1,6 +1,8 @@ package io.tolgee.development.testDataBuilder.data +import io.tolgee.development.testDataBuilder.builders.KeyBuilder import io.tolgee.model.enums.Scope +import io.tolgee.model.enums.TranslationState import io.tolgee.model.key.Key class BatchJobsTestData : BaseTestData() { @@ -16,7 +18,21 @@ class BatchJobsTestData : BaseTestData() { } fun addTranslationOperationData(keyCount: Int = 100): List { - this.projectBuilder.addKey { + addAKey() + return (1..keyCount).map { + this.projectBuilder.addKey { + name = "key$it" + }.build { + addTranslation { + language = englishLanguage + text = "en" + } + }.self + } + } + + private fun addAKey(): KeyBuilder { + return this.projectBuilder.addKey { name = "a-key" }.build { addTranslation { @@ -32,16 +48,14 @@ class BatchJobsTestData : BaseTestData() { text = "de" } } - return (1..keyCount).map { - this.projectBuilder.addKey { - name = "key$it" - }.build { - addTranslation { - language = englishLanguage - text = "en" - } - }.self + } + + fun addKeyWithTranslationsReviewed(): Key { + val aKey = addAKey() + aKey.translations.forEach { + it.self.state = TranslationState.REVIEWED } + return aKey.self } fun addStateChangeData(keyCount: Int = 100): List { diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/SingleStepImportTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/SingleStepImportTestData.kt index d67d4dd75a..f6d0c0e2fe 100644 --- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/SingleStepImportTestData.kt +++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/SingleStepImportTestData.kt @@ -2,6 +2,7 @@ package io.tolgee.development.testDataBuilder.data import io.tolgee.model.enums.OrganizationRoleType import io.tolgee.model.enums.Scope +import io.tolgee.model.enums.TranslationState class SingleStepImportTestData : BaseTestData() { val germanLanguage = projectBuilder.addGerman() @@ -18,6 +19,19 @@ class SingleStepImportTestData : BaseTestData() { } } + fun addReviewedTranslation() { + val key = + projectBuilder.addKey { + this.name = "test" + } + projectBuilder.addTranslation { + this.key = key.self + this.text = "conflict!" + state = TranslationState.REVIEWED + this.language = germanLanguage.self + } + } + fun setUserScopes(scopes: Array) { userAccountBuilder.defaultOrganizationBuilder.data.roles.first().self.type = OrganizationRoleType.MEMBER projectBuilder.data.permissions.first().self.scopes = scopes diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index e1ea81f974..8e07052757 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -279,8 +279,7 @@ class StoredDataImporter( if (language == language.project.baseLanguage && translation.text != this.text) { outdatedFlagKeys.add(translation.key.id) } - translation.text = this@doImport.text - translation.resetFlags() + translationService.setTextNoSave(translation, text) translationsToSave.add(this to translation) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index 60e054fc14..aa5b61962d 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -186,6 +186,15 @@ class TranslationService( translation: Translation, text: String?, ): Translation { + setTextNoSave(translation, text) + + return save(translation) + } + + fun setTextNoSave( + translation: Translation, + text: String?, + ) { val hasTextChanged = translation.text != text if (hasTextChanged) { @@ -203,8 +212,6 @@ class TranslationService( text.isNullOrEmpty() -> TranslationState.UNTRANSLATED else -> translation.state } - - return save(translation) } fun save(translation: Translation): Translation { From 10ec7ef44583eac436f8101eef388cf021fc6969 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 31 Jan 2025 14:39:25 +0100 Subject: [PATCH 3/7] chore: Move translation setting methods to separate class --- .../TranslationsControllerCachingTest.kt | 2 +- .../demoProject/DemoProjectCreator.kt | 2 +- .../service/dataImport/StoredDataImporter.kt | 2 +- .../service/key/ResolvingKeyImporter.kt | 2 +- .../tolgee/service/key/utils/KeysImporter.kt | 2 +- .../translation/SetTranslationTextUtil.kt | 120 +++++++++++++ .../service/translation/TranslationService.kt | 163 +++++------------- 7 files changed, 172 insertions(+), 121 deletions(-) create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt index a45c94f8d2..baab4330cc 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/translations/v2TranslationsController/TranslationsControllerCachingTest.kt @@ -68,7 +68,7 @@ class TranslationsControllerCachingTest : ProjectAuthControllerTest("/v2/project val newNow = Date(Date().time + 50000) setForcedDate(newNow) - translationService.setTranslation(testData.aKey, testData.englishLanguage, "This was changed!") + translationService.setTranslationText(testData.aKey, testData.englishLanguage, "This was changed!") val newLastModified = performWithIsModifiedSince(lastModified).andIsOk.lastModified() assertEqualsDate(newLastModified, newNow) diff --git a/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt b/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt index c584738e00..d75184cf61 100644 --- a/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt +++ b/backend/data/src/main/kotlin/io/tolgee/component/demoProject/DemoProjectCreator.kt @@ -113,7 +113,7 @@ class DemoProjectCreator( translation: String, ): Translation { val language = languages[languageTag]!! - return translationService.setTranslation(getOrCreateKey(keyName), language, translation).also { + return translationService.setTranslationText(getOrCreateKey(keyName), language, translation).also { it.state = TranslationState.REVIEWED } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt index 8e07052757..9f92f5f0e5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/dataImport/StoredDataImporter.kt @@ -279,7 +279,7 @@ class StoredDataImporter( if (language == language.project.baseLanguage && translation.text != this.text) { outdatedFlagKeys.add(translation.key.id) } - translationService.setTextNoSave(translation, text) + translationService.setTranslationTextNoSave(translation, text) translationsToSave.add(this to translation) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt index b453572574..4e6073301c 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/ResolvingKeyImporter.kt @@ -183,7 +183,7 @@ class ResolvingKeyImporter( private fun List.save() { this.forEach { - translationService.setTranslation(it.translation, it.text) + translationService.setTranslationText(it.translation, it.text) updatedTranslationIds.add(it.translation.id) } } diff --git a/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt b/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt index f50de0fe5b..bc16e23637 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/key/utils/KeysImporter.kt @@ -68,7 +68,7 @@ class KeysImporter( val translations = convertedToPlurals ?: keyDto.translations translations.entries.forEach { (languageTag, value) -> languages[languageTag]?.let { language -> - translationService.setTranslation(key, language, value) + translationService.setTranslationText(key, language, value) } } existing[safeNamespace to keyDto.name] = key diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt new file mode 100644 index 0000000000..33a9d6b100 --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt @@ -0,0 +1,120 @@ +package io.tolgee.service.translation + +import io.tolgee.constants.Message +import io.tolgee.events.OnTranslationsSet +import io.tolgee.exceptions.NotFoundException +import io.tolgee.model.Language +import io.tolgee.model.enums.TranslationState +import io.tolgee.model.key.Key +import io.tolgee.model.translation.Translation +import io.tolgee.service.language.LanguageService +import org.springframework.context.ApplicationContext + +class SetTranslationTextUtil( + private val applicationContext: ApplicationContext, +) { + fun setForKey( + key: Key, + translations: Map, + ): Map { + val normalized = + translationService.validateAndNormalizePlurals(translations, key.isPlural, key.pluralArgName) + val languages = languageService.findEntitiesByTags(translations.keys, key.project.id) + val oldTranslations = + translationService.getKeyTranslations(languages, key.project, key).associate { + languageByIdFromLanguages( + it.language.id, + languages, + ) to it.text + } + + return setForKey( + key, + normalized.map { languageByTagFromLanguages(it.key, languages) to it.value } + .toMap(), + oldTranslations, + ).mapKeys { it.key.tag } + } + + fun setForKey( + key: Key, + translations: Map, + oldTranslations: Map, + ): Map { + val result = + translations.entries.associate { (language, value) -> + language to setTranslationText(key, language, value) + }.mapValues { it.value } + + applicationContext.publishEvent( + OnTranslationsSet( + source = this, + key = key, + oldValues = oldTranslations.map { it.key.tag to it.value }.toMap(), + translations = result.values.toList(), + ), + ) + + return result + } + + fun setTranslationText( + key: Key, + language: Language, + text: String?, + ): Translation { + val translation = translationService.getOrCreate(key, language) + setTranslationText(translation, text) + key.translations.add(translation) + return translation + } + + fun setTranslationText( + translation: Translation, + text: String?, + ): Translation { + setTranslationTextNoSave(translation, text) + return translationService.save(translation) + } + + fun setTranslationTextNoSave( + translation: Translation, + text: String?, + ) { + val hasTextChanged = translation.text != text + + if (hasTextChanged) { + translation.resetFlags() + } + + translation.text = text + + val hasText = !text.isNullOrEmpty() + + translation.state = + when { + translation.isUntranslated && hasText -> TranslationState.TRANSLATED + hasTextChanged -> TranslationState.TRANSLATED + text.isNullOrEmpty() -> TranslationState.UNTRANSLATED + else -> translation.state + } + } + + private val translationService by lazy { + applicationContext.getBean(TranslationService::class.java) + } + + private val languageService by lazy { + applicationContext.getBean(LanguageService::class.java) + } + + private fun languageByIdFromLanguages( + id: Long, + languages: Set, + ) = languages.find { it.id == id } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) + + private fun languageByTagFromLanguages( + tag: String, + languages: Collection, + ) = languages.find { it.tag == tag } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) +} diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt index aa5b61962d..6f99e99df2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/TranslationService.kt @@ -5,7 +5,6 @@ import io.tolgee.constants.Message import io.tolgee.dtos.cacheable.LanguageDto import io.tolgee.dtos.request.translation.GetTranslationsParams import io.tolgee.dtos.request.translation.TranslationFilters -import io.tolgee.events.OnTranslationsSet import io.tolgee.exceptions.BadRequestException import io.tolgee.exceptions.NotFoundException import io.tolgee.formats.* @@ -30,7 +29,7 @@ import io.tolgee.service.queryBuilders.translationViewBuilder.TranslationViewDat import io.tolgee.util.nullIfEmpty import jakarta.persistence.EntityManager import org.springframework.beans.factory.annotation.Autowired -import org.springframework.context.ApplicationEventPublisher +import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Lazy import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable @@ -45,7 +44,7 @@ class TranslationService( private val translationRepository: TranslationRepository, private val importService: ImportService, private val tolgeeProperties: TolgeeProperties, - private val applicationEventPublisher: ApplicationEventPublisher, + private val applicationContext: ApplicationContext, private val translationViewDataProvider: TranslationViewDataProvider, private val entityManager: EntityManager, private val translationCommentService: TranslationCommentService, @@ -171,47 +170,43 @@ class TranslationService( return translationViewDataProvider.getSelectAllKeys(projectId, languages, params) } - fun setTranslation( + @Transactional + fun setForKey( + key: Key, + translations: Map, + ): Map { + return SetTranslationTextUtil(applicationContext).setForKey(key, translations) + } + + @Transactional + fun setForKey( + key: Key, + translations: Map, + oldTranslations: Map, + ): Map { + return SetTranslationTextUtil(applicationContext).setForKey(key, translations, oldTranslations) + } + + fun setTranslationText( key: Key, language: Language, text: String?, ): Translation { - val translation = getOrCreate(key, language) - setTranslation(translation, text) - key.translations.add(translation) - return translation + return SetTranslationTextUtil(applicationContext).setTranslationText(key, language, text) } - fun setTranslation( + fun setTranslationText( translation: Translation, text: String?, ): Translation { - setTextNoSave(translation, text) - - return save(translation) + return SetTranslationTextUtil(applicationContext).setTranslationText(translation, text) } - fun setTextNoSave( + fun setTranslationTextNoSave( translation: Translation, text: String?, ) { - val hasTextChanged = translation.text != text - - if (hasTextChanged) { - translation.resetFlags() - } - - translation.text = text - - val hasText = !text.isNullOrEmpty() - - translation.state = - when { - translation.isUntranslated && hasText -> TranslationState.TRANSLATED - hasTextChanged -> TranslationState.TRANSLATED - text.isNullOrEmpty() -> TranslationState.UNTRANSLATED - else -> translation.state - } + return SetTranslationTextUtil(applicationContext).setTranslationTextNoSave(translation, text) } fun save(translation: Translation): Translation { @@ -222,30 +217,6 @@ class TranslationService( return translationRepository.save(translation) } - @Transactional - fun setForKey( - key: Key, - translations: Map, - ): Map { - val normalized = - validateAndNormalizePlurals(translations, key.isPlural, key.pluralArgName) - val languages = languageService.findEntitiesByTags(translations.keys, key.project.id) - val oldTranslations = - getKeyTranslations(languages, key.project, key).associate { - languageByIdFromLanguages( - it.language.id, - languages, - ) to it.text - } - - return setForKey( - key, - normalized.map { languageByTagFromLanguages(it.key, languages) to it.value } - .toMap(), - oldTranslations, - ).mapKeys { it.key.tag } - } - fun findForKeyByLanguages( key: Key, languageTags: Collection, @@ -253,39 +224,6 @@ class TranslationService( return translationRepository.findForKeyByLanguages(key, languageTags) } - private fun languageByTagFromLanguages( - tag: String, - languages: Collection, - ) = languages.find { it.tag == tag } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) - - private fun languageByIdFromLanguages( - id: Long, - languages: Set, - ) = languages.find { it.id == id } ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND) - - @Transactional - fun setForKey( - key: Key, - translations: Map, - oldTranslations: Map, - ): Map { - val result = - translations.entries.associate { (language, value) -> - language to setTranslation(key, language, value) - }.mapValues { it.value } - - applicationEventPublisher.publishEvent( - OnTranslationsSet( - source = this, - key = key, - oldValues = oldTranslations.map { it.key.tag to it.value }.toMap(), - translations = result.values.toList(), - ), - ) - - return result - } - @Suppress("UNCHECKED_CAST") private fun addToMap( translation: SimpleTranslationView, @@ -433,11 +371,28 @@ class TranslationService( return translationRepository.getForKeys(keyIds, languageTags) } - fun findAllByKeyIdsAndLanguageIds( - keyIds: List, - languageIds: List, - ): List { - return translationRepository.findAllByKeyIdInAndLanguageIdIn(keyIds, languageIds) + fun validateAndNormalizePlurals( + texts: Map, + isKeyPlural: Boolean, + pluralArgName: String?, + ): Map { + if (isKeyPlural) { + return validateAndNormalizePlurals(texts, pluralArgName) + } + + return texts + } + + fun validateAndNormalizePlurals( + texts: Map, + pluralArgName: String?, + ): Map { + @Suppress("UNCHECKED_CAST") + return try { + normalizePlurals(texts, pluralArgName) + } catch (e: StringIsNotPluralException) { + throw BadRequestException(Message.INVALID_PLURAL_FORM, listOf(e.invalidStrings) as List) + } } @Transactional @@ -569,28 +524,4 @@ class TranslationService( throw BadRequestException(Message.PLURAL_FORMS_DATA_LOSS, listOf(text)) } } - - fun validateAndNormalizePlurals( - texts: Map, - isKeyPlural: Boolean, - pluralArgName: String?, - ): Map { - if (isKeyPlural) { - return validateAndNormalizePlurals(texts, pluralArgName) - } - - return texts - } - - fun validateAndNormalizePlurals( - texts: Map, - pluralArgName: String?, - ): Map { - @Suppress("UNCHECKED_CAST") - return try { - normalizePlurals(texts, pluralArgName) - } catch (e: StringIsNotPluralException) { - throw BadRequestException(Message.INVALID_PLURAL_FORM, listOf(e.invalidStrings) as List) - } - } } From daf20eeb7ebdcd9cf300013f393d8e537a8e8cc7 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 31 Jan 2025 16:18:20 +0100 Subject: [PATCH 4/7] fix: Fix complex update issue --- .../io/tolgee/component/KeyComplexEditHelper.kt | 13 +++++++++++-- .../v2KeyController/KeyControllerComplexEditTest.kt | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt b/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt index 0365f21364..0507b770ed 100644 --- a/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt +++ b/backend/api/src/main/kotlin/io/tolgee/component/KeyComplexEditHelper.kt @@ -317,7 +317,7 @@ class KeyComplexEditHelper( return !currentTagsContainAllNewTags || !newTagsContainAllCurrentTags } - val requireKeyEditPermission + private val requireKeyEditPermission get() = isKeyNameModified || isNamespaceChanged || @@ -333,7 +333,16 @@ class KeyComplexEditHelper( private fun prepareModifiedStates() { modifiedStates = - dto.states?.filter { it.value.translationState != existingTranslations[it.key]?.state } + dto.states?.filter { + // When updating translation, we automatically set it to TRANSLATED state + // While executing complex edit, user can prevent this by setting the state to REVIEWED + // In that case, we have to add it to modified states too + val tryingToKeepReviewState = + modifiedTranslations?.get(languageByTag(it.key).id) != null && + it.value.translationState == TranslationState.REVIEWED + + it.value.translationState != existingTranslations[it.key]?.state || tryingToKeepReviewState + } ?.map { languageByTag(it.key).id to it.value.translationState }?.toMap() } diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexEditTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexEditTest.kt index 12bbd94615..7e091ff273 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexEditTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerComplexEditTest.kt @@ -171,10 +171,10 @@ class KeyControllerComplexEditTest : ProjectAuthControllerTest("/v2/projects/") private fun verifyStates(statesToVerify: Map) { executeInNewTransaction { - val key = keyService.find(testData.keyWithReferences.id) + val key = keyService.get(testData.keyWithReferences.id) assertThat(key).isNotNull statesToVerify.forEach { - val state = key!!.translations.find { translation -> translation.language.tag == it.key }!!.state + val state = key.translations.find { translation -> translation.language.tag == it.key }!!.state assertThat(state) .describedAs("State for ${it.key} is not ${it.value}") .isEqualTo(it.value) From 74596cf114d2622024ad70e05e07ddfc27d176a6 Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Fri, 31 Jan 2025 16:44:10 +0100 Subject: [PATCH 5/7] fix: Fix machine translation properties issue --- .../batch/BatchChangeTranslationStateTest.kt | 6 ---- .../batch/BatchClearTranslationsTest.kt | 6 ---- .../batch/BatchCopyTranslationsTest.kt | 6 ---- .../controllers/batch/BatchDeleteKeysTest.kt | 6 ---- .../v2/controllers/batch/BatchJobTestBase.kt | 33 ++++++++++++------- .../batch/BatchMoveToNamespaceTest.kt | 6 ---- .../controllers/batch/BatchMtTranslateTest.kt | 6 ---- .../batch/BatchPreTranslateByMtTest.kt | 6 ---- .../v2/controllers/batch/BatchTagKeysTest.kt | 6 ---- .../translation/SetTranslationTextUtil.kt | 1 + 10 files changed, 23 insertions(+), 59 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt index 60ee5bba5f..cd119b500f 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchChangeTranslationStateTest.kt @@ -6,7 +6,6 @@ import io.tolgee.fixtures.waitForNotThrowing import io.tolgee.model.enums.TranslationState import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -20,11 +19,6 @@ class BatchChangeTranslationStateTest : ProjectAuthControllerTest("/v2/projects/ batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt index d4aa74ca26..e7d5756552 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchClearTranslationsTest.kt @@ -6,7 +6,6 @@ import io.tolgee.fixtures.waitForNotThrowing import io.tolgee.model.enums.TranslationState import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -20,11 +19,6 @@ class BatchClearTranslationsTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt index 93e147b06f..7fa38c601f 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchCopyTranslationsTest.kt @@ -7,7 +7,6 @@ import io.tolgee.model.enums.TranslationState import io.tolgee.model.key.Key import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -21,11 +20,6 @@ class BatchCopyTranslationsTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt index adbeec5c58..0d9a60cedf 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchDeleteKeysTest.kt @@ -6,7 +6,6 @@ import io.tolgee.fixtures.waitForNotThrowing import io.tolgee.model.batch.BatchJob import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -20,11 +19,6 @@ class BatchDeleteKeysTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt index 4c0e5020a9..391f6b6ed2 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobTestBase.kt @@ -4,6 +4,8 @@ import io.tolgee.ProjectAuthControllerTest import io.tolgee.batch.BatchJobChunkExecutionQueue import io.tolgee.batch.BatchJobService import io.tolgee.configuration.tolgee.InternalProperties +import io.tolgee.configuration.tolgee.machineTranslation.AwsMachineTranslationProperties +import io.tolgee.configuration.tolgee.machineTranslation.GoogleMachineTranslationProperties import io.tolgee.configuration.tolgee.machineTranslation.MachineTranslationProperties import io.tolgee.development.testDataBuilder.TestDataService import io.tolgee.development.testDataBuilder.data.BatchJobsTestData @@ -13,7 +15,10 @@ import io.tolgee.fixtures.waitForNotThrowing import io.tolgee.model.translation.Translation import io.tolgee.testing.assert import jakarta.persistence.EntityManager +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.mock.mockito.SpyBean import org.springframework.stereotype.Component import org.springframework.test.web.servlet.ResultActions import java.util.function.Consumer @@ -29,6 +34,7 @@ class BatchJobTestBase { lateinit var batchJobService: BatchJobService @Autowired + @SpyBean lateinit var machineTranslationProperties: MachineTranslationProperties @Autowired @@ -37,6 +43,7 @@ class BatchJobTestBase { var fakeBefore: Boolean = false @Autowired + @SpyBean private lateinit var internalProperties: InternalProperties @Autowired @@ -45,18 +52,22 @@ class BatchJobTestBase { fun setup() { batchJobOperationQueue.clear() testData = BatchJobsTestData() - fakeBefore = internalProperties.fakeMtProviders - internalProperties.fakeMtProviders = true - machineTranslationProperties.google.apiKey = "mock" - machineTranslationProperties.google.defaultEnabled = true - machineTranslationProperties.google.defaultPrimary = true - machineTranslationProperties.aws.defaultEnabled = false - machineTranslationProperties.aws.accessKey = "mock" - machineTranslationProperties.aws.secretKey = "mock" - } - fun after() { - internalProperties.fakeMtProviders = fakeBefore + whenever(internalProperties.fakeMtProviders).thenReturn(true) + + val googleMock = mock() + whenever(googleMock.apiKey).thenReturn("mock") + whenever(googleMock.defaultEnabled).thenReturn(true) + whenever(googleMock.defaultPrimary).thenReturn(true) + + whenever(machineTranslationProperties.google).thenReturn(googleMock) + + val awsMock = mock() + whenever(awsMock.defaultEnabled).thenReturn(false) + whenever(awsMock.accessKey).thenReturn("mock") + whenever(awsMock.secretKey).thenReturn("mock") + + whenever(machineTranslationProperties.aws).thenReturn(awsMock) } fun saveAndPrepare(testClass: ProjectAuthControllerTest) { diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt index 294e3127a8..cbc511a895 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMoveToNamespaceTest.kt @@ -8,7 +8,6 @@ import io.tolgee.model.batch.BatchJobStatus import io.tolgee.testing.ContextRecreatingTest import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -33,11 +32,6 @@ class BatchMoveToNamespaceTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt index 0411c47e1d..af587bcaba 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchMtTranslateTest.kt @@ -8,7 +8,6 @@ import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobStatus import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -22,11 +21,6 @@ class BatchMtTranslateTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt index 07fdfb2af7..b8a285dfb6 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt @@ -8,7 +8,6 @@ import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobStatus import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -22,11 +21,6 @@ class BatchPreTranslateByMtTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt index 4c94a6863e..954efae93f 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchTagKeysTest.kt @@ -4,7 +4,6 @@ import io.tolgee.ProjectAuthControllerTest import io.tolgee.fixtures.* import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -18,11 +17,6 @@ class BatchTagKeysTest : ProjectAuthControllerTest("/v2/projects/") { batchJobTestBase.setup() } - @AfterEach - fun after() { - batchJobTestBase.after() - } - val testData get() = batchJobTestBase.testData diff --git a/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt b/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt index 33a9d6b100..9d444ebeb4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt +++ b/backend/data/src/main/kotlin/io/tolgee/service/translation/SetTranslationTextUtil.kt @@ -93,6 +93,7 @@ class SetTranslationTextUtil( translation.state = when { + translation.state == TranslationState.DISABLED -> TranslationState.DISABLED translation.isUntranslated && hasText -> TranslationState.TRANSLATED hasTextChanged -> TranslationState.TRANSLATED text.isNullOrEmpty() -> TranslationState.UNTRANSLATED From 651d370d955ea589eaead05ac9a27eedfe14c34a Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 1 Feb 2025 12:46:06 +0100 Subject: [PATCH 6/7] chore: Refactor batch operation and fix the batch operation tests --- .../batch/BatchJobRunnerStopTest.kt | 16 +++ ...MtTest.kt => BatchPreTranslateByTmTest.kt} | 9 +- .../app/src/test/resources/application.yaml | 12 -- .../tolgee/batch/ApplicationBatchJobRunner.kt | 72 ++++++++++ .../io/tolgee/batch/BatchJobActionService.kt | 132 ++++++++---------- .../batch/BatchJobCancellationManager.kt | 14 +- .../batch/BatchJobConcurrentLauncher.kt | 37 ++--- .../io/tolgee/BatchJobCleanerListener.kt | 13 -- .../kotlin/io/tolgee/BatchJobTestListener.kt | 58 ++++++++ .../main/kotlin/io/tolgee/BatchJobTestUtil.kt | 24 ---- .../testing/AbstractTransactionalTest.kt | 4 +- 11 files changed, 234 insertions(+), 157 deletions(-) create mode 100644 backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobRunnerStopTest.kt rename backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/{BatchPreTranslateByMtTest.kt => BatchPreTranslateByTmTest.kt} (86%) create mode 100644 backend/data/src/main/kotlin/io/tolgee/batch/ApplicationBatchJobRunner.kt delete mode 100644 backend/testing/src/main/kotlin/io/tolgee/BatchJobCleanerListener.kt create mode 100644 backend/testing/src/main/kotlin/io/tolgee/BatchJobTestListener.kt delete mode 100644 backend/testing/src/main/kotlin/io/tolgee/BatchJobTestUtil.kt diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobRunnerStopTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobRunnerStopTest.kt new file mode 100644 index 0000000000..d647139919 --- /dev/null +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchJobRunnerStopTest.kt @@ -0,0 +1,16 @@ +package io.tolgee.api.v2.controllers.batch + +import io.tolgee.AbstractSpringTest +import io.tolgee.batch.ApplicationBatchJobRunner +import io.tolgee.testing.assert +import org.junit.jupiter.api.Test +import org.springframework.boot.test.context.SpringBootTest + +@SpringBootTest +class BatchJobRunnerStopTest : AbstractSpringTest() { + @Test + fun `it stops`() { + ApplicationBatchJobRunner.stopAll() + ApplicationBatchJobRunner.runningInstances.assert.hasSize(0) + } +} diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByTmTest.kt similarity index 86% rename from backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt rename to backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByTmTest.kt index b8a285dfb6..42d3976ff3 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByMtTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/batch/BatchPreTranslateByTmTest.kt @@ -1,6 +1,7 @@ package io.tolgee.api.v2.controllers.batch import io.tolgee.ProjectAuthControllerTest +import io.tolgee.batch.ApplicationBatchJobRunner import io.tolgee.fixtures.andAssertThatJson import io.tolgee.fixtures.andIsOk import io.tolgee.fixtures.isValidId @@ -8,14 +9,18 @@ import io.tolgee.model.batch.BatchJob import io.tolgee.model.batch.BatchJobStatus import io.tolgee.testing.annotations.ProjectJWTAuthTestMethod import io.tolgee.testing.assert +import io.tolgee.util.Logging import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired -class BatchPreTranslateByMtTest : ProjectAuthControllerTest("/v2/projects/") { +class BatchPreTranslateByTmTest : Logging, ProjectAuthControllerTest("/v2/projects/") { @Autowired lateinit var batchJobTestBase: BatchJobTestBase + @Autowired + lateinit var applicationBatchJobRunner: ApplicationBatchJobRunner + @BeforeEach fun setup() { batchJobTestBase.setup() @@ -26,7 +31,7 @@ class BatchPreTranslateByMtTest : ProjectAuthControllerTest("/v2/projects/") { @Test @ProjectJWTAuthTestMethod - fun `it pre-translates by mt`() { + fun `it pre-translates by tm`() { val keyCount = 1000 val keys = testData.addTranslationOperationData(keyCount) batchJobTestBase.saveAndPrepare(this) diff --git a/backend/app/src/test/resources/application.yaml b/backend/app/src/test/resources/application.yaml index c67fbea1e5..784c8169cb 100644 --- a/backend/app/src/test/resources/application.yaml +++ b/backend/app/src/test/resources/application.yaml @@ -4,7 +4,6 @@ spring: - org.redisson.spring.starter.RedissonAutoConfigurationV2 - org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration jpa: - show-sql: true properties: hibernate: order_by: @@ -84,14 +83,3 @@ tolgee: server: http://localhost:8080 batch: concurrency: 10 - -logging: - level: - io.tolgee.billing.api.v2.OrganizationInvoicesController: DEBUG - io.tolgee.batch.BatchJobActionService: DEBUG - io.tolgee.component.atomicLong.AtomicLongProvider: DEBUG - io.tolgee: DEBUG - io.tolgee.component.CurrentDateProvider: DEBUG - io.tolgee.component.reporting.BusinessEventPublisher: DEBUG - io.tolgee.ExceptionHandlers: DEBUG - io.tolgee.component.reporting.ReportingService: DEBUG diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/ApplicationBatchJobRunner.kt b/backend/data/src/main/kotlin/io/tolgee/batch/ApplicationBatchJobRunner.kt new file mode 100644 index 0000000000..df0d0a486f --- /dev/null +++ b/backend/data/src/main/kotlin/io/tolgee/batch/ApplicationBatchJobRunner.kt @@ -0,0 +1,72 @@ +package io.tolgee.batch + +import jakarta.annotation.PreDestroy +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.ApplicationContext +import org.springframework.context.event.EventListener +import org.springframework.stereotype.Component +import java.util.concurrent.ConcurrentHashMap + +/** + * This class handles running of batch jobs when application context is ready + * + * - It handles the stopping when application is being destroyed + * - It helps us to stop all running instances of this runner for testing purposes, where we can + * get to unexpected situations + */ +@Component +class ApplicationBatchJobRunner( + private val applicationContext: ApplicationContext, +) { + companion object { + /** + * This helps us to keep track of all running instances across all existing application contexts + * + * In tests, spring caches the application context and creates a may create a new instance + * of ApplicationBatchJobRunner + * + * We need to prevent this machine running in cached contexts, so we need this property to keep track + * of all running instances to be safe that we can stop them all + */ + val runningInstances: ConcurrentHashMap.KeySetView = + ConcurrentHashMap.newKeySet() + + fun stopAll() { + runningInstances.forEach { it.stop() } + } + } + + var isRunning = false + + @EventListener(ApplicationReadyEvent::class) + fun run() { + if (isRunning) { + return + } + + stopAll() + isRunning = true + runningInstances.add(this) + batchJobConcurrentLauncher.run() + } + + fun stop() { + runningInstances.remove(this) + batchJobConcurrentLauncher.stop() + isRunning = false + } + + @PreDestroy + fun preDestroy() { + stop() + + // To be super safe, rather stop them all if by any chance there is more than one instance running + stopAll() + } + + // We want to keep the same instance of BatchJobConcurrentLauncher for all instances of ApplicationBatchJobRunner + // This should prevent spring from magically giving us different instances + private val batchJobConcurrentLauncher: BatchJobConcurrentLauncher by lazy { + applicationContext.getBean(BatchJobConcurrentLauncher::class.java) + } +} diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt index 168712a90f..3480efea8e 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt @@ -21,14 +21,13 @@ import io.tolgee.util.logger import jakarta.persistence.EntityManager import jakarta.persistence.LockModeType import org.hibernate.LockOptions -import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.context.ApplicationContext import org.springframework.context.annotation.Lazy -import org.springframework.context.event.EventListener import org.springframework.data.redis.core.StringRedisTemplate import org.springframework.stereotype.Service import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.UnexpectedRollbackException +import kotlin.coroutines.coroutineContext @Service class BatchJobActionService( @@ -43,81 +42,73 @@ class BatchJobActionService( private val batchJobChunkExecutionQueue: BatchJobChunkExecutionQueue, @Lazy private val redisTemplate: StringRedisTemplate, - private val concurrentExecutionLauncher: BatchJobConcurrentLauncher, private val savePointManager: SavePointManager, private val currentDateProvider: CurrentDateProvider, private val activityHolder: ActivityHolder, private val metrics: Metrics, ) : Logging { - companion object { - const val MIN_TIME_BETWEEN_OPERATIONS = 100 - } + suspend fun handleItem(executionItem: ExecutionQueueItem) { + val coroutineContext = coroutineContext + var retryExecution: BatchJobChunkExecution? = null + try { + val execution = + catchingExceptions(executionItem) { + executeInNewTransaction(transactionManager) { transactionStatus -> + + val lockedExecution = + getPendingUnlockedExecutionItem(executionItem) + ?: return@executeInNewTransaction null + + publishRemoveConsuming(executionItem) + + progressManager.handleJobRunning(lockedExecution.batchJob.id) + val batchJobDto = batchJobService.getJobDto(lockedExecution.batchJob.id) + + logger.debug("Job ${batchJobDto.id}: 🟡 Processing chunk ${lockedExecution.id}") + val savepoint = savePointManager.setSavepoint() + val util = ChunkProcessingUtil(lockedExecution, applicationContext, coroutineContext) + util.processChunk() + + if (transactionStatus.isRollbackOnly) { + logger.debug("Job ${batchJobDto.id}: 🛑 Rollbacking chunk ${lockedExecution.id}") + savePointManager.rollbackSavepoint(savepoint) + // we have rolled back the transaction, so no targets were actually successfull + lockedExecution.successTargets = listOf() + entityManager.clear() + rollbackActivity() + } + + progressManager.handleProgress(lockedExecution) + entityManager.persist(entityManager.merge(lockedExecution)) - @EventListener(ApplicationReadyEvent::class) - fun run() { - println("Application ready") - concurrentExecutionLauncher.run { executionItem, coroutineContext -> - var retryExecution: BatchJobChunkExecution? = null - try { - val execution = - catchingExceptions(executionItem) { - executeInNewTransaction(transactionManager) { transactionStatus -> - - val lockedExecution = - getPendingUnlockedExecutionItem(executionItem) - ?: return@executeInNewTransaction null - - publishRemoveConsuming(executionItem) - - progressManager.handleJobRunning(lockedExecution.batchJob.id) - val batchJobDto = batchJobService.getJobDto(lockedExecution.batchJob.id) - - logger.debug("Job ${batchJobDto.id}: 🟡 Processing chunk ${lockedExecution.id}") - val savepoint = savePointManager.setSavepoint() - val util = ChunkProcessingUtil(lockedExecution, applicationContext, coroutineContext) - util.processChunk() - - if (transactionStatus.isRollbackOnly) { - logger.debug("Job ${batchJobDto.id}: 🛑 Rollbacking chunk ${lockedExecution.id}") - savePointManager.rollbackSavepoint(savepoint) - // we have rolled back the transaction, so no targets were actually successfull - lockedExecution.successTargets = listOf() - entityManager.clear() - rollbackActivity() - } - - progressManager.handleProgress(lockedExecution) - entityManager.persist(entityManager.merge(lockedExecution)) - - if (lockedExecution.retry) { - retryExecution = util.retryExecution - entityManager.persist(util.retryExecution) - entityManager.flush() - } - - logger.debug("Job ${batchJobDto.id}: ✅ Processed chunk ${lockedExecution.id}") - return@executeInNewTransaction lockedExecution + if (lockedExecution.retry) { + retryExecution = util.retryExecution + entityManager.persist(util.retryExecution) + entityManager.flush() } + + logger.debug("Job ${batchJobDto.id}: ✅ Processed chunk ${lockedExecution.id}") + return@executeInNewTransaction lockedExecution } - execution?.let { - logger.debug("Job: ${it.batchJob.id} - Handling execution committed ${it.id} (standard flow)") - progressManager.handleChunkCompletedCommitted(it) } - addRetryExecutionToQueue(retryExecution, jobCharacter = executionItem.jobCharacter) - } catch (e: Throwable) { - progressManager.rollbackSetToRunning(executionItem.chunkExecutionId, executionItem.jobId) - when (e) { - is UnexpectedRollbackException -> { - logger.debug( - "Job ${executionItem.jobId}: ⚠️ Chunk ${executionItem.chunkExecutionId}" + - " thrown UnexpectedRollbackException", - ) - } + execution?.let { + logger.debug("Job: ${it.batchJob.id} - Handling execution committed ${it.id} (standard flow)") + progressManager.handleChunkCompletedCommitted(it) + } + addRetryExecutionToQueue(retryExecution, jobCharacter = executionItem.jobCharacter) + } catch (e: Throwable) { + progressManager.rollbackSetToRunning(executionItem.chunkExecutionId, executionItem.jobId) + when (e) { + is UnexpectedRollbackException -> { + logger.debug( + "Job ${executionItem.jobId}: ⚠️ Chunk ${executionItem.chunkExecutionId}" + + " thrown UnexpectedRollbackException", + ) + } - else -> { - logger.error("Job ${executionItem.jobId}: ⚠️ Chunk ${executionItem.chunkExecutionId} thrown error", e) - Sentry.captureException(e) - } + else -> { + logger.error("Job ${executionItem.jobId}: ⚠️ Chunk ${executionItem.chunkExecutionId} thrown error", e) + Sentry.captureException(e) } } } @@ -223,11 +214,4 @@ class BatchJobActionService( LockOptions.SKIP_LOCKED, ).resultList.singleOrNull() } - - fun cancelLocalJob(jobId: Long) { - batchJobChunkExecutionQueue.removeJobExecutions(jobId) - concurrentExecutionLauncher.runningJobs.filter { it.value.first.id == jobId }.forEach { - it.value.second.cancel() - } - } } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt index 2726ce837e..649dec4db5 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobCancellationManager.kt @@ -29,11 +29,12 @@ class BatchJobCancellationManager( private val redisTemplate: StringRedisTemplate, private val entityManager: EntityManager, private val transactionManager: PlatformTransactionManager, - private val batchJobActionService: BatchJobActionService, private val progressManager: ProgressManager, private val activityHolder: ActivityHolder, private val batchJobService: BatchJobService, private val batchJobStatusProvider: BatchJobStatusProvider, + private val batchJobChunkExecutionQueue: BatchJobChunkExecutionQueue, + private val concurrentExecutionLauncher: BatchJobConcurrentLauncher, ) : Logging { @Transactional fun cancel(id: Long) { @@ -63,12 +64,12 @@ class BatchJobCancellationManager( ) return } - batchJobActionService.cancelLocalJob(id) + cancelLocalJob(id) } @EventListener(JobCancelEvent::class) fun cancelJobListener(event: JobCancelEvent) { - batchJobActionService.cancelLocalJob(event.jobId) + cancelLocalJob(event.jobId) } fun cancelJob(jobId: Long) { @@ -185,4 +186,11 @@ class BatchJobCancellationManager( val current = activityHolder.activityRevision.cancelledBatchJobExecutionCount ?: 0 activityHolder.activityRevision.cancelledBatchJobExecutionCount = current + 1 } + + fun cancelLocalJob(jobId: Long) { + batchJobChunkExecutionQueue.removeJobExecutions(jobId) + concurrentExecutionLauncher.runningJobs.filter { it.value.first.id == jobId }.forEach { + it.value.second.cancel() + } + } } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt index 7516fa1ef0..252ff348e2 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobConcurrentLauncher.kt @@ -10,16 +10,9 @@ import io.tolgee.model.batch.BatchJobChunkExecutionStatus import io.tolgee.util.Logging import io.tolgee.util.logger import io.tolgee.util.trace -import jakarta.annotation.PreDestroy -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.* import org.springframework.stereotype.Component import java.util.concurrent.ConcurrentHashMap -import kotlin.coroutines.CoroutineContext import kotlin.math.ceil @Component @@ -30,10 +23,10 @@ class BatchJobConcurrentLauncher( private val batchJobProjectLockingManager: BatchJobProjectLockingManager, private val batchJobService: BatchJobService, private val progressManager: ProgressManager, + private val batchJobActionService: BatchJobActionService, ) : Logging { companion object { - val runningInstances: ConcurrentHashMap.KeySetView = - ConcurrentHashMap.newKeySet() + const val MIN_TIME_BETWEEN_OPERATIONS = 100 } /** @@ -63,18 +56,9 @@ class BatchJobConcurrentLauncher( masterRunJob?.join() } logger.trace("Batch job launcher stopped ${System.identityHashCode(this)}") - runningInstances.remove(this) - } - - @PreDestroy - fun preDestroy() { - this.stop() } fun repeatForever(fn: () -> Boolean) { - runningInstances.forEach { it.stop() } - runningInstances.add(this) - logger.trace("Started batch job action service ${System.identityHashCode(this)}") while (run) { try { @@ -98,10 +82,12 @@ class BatchJobConcurrentLauncher( if (!batchJobChunkExecutionQueue.isEmpty() && jobsToLaunch > 0 && somethingHandled) { return 0 } - return BatchJobActionService.MIN_TIME_BETWEEN_OPERATIONS - (System.currentTimeMillis() - startTime) + return MIN_TIME_BETWEEN_OPERATIONS - (System.currentTimeMillis() - startTime) } - fun run(processExecution: (executionItem: ExecutionQueueItem, coroutineContext: CoroutineContext) -> Unit) { + fun run() { + run = true + pause = false @Suppress("OPT_IN_USAGE") masterRunJob = GlobalScope.launch(Dispatchers.IO) { @@ -124,7 +110,7 @@ class BatchJobConcurrentLauncher( // when something handled, return true items.map { executionItem -> - handleItem(executionItem, processExecution) + handleItem(executionItem) }.any() } } @@ -147,10 +133,7 @@ class BatchJobConcurrentLauncher( /** * Returns true if item was handled */ - private fun CoroutineScope.handleItem( - executionItem: ExecutionQueueItem, - processExecution: (executionItem: ExecutionQueueItem, coroutineContext: CoroutineContext) -> Unit, - ): Boolean { + private fun CoroutineScope.handleItem(executionItem: ExecutionQueueItem): Boolean { logger.trace("Trying to run execution ${executionItem.chunkExecutionId}") if (!executionItem.isTimeToExecute()) { logger.trace { @@ -209,7 +192,7 @@ class BatchJobConcurrentLauncher( val job = launch { - processExecution(executionItem, this.coroutineContext) + batchJobActionService.handleItem(executionItem) } val batchJobDto = batchJobService.getJobDto(executionItem.jobId) diff --git a/backend/testing/src/main/kotlin/io/tolgee/BatchJobCleanerListener.kt b/backend/testing/src/main/kotlin/io/tolgee/BatchJobCleanerListener.kt deleted file mode 100644 index 0522957e08..0000000000 --- a/backend/testing/src/main/kotlin/io/tolgee/BatchJobCleanerListener.kt +++ /dev/null @@ -1,13 +0,0 @@ -package io.tolgee - -import io.tolgee.BatchJobTestUtil.pauseAndClearBatchJobs -import io.tolgee.BatchJobTestUtil.resumeBatchJobs -import org.springframework.test.context.TestContext -import org.springframework.test.context.TestExecutionListener - -class BatchJobCleanerListener : TestExecutionListener { - override fun beforeTestMethod(testContext: TestContext) { - pauseAndClearBatchJobs(testContext) - resumeBatchJobs(testContext) - } -} diff --git a/backend/testing/src/main/kotlin/io/tolgee/BatchJobTestListener.kt b/backend/testing/src/main/kotlin/io/tolgee/BatchJobTestListener.kt new file mode 100644 index 0000000000..46ccd50e1c --- /dev/null +++ b/backend/testing/src/main/kotlin/io/tolgee/BatchJobTestListener.kt @@ -0,0 +1,58 @@ +package io.tolgee + +import io.tolgee.batch.ApplicationBatchJobRunner +import io.tolgee.batch.BatchJobChunkExecutionQueue +import io.tolgee.batch.BatchJobConcurrentLauncher +import io.tolgee.util.Logging +import io.tolgee.util.logger +import org.springframework.test.context.TestContext +import org.springframework.test.context.TestExecutionListener + +/** + * This listener us used to run application batch job runner always once and with the current application context + * + * Spring caches application context in tests and may create a new instance of ApplicationBatchJobRunner + * + * To not run the batch jobs with different test context, this class helps us to clean the context and + * run the batch job runner correctly + */ +class BatchJobTestListener : Logging, TestExecutionListener { + override fun beforeTestMethod(testContext: TestContext) { + logger.info("Pausing and clearing batch jobs") + pauseAndClearBatchJobs(testContext) + + logStopping() + ApplicationBatchJobRunner.stopAll() + + logger.info("Running application batch job runner for current context") + testContext.applicationBatchJobRunner.run() + } + + override fun afterTestMethod(testContext: TestContext) { + logStopping() + // to be super safe, rather stop them all if by any chance there is more than one instance running + ApplicationBatchJobRunner.stopAll() + } + + private fun logStopping() { + logger.info("Stopping all application batch job runners") + } + + private inline fun TestContext.getBean(): T { + return this.applicationContext.getBean(T::class.java) + } + + private val TestContext.applicationBatchJobRunner: ApplicationBatchJobRunner + get() = this.getBean() + + private fun pauseAndClearBatchJobs(testContext: TestContext) { + testContext.batchJobConcurrentLauncher.pause = true + testContext.batchJobChunkExecutionQueue.clear() + } + + private val TestContext.batchJobChunkExecutionQueue + get() = this.getBean() + + private val TestContext.batchJobConcurrentLauncher + get() = this.getBean() +} diff --git a/backend/testing/src/main/kotlin/io/tolgee/BatchJobTestUtil.kt b/backend/testing/src/main/kotlin/io/tolgee/BatchJobTestUtil.kt deleted file mode 100644 index 3d4264c279..0000000000 --- a/backend/testing/src/main/kotlin/io/tolgee/BatchJobTestUtil.kt +++ /dev/null @@ -1,24 +0,0 @@ -package io.tolgee - -import io.tolgee.batch.BatchJobChunkExecutionQueue -import io.tolgee.batch.BatchJobConcurrentLauncher -import org.springframework.test.context.TestContext - -object BatchJobTestUtil { - fun resumeBatchJobs(testContext: TestContext) { - getBatchJobConcurrentLauncher(testContext).pause = false - } - - fun pauseAndClearBatchJobs(testContext: TestContext) { - getBatchJobConcurrentLauncher(testContext).pause = true - getBatchJobExecutionQueue(testContext).clear() - } - - private fun getBatchJobExecutionQueue(testContext: TestContext): BatchJobChunkExecutionQueue { - return testContext.applicationContext.getBean(BatchJobChunkExecutionQueue::class.java) - } - - private fun getBatchJobConcurrentLauncher(testContext: TestContext): BatchJobConcurrentLauncher { - return testContext.applicationContext.getBean(BatchJobConcurrentLauncher::class.java) - } -} diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt index c358039252..a396c734b6 100644 --- a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt +++ b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt @@ -1,6 +1,6 @@ package io.tolgee.testing -import io.tolgee.BatchJobCleanerListener +import io.tolgee.BatchJobTestListener import io.tolgee.CleanDbTestListener import jakarta.persistence.EntityManager import org.springframework.beans.factory.annotation.Autowired @@ -17,7 +17,7 @@ import org.springframework.test.context.transaction.TransactionalTestExecutionLi DependencyInjectionTestExecutionListener::class, CleanDbTestListener::class, DirtiesContextTestExecutionListener::class, - BatchJobCleanerListener::class, + BatchJobTestListener::class, ], ) @ActiveProfiles(profiles = ["local"]) From 7e9643e856492610622aca56047df9bc24846a9d Mon Sep 17 00:00:00 2001 From: Jan Cizmar Date: Sat, 1 Feb 2025 14:03:11 +0100 Subject: [PATCH 7/7] chore: Final touch --- .../KeyControllerResolvableImportAutomationsTest.kt | 3 +-- .../src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerResolvableImportAutomationsTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerResolvableImportAutomationsTest.kt index 79a754ac1a..433b71ee15 100644 --- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerResolvableImportAutomationsTest.kt +++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/v2KeyController/KeyControllerResolvableImportAutomationsTest.kt @@ -12,7 +12,6 @@ import org.junit.jupiter.api.Test class KeyControllerResolvableImportAutomationsTest : MachineTranslationTest() { companion object { private const val INITIAL_BUCKET_CREDITS = 150000L - private const val TRANSLATED_WITH_GOOGLE_RESPONSE = "changed translated with GOOGLE from en to de" } lateinit var testData: ResolvableImportTestData @@ -55,7 +54,7 @@ class KeyControllerResolvableImportAutomationsTest : MachineTranslationTest() { Assertions.assertThat( translatedText, - ).isEqualTo(TRANSLATED_WITH_GOOGLE_RESPONSE) + ).isEqualTo("Translated with Google") } } } diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt index 3480efea8e..26a984c1f4 100644 --- a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt +++ b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobActionService.kt @@ -47,7 +47,7 @@ class BatchJobActionService( private val activityHolder: ActivityHolder, private val metrics: Metrics, ) : Logging { - suspend fun handleItem(executionItem: ExecutionQueueItem) { + suspend fun handleItem(executionItem: ExecutionQueueItem) { val coroutineContext = coroutineContext var retryExecution: BatchJobChunkExecution? = null try {