From 72544469104529d0aaedbd4a673b41eaf58929c6 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Tue, 18 Jun 2024 10:06:26 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat:=20=E8=B0=83=E6=95=B4pypi=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E7=89=88=E6=9C=AC=E9=94=99=E8=AF=AF=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=AD=89=E7=BA=A7=20#2276?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/pypi/artifact/repository/PypiLocalRepository.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt index c42bf48314..65c1401726 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt @@ -433,7 +433,7 @@ class PypiLocalRepository( val pypiPackagePojo = try { artifactInfo.getArtifactFullPath().toPypiPackagePojo() } catch (e: Exception) { - logger.error("parse pypi package failed", e) + logger.info("parse path[${artifactInfo.getArtifactFullPath()}] to pypi package version failed: ", e) null } ?: return null From bd6da849dad7933a15facd12690e512c8754e779 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Tue, 18 Jun 2024 15:20:14 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20=E6=B5=81=E6=B0=B4=E7=BA=BF?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E3=80=81=E8=87=AA=E5=AE=9A=E4=B9=89=E4=BB=93?= =?UTF-8?q?=E5=BA=93=E9=99=90=E5=88=B6=E4=B8=8A=E4=BC=A0=E6=B8=A0=E9=81=93?= =?UTF-8?q?=20#2183?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/common/security/manager/ci/CIPermissionManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/ci/CIPermissionManager.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/ci/CIPermissionManager.kt index 957f56c369..cf6cbb55d8 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/ci/CIPermissionManager.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/ci/CIPermissionManager.kt @@ -61,7 +61,7 @@ open class CIPermissionManager( buildId: String?, taskId: String? = null ): PipelineBuildStatus { - if (!ciPermissionProperties.enabled) { + if (!ciPermissionProperties.enabled || whiteListRequest()) { return PipelineBuildStatus(SecurityUtils.getUserId(), false, "RUNNING") } var url = "${ciPermissionProperties.host}/ms/process/api/open/service/pipeline/get_build_status?" + From 838d59f8315d91ac82cdf1be4e1f2ed95e7314c3 Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 18 Jun 2024 15:44:02 +0800 Subject: [PATCH 3/8] =?UTF-8?q?bug:=20auth=20endpoint=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E5=8D=95=E4=BD=93=E9=83=A8=E7=BD=B2=20#2280?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bug: auth endpoint校验兼容单体部署 #2280 * bug: auth endpoint校验兼容单体部署 #2280 --- .../com/tencent/bkrepo/auth/interceptor/AuthInterceptor.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/interceptor/AuthInterceptor.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/interceptor/AuthInterceptor.kt index 8e9bda5317..ebcbb2c25a 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/interceptor/AuthInterceptor.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/interceptor/AuthInterceptor.kt @@ -121,7 +121,7 @@ class AuthInterceptor( } private fun checkUserFromBasic(request: HttpServletRequest, authHeader: String): Boolean { - val userAccess = userAccessApiSet.any { request.requestURI.startsWith(it) } + val userAccess = userAccessApiSet.any { request.requestURI.contains(it) } val encodedCredentials = authHeader.removePrefix(BASIC_AUTH_HEADER_PREFIX) val decodedHeader = String(Base64.getDecoder().decode(encodedCredentials)) val parts = decodedHeader.split(COLON) @@ -188,8 +188,8 @@ class AuthInterceptor( } private fun setAuthAttribute(userId: String, appId: String, request: HttpServletRequest) { - val userAccess = userAccessApiSet.any { request.requestURI.startsWith(it) } - val anonymousAccess = anonymousAccessApiSet.any { request.requestURI.startsWith(it) } + val userAccess = userAccessApiSet.any { request.requestURI.contains(it) } + val anonymousAccess = anonymousAccessApiSet.any { request.requestURI.contains(it) } val userInfo = userService.getUserInfoById(userId) val isAdmin: Boolean = userInfo?.admin ?: false if (userId.isNotEmpty() && userInfo == null && userId != ANONYMOUS_USER) { From ecb3da4dabbaa4faa706750d126fef1c9e6e3430 Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 18 Jun 2024 21:14:00 +0800 Subject: [PATCH 4/8] =?UTF-8?q?feat:=20=E5=8D=87=E7=BA=A7=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E7=9A=84gradle=E4=BE=9D=E8=B5=96=E5=9C=B0=E5=9D=80=20?= =?UTF-8?q?#2283?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 升级依赖的gradle小版本 #2283 * feat: 升级依赖的gradle小版本 #2283 --- src/backend/gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/gradle/wrapper/gradle-wrapper.properties b/src/backend/gradle/wrapper/gradle-wrapper.properties index aa991fceae..47154cee76 100644 --- a/src/backend/gradle/wrapper/gradle-wrapper.properties +++ b/src/backend/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://mirrors.cloud.tencent.com/gradle/gradle-7.4.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 527d71d3df3bd52f5c6aa179287b980845a27b56 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Wed, 19 Jun 2024 10:19:53 +0800 Subject: [PATCH 5/8] =?UTF-8?q?feat:=20=20=E7=A9=BA=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E6=B8=85=E7=90=86=E4=BB=BB=E5=8A=A1=E6=A0=B9=E6=8D=AE=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=88=A4=E6=96=AD=E6=98=AF=E5=90=A6=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E5=88=A0=E9=99=A4#2284?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 空目录清理任务根据配置判断是否进行删除#2284 * feat: 代码调整 #2284 --- .../com/tencent/bkrepo/job/Constants.kt | 2 ++ .../ActiveProjectEmptyFolderCleanupJob.kt | 4 ++-- .../job/batch/task/stat/EmptyFolderCleanup.kt | 20 +++++++++++++------ .../InactiveProjectEmptyFolderCleanupJob.kt | 5 ++++- ...eProjectEmptyFolderCleanupJobProperties.kt | 1 + ...eProjectEmptyFolderCleanupJobProperties.kt | 3 ++- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt index d634394162..88abc460f2 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt @@ -63,6 +63,8 @@ const val RECORD_RESERVE_DAYS = "recordReserveDays" const val REPLICA_TYPE = "replicaType" const val NAME = "name" const val SIZE = "size" +const val NODE_NUM = "nodeNum" + /** * 缓存类型 diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt index 7c7de44b7a..28920186d0 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt @@ -51,7 +51,7 @@ import java.time.Duration @Component @EnableConfigurationProperties(ActiveProjectEmptyFolderCleanupJobProperties::class) class ActiveProjectEmptyFolderCleanupJob( - properties: ActiveProjectEmptyFolderCleanupJobProperties, + private val properties: ActiveProjectEmptyFolderCleanupJobProperties, executor: ThreadPoolTaskExecutor, private val activeProjectService: ActiveProjectService, private val mongoTemplate: MongoTemplate, @@ -104,7 +104,7 @@ class ActiveProjectEmptyFolderCleanupJob( override fun onRunProjectFinished(collection: String, projectId: String, context: JobContext) { require(context is EmptyFolderCleanupJobContext) logger.info("will filter empty folder in project $projectId") - emptyFolderCleanup.emptyFolderHandler(collection, context, projectId) + emptyFolderCleanup.emptyFolderHandler(collection, context, properties.deletedEmptyFolder, projectId) } companion object { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt index 8c27fe5880..01f1aa1ab3 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt @@ -34,8 +34,10 @@ import com.tencent.bkrepo.job.DELETED_DATE import com.tencent.bkrepo.job.FOLDER import com.tencent.bkrepo.job.FULL_PATH import com.tencent.bkrepo.job.LAST_MODIFIED_DATE +import com.tencent.bkrepo.job.NODE_NUM import com.tencent.bkrepo.job.PROJECT import com.tencent.bkrepo.job.REPO +import com.tencent.bkrepo.job.SIZE import com.tencent.bkrepo.job.batch.context.EmptyFolderCleanupJobContext import com.tencent.bkrepo.job.batch.utils.FolderUtils import com.tencent.bkrepo.job.batch.utils.FolderUtils.extractFolderInfoFromCacheKey @@ -108,6 +110,7 @@ class EmptyFolderCleanup( fun emptyFolderHandler( collection: String, context: EmptyFolderCleanupJobContext, + deletedEmptyFolder: Boolean, projectId: String = StringPool.EMPTY, runCollection: Boolean = false, ) { @@ -128,9 +131,10 @@ class EmptyFolderCleanup( )) { logger.info( "will delete empty folder ${folderInfo.fullPath}" + - " in repo ${folderInfo.projectId}|${folderInfo.repoName}" + " in repo ${folderInfo.projectId}|${folderInfo.repoName} " + + "with config deletedEmptyFolder: $deletedEmptyFolder" ) - doEmptyFolderDelete(entry.value.id, collection) + doEmptyFolderDelete(entry.value.id, collection, deletedEmptyFolder) context.totalDeletedNum.increment() } } @@ -163,7 +167,8 @@ class EmptyFolderCleanup( */ private fun doEmptyFolderDelete( objectId: String?, - collectionName: String + collectionName: String, + deletedEmptyFolder: Boolean, ) { if (objectId.isNullOrEmpty()) return val query = Query( @@ -171,9 +176,12 @@ class EmptyFolderCleanup( .and(FOLDER).isEqualTo(true) ) val deleteTime = LocalDateTime.now() - val update = Update() - .set(LAST_MODIFIED_DATE, deleteTime) - .set(DELETED_DATE, deleteTime) + val update = Update().set(LAST_MODIFIED_DATE, deleteTime) + if (deletedEmptyFolder) { + update.set(DELETED_DATE, deleteTime) + } else { + update.set(SIZE, 0).set(NODE_NUM, 0) + } mongoTemplate.updateFirst(query, update, collectionName) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt index ebe2a3a7ad..c65bf81637 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt @@ -129,7 +129,10 @@ class InactiveProjectEmptyFolderCleanupJob( require(context is EmptyFolderCleanupJobContext) super.onRunCollectionFinished(collectionName, context) logger.info("will filter empty folder in $collectionName") - emptyFolderCleanup.emptyFolderHandler(collectionName, context, runCollection = true) + emptyFolderCleanup.emptyFolderHandler( + collectionName, context, + properties.deletedEmptyFolder, runCollection = true, + ) } data class Node( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectEmptyFolderCleanupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectEmptyFolderCleanupJobProperties.kt index 524528012d..d23f8f15ba 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectEmptyFolderCleanupJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectEmptyFolderCleanupJobProperties.kt @@ -36,4 +36,5 @@ import org.springframework.boot.context.properties.ConfigurationProperties class ActiveProjectEmptyFolderCleanupJobProperties( override var enabled: Boolean = true, override var cron: String = "0 0/30 * * * ?", + var deletedEmptyFolder: Boolean = false, ) : StatJobProperties() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectEmptyFolderCleanupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectEmptyFolderCleanupJobProperties.kt index 2a90ca0efe..d48ce53608 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectEmptyFolderCleanupJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectEmptyFolderCleanupJobProperties.kt @@ -36,4 +36,5 @@ import org.springframework.boot.context.properties.ConfigurationProperties class InactiveProjectEmptyFolderCleanupJobProperties( override var enabled: Boolean = true, override var cron: String = "0 0 0/12 * * ?", -) : StatJobProperties() + var deletedEmptyFolder: Boolean = false, + ) : StatJobProperties() From ac52fe2ec22434d9f4ca262d9a91a5b1ce3c0995 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:08:22 +0800 Subject: [PATCH 6/8] =?UTF-8?q?feat:=20=20=E5=8E=86=E5=8F=B2=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E8=8A=82=E7=82=B9=E5=8F=AA=E6=9F=A5=E8=AF=A2=E6=9C=80?= =?UTF-8?q?=E6=96=B0=E4=B8=80=E6=9D=A1=E8=AE=B0=E5=BD=95=20#2289?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/service/node/impl/NodeRestoreSupport.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index bf3a7ef71c..1ff2302842 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -78,7 +78,12 @@ open class NodeRestoreSupport( override fun getDeletedNodeDetail(artifact: ArtifactInfo): List { with(artifact) { val query = nodeDeletedPointListQuery(projectId, repoName, getArtifactFullPath()) - return nodeDao.find(query).map { convertToDetail(it)!! } + val deletedNode = nodeDao.findOne(query) + return if (deletedNode == null) { + emptyList() + } else { + listOf(convertToDetail(deletedNode)!!) + } } } From 510408c80960f4c8786b71e9c9f6027316bcec44 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Fri, 21 Jun 2024 10:05:36 +0800 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20CosClient=20copy=E6=9B=BF=E6=8D=A2?= =?UTF-8?q?=E4=B8=BAmigrate=20#2293?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/storage/innercos/InnerCosFileStorage.kt | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt index 6af0a4c3e6..c320ad988e 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt @@ -36,7 +36,6 @@ import com.tencent.bkrepo.common.storage.core.AbstractEncryptorFileStorage import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials import com.tencent.bkrepo.common.storage.innercos.client.CosClient import com.tencent.bkrepo.common.storage.innercos.request.CheckObjectExistRequest -import com.tencent.bkrepo.common.storage.innercos.request.CopyObjectRequest import com.tencent.bkrepo.common.storage.innercos.request.DeleteObjectRequest import com.tencent.bkrepo.common.storage.innercos.request.GetObjectRequest import com.tencent.bkrepo.common.storage.innercos.request.MigrateObjectRequest @@ -88,15 +87,7 @@ open class InnerCosFileStorage : AbstractEncryptorFileStorage Date: Fri, 21 Jun 2024 18:04:13 +0800 Subject: [PATCH 8/8] =?UTF-8?q?bug=EF=BC=9A=E5=89=8D=E7=AB=AF=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8C=96=E9=94=99=E8=AF=AF=E7=BF=BB=E8=AF=91=EF=BC=88?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E7=AE=A1=E7=90=86=E8=8A=82=E7=82=B9=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E9=94=99=E8=AF=AF=E6=98=BE=E7=A4=BA=EF=BC=89=20#2295?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/devops-repository/src/views/nodeManage/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/devops-repository/src/views/nodeManage/index.vue b/src/frontend/devops-repository/src/views/nodeManage/index.vue index e95307d1f5..1799db5ac0 100644 --- a/src/frontend/devops-repository/src/views/nodeManage/index.vue +++ b/src/frontend/devops-repository/src/views/nodeManage/index.vue @@ -35,7 +35,7 @@