feat: Proxy支持删除未同步的文件存储 #1836

* feat: Proxy支持删除未同步的文件存储 #1836

* feat: Proxy支持删除未同步的文件存储 #1836

* feat: Proxy支持删除未同步的文件存储 #1836

* feat: Proxy支持删除未同步的文件存储 #1836

* feat: Proxy支持删除未同步的文件存储 #1836

* feat: Proxy支持删除未同步的文件存储 #1836

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractFileStorage.kt

@@ -159,6 +159,19 @@ abstract class AbstractFileStorage<Credentials : StorageCredentials, Client> : F
         copy(path, name, fromClient, toClient)
     }
 
+    override fun move(
+        fromPath: String,
+        fromName: String,
+        toPath: String,
+        toName: String,
+        fromCredentials: StorageCredentials,
+        toCredentials: StorageCredentials
+    ) {
+        val fromClient = getClient(fromCredentials)
+        val toClient = getClient(toCredentials)
+        move(fromPath, fromName, toPath, toName, fromClient, toClient)
+    }
+
     private fun getClient(storageCredentials: StorageCredentials): Client {
         return if (storageCredentials == storageProperties.defaultStorageCredentials()) {
             defaultClient
@@ -177,6 +190,17 @@ abstract class AbstractFileStorage<Credentials : StorageCredentials, Client> : F
         throw UnsupportedOperationException("Copy operation unsupported")
     }
 
+    open fun move(
+        fromPath: String,
+        fromName: String,
+        toPath: String,
+        toName: String,
+        fromClient: Client,
+        toClient: Client
+    ) {
+        throw UnsupportedOperationException("Move operation unsupported")
+    }
+
     companion object {
         private val logger = LoggerFactory.getLogger(AbstractFileStorage::class.java)
         private const val MAX_CACHE_CLIENT = 10L

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/FileStorage.kt

@@ -95,6 +95,24 @@ interface FileStorage {
      */
     fun copy(path: String, name: String, fromCredentials: StorageCredentials, toCredentials: StorageCredentials)
 
+    /**
+     * 在不同存储实例之间移动文件
+     * @param fromPath 文件源路径
+     * @param fromName 文件源名称
+     * @param toPath 文件目标路径
+     * @param toName 文件目标名称
+     * @param fromCredentials 源存储凭证
+     * @param toCredentials 目的存储凭证
+     */
+    fun move(
+        fromPath: String,
+        fromName: String,
+        toPath: String,
+        toName: String,
+        fromCredentials: StorageCredentials,
+        toCredentials: StorageCredentials
+    )
+
     /**
      * 获取存储的临时目录，默认实现返回`java.io.tmpdir`目录
      * @param storageCredentials 存储凭证

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/FileSystemClient.kt

@@ -43,7 +43,7 @@ import java.nio.file.Paths
 /**
  * 本地文件存储客户端
  */
-class FileSystemClient(private val root: String) {
+class FileSystemClient(val root: String) {
 
     constructor(path: Path) : this(path.toString())
 

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/FileSystemStorage.kt

@@ -39,7 +39,9 @@ import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 import java.io.File
 import java.io.InputStream
+import java.nio.file.Files
 import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 
 /**
  * 文件系统存储
@@ -76,4 +78,17 @@ open class FileSystemStorage : AbstractEncryptorFileStorage<FileSystemCredential
         require(storageCredentials is FileSystemCredentials)
         return Paths.get(storageCredentials.path, TEMP).toString()
     }
+
+    override fun move(
+        fromPath: String,
+        fromName: String,
+        toPath: String,
+        toName: String,
+        fromClient: FileSystemClient,
+        toClient: FileSystemClient
+    ) {
+        val fromFullPath = Paths.get(fromClient.root, fromPath, fromName)
+        val toFullPath = Paths.get(toClient.root, toPath, toName)
+        Files.move(fromFullPath, toFullPath, StandardCopyOption.REPLACE_EXISTING)
+    }
 }

src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/proxy/ProxyFileReferenceClient.kt

@@ -0,0 +1,45 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ *  A copy of the MIT License is included in this file.
+ *
+ *
+ *  Terms of the MIT License:
+ *  ---------------------------------------------------
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ *  documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ *  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ *  permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ *  the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ *  LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ *  NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ *  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.repository.api.proxy
+
+import com.tencent.bkrepo.common.api.constant.REPOSITORY_SERVICE_NAME
+import com.tencent.bkrepo.common.api.pojo.Response
+import io.swagger.annotations.Api
+import org.springframework.cloud.openfeign.FeignClient
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+
+@Api("文件引用服务接口")
+@FeignClient(REPOSITORY_SERVICE_NAME, contextId = "ProxyFileReferenceClient", primary = false)
+@RequestMapping("/proxy/fileReference")
+interface ProxyFileReferenceClient {
+
+    @GetMapping("/count")
+    fun count(@RequestParam sha256: String, @RequestParam credentialsKey: String?): Response<Long>
+}

src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyFileReferenceController.kt

@@ -0,0 +1,43 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ *  A copy of the MIT License is included in this file.
+ *
+ *
+ *  Terms of the MIT License:
+ *  ---------------------------------------------------
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ *  documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ *  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ *  permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ *  the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ *  LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ *  NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ *  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.repository.controller.proxy
+
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import com.tencent.bkrepo.repository.api.proxy.ProxyFileReferenceClient
+import com.tencent.bkrepo.repository.service.file.FileReferenceService
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+class ProxyFileReferenceController(
+    private val fileReferenceService: FileReferenceService
+) : ProxyFileReferenceClient {
+    override fun count(sha256: String, credentialsKey: String?): Response<Long> {
+        return ResponseBuilder.success(fileReferenceService.count(sha256, credentialsKey))
+    }
+}

src/proxy/biz-proxy/src/main/kotlin/com/tencent/bkrepo/proxy/artifact/storage/ProxyCleanFileVisitor.kt

@@ -0,0 +1,99 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.proxy.artifact.storage
+
+import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException
+import com.tencent.bkrepo.common.storage.filesystem.ArtifactFileVisitor
+import com.tencent.bkrepo.repository.api.proxy.ProxyFileReferenceClient
+import org.slf4j.LoggerFactory
+import java.io.File
+import java.nio.file.FileVisitResult
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.concurrent.TimeUnit
+
+/**
+ * 不同步时，清理本地存储的文件遍历器
+ */
+class ProxyCleanFileVisitor(
+    private val proxyFileReferenceClient: ProxyFileReferenceClient,
+    private val cacheExpireDays: Int
+) : ArtifactFileVisitor() {
+    override fun needWalk(): Boolean {
+        return true
+    }
+
+    override fun visitFile(filePath: Path, attrs: BasicFileAttributes?): FileVisitResult {
+        try {
+            if (filePath.toString().endsWith(".sync")) {
+                val syncFile = filePath.toFile()
+                val dataFile = File(filePath.toString().removeSuffix(".sync"))
+                cleanFile(dataFile, syncFile)
+            } else {
+                ProxyStorageUtils.deleteCacheFile(filePath, cacheExpireDays)
+            }
+        } catch (e: Exception) {
+            logger.error("clean file error: ", e)
+        }
+        return FileVisitResult.CONTINUE
+    }
+
+    /**
+     * 当文件索引为0时，删除本地存储
+     */
+    private fun cleanFile(dataFile: File, syncFile: File) {
+        val attrs = Files.readAttributes(dataFile.toPath(), BasicFileAttributes::class.java)
+        val creationTime = attrs.creationTime().toMillis()
+        // 防止刚存储，未创建文件引用时被误删
+        if (System.currentTimeMillis() - creationTime < TimeUnit.MINUTES.toMillis(1)) {
+            return
+        }
+        val sha256 = dataFile.name
+        val credentialKeys = ProxyStorageUtils.readStorageCredentialKeys(syncFile)
+        try {
+            credentialKeys.forEach {
+                val fileReference = proxyFileReferenceClient.count(sha256, it).data!!
+                if (fileReference > 0) {
+                    logger.info("skip clean file[$sha256] which file reference is $fileReference on storage[$it]")
+                    return
+                }
+            }
+        } catch (e: RemoteErrorCodeException) {
+            logger.warn("get sha256[$sha256] file reference failed: ${e.errorMessage}")
+            return
+        }
+        dataFile.delete()
+        syncFile.delete()
+        logger.info("clean file[$sha256] success")
+    }
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(ProxyCleanFileVisitor::class.java)
+    }
+}