From 207308fdbf89fc0c4f31e2f3d41fda21223d1358 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 6 Jan 2025 17:09:26 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83=E6=95=B4#28?= =?UTF-8?q?37?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/job/backup/dao/BackupTaskDao.kt | 4 +- .../pojo/query/BackupPackageVersionInfo.kt | 1 + .../BackupPackageVersionInfoWithKeyInfo.kt | 1 + .../service/impl/DataBackupServiceImpl.kt | 4 +- .../impl/base/BackupNodeDataHandler.kt | 65 +++++++++++++------ .../base/BackupPackageVersionDataHandler.kt | 2 + 6 files changed, 52 insertions(+), 25 deletions(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt index ab6fa92eb5..6db4bf47f0 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt @@ -69,9 +69,9 @@ import java.time.LocalDateTime @Repository class BackupTaskDao : SimpleMongoDao() { - fun findTasksById(taskId: String): List { + fun findTasksById(taskId: String): TBackupTask? { val criteria = Criteria().and(TBackupTask::id.name).isEqualTo(taskId) - return find(Query(criteria)) + return findOne(Query(criteria)) } fun find(state: String?, pageRequest: PageRequest): List { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt index 9b97b8d263..524d47d538 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt @@ -71,6 +71,7 @@ data class BackupPackageVersionInfo( var downloads: Long, var manifestPath: String? = null, var artifactPath: String? = null, + var artifactPaths: MutableSet? = null, var stageTag: List, var metadata: List, var tags: List? = null, diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt index e20ff04093..8ba74ea317 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt @@ -71,6 +71,7 @@ data class BackupPackageVersionInfoWithKeyInfo( var downloads: Long, var manifestPath: String? = null, var artifactPath: String? = null, + var artifactPaths: MutableSet? = null, var stageTag: List, var metadata: List, var tags: List? = null, diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt index 4f89655da4..d1b12b93b4 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt @@ -39,8 +39,8 @@ class DataBackupServiceImpl( } override fun executeTask(taskId: String) { - val records = backupTaskDao.findTasksById(taskId) - val task = records.firstOrNull() ?: throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "taskId") + val task = backupTaskDao.findTasksById(taskId) + ?: throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "taskId") if (task.state != BackupTaskState.PENDING.name) throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "state") if (task.type == DATA_RECORDS_BACKUP) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt index 7b967656f3..09a90009b7 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt @@ -23,6 +23,7 @@ import com.tencent.bkrepo.job.separation.util.SeparationUtils import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException import org.springframework.data.mongodb.core.MongoTemplate @@ -72,8 +73,9 @@ class BackupNodeDataHandler( override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { val record = record as BackupNodeInfo? val collectionName = SeparationUtils.getNodeCollectionName(record!!.projectId) - uploadFile(record, context) - val existRecord = findExistNode(record) + val repo = RepositoryCommonUtils.getRepositoryDetail(record.projectId, record.repoName) + uploadFile(record, context, repo) + var existRecord = findExistNode(record) if (existRecord != null) { if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { return @@ -82,17 +84,29 @@ class BackupNodeDataHandler( updateExistNode(record, collectionName) } catch (e: DuplicateKeyException) { updateDuplicateNode(record, collectionName) + if (sha256Check(record.folder, record.sha256)) { + increment(record.sha256!!, repo.storageCredentials?.key) + increment(existRecord.sha256!!, repo.storageCredentials?.key, -1) + } } } else { try { mongoTemplate.save(record, collectionName) + if (sha256Check(record.folder, record.sha256)) { + increment(record.sha256!!, repo.storageCredentials?.key) + } logger.info("Create node ${record.fullPath} in ${record.projectId}|${record.repoName} success!") } catch (exception: DuplicateKeyException) { if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { return } + existRecord = findExistNode(record, false) // 可能存在已经上传的节点记录不在备份数据里 updateDuplicateNode(record, collectionName) + if (sha256Check(record.folder, record.sha256)) { + increment(record.sha256!!, repo.storageCredentials?.key) + increment(existRecord!!.sha256!!, repo.storageCredentials?.key, -1) + } } } } @@ -172,13 +186,12 @@ class BackupNodeDataHandler( return metadataList?.associate { it.key to it.value }.orEmpty() } - private fun findExistNode(record: BackupNodeInfo): BackupNodeInfo? { - val existNodeQuery = Query( - Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) - .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) - .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) - .and(BackupNodeInfo::id.name).isEqualTo(record.id) - ) + private fun findExistNode(record: BackupNodeInfo, findById: Boolean = true): BackupNodeInfo? { + val existNodeQuery = if (findById) { + buildIdQuery(record) + } else { + buildQuery(record) + } val collectionName = SeparationUtils.getNodeCollectionName(record.projectId) return mongoTemplate.findOne(existNodeQuery, BackupNodeInfo::class.java, collectionName) } @@ -206,15 +219,12 @@ class BackupNodeDataHandler( } - fun uploadFile(record: BackupNodeInfo, context: BackupContext) { + fun uploadFile(record: BackupNodeInfo, context: BackupContext, repo: RepositoryDetail) { if (!sha256Check(record.folder, record.sha256)) return - val repo = RepositoryCommonUtils.getRepositoryDetail(record.projectId, record.repoName) val filePath = generateRandomPath(context.targertPath, record.sha256!!) val artifactFile = filePath.toFile().toArtifactFile() // TODO 增加重试以及异常捕获 storageService.store(record.sha256!!, artifactFile, repo.storageCredentials) - // 只有新增的时候才去尽显文件索引新增 - increment(record.sha256!!, repo.storageCredentials?.key) } fun sha256Check(folder: Boolean, sha256: String?): Boolean { @@ -225,12 +235,12 @@ class BackupNodeDataHandler( return true } - fun increment(sha256: String, credentialsKey: String?) { + fun increment(sha256: String, credentialsKey: String?, inc: Long = 1) { val collectionName = SeparationUtils.getFileReferenceCollectionName(sha256) val criteria = Criteria.where(BackupFileReferenceInfo::sha256.name).`is`(sha256) .and(BackupFileReferenceInfo::credentialsKey.name).`is`(credentialsKey) val query = Query(criteria) - val update = Update().inc(BackupFileReferenceInfo::count.name, 1) + val update = Update().inc(BackupFileReferenceInfo::count.name, inc) try { mongoTemplate.upsert(query, update, collectionName) } catch (exception: DuplicateKeyException) { @@ -241,12 +251,7 @@ class BackupNodeDataHandler( } fun updateDuplicateNode(record: BackupNodeInfo, collectionName: String) { - val existNodeQuery = Query( - Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) - .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) - .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) - .and(BackupNodeInfo::deleted.name).isEqualTo(record.deleted) - ) + val existNodeQuery = buildQuery(record) val update = Update() .set(NodeDetailInfo::createdBy.name, record.createdBy) .set(NodeDetailInfo::createdDate.name, record.createdDate) @@ -266,6 +271,24 @@ class BackupNodeDataHandler( mongoTemplate.updateFirst(existNodeQuery, update, collectionName) } + private fun buildQuery(record: BackupNodeInfo): Query { + return Query( + Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) + .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) + .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) + .and(BackupNodeInfo::deleted.name).isEqualTo(record.deleted) + ) + } + + private fun buildIdQuery(record: BackupNodeInfo): Query { + return Query( + Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) + .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) + .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) + .and(BackupNodeInfo::id.name).isEqualTo(record.id) + ) + } + /** * 生成随机文件路径 * */ diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt index 51590e48c6..de8ec041e3 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt @@ -100,6 +100,7 @@ class BackupPackageVersionDataHandler( .set(BackupPackageVersionInfo::downloads.name, versionInfo.downloads) .set(BackupPackageVersionInfo::manifestPath.name, versionInfo.manifestPath) .set(BackupPackageVersionInfo::artifactPath.name, versionInfo.artifactPath) + .set(BackupPackageVersionInfo::artifactPaths.name, versionInfo.artifactPaths) .set(BackupPackageVersionInfo::stageTag.name, versionInfo.stageTag) .set(BackupPackageVersionInfo::metadata.name, versionInfo.metadata) .set(BackupPackageVersionInfo::tags.name, versionInfo.tags) @@ -129,6 +130,7 @@ class BackupPackageVersionDataHandler( downloads = downloads, manifestPath = manifestPath, artifactPath = artifactPath, + artifactPaths = artifactPaths, stageTag = stageTag, metadata = metadata, tags = tags,