Skip to content

Commit

Permalink
bug: 修复部分helm remote仓库拉取404问题 #2884
Browse files Browse the repository at this point in the history 
* bug: 修复部分helm remote仓库拉取404问题 #2884

* bug: 增加日志 #2884
  • Loading branch information
@zacYL
zacYL authored Jan 8, 2025
1 parent c4fbfad commit beb0de2
Show file tree
Hide file tree
Showing 7 changed files with 87 additions and 9 deletions.
31 changes: 30 additions & 1 deletion ...-helm/src/main/kotlin/com/tencent/bkrepo/helm/artifact/repository/HelmRemoteRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

package com.tencent.bkrepo.helm.artifact.repository

import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.util.readYamlString
import com.tencent.bkrepo.common.api.util.toYamlString
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
Expand All @@ -47,6 +48,8 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.artifact.stream.artifactStream
import com.tencent.bkrepo.common.artifact.util.FileNameParser
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.helm.config.HelmProperties
import com.tencent.bkrepo.helm.constants.CHART
import com.tencent.bkrepo.helm.constants.FILE_TYPE
Expand All @@ -66,6 +69,7 @@ import com.tencent.bkrepo.helm.utils.ChartParserUtil
import com.tencent.bkrepo.helm.utils.HelmMetadataUtils
import com.tencent.bkrepo.helm.utils.HelmUtils
import com.tencent.bkrepo.helm.utils.ObjectBuilderUtil
import com.tencent.bkrepo.repository.constant.PROXY_DOWNLOAD_URL
import com.tencent.bkrepo.repository.pojo.download.PackageDownloadRecord
import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel
import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
Expand Down Expand Up @@ -161,10 +165,35 @@ class HelmRemoteRepository(
val remoteDomain = remoteConfiguration.url.trimEnd('/')
return when (fullPath) {
HelmUtils.getIndexCacheYamlFullPath() -> INDEX_REQUEST_URL.format(remoteDomain, INDEX_YAML)
else -> CHART_REQUEST_URL.format(remoteDomain, fullPath)
else -> buildDownloadUrl(context, fullPath, remoteDomain)
}
}

private fun buildDownloadUrl(context: ArtifactContext, fullPath: String?, remoteDomain: String): String {
val map = FileNameParser.parseNameAndVersionWithRegex(fullPath!!)
val name = map[NAME].toString()
val version = map[VERSION].toString()
val packageVersion = packageService.findVersionByName(
projectId = context.projectId,
repoName = context.repoName,
packageKey = PackageKeys.ofHelm(name),
versionName = version
)
var downloadUrl = CHART_REQUEST_URL.format(remoteDomain, fullPath)
if (packageVersion != null) {
val proxyDownloadUrl = packageVersion.metadata[PROXY_DOWNLOAD_URL]?.toString()
if (proxyDownloadUrl != null) {
downloadUrl = if (!proxyDownloadUrl.contains(remoteDomain)) {
remoteDomain + StringPool.SLASH + proxyDownloadUrl
} else {
proxyDownloadUrl
}
}
}
logger.info("remote chart download url is $downloadUrl")
return downloadUrl
}

/**
* 远程下载响应回调
*/
Expand Down
3 changes: 3 additions & 0 deletions .../helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/pojo/metadata/HelmChartMetadata.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

package com.tencent.bkrepo.helm.pojo.metadata

import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonInclude
import com.github.zafarkhaja.semver.Version
Expand All @@ -52,6 +53,8 @@ data class HelmChartMetadata(
var name: String,
var sources: List<String> = emptyList(),
var urls: List<String> = emptyList(),
@JsonIgnore
var proxyDownloadUrl: String? = null,
var version: String,
var type: String?,
var annotations: Map<String, Any>?
Expand Down
6 changes: 5 additions & 1 deletion ...elm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/AbstractChartService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -602,10 +602,14 @@ open class AbstractChartService : ArtifactService() {
domain: String,
projectId: String,
repoName: String,
helmIndexYamlMetadata: HelmIndexYamlMetadata
helmIndexYamlMetadata: HelmIndexYamlMetadata,
initProxyUrls: Boolean = true
) {
helmIndexYamlMetadata.entries.values.forEach {
it.forEach { chart ->
if (initProxyUrls) {
chart.proxyDownloadUrl = chart.urls.firstOrNull()
}
chart.urls = listOf(
UrlFormatter.format(
domain, "$projectId/$repoName/charts/${chart.name}-${chart.version}.tgz"
Expand Down
11 changes: 6 additions & 5 deletions ...elm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,10 @@ class HelmOperationService(
// 删除index文件
nodeService.deleteNode(
NodeDeleteRequest(
projectId = projectId,
repoName = repoName,
fullPath = HelmUtils.getIndexCacheYamlFullPath(),
operator = SecurityUtils.getUserId()
projectId = projectId,
repoName = repoName,
fullPath = HelmUtils.getIndexCacheYamlFullPath(),
operator = SecurityUtils.getUserId()
)
)
}
Expand Down Expand Up @@ -213,7 +213,8 @@ class HelmOperationService(
domain = helmProperties.domain,
projectId = repoDetail.projectId,
repoName = repoDetail.name,
helmIndexYamlMetadata = oldIndex
helmIndexYamlMetadata = oldIndex,
initProxyUrls = false
)
// 存储新index文件
storeIndex(
Expand Down
4 changes: 4 additions & 0 deletions src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/utils/HelmMetadataUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.api.util.toJsonString
import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
import com.tencent.bkrepo.helm.pojo.metadata.HelmChartMetadata
import com.tencent.bkrepo.repository.constant.PROXY_DOWNLOAD_URL
import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel

object HelmMetadataUtils {
Expand All @@ -53,6 +54,9 @@ object HelmMetadataUtils {
sourceType?.let {
mutableMap.add(MetadataModel(SOURCE_TYPE, sourceType))
}
if (!chartInfo.proxyDownloadUrl.isNullOrEmpty()) {
mutableMap.add(MetadataModel(PROXY_DOWNLOAD_URL, chartInfo.proxyDownloadUrl!!))
}
return mutableMap
}

Expand Down
2 changes: 2 additions & 0 deletions ...sitory/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/constant/Constants.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,7 @@ const val NODE_METADATA = "nodeMetadata"
const val MD5 = "md5"
const val SHA256 = "sha256"
const val PACKAGE_KEY = "key"
const val VERSION = "version"

const val NODE_DETAIL_LIST_KEY = "nodeDetailList"
const val PROXY_DOWNLOAD_URL = "proxyDownloadUrl"
39 changes: 37 additions & 2 deletions ...repository/src/main/kotlin/com/tencent/bkrepo/repository/config/CommonRemoteRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,17 @@

package com.tencent.bkrepo.repository.config

import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository
import com.tencent.bkrepo.common.artifact.util.FileNameParser
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.repository.constant.NAME
import com.tencent.bkrepo.repository.constant.PROXY_DOWNLOAD_URL
import com.tencent.bkrepo.repository.constant.VERSION
import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel
import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
import org.slf4j.Logger
Expand All @@ -56,8 +62,7 @@ class CommonRemoteRepository : RemoteRepository() {
return if (RepositoryType.HELM != type) {
super.createRemoteDownloadUrl(context)
} else {
val remoteConfiguration = context.getRemoteConfiguration()
CHART_REQUEST_URL.format(remoteConfiguration.url, context.artifactInfo.getArtifactFullPath())
buildChartDownloadUrl(context)
}
}

Expand Down Expand Up @@ -91,6 +96,36 @@ class CommonRemoteRepository : RemoteRepository() {
return redirectManager.redirect(context)
}

private fun buildChartDownloadUrl(context: ArtifactContext): String {
val remoteConfiguration = context.getRemoteConfiguration()
val remoteDomain = remoteConfiguration.url.trimEnd('/')
val map = FileNameParser.parseNameAndVersionWithRegex(context.artifactInfo.getArtifactFullPath())
val name = map[NAME].toString()
val version = map[VERSION].toString()
val packageVersion = packageService.findVersionByName(
projectId = context.projectId,
repoName = context.repoName,
packageKey = PackageKeys.ofHelm(name),
versionName = version
)
var downloadUrl = CHART_REQUEST_URL.format(
remoteConfiguration.url,
context.artifactInfo.getArtifactFullPath()
)
if (packageVersion != null) {
val proxyDownloadUrl = packageVersion.metadata[PROXY_DOWNLOAD_URL]?.toString()
if (proxyDownloadUrl != null) {
downloadUrl = if (!proxyDownloadUrl.contains(remoteDomain)) {
remoteDomain + StringPool.SLASH + proxyDownloadUrl
} else {
proxyDownloadUrl
}
}
}
logger.info("remote chart download url is $downloadUrl")
return downloadUrl
}

/**
* 如果fullPath已经是完整的url，则直接使用，否则进行拼接
*/
Expand Down

0 comments on commit beb0de2

Please sign in to comment.