diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f5f2a6..352050ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: image: uptane/tuf-nginx:latest db: - image: mariadb:10.4 + image: mariadb:10.11 env: MYSQL_ROOT_PASSWORD: "root" MYSQL_DATABASE: "ota_tuf" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8031b333..78cc9110 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ test: services: - name: uptane/tuf-nginx:latest alias: tuf-nginx - - name: mariadb:10.4 + - name: mariadb:10.11 alias: db command: - --character-set-server=utf8 diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala index 2f8b7f79..75dc5c87 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala @@ -105,8 +105,7 @@ class PackageSearch()(implicit db: Database) { IF(${searchParams.name.isEmpty}, true, name = ${searchParams.name}) AND IF(${searchParams.version.isEmpty}, true, version = ${searchParams.version}) AND IF(${searchParams.nameContains.isEmpty}, true, LOCATE(${searchParams.nameContains}, name) > 0) AND - IF(${searchParams.hardwareIds.isEmpty}, true, JSON_CONTAINS(hardwareids, JSON_QUOTE(${searchParams.hardwareIds.headOption - .map(_.value)}))) AND + IF(${searchParams.hardwareIds.isEmpty}, true, JSON_OVERLAPS(${searchParams.hardwareIds}, hardwareids) = 1) AND IF(${searchParams.hashes.isEmpty}, true, FIND_IN_SET(JSON_UNQUOTE(JSON_EXTRACT(checksum, '$$.hash')), ${searchParams.hashes .map(_.value)}) > 0) ORDER BY @@ -115,11 +114,6 @@ class PackageSearch()(implicit db: Database) { length LIMIT $limit OFFSET $offset""".as[Q] - // TODO: This needs a newer mariadb version - // hardwareId filter should be: - // IF(${searchParams.hardwareIds.isEmpty}, true, JSON_LENGTH(JSON_ARRAY_INTERSECT(hardwareids, ${searchParams.hardwareIds})) > 0) - // but JSON_ARRAY_INTERSECT is not supported in our mariadb version - querySqlAction } @@ -253,8 +247,7 @@ class PackageSearch()(implicit db: Database) { IF(${searchParams.origin.isEmpty}, true, FIND_IN_SET(origin, ${searchParams.origin}) > 0) AND IF(${searchParams.nameContains.isEmpty}, true, LOCATE(${searchParams.nameContains}, name) > 0) AND IF(${searchParams.version.isEmpty}, true, version = ${searchParams.version}) AND - IF(${searchParams.hardwareIds.isEmpty}, true, JSON_CONTAINS(hardwareids, JSON_QUOTE(${searchParams.hardwareIds.headOption - .map(_.value)}))) AND + IF(${searchParams.hardwareIds.isEmpty}, true, JSON_OVERLAPS(${searchParams.hardwareIds}, hardwareids) = 1) AND IF(${searchParams.hashes.isEmpty}, true, FIND_IN_SET(JSON_UNQUOTE(JSON_EXTRACT(checksum, '$$.hash')), ${searchParams.hashes .map(_.value)}) > 0) GROUP BY name @@ -264,6 +257,25 @@ class PackageSearch()(implicit db: Database) { LIMIT $limit OFFSET $offset""".as[Q] db.run(q) + + } + + def hardwareIdsWithPackages(repoId: RepoId): Future[Seq[HardwareIdentifier]] = { + implicit val getResult: GetResult[HardwareIdentifier] = GetResult.GetString.andThen { str => + refineV[ValidHardwareIdentifier](str) + .valueOr(msg => + throw new IllegalArgumentException(s"hardwareid not properly formatted: $str: $msg") + ) + } + + val q = + sql""" + select distinct hwid from aggregated_items a, + json_table(hardwareids, '$$[*]' columns(hwid varchar(255) path '$$')) t1 + where a.repo_id = ${repoId.show} + """.as[HardwareIdentifier] + + db.run(q) } } diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala index 30daed12..f03f77a5 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala @@ -25,6 +25,7 @@ import eu.timepit.refined.refineV import slick.jdbc.MySQLProfile.api.* import com.advancedtelematic.libats.http.RefinedMarshallingSupport.* import scala.concurrent.ExecutionContext +import com.advancedtelematic.libats.codecs.CirceRefined.* case class PackageSearchParameters(origin: Seq[String], nameContains: Option[String], @@ -84,16 +85,28 @@ class RepoTargetsResource(namespaceValidation: NamespaceValidation)( ) } + private var packageSearch = new PackageSearch() + // format: off + val route = (pathPrefix("user_repo") & NamespaceRepoId(namespaceValidation, repoNamespaceRepo.findFor)) { repoId => + packageSearch = new PackageSearch() concat( + path("hardwareids-packages") { + get { + val f = packageSearch.hardwareIdsWithPackages(repoId).map { values => + PaginationResult(values, values.length, 0, values.length) + } + complete(f) + } + }, path("search") { (get & PaginationParams & SearchParams & SortByTargetItemsParam) { case (offset, limit, searchParams, sortBy, sortDirection) => val f = for { - count <- (new PackageSearch()).count(repoId, searchParams) - values <- (new PackageSearch()).find(repoId, offset, limit, searchParams, sortBy, sortDirection) + count <- packageSearch.count(repoId, searchParams) + values <- packageSearch.find(repoId, offset, limit, searchParams, sortBy, sortDirection) } yield { PaginationResult(values.map(_.transformInto[ClientPackage]), count, offset, limit) } @@ -104,8 +117,8 @@ class RepoTargetsResource(namespaceValidation: NamespaceValidation)( path("grouped-search") { (get & PaginationParams & SearchParams & SortByAggregatedTargetItemsParam) { case (offset, limit, searchParams, sortBy, sortDirection) => val f = for { - count <- (new PackageSearch()).findAggregatedCount(repoId, searchParams) - values <- (new PackageSearch()).findAggregated(repoId, offset, limit, searchParams, sortBy, sortDirection) + count <- packageSearch.findAggregatedCount(repoId, searchParams) + values <- packageSearch.findAggregated(repoId, offset, limit, searchParams, sortBy, sortDirection) } yield PaginationResult(values.map(_.transformInto[ClientAggregatedPackage]), count, offset, limit) diff --git a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala index c7ed1def..cd3c1a59 100644 --- a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala +++ b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala @@ -8,21 +8,18 @@ import org.scalatest.OptionValues.* import akka.http.scaladsl.model.{HttpEntity, StatusCodes} import akka.util.ByteString import com.advancedtelematic.libats.data.PaginationResult -import com.advancedtelematic.tuf.reposerver.util.{ - RepoResourceDelegationsSpecUtil, - ResourceSpec, - TufReposerverSpec -} +import com.advancedtelematic.tuf.reposerver.util.{RepoResourceDelegationsSpecUtil, ResourceSpec, TufReposerverSpec} import com.advancedtelematic.tuf.reposerver.util.NamespaceSpecOps.* import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* import com.advancedtelematic.tuf.reposerver.data.RepoDataType.* import com.advancedtelematic.tuf.reposerver.data.RepoDataType.Package.* import com.advancedtelematic.libats.data.DataType.HashMethod import com.advancedtelematic.libtuf.data.ClientDataType.ClientTargetItem -import com.advancedtelematic.libtuf.data.TufDataType.TargetFilename +import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, TargetFilename} import com.advancedtelematic.libtuf_server.crypto.Sha256Digest import eu.timepit.refined.api.Refined import io.circe.Json +import com.advancedtelematic.libats.codecs.CirceRefined.* class RepoTargetsResourceSpec extends TufReposerverSpec @@ -33,6 +30,10 @@ class RepoTargetsResourceSpec The library will endure; it is the universe. As for us, everything has not been written; we are not turning into phantoms. We walk the corridors, searching the shelves and rearranging them, looking for lines of meaning amid leagues of cacophony and incoherence, reading the history of the past and our future, collecting our thoughts and collecting the thoughts of others, and every so often glimpsing mirrors, in which we may recognize creatures of the information.” """.stripMargin)) + val testEntity2 = HttpEntity(ByteString(""" + If honor and wisdom and happiness are not for me, let them be for others. Let heaven exist, though my place be in hell + """.stripMargin)) + testWithRepo("GET returns delegation items ") { implicit ns => implicit repoId => addTargetToRepo(repoId) @@ -378,7 +379,6 @@ class RepoTargetsResourceSpec } } - // TODO: Currently only filters by single hardwareid testWithRepo("filters by hardwareIds") { implicit ns => implicit repoId => addTargetToRepo(repoId) @@ -414,6 +414,14 @@ class RepoTargetsResourceSpec val values = responseAs[PaginationResult[Package]].values values shouldBe empty } + + Get( + apiUriV2(s"user_repo/search?hardwareIds=delegated-hardware-id-001,myid001") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val values = responseAs[PaginationResult[Package]].values + values should have size 2 + } } testWithRepo("grouped-search gets packages aggregated by version") { @@ -480,6 +488,14 @@ class RepoTargetsResourceSpec values should have size 1 } + Get( + apiUriV2(s"user_repo/grouped-search?hardwareIds=delegated-hardware-id-001,myid001") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val values = responseAs[PaginationResult[AggregatedPackage]].values + values should have size 2 + } + Get( apiUriV2(s"user_repo/grouped-search?hardwareIds=somethingelse") ).namespaced ~> routes ~> check { @@ -577,4 +593,31 @@ class RepoTargetsResourceSpec } } + testWithRepo("GET hardwareids-packages returns hwids for which there are packages") { implicit ns => implicit repoId => + addTargetToRepo(repoId) + + Put( + apiUri("user_repo/targets/mypkg_file?name=library&version=0.0.1&hardwareIds=myid001,myid002"), + testEntity + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + + Put( + apiUri("user_repo/targets/mypkg_file2?name=library&version=0.0.2&hardwareIds=myid002,myid003"), + testEntity2 + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + + Get(apiUriV2(s"user_repo/hardwareids-packages")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val result = responseAs[PaginationResult[HardwareIdentifier]] + result.total shouldBe 3 + result.offset shouldBe 0 + result.limit shouldBe 3 + result.values.map(_.value) should contain theSameElementsAs List("myid001", "myid002", "myid003") + } + } + }