diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java index d70c3e43e..daa634eed 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java @@ -117,6 +117,7 @@ public void updateExtension(Extension extension) { cache.evictNamespaceDetails(extension); cache.evictLatestExtensionVersion(extension); cache.evictExtensionJsons(extension); + cache.evictSearchEntryJsons(extension); if (extension.getVersions().stream().anyMatch(ExtensionVersion::isActive)) { // There is at least one active version => activate the extension diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index 88df9c996..8303b136c 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -29,7 +29,6 @@ import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.json.*; import org.eclipse.openvsx.repositories.RepositoryService; -import org.eclipse.openvsx.search.ExtensionSearch; import org.eclipse.openvsx.search.ISearchService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.storage.StorageUtilService; @@ -39,8 +38,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.data.elasticsearch.core.SearchHit; -import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.retry.annotation.Retryable; @@ -82,6 +79,9 @@ public class LocalRegistryService implements IExtensionRegistry { @Autowired CacheService cache; + @Autowired + SearchEntryService searchEntries; + @Override public NamespaceJson getNamespace(String namespaceName) { var namespace = repositories.findNamespace(namespaceName); @@ -211,7 +211,22 @@ public SearchResultJson search(ISearchService.Options options) { } var searchHits = search.search(options); - json.extensions = toSearchEntries(searchHits, options); + var extensions = new ArrayList(); + for (var searchHit : searchHits) { + var searchEntry = searchEntries.toJson(searchHit, options.includeAllVersions); + if(searchEntry != null) { + // use averageRating, reviewCount and downloadCount from ElasticSearch response, + // so that cached SearchEntryJson doesn't have to be evicted every time + // averageRating, reviewCount or downloadCount are updated. + var extensionSearch = searchHit.getContent(); + searchEntry.averageRating = extensionSearch.averageRating; + searchEntry.reviewCount = extensionSearch.reviewCount; + searchEntry.downloadCount = extensionSearch.downloadCount; + extensions.add(searchEntry); + } + } + + json.extensions = extensions; json.offset = options.requestedOffset; json.totalSize = (int) searchHits.getTotalHits(); return json; @@ -741,81 +756,6 @@ public ResultJson deleteReview(String namespace, String extensionName) { return ResultJson.success("Deleted review for " + extension.getNamespace().getName() + "." + extension.getName()); } - private Extension getExtension(SearchHit searchHit) { - var searchItem = searchHit.getContent(); - var extension = entityManager.find(Extension.class, searchItem.id); - if (extension == null || !extension.isActive()) { - extension = new Extension(); - extension.setId(searchItem.id); - search.removeSearchEntry(extension); - return null; - } - - return extension; - } - - private List toSearchEntries(SearchHits searchHits, ISearchService.Options options) { - var serverUrl = UrlUtil.getBaseUrl(); - var extensions = searchHits.stream() - .map(this::getExtension) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - var latestVersions = extensions.stream() - .map(e -> { - var latest = versions.getLatestTrxn(e, null, false, true); - return new AbstractMap.SimpleEntry<>(e.getId(), latest); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - var searchEntries = latestVersions.entrySet().stream() - .map(e -> { - var entry = e.getValue().toSearchEntryJson(); - entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name); - return new AbstractMap.SimpleEntry<>(e.getKey(), entry); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - var fileUrls = storageUtil.getFileUrls(latestVersions.values(), serverUrl, DOWNLOAD, ICON); - searchEntries.forEach((extensionId, searchEntry) -> searchEntry.files = fileUrls.get(latestVersions.get(extensionId).getId())); - if (options.includeAllVersions) { - var allActiveVersions = repositories.findActiveVersions(extensions).stream() - .sorted(ExtensionVersion.SORT_COMPARATOR) - .collect(Collectors.toList()); - - var activeVersionsByExtensionId = allActiveVersions.stream() - .collect(Collectors.groupingBy(ev -> ev.getExtension().getId())); - - var versionUrls = storageUtil.getFileUrls(allActiveVersions, serverUrl, DOWNLOAD); - for(var extension : extensions) { - var activeVersions = activeVersionsByExtensionId.get(extension.getId()); - var searchEntry = searchEntries.get(extension.getId()); - searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl); - } - } - - return extensions.stream() - .map(Extension::getId) - .map(searchEntries::get) - .collect(Collectors.toList()); - } - - private List getAllVersionReferences( - List extVersions, - Map> versionUrls, - String serverUrl - ) { - Collections.sort(extVersions, ExtensionVersion.SORT_COMPARATOR); - return extVersions.stream().map(extVersion -> { - var ref = new SearchEntryJson.VersionReference(); - ref.version = extVersion.getVersion(); - ref.engines = extVersion.getEnginesMap(); - ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion); - ref.files = versionUrls.get(extVersion.getId()); - return ref; - }).collect(Collectors.toList()); - } - public ExtensionJson toExtensionVersionJson(ExtensionVersion extVersion, String targetPlatform, boolean onlyActive, boolean inTransaction) { var extension = extVersion.getExtension(); var latest = inTransaction diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java index ddea14674..53d338897 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java @@ -12,11 +12,14 @@ import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.UserData; +import org.eclipse.openvsx.json.SearchEntryJson; import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.search.ExtensionSearch; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionAlias; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.CacheManager; +import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.stereotype.Component; import java.util.ArrayList; @@ -27,6 +30,7 @@ public class CacheService { public static final String CACHE_DATABASE_SEARCH = "database.search"; public static final String CACHE_EXTENSION_JSON = "extension.json"; + public static final String CACHE_SEARCH_ENTRY_JSON = "search.entry.json"; public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version"; public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json"; public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating"; @@ -43,6 +47,9 @@ public class CacheService { @Autowired ExtensionJsonCacheKeyGenerator extensionJsonCacheKey; + @Autowired + SearchEntryJsonCacheKeyGenerator searchEntryJsonCacheKeyGenerator; + @Autowired LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKey; @@ -92,6 +99,33 @@ public void evictExtensionJsons(Extension extension) { } } + public SearchEntryJson getSearchEntryJson(SearchHit searchHit, boolean includeAllVersions) { + var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON); + return cache != null + ? cache.get(searchEntryJsonCacheKeyGenerator.generate(searchHit.getContent().id, includeAllVersions), SearchEntryJson.class) + : null; + } + + public void putSearchEntryJson(SearchEntryJson searchEntry, SearchHit searchHit, boolean includeAllVersions) { + var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON); + if(cache != null) { + cache.put(searchEntryJsonCacheKeyGenerator.generate(searchHit.getContent().id, includeAllVersions), searchEntry); + } + } + + public void evictSearchEntryJsons(Extension extension) { + var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON); + if(cache == null) { + return; // cache is not created + } + + var includeAllVersionsList = List.of(true, false); + for(var includeAllVersions : includeAllVersionsList) { + var key = searchEntryJsonCacheKeyGenerator.generate(extension.getId(), includeAllVersions); + cache.evictIfPresent(key); + } + } + public void evictLatestExtensionVersion(Extension extension) { var cache = cacheManager.getCache(CACHE_LATEST_EXTENSION_VERSION); if(cache != null) { diff --git a/server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java b/server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java new file mode 100644 index 000000000..ae5e1144b --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java @@ -0,0 +1,20 @@ +/** ****************************************************************************** + * Copyright (c) 2022 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import org.springframework.stereotype.Component; + +@Component +public class SearchEntryJsonCacheKeyGenerator { + + public Object generate(long extensionId, boolean includeAllVersions) { + return "extensionId=" + extensionId + ",includeAllVersions=" + includeAllVersions; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/entities/Extension.java b/server/src/main/java/org/eclipse/openvsx/entities/Extension.java index 0ce37ea83..19cab8c9d 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/Extension.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/Extension.java @@ -62,6 +62,7 @@ public ExtensionSearch toSearch(ExtensionVersion latest) { search.name = this.getName(); search.namespace = this.getNamespace().getName(); search.extensionId = search.namespace + "." + search.name; + search.averageRating = this.getAverageRating(); search.downloadCount = this.getDownloadCount(); search.targetPlatforms = this.getVersions().stream() .map(ExtensionVersion::getTargetPlatform) diff --git a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java index f8ff23603..cea7d8048 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java @@ -78,7 +78,7 @@ public class SearchEntryJson implements Serializable { name = "VersionReference", description = "Essential metadata of an extension version" ) - public static class VersionReference { + public static class VersionReference implements Serializable { @Schema(description = "URL to get the full metadata of this version") public String url; diff --git a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java new file mode 100644 index 000000000..13eb3bebe --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java @@ -0,0 +1,118 @@ +/** ****************************************************************************** + * Copyright (c) 2022 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.json; + +import org.eclipse.openvsx.cache.CacheService; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.search.ExtensionSearch; +import org.eclipse.openvsx.search.SearchUtilService; +import org.eclipse.openvsx.storage.StorageUtilService; +import org.eclipse.openvsx.util.UrlUtil; +import org.eclipse.openvsx.util.VersionService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.elasticsearch.core.SearchHit; +import org.springframework.stereotype.Component; + +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD; +import static org.eclipse.openvsx.entities.FileResource.ICON; +import static org.eclipse.openvsx.util.UrlUtil.createApiUrl; + +@Component +public class SearchEntryService { + + @Autowired + EntityManager entityManager; + + @Autowired + VersionService versions; + + @Autowired + StorageUtilService storageUtil; + + @Autowired + SearchUtilService search; + + @Autowired + RepositoryService repositories; + + @Autowired + CacheService cache; + + @Transactional + public SearchEntryJson toJson(SearchHit searchHit, boolean includeAllVersions) { + var searchEntry = cache.getSearchEntryJson(searchHit, includeAllVersions); + if(searchEntry != null) { + return searchEntry; + } + + var extension = getExtension(searchHit); + if(extension == null) { + return null; + } + + var serverUrl = UrlUtil.getBaseUrl(); + if(includeAllVersions && cache != null) { + searchEntry = cache.getSearchEntryJson(searchHit, false); + } + if(searchEntry == null) { + var latest = versions.getLatest(extension, null, false, true); + searchEntry = latest.toSearchEntryJson(); + searchEntry.url = createApiUrl(serverUrl, "api", searchEntry.namespace, searchEntry.name); + searchEntry.files = storageUtil.getFileUrls(latest, serverUrl, DOWNLOAD, ICON); + cache.putSearchEntryJson(searchEntry, searchHit, false); + } + if (includeAllVersions) { + var activeVersions = repositories.findActiveVersions(extension).toList(); + var versionUrls = storageUtil.getFileUrls(activeVersions, serverUrl, DOWNLOAD); + searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl); + cache.putSearchEntryJson(searchEntry, searchHit, true); + } + + return searchEntry; + } + + private Extension getExtension(SearchHit searchHit) { + var searchItem = searchHit.getContent(); + var extension = entityManager.find(Extension.class, searchItem.id); + if (extension == null || !extension.isActive()) { + extension = new Extension(); + extension.setId(searchItem.id); + search.removeSearchEntry(extension); + return null; + } + + return extension; + } + + private List getAllVersionReferences( + List extVersions, + Map> versionUrls, + String serverUrl + ) { + return extVersions.stream() + .sorted(ExtensionVersion.SORT_COMPARATOR) + .map(extVersion -> { + var ref = new SearchEntryJson.VersionReference(); + ref.version = extVersion.getVersion(); + ref.engines = extVersion.getEnginesMap(); + ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion); + ref.files = versionUrls.get(extVersion.getId()); + return ref; + }).collect(Collectors.toList()); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java index bb5a3dbbf..2b4e02be3 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java @@ -44,6 +44,14 @@ public class ExtensionSearch implements Serializable { @Field(index = false) public long timestamp; + @Nullable + @Field(index = false, type = FieldType.Float) + public Double averageRating; + + @Nullable + @Field(index = false, type = FieldType.Float) + public Long reviewCount; + @Nullable @Field(index = false, type = FieldType.Float) public Double rating; diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml index 3a37f5f9d..6b72da337 100644 --- a/server/src/main/resources/ehcache.xml +++ b/server/src/main/resources/ehcache.xml @@ -30,6 +30,16 @@ 1024 + + + 3600 + + + 1024 + 32 + 128 + + 3600 diff --git a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java index 3786f324c..1b05fc5ca 100644 --- a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java @@ -68,7 +68,7 @@ ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, PublishExtensionVersionHandler.class, SearchUtilService.class, - EclipseService.class, SimpleMeterRegistry.class + EclipseService.class, SimpleMeterRegistry.class, SearchEntryService.class }) public class AdminAPITest { diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index 8fd02aef1..3364eb882 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -36,6 +36,7 @@ import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.ExtensionJsonCacheKeyGenerator; import org.eclipse.openvsx.cache.LatestExtensionVersionCacheKeyGenerator; +import org.eclipse.openvsx.cache.SearchEntryJsonCacheKeyGenerator; import org.eclipse.openvsx.eclipse.EclipseService; import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.json.*; @@ -2172,5 +2173,15 @@ LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKeyGenerator( PublishExtensionVersionHandler publishExtensionVersionHandler() { return new PublishExtensionVersionHandler(); } + + @Bean + SearchEntryService searchEntryService() { + return new SearchEntryService(); + } + + @Bean + SearchEntryJsonCacheKeyGenerator searchEntryJsonCacheKeyGenerator() { + return new SearchEntryJsonCacheKeyGenerator(); + } } } \ No newline at end of file