From 1acb5071ad0cc019b1776a34b61d91c0235d6a7a Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Fri, 29 Mar 2024 21:22:48 +0200 Subject: [PATCH] Limit amount of extension versions --- .../eclipse/openvsx/admin/AdminService.java | 11 ++ .../FixTargetPlatformsJobRequestHandler.java | 6 +- .../migration/FixTargetPlatformsService.java | 40 ----- .../openvsx/mirror/DataMirrorService.java | 9 +- .../PublishExtensionVersionHandler.java | 4 +- .../PublishExtensionVersionService.java | 20 +++ .../ExtensionVersionJooqRepository.java | 152 ++++++++++++++---- .../repositories/RepositoryService.java | 8 + .../LimitStoredVersionsJobRequest.java | 61 +++++++ .../LimitStoredVersionsJobRequestHandler.java | 76 +++++++++ .../storage/StoredVersionsLimiter.java | 53 ++++++ .../RepositoryServiceSmokeTest.java | 4 +- 12 files changed, 359 insertions(+), 85 deletions(-) delete mode 100644 server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsService.java create mode 100644 server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequest.java create mode 100644 server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequestHandler.java create mode 100644 server/src/main/java/org/eclipse/openvsx/storage/StoredVersionsLimiter.java diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java index a4902c97f..2ed4301d4 100644 --- a/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java @@ -371,4 +371,15 @@ private void validateYearAndMonth(int year, int month) { throw new ErrorResultException("Combination of year and month lies in the future", HttpStatus.BAD_REQUEST); } } + + @Transactional + public UserData createSystemUser(String userName) { + var user = repositories.findUserByLoginName(null, userName); + if(user == null) { + user = new UserData(); + user.setLoginName(userName); + entityManager.persist(user); + } + return user; + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java index 4f2751137..4f41d3380 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java @@ -41,9 +41,6 @@ public class FixTargetPlatformsJobRequestHandler implements JobRequestHandler getExtensionTargetVersions(String namespaceName, S return extension.getVersions().stream().filter(v -> targetPlatform.equals(v.getTargetPlatform())).collect(Collectors.toList()); } - @Transactional public UserData createMirrorUser() { - var user = repositories.findUserByLoginName(null, userName); - if(user == null) { - user = new UserData(); - user.setLoginName(userName); - entityManager.persist(user); - } - return user; + return admin.createSystemUser(userName); } @Transactional diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java index 5ad652fc2..d7a24f1fa 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java @@ -10,6 +10,8 @@ package org.eclipse.openvsx.publish; import com.google.common.base.Joiner; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; import org.apache.commons.lang3.StringUtils; import org.eclipse.openvsx.ExtensionProcessor; import org.eclipse.openvsx.ExtensionService; @@ -29,8 +31,6 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; import java.io.IOException; import java.time.LocalDateTime; import java.util.function.Consumer; diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java index 22632e15c..3b3bd2743 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java @@ -14,9 +14,12 @@ import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.storage.LimitStoredVersionsJobRequest; import org.eclipse.openvsx.storage.StorageUtilService; +import org.eclipse.openvsx.storage.StoredVersionsLimiter; import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.TempFile; +import org.jobrunr.scheduling.JobRequestScheduler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; import org.springframework.retry.annotation.Retryable; @@ -41,6 +44,12 @@ public class PublishExtensionVersionService { @Autowired StorageUtilService storageUtil; + @Autowired + StoredVersionsLimiter storedVersionsLimiter; + + @Autowired + JobRequestScheduler scheduler; + @Transactional public void deleteFileResources(ExtensionVersion extVersion) { repositories.findFiles(extVersion).forEach(entityManager::remove); @@ -92,6 +101,17 @@ public void activateExtension(ExtensionVersion extVersion, ExtensionService exte extVersion.setActive(true); extVersion = entityManager.merge(extVersion); extensions.updateExtension(extVersion.getExtension()); + scheduleLimitStoredVersionsJob(extVersion); + } + + private void scheduleLimitStoredVersionsJob(ExtensionVersion extVersion) { + if(!storedVersionsLimiter.isEnabled()) { + return; + } + + var extension = extVersion.getExtension(); + var namespace = extension.getNamespace(); + scheduler.enqueue(new LimitStoredVersionsJobRequest(namespace.getName(), extension.getName())); } @Transactional(Transactional.TxType.REQUIRES_NEW) diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java index b2aa00d74..779944fa5 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java @@ -28,6 +28,7 @@ import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.eclipse.openvsx.jooq.Tables.*; @@ -543,42 +544,46 @@ private List toList(String raw, ListOfStringConverter converter) { return converter.convertToEntityAttribute(raw); } + private SelectQuery findGroupedByVersionQuery() { + var query = dsl.selectQuery(); + query.addSelect( + EXTENSION_VERSION.SEMVER_MAJOR, + EXTENSION_VERSION.SEMVER_MINOR, + EXTENSION_VERSION.SEMVER_PATCH, + EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE, + EXTENSION_VERSION.VERSION + ); + query.addFrom(EXTENSION_VERSION); + query.addGroupBy( + EXTENSION_VERSION.SEMVER_MAJOR, + EXTENSION_VERSION.SEMVER_MINOR, + EXTENSION_VERSION.SEMVER_PATCH, + EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE, + EXTENSION_VERSION.VERSION + ); + query.addOrderBy( + EXTENSION_VERSION.SEMVER_MAJOR.desc(), + EXTENSION_VERSION.SEMVER_MINOR.desc(), + EXTENSION_VERSION.SEMVER_PATCH.desc(), + EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE.asc(), + EXTENSION_VERSION.VERSION.asc() + ); + return query; + } + public List findTargetPlatformsGroupedByVersion(Extension extension) { + var query = findGroupedByVersionQuery(); var targetPlatforms = DSL.arrayAgg(EXTENSION_VERSION.TARGET_PLATFORM) .orderBy( EXTENSION_VERSION.UNIVERSAL_TARGET_PLATFORM.desc(), EXTENSION_VERSION.TARGET_PLATFORM.asc() ); - - return dsl.select( - EXTENSION_VERSION.SEMVER_MAJOR, - EXTENSION_VERSION.SEMVER_MINOR, - EXTENSION_VERSION.SEMVER_PATCH, - EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE, - EXTENSION_VERSION.VERSION, - targetPlatforms - ) - .from(EXTENSION_VERSION) - .where(EXTENSION_VERSION.EXTENSION_ID.eq(extension.getId())) - .groupBy( - EXTENSION_VERSION.SEMVER_MAJOR, - EXTENSION_VERSION.SEMVER_MINOR, - EXTENSION_VERSION.SEMVER_PATCH, - EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE, - EXTENSION_VERSION.VERSION - ) - .orderBy( - EXTENSION_VERSION.SEMVER_MAJOR.desc(), - EXTENSION_VERSION.SEMVER_MINOR.desc(), - EXTENSION_VERSION.SEMVER_PATCH.desc(), - EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE.asc(), - EXTENSION_VERSION.VERSION.asc() - ) - .fetch() - .map(record -> new VersionTargetPlatformsJson( - record.get(EXTENSION_VERSION.VERSION), - record.get(targetPlatforms) - )); + query.addSelect(targetPlatforms); + query.addConditions(EXTENSION_VERSION.EXTENSION_ID.eq(extension.getId())); + return query.fetch(record -> new VersionTargetPlatformsJson( + record.get(EXTENSION_VERSION.VERSION), + record.get(targetPlatforms) + )); } @Observed @@ -792,4 +797,91 @@ public List findDistinctTargetPlatforms(Extension extension) { .and(EXTENSION_VERSION.ACTIVE.eq(true)) .fetch(EXTENSION_VERSION.TARGET_PLATFORM); } + + public List findExcess(String namespaceName, String extensionName, int limit) { + var TARGET_PLATFORMS = DSL.arrayAgg(EXTENSION_VERSION.TARGET_PLATFORM); + var query = findGroupedByVersionQuery(); + query.addSelect(TARGET_PLATFORMS); + query.addJoin(EXTENSION, EXTENSION.ID.eq(EXTENSION_VERSION.EXTENSION_ID)); + query.addJoin(NAMESPACE, NAMESPACE.ID.eq(EXTENSION.NAMESPACE_ID)); + query.addConditions( + NAMESPACE.NAME.eq(namespaceName), + EXTENSION.NAME.eq(extensionName) + ); + query.addOffset(limit); + return query.fetch(record -> { + var list = new ArrayList(); + var version = record.get(EXTENSION_VERSION.VERSION); + var targetPlatforms = record.get(TARGET_PLATFORMS); + for(var targetPlatform : targetPlatforms) { + var namespace = new Namespace(); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setName(extensionName); + extension.setNamespace(namespace); + + var extVersion = new ExtensionVersion(); + extVersion.setVersion(version); + extVersion.setTargetPlatform(targetPlatform); + extVersion.setExtension(extension); + list.add(extVersion); + } + return list; + }) + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + public List findAllExcess(int limit) { + var namespaceNameAlias = NAMESPACE.NAME.as("namespaceName"); + var extensionNameAlias = EXTENSION.NAME.as("extensionName"); + var table = dsl.select(namespaceNameAlias, extensionNameAlias, EXTENSION.ID) + .from(EXTENSION) + .join(NAMESPACE).on(NAMESPACE.ID.eq(EXTENSION.NAMESPACE_ID)) + .asTable(); + + var namespaceNameField = table.field(namespaceNameAlias); + var extensionNameField = table.field(extensionNameAlias); + + var TARGET_PLATFORMS = DSL.arrayAgg(EXTENSION_VERSION.TARGET_PLATFORM); + var lateralQuery = findGroupedByVersionQuery(); + lateralQuery.addSelect(TARGET_PLATFORMS); + lateralQuery.addConditions(EXTENSION_VERSION.EXTENSION_ID.eq(table.field(EXTENSION.ID))); + lateralQuery.addOffset(limit); + var lateralTable = lateralQuery.asTable(); + var versionField = lateralTable.field(EXTENSION_VERSION.VERSION); + var targetPlatformsField = lateralTable.field(TARGET_PLATFORMS); + + var query = dsl.selectQuery(); + query.addSelect(namespaceNameField, extensionNameField, versionField, targetPlatformsField); + query.addFrom(table, DSL.lateral(lateralTable)); + + return query.fetch(record -> { + var list = new ArrayList(); + var namespaceName = record.get(namespaceNameField); + var extensionName = record.get(extensionNameField); + var version = record.get(versionField); + var targetPlatforms = record.get(targetPlatformsField); + for(var targetPlatform : targetPlatforms) { + var namespace = new Namespace(); + namespace.setName(namespaceName); + + var extension = new Extension(); + extension.setName(extensionName); + extension.setNamespace(namespace); + + var extVersion = new ExtensionVersion(); + extVersion.setVersion(version); + extVersion.setTargetPlatform(targetPlatform); + extVersion.setExtension(extension); + list.add(extVersion); + } + return list; + }) + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + } } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index d73caa1f7..4562a2184 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -555,4 +555,12 @@ public ExtensionVersion findLatestVersion(Extension extension, String targetPlat public List findExtensionTargetPlatforms(Extension extension) { return extensionVersionJooqRepo.findDistinctTargetPlatforms(extension); } + + public List findExcessExtensionVersions(String namespace, String extension, int limit) { + return extensionVersionJooqRepo.findExcess(namespace, extension, limit); + } + + public List findAllExcessExtensionVersions(int limit) { + return extensionVersionJooqRepo.findAllExcess(limit); + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequest.java b/server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequest.java new file mode 100644 index 000000000..2578292aa --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequest.java @@ -0,0 +1,61 @@ +/** ****************************************************************************** + * Copyright (c) 2024 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.storage; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +public class LimitStoredVersionsJobRequest implements JobRequest { + + private String namespace; + private String extension; + private boolean findAll; + + public LimitStoredVersionsJobRequest() {} + + public LimitStoredVersionsJobRequest(boolean findAll) { + this.findAll = findAll; + } + + public LimitStoredVersionsJobRequest(String namespace, String extension) { + this.namespace = namespace; + this.extension = extension; + } + + @Override + public Class getJobRequestHandler() { + return LimitStoredVersionsJobRequestHandler.class; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getExtension() { + return extension; + } + + public void setExtension(String extension) { + this.extension = extension; + } + + public boolean isFindAll() { + return findAll; + } + + public void setFindAll(boolean findAll) { + this.findAll = findAll; + } +} + diff --git a/server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequestHandler.java new file mode 100644 index 000000000..33232fba0 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/storage/LimitStoredVersionsJobRequestHandler.java @@ -0,0 +1,76 @@ +/** ****************************************************************************** + * Copyright (c) 2024 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.storage; + +import org.apache.commons.lang3.StringUtils; +import org.eclipse.openvsx.admin.AdminService; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class LimitStoredVersionsJobRequestHandler implements JobRequestHandler { + + protected final Logger logger = LoggerFactory.getLogger(LimitStoredVersionsJobRequestHandler.class); + + @Autowired + AdminService admins; + + @Autowired + RepositoryService repositories; + + @Autowired + StoredVersionsLimiter service; + + @Override + public void run(LimitStoredVersionsJobRequest jobRequest) throws Exception { + if(!service.isEnabled()) { + return; + } + + var extVersions = findExtensionVersions(jobRequest); + if(extVersions.isEmpty()) { + return; + } + + var user = admins.createSystemUser(service.getUserName()); + for(var extVersion : extVersions) { + var extension = extVersion.getExtension(); + var namespace = extension.getNamespace(); + var result = admins.deleteExtension(namespace.getName(), extension.getName(), extVersion.getTargetPlatform(), extVersion.getVersion(), user); + if(StringUtils.isNotEmpty(result.error)) { + logger.error(result.error); + } + if(StringUtils.isNotEmpty(result.warning)) { + logger.warn(result.warning); + } + if(StringUtils.isNotEmpty(result.success)) { + logger.info(result.success); + } + } + } + + private List findExtensionVersions(LimitStoredVersionsJobRequest jobRequest) { + var limit = service.getLimit(); + if(jobRequest.isFindAll()) { + return repositories.findAllExcessExtensionVersions(limit); + } else { + var namespace = jobRequest.getNamespace(); + var extension = jobRequest.getExtension(); + return repositories.findExcessExtensionVersions(namespace, extension, limit); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StoredVersionsLimiter.java b/server/src/main/java/org/eclipse/openvsx/storage/StoredVersionsLimiter.java new file mode 100644 index 000000000..7eef9bb94 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/storage/StoredVersionsLimiter.java @@ -0,0 +1,53 @@ +/** ****************************************************************************** + * Copyright (c) 2024 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.storage; + +import org.jobrunr.scheduling.JobRequestScheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +@Component +public class StoredVersionsLimiter { + + @Autowired + JobRequestScheduler scheduler; + + @Value("${ovsx.storage.versions-limiter.run-on-start:false}") + boolean runOnStart; + + @Value("${ovsx.storage.versions-limiter.user-name:versions-limiter}") + String userName; + + @Value("${ovsx.storage.versions-limiter.limit:0}") + int limit; + + @EventListener + public void applicationStarted(ApplicationStartedEvent event) { + if(!runOnStart || !isEnabled()) { + return; + } + + scheduler.enqueue(new LimitStoredVersionsJobRequest(true)); + } + + public boolean isEnabled() { + return limit > 0; + } + public int getLimit() { + return limit; + } + + public String getUserName() { + return userName; + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index 66e748731..9e1cedd08 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -200,7 +200,9 @@ void testExecuteQueries() { () -> repositories.findExtensionVersion("namespaceName", "extensionName", "targetPlatform", "version"), () -> repositories.findLatestVersionForAllUrls(extension, "targetPlatform", false, false), () -> repositories.findLatestVersion(extension, "targetPlatform", false, false), - () -> repositories.findExtensionTargetPlatforms(extension) + () -> repositories.findExtensionTargetPlatforms(extension), + () -> repositories.findExcessExtensionVersions("namespaceName", "extensionName",1), + () -> repositories.findAllExcessExtensionVersions(1) ); // check that we did not miss anything