diff --git a/.github/workflows/dockerhub-release.yaml b/.github/workflows/dockerhub-release.yaml new file mode 100644 index 0000000..f874986 --- /dev/null +++ b/.github/workflows/dockerhub-release.yaml @@ -0,0 +1,71 @@ +name: Retag RC Docker image + +on: + pull_request_review: + types: [submitted] + +env: + AWS_REGION: ${{ vars.AWS_REGION }} # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} # set this to your Amazon ECR repository name + TARGET_REGISTRY: ${{ vars.TARGET_REGISTRY }} # set to target regestry (DockerHub, GitHub & etc) + TARGET_REPOSITORY: ${{ vars.TARGET_REPOSITORY }} # set to target repository + PLATFORMS: ${{ vars.BUILD_PLATFORMS }} # set target build platforms. By default linux/amd64 + RELEASE_MODE: ${{ vars.RELEASE_MODE }} + +jobs: + retag-image: + name: Retag and push image + runs-on: ubuntu-latest + environment: rc + if: github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main' + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + # role-to-assume: arn:aws:iam::123456789012:role/my-github-actions-role + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.REGESTRY_USERNAME }} + password: ${{ secrets.REGESTRY_PASSWORD }} + + - name: Create variables + id: vars + run: | + echo "tag=$(echo '${{ github.event.pull_request.title }}' | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Retag and Push Docker Image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + docker buildx imagetools create $ECR_REGISTRY/$ECR_REPOSITORY:latest --tag $TARGET_REGISTRY/$TARGET_REPOSITORY:$IMAGE_TAG --tag $TARGET_REGISTRY/$TARGET_REPOSITORY:latest + + - name: Summarize + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- :whale: Docker image: $TARGET_REGISTRY/$TARGET_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY + echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/manually-release.yml b/.github/workflows/manually-release.yml index 0af5b9f..9af4925 100644 --- a/.github/workflows/manually-release.yml +++ b/.github/workflows/manually-release.yml @@ -6,17 +6,9 @@ on: version: description: 'Release version' required: true - scripts_version: - description: 'Gradle scripts version' - required: true - bom_version: - description: 'Commons bom version' - required: true env: GH_USER_NAME: github.actor - SCRIPTS_VERSION: ${{ github.event.inputs.scripts_version }} - BOM_VERSION: ${{ github.event.inputs.bom_version }} RELEASE_VERSION: ${{ github.event.inputs.version }} REPOSITORY_URL: 'https://maven.pkg.github.com/' @@ -46,7 +38,6 @@ jobs: - name: Release with Gradle id: release run: | - ./gradlew release -PreleaseMode -Pscripts.version=${{env.SCRIPTS_VERSION}} \ - -Pbom.version=${{env.BOM_VERSION}} \ + ./gradlew release -PreleaseMode \ -PgithubUserName=${{env.GH_USER_NAME}} -PgithubToken=${{secrets.GITHUB_TOKEN}} \ -PgpgPassphrase=${{secrets.GPG_PASSPHRASE}} -PgpgPrivateKey="${{secrets.GPG_PRIVATE_KEY}}" diff --git a/.github/workflows/rc.yaml b/.github/workflows/rc.yaml new file mode 100644 index 0000000..056c033 --- /dev/null +++ b/.github/workflows/rc.yaml @@ -0,0 +1,93 @@ +name: Build RC Docker image + +on: + push: + branches: + - "rc/*" + - "hotfix/*" + +env: + AWS_REGION: ${{ vars.AWS_REGION }} # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} # set this to your Amazon ECR repository name + PLATFORMS: ${{ vars.BUILD_PLATFORMS }} # set target build platforms. By default linux/amd64 + RELEASE_MODE: ${{ vars.RELEASE_MODE }} + +jobs: + build-and-export: + name: Build and export to AWS ECR + runs-on: ubuntu-latest + environment: rc + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + # role-to-assume: arn:aws:iam::123456789012:role/my-github-actions-role + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Create variables + id: vars + run: | + echo "tag=$(echo ${{ github.ref_name }} | tr '/' '-')" >> $GITHUB_OUTPUT + echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + echo "version=$(echo '${{ github.ref_name }}' | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build + uses: docker/build-push-action@v4 + env: + VERSION: ${{ steps.vars.outputs.version }} + DATE: ${{ steps.vars.outputs.date }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + push: true + build-args: | + APP_VERSION=${{ env.VERSION }} + BUILD_DATE=${{ env.DATE }} + GITHUB_USER=${{ secrets.GH_USER }} + GITHUB_TOKEN=${{ secrets.GH_TOKEN }} + RELEASE_MODE=${{ env.RELEASE_MODE }} + platforms: ${{ env.PLATFORMS }} + tags: | + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + env: + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + image-ref: '${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}' + format: 'table' + exit-code: '0' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + + - name: Summarize + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- :gift: Docker image in Amazon ECR: ecr/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY + echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f8840b0..80cbee9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,6 @@ on: env: GH_USER_NAME: github.actor - SCRIPTS_VERSION: 5.8.0 - BOM_VERSION: 5.7.6 REPOSITORY_URL: 'https://maven.pkg.github.com/' jobs: @@ -41,7 +39,6 @@ jobs: - name: Release with Gradle id: release run: | - ./gradlew release -PreleaseMode -Pscripts.version=${{env.SCRIPTS_VERSION}} \ - -Pbom.version=${{env.BOM_VERSION}} \ + ./gradlew release -PreleaseMode \ -PgithubUserName=${{env.GH_USER_NAME}} -PgithubToken=${{secrets.GITHUB_TOKEN}} \ -PgpgPassphrase=${{secrets.GPG_PASSPHRASE}} -PgpgPrivateKey="${{secrets.GPG_PRIVATE_KEY}}" diff --git a/Dockerfile b/Dockerfile index f7a3544..ac5e088 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,26 @@ -FROM alpine:latest -LABEL version=5.8.2 description="EPAM Report portal. Service jobs" maintainer="Andrei Varabyeu , Hleb Kanonik " -ARG GH_TOKEN -RUN echo 'exec java ${JAVA_OPTS} -jar service-jobs-5.8.2-exec.jar' > /start.sh && chmod +x /start.sh && \ - wget --header="Authorization: Bearer ${GH_TOKEN}" -q https://maven.pkg.github.com/reportportal/service-jobs/com/epam/reportportal/service-jobs/5.8.2/service-jobs-5.8.2-exec.jar -ENV JAVA_OPTS="-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" +FROM gradle:6.8.3-jdk11 AS build +ARG RELEASE_MODE +ARG APP_VERSION +ARG GITHUB_USER +ARG GITHUB_TOKEN +WORKDIR /usr/app +COPY . /usr/app +RUN if [ "${RELEASE_MODE}" = true ]; then \ + gradle build --exclude-task test \ + -PreleaseMode=true \ + -PgithubUserName=${GITHUB_USER} \ + -PgithubToken=${GITHUB_TOKEN} \ + -Dorg.gradle.project.version=${APP_VERSION}; \ + else gradle build --exclude-task test -Dorg.gradle.project.version=${APP_VERSION}; fi + +# For ARM build use flag: `--platform linux/arm64` +FROM --platform=$BUILDPLATFORM amazoncorretto:11.0.20 +LABEL version=${APP_VERSION} description="EPAM Report portal. Jobs Service" maintainer="Andrei Varabyeu , Hleb Kanonik " +ARG APP_VERSION=${APP_VERSION} +ENV APP_DIR=/usr/app +ENV JAVA_OPTS="-Xmx1g -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom" +WORKDIR $APP_DIR +COPY --from=build $APP_DIR/build/libs/service-jobs-*exec.jar . VOLUME ["/tmp"] EXPOSE 8080 -ENTRYPOINT ./start.sh +ENTRYPOINT exec java ${JAVA_OPTS} -jar ${APP_DIR}/service-jobs-*exec.jar diff --git a/build.gradle b/build.gradle index 0ad29c8..0e4fc16 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ project.ext { releaseMode = project.hasProperty("releaseMode") } -def scriptsUrl = 'https://raw.githubusercontent.com/reportportal/gradle-scripts/' + (releaseMode ? getProperty('scripts.version') : 'master') +def scriptsUrl = 'https://raw.githubusercontent.com/reportportal/gradle-scripts/' + (releaseMode ? '5.10.0' : 'master') apply from: "$scriptsUrl/build-docker.gradle" apply from: "$scriptsUrl/build-commons.gradle" @@ -54,9 +54,9 @@ ext['snakeyaml.version'] = '1.31' dependencies { if (releaseMode) { - implementation 'com.github.reportportal:commons-events:5.7.3' + implementation 'com.github.reportportal:commons-events:5.10.0' } else { - implementation 'com.github.reportportal:commons-events:35a1246f' + implementation 'com.github.reportportal:commons-events:e337f8b7be' } implementation group: 'org.json', name: 'json', version: '20220320' @@ -68,6 +68,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + implementation 'org.springframework.boot:spring-boot-starter-aop' implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-web' @@ -98,7 +99,5 @@ test { useJUnitPlatform() } -addDockerfileToGit.dependsOn createDockerfile -beforeReleaseBuild.dependsOn addDockerfileToGit publish.dependsOn build publish.mustRunAfter build diff --git a/gradle.properties b/gradle.properties index 007f5df..e5e7658 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -version=5.8.3 +version=5.10.0 description=EPAM Report portal. Service jobs dockerServerUrl=unix:///var/run/docker.sock dockerPrepareEnvironment= -dockerJavaOpts=-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom \ No newline at end of file +dockerJavaOpts=-Xmx512m -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=70 -Djava.security.egd=file:/dev/./urandom diff --git a/src/main/java/com/epam/reportportal/ServiceJobApplication.java b/src/main/java/com/epam/reportportal/ServiceJobApplication.java index e79fe9f..75bf53a 100644 --- a/src/main/java/com/epam/reportportal/ServiceJobApplication.java +++ b/src/main/java/com/epam/reportportal/ServiceJobApplication.java @@ -18,8 +18,10 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration; -@SpringBootApplication(scanBasePackages = { "com.epam.reportportal" }) +@SpringBootApplication(scanBasePackages = { "com.epam.reportportal" }, exclude = { + FlywayAutoConfiguration.class}) public class ServiceJobApplication { public static void main(String[] args) { diff --git a/src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java b/src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java new file mode 100644 index 0000000..f1393d3 --- /dev/null +++ b/src/main/java/com/epam/reportportal/analyzer/AnalyzerUtils.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.analyzer; + +import static java.util.Optional.ofNullable; + +import com.rabbitmq.http.client.domain.ExchangeInfo; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.math.NumberUtils; + +/** + * @author Andrei Piankouski + */ +public final class AnalyzerUtils { + + static final String ANALYZER_KEY = "analyzer"; + static final String ANALYZER_PRIORITY = "analyzer_priority"; + static final String ANALYZER_INDEX = "analyzer_index"; + static final String ANALYZER_LOG_SEARCH = "analyzer_log_search"; + static final String ANALYZER_SUGGEST = "analyzer_suggest"; + static final String ANALYZER_CLUSTER = "analyzer_cluster"; + + /** + * Comparing by client service priority + */ + public static final ToIntFunction EXCHANGE_PRIORITY = it -> ofNullable(it.getArguments() + .get(ANALYZER_PRIORITY)).map(val -> NumberUtils.toInt(val.toString(), Integer.MAX_VALUE)).orElse(Integer.MAX_VALUE); + + /** + * Checks if service support items indexing. false + * by default + */ + public static final Predicate DOES_SUPPORT_INDEX = it -> ofNullable(it.getArguments() + .get(ANALYZER_INDEX)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + + /** + * Checks if service support logs searching. false + * by default + */ + public static final Predicate DOES_SUPPORT_SEARCH = it -> ofNullable(it.getArguments() + .get(ANALYZER_LOG_SEARCH)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + + /** + * Checks if service support logs searching. false + * by default + */ + public static final Predicate DOES_SUPPORT_SUGGEST = it -> ofNullable(it.getArguments() + .get(ANALYZER_SUGGEST)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + + /** + * Checks if service support logs cluster creation. false + * by default + */ + public static final Predicate DOES_SUPPORT_CLUSTER = it -> ofNullable(it.getArguments() + .get(ANALYZER_CLUSTER)).map(val -> BooleanUtils.toBoolean(val.toString())).orElse(false); + +} diff --git a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java index b0e1940..9e11932 100644 --- a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java +++ b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClient.java @@ -45,4 +45,18 @@ public interface IndexerServiceClient { */ void removeFromIndexLessThanLaunchDate(Long index, LocalDateTime lessThanDate); + /** + * Delete index + * + * @param index Index to be deleted + */ + void deleteIndex(Long index); + + /** + * Removes suggest index + * + * @param projectId Project/index id + */ + void removeSuggest(Long projectId); + } diff --git a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java index fe17e0e..363dd5c 100644 --- a/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java +++ b/src/main/java/com/epam/reportportal/analyzer/index/IndexerServiceClientImpl.java @@ -3,6 +3,12 @@ import com.epam.reportportal.analyzer.RabbitMqManagementClient; import com.epam.reportportal.model.index.CleanIndexByDateRangeRq; import com.epam.reportportal.model.index.CleanIndexRq; +import com.rabbitmq.http.client.domain.ExchangeInfo; +import java.util.Comparator; +import java.util.Optional; +import java.util.function.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,6 +21,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static com.epam.reportportal.analyzer.AnalyzerUtils.DOES_SUPPORT_SUGGEST; import static com.epam.reportportal.analyzer.RabbitMqManagementClientTemplate.EXCHANGE_PRIORITY; /** @@ -23,10 +30,14 @@ @Service public class IndexerServiceClientImpl implements IndexerServiceClient { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexerServiceClient.class); private static final String CLEAN_ROUTE = "clean"; private static final String CLEAN_BY_LOG_DATE_ROUTE = "remove_by_log_time"; private static final String CLEAN_BY_LAUNCH_DATE_ROUTE = "remove_by_launch_start_time"; private static final String EXCHANGE_NAME = "analyzer-default"; + private static final String REMOVE_SUGGEST_ROUTE = "remove_suggest_info"; + static final String DELETE_ROUTE = "delete"; + private static final Integer DELETE_INDEX_SUCCESS_CODE = 1; // need to be in line with analyzer API, better to fix api and remove it in future. private static final LocalDateTime OLDEST_DATE = LocalDateTime.now().minusYears(10L); @@ -69,9 +80,42 @@ public void removeFromIndexLessThanLaunchDate(Long index, LocalDateTime lessThan sendRangeRemovingMessageToRoute(index, lessThanDate, CLEAN_BY_LAUNCH_DATE_ROUTE); } + @Override + public void deleteIndex(Long index) { + rabbitMqManagementClient.getAnalyzerExchangesInfo() + .stream() + .map(exchange -> rabbitTemplate.convertSendAndReceiveAsType(exchange.getName(), + DELETE_ROUTE, + index, + new ParameterizedTypeReference() { + } + )) + .forEach(it -> { + if (DELETE_INDEX_SUCCESS_CODE.equals(it)) { + LOGGER.info("Successfully deleted index '{}'", index); + } else { + LOGGER.error("Error deleting index '{}'", index); + } + }); + } + private void sendRangeRemovingMessageToRoute(Long index, LocalDateTime lessThanDate, String route) { CleanIndexByDateRangeRq message = new CleanIndexByDateRangeRq(index, OLDEST_DATE, lessThanDate); rabbitTemplate.convertAndSend(EXCHANGE_NAME, route, message); } + @Override + public void removeSuggest(Long projectId) { + resolveExchangeName(DOES_SUPPORT_SUGGEST) + .ifPresent(suggestExchange -> rabbitTemplate.convertAndSend(suggestExchange, REMOVE_SUGGEST_ROUTE, projectId)); + } + + private Optional resolveExchangeName(Predicate supportCondition) { + return rabbitMqManagementClient.getAnalyzerExchangesInfo() + .stream() + .filter(supportCondition) + .min(Comparator.comparingInt(EXCHANGE_PRIORITY)) + .map(ExchangeInfo::getName); + } + } diff --git a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java index 386e042..40894b6 100644 --- a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java +++ b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java @@ -166,6 +166,7 @@ public BlobStore minioBlobStore(@Value("${datastore.accessKey}") String accessKe * * @param blobStore {@link BlobStore} object * @param bucketPrefix Prefix for bucket name + * @param bucketPostfix Postfix for bucket name * @param defaultBucketName Name of default bucket to use * @param featureFlagHandler Instance of {@link FeatureFlagHandler} to check enabled features * @return {@link DataStorageService} object diff --git a/src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java new file mode 100644 index 0000000..c72013e --- /dev/null +++ b/src/main/java/com/epam/reportportal/config/rabbit/InternalConfiguration.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.config.rabbit; + +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.DirectExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * RabbitMQ queue and exchange configuration. + * + * @author Andrei Piankouski + */ +@Configuration +public class InternalConfiguration { + + /** + * Exchanges. + */ + public static final String EXCHANGE_NOTIFICATION = "notification"; + + /** + * Queues. + */ + public static final String QUEUE_EMAIL = "notification.email"; + + @Bean + Queue emailNotificationQueue() { + return new Queue(QUEUE_EMAIL); + } + + @Bean + DirectExchange notificationExchange() { + return new DirectExchange(EXCHANGE_NOTIFICATION); + } + + @Bean + public Binding emailNotificationBinding() { + return BindingBuilder.bind(emailNotificationQueue()).to(notificationExchange()) + .with(QUEUE_EMAIL); + } + +} diff --git a/src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java deleted file mode 100644 index 25bea9e..0000000 --- a/src/main/java/com/epam/reportportal/config/rabbit/ProcessingRabbitMqConfiguration.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 EPAM Systems - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.reportportal.config.rabbit; - -import org.springframework.amqp.rabbit.annotation.EnableRabbit; -import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory; -import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; -import org.springframework.amqp.rabbit.connection.ConnectionFactory; -import org.springframework.amqp.rabbit.core.RabbitAdmin; -import org.springframework.amqp.support.converter.MessageConverter; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.amqp.SimpleRabbitListenerContainerFactoryConfigurer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.net.URI; - -/** - * @author Pavel Bortnik - */ -@EnableRabbit -@Configuration -public class ProcessingRabbitMqConfiguration { - - @Bean(name = "processingConnectionFactory") - public ConnectionFactory processingConnectionFactory(@Value("${rp.amqp.addresses}") URI addresses, - @Value("${rp.amqp.base-vhost}") String virtualHost) { - CachingConnectionFactory factory = new CachingConnectionFactory(addresses); - factory.setVirtualHost(virtualHost); - return factory; - } - - @Bean - public SimpleRabbitListenerContainerFactory processingRabbitListenerContainerFactory( - @Qualifier("processingConnectionFactory") ConnectionFactory connectionFactory, MessageConverter jsonMessageConverter, - @Value("${rp.amqp.maxLogConsumer}") int maxLogConsumer) { - SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); - factory.setConnectionFactory(connectionFactory); - factory.setMaxConcurrentConsumers(maxLogConsumer); - factory.setMessageConverter(jsonMessageConverter); - return factory; - } - - @Bean - public RabbitAdmin processingRabbitAdmin(@Qualifier("processingConnectionFactory") ConnectionFactory connectionFactory) { - return new RabbitAdmin(connectionFactory); - } - - @Bean - SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory( - SimpleRabbitListenerContainerFactoryConfigurer configurer, @Qualifier("processingConnectionFactory") ConnectionFactory connectionFactory) { - SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); - configurer.configure(factory, connectionFactory); - return factory; - } -} diff --git a/src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java new file mode 100644 index 0000000..7c4c1f4 --- /dev/null +++ b/src/main/java/com/epam/reportportal/config/rabbit/RabbitMqConfiguration.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.config.rabbit; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; +import org.springframework.amqp.rabbit.annotation.EnableRabbit; +import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.amqp.SimpleRabbitListenerContainerFactoryConfigurer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +/** + * @author Pavel Bortnik + */ +@EnableRabbit +@Configuration +public class RabbitMqConfiguration { + + private final ObjectMapper objectMapper; + + @Autowired + public RabbitMqConfiguration(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Bean(name = "connectionFactory") + public ConnectionFactory connectionFactory(@Value("${rp.amqp.addresses}") URI addresses, + @Value("${rp.amqp.base-vhost}") String virtualHost) { + CachingConnectionFactory factory = new CachingConnectionFactory(addresses); + factory.setVirtualHost(virtualHost); + return factory; + } + + @Bean + public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory( + @Qualifier("connectionFactory") ConnectionFactory connectionFactory, + MessageConverter jsonMessageConverter, + @Value("${rp.amqp.maxLogConsumer}") int maxLogConsumer) { + SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); + factory.setConnectionFactory(connectionFactory); + factory.setMaxConcurrentConsumers(maxLogConsumer); + factory.setMessageConverter(jsonMessageConverter); + return factory; + } + + @Bean + public RabbitAdmin rabbitAdmin( + @Qualifier("connectionFactory") ConnectionFactory connectionFactory) { + return new RabbitAdmin(connectionFactory); + } + + @Bean + SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory( + SimpleRabbitListenerContainerFactoryConfigurer configurer, + @Qualifier("connectionFactory") ConnectionFactory connectionFactory) { + SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); + configurer.configure(factory, connectionFactory); + return factory; + } + + @Bean(name = "rabbitTemplate") + public RabbitTemplate rabbitTemplate( + @Autowired @Qualifier("connectionFactory") ConnectionFactory connectionFactory, + @Value("${rp.amqp.reply-timeout}") long replyTimeout) { + RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory); + rabbitTemplate.setMessageConverter(new Jackson2JsonMessageConverter(objectMapper)); + rabbitTemplate.setReplyTimeout(replyTimeout); + return rabbitTemplate; + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/BaseJob.java b/src/main/java/com/epam/reportportal/jobs/BaseJob.java index 98fc4ad..8f5ad0f 100644 --- a/src/main/java/com/epam/reportportal/jobs/BaseJob.java +++ b/src/main/java/com/epam/reportportal/jobs/BaseJob.java @@ -12,15 +12,5 @@ public BaseJob(JdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; } - protected void logStart() { - LOGGER.info("Job {} has been started.", this.getClass().getSimpleName()); - } - - protected void logFinish(Object result) { - LOGGER.info("Job {} has been finished. Result {}", this.getClass().getSimpleName(), result); - } - - protected void logFinish() { - logFinish(null); - } + public abstract void execute(); } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java b/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java index aa775be..af9ba03 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/BaseCleanJob.java @@ -12,7 +12,7 @@ /** * @author Pavel Bortnik */ -public class BaseCleanJob extends BaseJob { +public abstract class BaseCleanJob extends BaseJob { protected static final String KEEP_LAUNCHES = "job.keepLaunches"; protected static final String KEEP_LOGS = "job.keepLogs"; diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java index 34f9bc8..b070f4b 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java @@ -27,6 +27,7 @@ public CleanAttachmentJob(JdbcTemplate jdbcTemplate) { super(jdbcTemplate); } + @Override @Scheduled(cron = "${rp.environment.variable.clean.attachment.cron}") @SchedulerLock(name = "cleanAttachment", lockAtMostFor = "24h") public void execute() { @@ -34,7 +35,6 @@ public void execute() { } void moveAttachments() { - logStart(); AtomicInteger counter = new AtomicInteger(0); getProjectsWithAttribute(KEEP_SCREENSHOTS).forEach((projectId, duration) -> { LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); @@ -42,6 +42,5 @@ void moveAttachments() { counter.addAndGet(movedCount); LOGGER.info("Moved {} attachments to the deletion table for project {}, lessThanDate {} ", movedCount, projectId, lessThanDate); }); - logFinish(counter.get()); } } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java index 2904a19..13da6e8 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java @@ -53,6 +53,7 @@ public CleanLaunchJob(@Value("${rp.environment.variable.elements-counter.batch-s this.elasticSearchClient = elasticSearchClient; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.launch.cron}") @SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h") public void execute() { @@ -61,7 +62,6 @@ public void execute() { } private void removeLaunches() { - logStart(); AtomicInteger counter = new AtomicInteger(0); getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> { final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); @@ -84,7 +84,6 @@ private void removeLaunches() { } } }); - logFinish(counter.get()); } private void deleteLogsFromElasticsearchByLaunchIdsAndProjectId(List launchIds, Long projectId) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java index 5428d24..4564435 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java @@ -45,6 +45,7 @@ public CleanLogJob(JdbcTemplate jdbcTemplate, CleanAttachmentJob cleanAttachment this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.log.cron}") @SchedulerLock(name = "cleanLog", lockAtMostFor = "24h") public void execute() { @@ -53,7 +54,6 @@ public void execute() { } void removeLogs() { - logStart(); AtomicInteger counter = new AtomicInteger(0); // TODO: Need to refactor Logs to keep real it's launchId and combine code with // CleanLaunch to avoid duplication @@ -76,8 +76,6 @@ void removeLogs() { // LOGGER.info("Send event with elements deleted number {} for project {}", deleted, projectId); } }); - - logFinish(counter.get()); } private void deleteLogsFromElasticsearchByLaunchIdsAndProjectId(List launchIds, Long projectId) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java index db4d963..7caa11b 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanMaterializedViewJob.java @@ -51,10 +51,10 @@ public CleanMaterializedViewJob(JdbcTemplate jdbcTemplate, @Value("${rp.environm this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; } + @Override @Scheduled(cron = "${rp.environment.variable.clean.view.cron}") @SchedulerLock(name = "cleanMaterializedView", lockAtMostFor = "24h") public void execute() { - logStart(); final AtomicInteger existingCounter = new AtomicInteger(0); final AtomicInteger staleCounter = new AtomicInteger(0); @@ -75,9 +75,6 @@ public void execute() { staleViews = getStaleViews(timeBound); } - - logFinish(String.format("Stale removed: %d, Existing removed: %d", staleCounter.get(), existingCounter.get())); - } private List getStaleViews(LocalDateTime timeBound) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java index aa226dd..16ebe84 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanStorageJob.java @@ -54,11 +54,11 @@ public CleanStorageJob(JdbcTemplate jdbcTemplate, DataStorageService storageServ /** * Deletes attachments, which are set to be deleted. */ + @Override @Scheduled(cron = "${rp.environment.variable.clean.storage.cron}") @SchedulerLock(name = "cleanStorage", lockAtMostFor = "24h") @Transactional public void execute() { - logStart(); AtomicInteger counter = new AtomicInteger(0); int batchNumber = 1; @@ -99,8 +99,6 @@ public void execute() { LOGGER.info("Iteration {}, deleted {} attachments", batchNumber, attachmentsSize); batchNumber++; } - - logFinish(counter.get()); } private String decode(String data) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java new file mode 100644 index 0000000..63668e9 --- /dev/null +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -0,0 +1,279 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.jobs.clean; + +import com.epam.reportportal.analyzer.index.IndexerServiceClient; +import com.epam.reportportal.jobs.BaseJob; +import com.epam.reportportal.model.EmailNotificationRequest; +import com.epam.reportportal.model.activity.event.ProjectDeletedEvent; +import com.epam.reportportal.model.activity.event.UnassignUserEvent; +import com.epam.reportportal.model.activity.event.UserDeletedEvent; +import com.epam.reportportal.service.MessageBus; +import com.epam.reportportal.utils.ValidationUtil; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.jclouds.blobstore.BlobStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +/** + * Deleting Users and their personal project by retention policy. + * + * @author Andrei Piankouski + */ +@Service +@ConditionalOnProperty(prefix = "rp.environment.variable", + name = "clean.expiredUser.retentionPeriod") +public class DeleteExpiredUsersJob extends BaseJob { + + public static final Logger LOGGER = LoggerFactory.getLogger(DeleteExpiredUsersJob.class); + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + private static final String RETENTION_PERIOD = "retentionPeriod"; + + private static final String USER_DELETION_TEMPLATE = "userDeletionNotification"; + + private static final String SELECT_EXPIRED_USERS = "SELECT u.id AS user_id, " + + "p.id AS project_id, u.email as user_email " + + "FROM users u " + + "LEFT JOIN api_keys ak ON u.id = ak.user_id " + + "LEFT JOIN project p ON u.login || '_personal' = p.name AND p.project_type = 'PERSONAL' " + + "WHERE (u.metadata->'metadata'->>'last_login')::BIGINT <= :retentionPeriod " + + "AND ( " + + "ak.user_id IS NULL " + + "OR (EXTRACT(EPOCH FROM ak.last_used_at) * 1000)::BIGINT <= :retentionPeriod " + + "OR NOT EXISTS (SELECT 1 FROM api_keys WHERE user_id = u.id AND last_used_at IS NOT NULL) " + + ") " + + "AND u.role != 'ADMINISTRATOR' " + + "GROUP BY u.id, p.id"; + + private static final String MOVE_ATTACHMENTS_TO_DELETE = + "WITH moved_rows AS (DELETE FROM attachment " + + "WHERE project_id = :projectId RETURNING id, file_id, thumbnail_id, creation_date) " + + "INSERT INTO attachment_deletion " + + "(id, file_id, thumbnail_id, creation_attachment_date, deletion_date) " + + "SELECT id, file_id, thumbnail_id, creation_date, NOW() FROM moved_rows"; + + private static final String DELETE_PROJECT_ISSUE_TYPES = + "DELETE FROM issue_type " + + "WHERE id IN (" + + " SELECT it.id " + + " FROM issue_type it " + + " JOIN issue_type_project itp ON it.id = itp.issue_type_id " + + " WHERE itp.project_id = :projectId " + + " AND it.locator NOT IN ('pb001', 'ab001', 'si001', 'ti001', 'nd001'))"; + + private static final String DELETE_USERS = "DELETE FROM users WHERE id IN (:userIds)"; + + private static final String DELETE_PROJECTS_BY_ID_LIST = + "DELETE FROM project WHERE id IN (:projectIds)"; + + private static final String FIND_NON_PERSONAL_PROJECTS_BY_USER_IDS = "SELECT p.id " + + "FROM project_user pu " + + "JOIN project p ON pu.project_id = p.id " + + "WHERE p.project_type != 'PERSONAL' AND pu.user_id IN (:userIds)"; + + @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") + private Long retentionPeriod; + + private final BlobStore blobStore; + + private final IndexerServiceClient indexerServiceClient; + + private final MessageBus messageBus; + + @Value("${datastore.bucketPrefix}") + private String bucketPrefix; + + @Autowired + public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, + NamedParameterJdbcTemplate namedParameterJdbcTemplate, + BlobStore blobStore, IndexerServiceClient indexerServiceClient, + MessageBus messageBus) { + super(jdbcTemplate); + this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + this.blobStore = blobStore; + this.indexerServiceClient = indexerServiceClient; + this.messageBus = messageBus; + } + + @Override + @Scheduled(cron = "${rp.environment.variable.clean.expiredUser.cron}") + @SchedulerLock(name = "deleteExpiredUsers", lockAtMostFor = "24h") + public void execute() { + if (ValidationUtil.isInvalidRetentionPeriod(retentionPeriod)) { + LOGGER.info("No users are deleted"); + return; + } + + List userProjects = findUsersAndPersonalProjects(); + List userIds = getUserIds(userProjects); + + List personalProjectIds = getProjectIds(userProjects); + List nonPersonalProjectsByUserIds = findNonPersonalProjectIdsByUserIds(userIds); + + deleteUsersByIds(userIds); + publishUnassignUserEvents(nonPersonalProjectsByUserIds); + personalProjectIds.forEach(this::deleteProjectAssociatedData); + deleteProjectsByIds(personalProjectIds); + + publishEmailNotificationEvents(getUserEmails(userProjects)); + + LOGGER.info("{} - users was deleted due to retention policy", userIds.size()); + } + + private void publishEmailNotificationEvents(List userEmails) { + List notifications = userEmails.stream() + .map(recipient -> new EmailNotificationRequest(recipient, USER_DELETION_TEMPLATE)) + .collect(Collectors.toList()); + messageBus.publishEmailNotificationEvents(notifications); + } + + private void publishUnassignUserEvents(List nonPersonalProjectsByUserIds) { + nonPersonalProjectsByUserIds.forEach( + projectId -> messageBus.publishActivity(new UnassignUserEvent(projectId))); + } + + private List findNonPersonalProjectIdsByUserIds(List userIds) { + return CollectionUtils.isEmpty(userIds) + ? Collections.emptyList() + : namedParameterJdbcTemplate.queryForList(FIND_NON_PERSONAL_PROJECTS_BY_USER_IDS, + Map.of("userIds", userIds), Long.class); + } + + private List findUsersAndPersonalProjects() { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue(RETENTION_PERIOD, lastLoginBorder()); + + RowMapper rowMapper = (rs, rowNum) -> { + UserProject userProject = new UserProject(); + userProject.setUserId(rs.getLong("user_id")); + userProject.setProjectId(rs.getLong("project_id")); + userProject.setEmail(rs.getString("user_email")); + return userProject; + }; + + return namedParameterJdbcTemplate.query(SELECT_EXPIRED_USERS, params, rowMapper); + } + + private void deleteProjectAssociatedData(Long projectId) { + deleteAttachmentsByProjectId(projectId); + deleteProjectIssueTypes(projectId); + indexerServiceClient.removeSuggest(projectId); + try { + blobStore.deleteContainer(bucketPrefix + projectId); + } catch (Exception e) { + LOGGER.warn("Cannot delete attachments bucket " + bucketPrefix + projectId); + } + indexerServiceClient.deleteIndex(projectId); + } + + private void deleteUsersByIds(List userIds) { + if (!userIds.isEmpty()) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("userIds", userIds); + namedParameterJdbcTemplate.update(DELETE_USERS, params); + messageBus.publishActivity(new UserDeletedEvent(userIds.size())); + } + } + + private void deleteProjectIssueTypes(Long projectId) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("projectId", projectId); + namedParameterJdbcTemplate.update(DELETE_PROJECT_ISSUE_TYPES, params); + } + + private void deleteAttachmentsByProjectId(Long projectId) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("projectId", projectId); + namedParameterJdbcTemplate.update(MOVE_ATTACHMENTS_TO_DELETE, params); + } + + private void deleteProjectsByIds(List projectIds) { + if (!projectIds.isEmpty()) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("projectIds", projectIds); + namedParameterJdbcTemplate.update(DELETE_PROJECTS_BY_ID_LIST, params); + messageBus.publishActivity(new ProjectDeletedEvent(projectIds.size())); + } + } + + private long lastLoginBorder() { + return LocalDateTime.now().minusDays(retentionPeriod).toInstant(ZoneOffset.UTC).toEpochMilli(); + } + + private List getUserIds(List userProjects) { + return userProjects.stream().map(UserProject::getUserId).collect(Collectors.toList()); + } + + private List getUserEmails(List userProjects) { + return userProjects.stream().map(UserProject::getEmail).collect(Collectors.toList()); + } + + private List getProjectIds(List userProjects) { + return userProjects.stream().filter(Objects::nonNull).map(UserProject::getProjectId) + .collect(Collectors.toList()); + } + + private static class UserProject { + + private long userId; + private long projectId; + private String email; + + public long getUserId() { + return userId; + } + + public void setUserId(long userId) { + this.userId = userId; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public long getProjectId() { + return projectId; + } + + public void setProjectId(long projectId) { + this.projectId = projectId; + } + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java new file mode 100644 index 0000000..2889f29 --- /dev/null +++ b/src/main/java/com/epam/reportportal/jobs/notification/NotifyUserExpirationJob.java @@ -0,0 +1,153 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.jobs.notification; + +import com.epam.reportportal.jobs.BaseJob; +import com.epam.reportportal.model.EmailNotificationRequest; +import com.epam.reportportal.service.MessageBus; +import com.epam.reportportal.utils.ValidationUtil; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * Notify users of oncoming deletion according to retention policy. + * + * @author Andrei Piankouski + */ +@Service +@ConditionalOnProperty(prefix = "rp.environment.variable", + name = "clean.expiredUser.retentionPeriod") +public class NotifyUserExpirationJob extends BaseJob { + + public static final Logger LOGGER = LoggerFactory.getLogger(NotifyUserExpirationJob.class); + + private static final String EMAIL = "email"; + private static final String USER_EXPIRATION_TEMPLATE = "userExpirationNotification"; + private static final String INACTIVITY_PERIOD = "inactivityPeriod"; + private static final String REMAINING_TIME = "remainingTime"; + private static final String DEADLINE_DATE = "deadlineDate"; + private static final String DAYS = " days"; + + private static final String SELECT_USERS_FOR_NOTIFY = "WITH user_last_action AS ( " + + "SELECT " + + " u.id as user_id, " + + " u.email as email, " + + "DATE_PART('day', NOW() - GREATEST(" + + " DATE(to_timestamp(CAST(u.metadata->'metadata'->>'last_login' AS bigint) / 1000)), " + + "MAX(ak.last_used_at))) AS inactivityPeriod " + + "FROM " + + " users u " + + " LEFT JOIN api_keys ak ON u.id = ak.user_id " + + "WHERE " + + " u.role != 'ADMINISTRATOR' " + + "GROUP BY " + + " u.id " + + ") " + + "SELECT " + + " user_last_action.user_id, " + + " user_last_action.email, " + + " user_last_action.inactivityPeriod, " + + " :retentionPeriod - inactivityPeriod as remainingTime " + + "FROM " + + " user_last_action " + + "WHERE " + + " :retentionPeriod - inactivityPeriod IN (1, 30, 60)"; + + private static final String RETENTION_PERIOD = "retentionPeriod"; + + @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") + private Long retentionPeriod; + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + private final MessageBus messageBus; + + @Autowired + public NotifyUserExpirationJob(JdbcTemplate jdbcTemplate, + NamedParameterJdbcTemplate namedParameterJdbcTemplate, + MessageBus messageBus) { + super(jdbcTemplate); + this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + this.messageBus = messageBus; + } + + @Override + @Scheduled(cron = "${rp.environment.variable.notification.expiredUser.cron}") + @SchedulerLock(name = "notifyUserExpiration", lockAtMostFor = "24h") + public void execute() { + if (ValidationUtil.isInvalidRetentionPeriod(retentionPeriod)) { + LOGGER.info("No notifications are send to users."); + return; + } + List notifications = getUsersForNotify(); + messageBus.publishEmailNotificationEvents(notifications); + } + + private List getUsersForNotify() { + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue(RETENTION_PERIOD, retentionPeriod); + return namedParameterJdbcTemplate.query( + SELECT_USERS_FOR_NOTIFY, parameters, (rs, rowNum) -> { + Map params = new HashMap<>(); + params.put(INACTIVITY_PERIOD, getInactivityPeriod(rs.getInt(INACTIVITY_PERIOD))); + params.put(REMAINING_TIME, getRemainingTime(rs.getInt(REMAINING_TIME))); + params.put(DEADLINE_DATE, getDeadlineDate(rs.getInt(REMAINING_TIME))); + EmailNotificationRequest emailNotificationRequest = + new EmailNotificationRequest(rs.getString(EMAIL), USER_EXPIRATION_TEMPLATE); + emailNotificationRequest.setParams(params); + return emailNotificationRequest; + }); + } + + private String getRemainingTime(int remainingTime) { + if (remainingTime == 1) { + return "tomorrow"; + } else if (remainingTime == 30) { + return "in 1 month"; + } else if (remainingTime == 60) { + return "in 2 months"; + } else { + return remainingTime + DAYS; + } + } + + private String getDeadlineDate(int remainingTime) { + return remainingTime == 1 + ? "today" + : "before " + LocalDate.now().plusDays(remainingTime) + ""; + } + + private String getInactivityPeriod(int inactivityPeriod) { + int inactivityMouths = inactivityPeriod / 30; + return retentionPeriod - inactivityPeriod == 1 ? "almost " + retentionPeriod / 30 + + " months" + : "the past " + inactivityMouths + " months"; + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java b/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java index e2c92fe..4a93e6b 100644 --- a/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java @@ -2,13 +2,12 @@ import com.epam.reportportal.log.LogMessage; import com.epam.reportportal.log.LogProcessing; +import java.util.Objects; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.messaging.handler.annotation.Payload; import org.springframework.stereotype.Service; -import java.util.Objects; - /** * Log consumer. * @@ -17,17 +16,19 @@ @Service @ConditionalOnProperty(prefix = "rp.elasticsearch", name = "host") public class SaveLogMessageJob { - public static final String LOG_MESSAGE_SAVING_QUEUE_NAME = "log_message_saving"; - private final LogProcessing logProcessing; - public SaveLogMessageJob(LogProcessing logProcessing) { - this.logProcessing = logProcessing; - } + public static final String LOG_MESSAGE_SAVING_QUEUE_NAME = "log_message_saving"; + private final LogProcessing logProcessing; + + public SaveLogMessageJob(LogProcessing logProcessing) { + this.logProcessing = logProcessing; + } - @RabbitListener(queues = LOG_MESSAGE_SAVING_QUEUE_NAME, containerFactory = "processingRabbitListenerContainerFactory") - public void execute(@Payload LogMessage logMessage) { - if (Objects.nonNull(logMessage)) { - this.logProcessing.add(logMessage); - } + @RabbitListener(queues = LOG_MESSAGE_SAVING_QUEUE_NAME, + containerFactory = "rabbitListenerContainerFactory") + public void execute(@Payload LogMessage logMessage) { + if (Objects.nonNull(logMessage)) { + this.logProcessing.add(logMessage); } + } } diff --git a/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java b/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java index 32bc71e..c145872 100644 --- a/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJob.java @@ -33,12 +33,10 @@ public CalculateAllocatedStorageJob(TaskExecutor projectAllocatedStorageExecutor @Scheduled(cron = "${rp.environment.variable.storage.project.cron}") @SchedulerLock(name = "calculateAllocatedStorage", lockAtMostFor = "24h") - public void calculate() { - logStart(); + public void execute() { CompletableFuture.allOf(getProjectIds().stream() .map(id -> CompletableFuture.runAsync(() -> updateAllocatedStorage(id), projectAllocatedStorageExecutor)) .toArray(CompletableFuture[]::new)).join(); - logFinish(); } private List getProjectIds() { diff --git a/src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java b/src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java new file mode 100644 index 0000000..d83f133 --- /dev/null +++ b/src/main/java/com/epam/reportportal/logging/ExecutionTimeAspect.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.logging; + +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * @author Andrei Piankouski + */ +@Aspect +@Component +public class ExecutionTimeAspect { + + public static final Logger LOGGER = LoggerFactory.getLogger(ExecutionTimeAspect.class); + + @Around("@annotation(annotation)") + public Object executionTime(ProceedingJoinPoint point, SchedulerLock annotation) throws Throwable { + String name = annotation.name(); + long startTime = System.currentTimeMillis(); + LOGGER.info("Job {} has been started.", name); + Object object = point.proceed(); + long endtime = System.currentTimeMillis(); + + LOGGER.info("Job {} has been finished. Time taken for Execution is : {} ms", name, (endtime-startTime)); + return object; + } +} \ No newline at end of file diff --git a/src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java b/src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java new file mode 100644 index 0000000..2aa4b8c --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/EmailNotificationRequest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model; + +import java.util.Map; + +/** + * EmailNotification model for rabbitMq topic. + * + * @author Andrei Piankouski + */ +public class EmailNotificationRequest { + + private String recipient; + + private String template; + + private Map params; + + public EmailNotificationRequest(String recipient, String template) { + this.recipient = recipient; + this.template = template; + } + + public String getRecipient() { + return recipient; + } + + public void setRecipient(String recipient) { + this.recipient = recipient; + } + + public String getTemplate() { + return template; + } + + public void setTemplate(String template) { + this.template = template; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + @Override + public String toString() { + return "EmailNotificationRequest{" + + "recipient='" + recipient + '\'' + + ", template='" + template + '\'' + + ", params=" + params + + '}'; + } +} diff --git a/src/main/java/com/epam/reportportal/model/activity/Activity.java b/src/main/java/com/epam/reportportal/model/activity/Activity.java new file mode 100644 index 0000000..0016b4e --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/Activity.java @@ -0,0 +1,238 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity; + +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; +import java.time.LocalDateTime; + +/** + * A model that represents the state of the Activity. + * + * @author Ryhor_Kukharenka + */ +public class Activity { + + private LocalDateTime createdAt; + private EventAction action; + private String eventName; + private EventPriority priority; + private Long objectId; + private String objectName; + private EventObject objectType; + private Long projectId; + private String projectName; + private Long subjectId; + private String subjectName; + private EventSubject subjectType; + private boolean isSavedEvent; + + public Activity() { + this.isSavedEvent = true; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(LocalDateTime createdAt) { + this.createdAt = createdAt; + } + + public EventAction getAction() { + return action; + } + + public void setAction(EventAction action) { + this.action = action; + } + + public String getEventName() { + return eventName; + } + + public void setEventName(String eventName) { + this.eventName = eventName; + } + + public EventPriority getPriority() { + return priority; + } + + public void setPriority(EventPriority priority) { + this.priority = priority; + } + + public Long getObjectId() { + return objectId; + } + + public void setObjectId(Long objectId) { + this.objectId = objectId; + } + + public String getObjectName() { + return objectName; + } + + public void setObjectName(String objectName) { + this.objectName = objectName; + } + + public EventObject getObjectType() { + return objectType; + } + + public void setObjectType(EventObject objectType) { + this.objectType = objectType; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Long getSubjectId() { + return subjectId; + } + + public void setSubjectId(Long subjectId) { + this.subjectId = subjectId; + } + + public String getSubjectName() { + return subjectName; + } + + public void setSubjectName(String subjectName) { + this.subjectName = subjectName; + } + + public EventSubject getSubjectType() { + return subjectType; + } + + public void setSubjectType(EventSubject subjectType) { + this.subjectType = subjectType; + } + + public boolean isSavedEvent() { + return isSavedEvent; + } + + public void setSavedEvent(boolean savedEvent) { + isSavedEvent = savedEvent; + } + + public static ActivityBuilder builder() { + return new ActivityBuilder(); + } + + + /** + * Activity builder. + * + * @author Ryhor_Kukharenka + */ + public static class ActivityBuilder { + + private final Activity activity; + + private ActivityBuilder() { + this.activity = new Activity(); + } + + public ActivityBuilder addCreatedNow() { + activity.setCreatedAt(LocalDateTime.now()); + return this; + } + + public ActivityBuilder addAction(EventAction action) { + activity.setAction(action); + return this; + } + + public ActivityBuilder addEventName(String eventName) { + activity.setEventName(eventName); + return this; + } + + public ActivityBuilder addPriority(EventPriority priority) { + activity.setPriority(priority); + return this; + } + + public ActivityBuilder addObjectId(Long objectId) { + activity.setObjectId(objectId); + return this; + } + + public ActivityBuilder addObjectName(String objectName) { + activity.setObjectName(objectName); + return this; + } + + public ActivityBuilder addObjectType(EventObject objectType) { + activity.setObjectType(objectType); + return this; + } + + public ActivityBuilder addProjectId(Long projectId) { + activity.setProjectId(projectId); + return this; + } + + public ActivityBuilder addProjectName(String projectName) { + activity.setProjectName(projectName); + return this; + } + + public ActivityBuilder addSubjectId(Long subjectId) { + activity.setSubjectId(subjectId); + return this; + } + + public ActivityBuilder addSubjectName(String subjectName) { + activity.setSubjectName(subjectName); + return this; + } + + public ActivityBuilder addSubjectType(EventSubject subjectType) { + activity.setSubjectType(subjectType); + return this; + } + + public Activity build() { + return activity; + } + + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java b/src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java new file mode 100644 index 0000000..3999535 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/ActivityEvent.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity; + +/** + * Activity Event. + * + * @author Ryhor_Kukharenka + */ +public interface ActivityEvent { + + /** + * Method for transform Event to Activity. + * + * @return Activity entity + */ + Activity toActivity(); + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java b/src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java new file mode 100644 index 0000000..62ed204 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/ActivityAction.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Activity Action Type (event_name). + * + * @author Ryhor_Kukharenka + */ +public enum ActivityAction { + + UNASSIGN_USER("unassignUser"), + + BULK_DELETE_USERS("bulkDeleteUsers"), + + BULK_DELETE_PROJECT("bulkDeleteProject"); + + private final String value; + + ActivityAction(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java new file mode 100644 index 0000000..8565523 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventAction.java @@ -0,0 +1,29 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Action Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventAction { + + UNASSIGN, + BULK_DELETE + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java new file mode 100644 index 0000000..ea4118b --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventObject.java @@ -0,0 +1,29 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Object Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventObject { + + USER, + PROJECT + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java new file mode 100644 index 0000000..1260740 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventPriority.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Priority Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventPriority { + + CRITICAL, + HIGH, + MEDIUM + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java b/src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java new file mode 100644 index 0000000..f03146f --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/enums/EventSubject.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.enums; + +/** + * Event Subject Type. + * + * @author Ryhor_Kukharenka + */ +public enum EventSubject { + + APPLICATION("Job Service"); + + private final String applicationName; + + EventSubject(String applicationName) { + this.applicationName = applicationName; + } + + public String getApplicationName() { + return applicationName; + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java new file mode 100644 index 0000000..19c6749 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/event/ProjectDeletedEvent.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.event; + +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.model.activity.enums.ActivityAction; +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; +import org.apache.commons.lang3.StringUtils; + +/** + * Publish an event when project is deleted. + * + * @author Ryhor_Kukharenka + */ +public class ProjectDeletedEvent implements ActivityEvent { + + private final int countProjectsDeleted; + + public ProjectDeletedEvent(int countProjectsDeleted) { + this.countProjectsDeleted = countProjectsDeleted; + } + + @Override + public Activity toActivity() { + return Activity.builder() + .addCreatedNow() + .addAction(EventAction.BULK_DELETE) + .addEventName(ActivityAction.BULK_DELETE_PROJECT.getValue()) + .addObjectName(getFormatText()) + .addObjectType(EventObject.PROJECT) + .addSubjectName(EventSubject.APPLICATION.getApplicationName()) + .addSubjectType(EventSubject.APPLICATION) + .addPriority(EventPriority.CRITICAL) + .build(); + } + + private String getFormatText() { + String projectWord = "project" + (countProjectsDeleted == 1 ? StringUtils.EMPTY : "s"); + return String.format("%d personal %s", countProjectsDeleted, projectWord); + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java new file mode 100644 index 0000000..e0c4861 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/event/UnassignUserEvent.java @@ -0,0 +1,55 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.event; + +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.model.activity.enums.ActivityAction; +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; + +/** + * Event publish when user is unassigned to a project. + * + * @author Ryhor_Kukharenka + */ +public class UnassignUserEvent implements ActivityEvent { + + private final Long projectId; + + public UnassignUserEvent(Long projectId) { + this.projectId = projectId; + } + + @Override + public Activity toActivity() { + return Activity.builder() + .addCreatedNow() + .addAction(EventAction.UNASSIGN) + .addEventName(ActivityAction.UNASSIGN_USER.getValue()) + .addObjectType(EventObject.USER) + .addObjectName("deleted_user") + .addProjectId(projectId) + .addSubjectName(EventSubject.APPLICATION.getApplicationName()) + .addSubjectType(EventSubject.APPLICATION) + .addPriority(EventPriority.MEDIUM) + .build(); + } + +} diff --git a/src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java b/src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java new file mode 100644 index 0000000..0a97510 --- /dev/null +++ b/src/main/java/com/epam/reportportal/model/activity/event/UserDeletedEvent.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.model.activity.event; + +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.model.activity.enums.ActivityAction; +import com.epam.reportportal.model.activity.enums.EventAction; +import com.epam.reportportal.model.activity.enums.EventObject; +import com.epam.reportportal.model.activity.enums.EventPriority; +import com.epam.reportportal.model.activity.enums.EventSubject; +import org.apache.commons.lang3.StringUtils; + +/** + * Publish an event when user is deleted. + * + * @author Ryhor_Kukharenka + */ +public class UserDeletedEvent implements ActivityEvent { + + private final int countUsersDeleted; + + public UserDeletedEvent(int countUsersDeleted) { + this.countUsersDeleted = countUsersDeleted; + } + + @Override + public Activity toActivity() { + return Activity.builder() + .addCreatedNow() + .addAction(EventAction.BULK_DELETE) + .addEventName(ActivityAction.BULK_DELETE_USERS.getValue()) + .addObjectName(getFormatText()) + .addObjectType(EventObject.USER) + .addSubjectName(EventSubject.APPLICATION.getApplicationName()) + .addSubjectType(EventSubject.APPLICATION) + .addPriority(EventPriority.HIGH) + .build(); + } + + private String getFormatText() { + String userWord = "user" + (countUsersDeleted == 1 ? StringUtils.EMPTY : "s"); + return String.format("%d deleted %s", countUsersDeleted, userWord); + } + +} diff --git a/src/main/java/com/epam/reportportal/service/MessageBus.java b/src/main/java/com/epam/reportportal/service/MessageBus.java new file mode 100644 index 0000000..ebfe191 --- /dev/null +++ b/src/main/java/com/epam/reportportal/service/MessageBus.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.service; + +import com.epam.reportportal.model.EmailNotificationRequest; +import com.epam.reportportal.model.activity.ActivityEvent; +import java.util.List; + +/** + * MessageBus is an abstraction for dealing with events over external event-streaming system. + * + * @author Ryhor_Kukharenka + */ +public interface MessageBus { + + void publishActivity(ActivityEvent event); + + void publishEmailNotificationEvents(List notifications); + +} diff --git a/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java b/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java new file mode 100644 index 0000000..f9882ac --- /dev/null +++ b/src/main/java/com/epam/reportportal/service/impl/MessageBusImpl.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.service.impl; + +import static com.epam.reportportal.config.rabbit.InternalConfiguration.EXCHANGE_NOTIFICATION; +import static com.epam.reportportal.config.rabbit.InternalConfiguration.QUEUE_EMAIL; + +import com.epam.reportportal.model.EmailNotificationRequest; +import com.epam.reportportal.model.activity.Activity; +import com.epam.reportportal.model.activity.ActivityEvent; +import com.epam.reportportal.service.MessageBus; +import java.util.List; +import java.util.Objects; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +/** + * MessageBus implementation using RabbitMQ for transfer message. + * + * @author Ryhor_Kukharenka + */ +@Service +public class MessageBusImpl implements MessageBus { + + private final RabbitTemplate rabbitTemplate; + + public MessageBusImpl(@Qualifier("rabbitTemplate") RabbitTemplate rabbitTemplate) { + this.rabbitTemplate = rabbitTemplate; + } + + /** + * Publishes activity to the queue with the following routing key. + * + * @param event Activity event to be converted to Activity object + * @author Ryhor_Kuharenka + */ + @Override + public void publishActivity(ActivityEvent event) { + final Activity activity = event.toActivity(); + if (Objects.nonNull(activity)) { + rabbitTemplate.convertAndSend("activity", generateKeyForActivity(activity), activity); + } + } + + private String generateKeyForActivity(Activity activity) { + return String.format("activity.%d.%s.%s", + activity.getProjectId(), + activity.getObjectType(), + activity.getEventName()); + } + + @Override + public void publishEmailNotificationEvents(List notifications) { + notifications.forEach(notification -> + rabbitTemplate.convertAndSend(EXCHANGE_NOTIFICATION, QUEUE_EMAIL, notification)); + } + +} diff --git a/src/main/java/com/epam/reportportal/utils/ValidationUtil.java b/src/main/java/com/epam/reportportal/utils/ValidationUtil.java new file mode 100644 index 0000000..6229bf5 --- /dev/null +++ b/src/main/java/com/epam/reportportal/utils/ValidationUtil.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.utils; + +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class which validate some information. + * + * @author Ryhor_Kukharenka + */ +public final class ValidationUtil { + + public static final Logger LOGGER = LoggerFactory.getLogger(ValidationUtil.class); + + private ValidationUtil() { + } + + /** + * Check retention period. + * + * @param retentionPeriod retentionPeriod + * @return boolean value + */ + public static boolean isInvalidRetentionPeriod(Long retentionPeriod) { + if (Objects.isNull(retentionPeriod) || retentionPeriod <= 0) { + LOGGER.warn("The parameter 'retentionPeriod' is not specified correctly.\n" + + "Must be greater than 0.\n" + + "The current value of the parameter = {}.", retentionPeriod); + return true; + } + return false; + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index cffed6a..5768f16 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -5,15 +5,19 @@ rp: variable: elements-counter: batch-size: 50 + notification: + expiredUser: + ## 24 hours + cron: '0 0 */24 * * *' clean: storage: - ## 30 seconds - cron: '*/30 * * * * *' + ## 24 hours + cron: '0 0 */24 * * *' chunkSize: 1000 batchSize: 100 attachment: - ## 2 minutes - cron: '0 */2 * * * *' + ## 24 hours + cron: '0 0 */24 * * *' log: ## 5 minutes cron: '0 */5 * * * *' @@ -23,9 +27,11 @@ rp: view: ## 24 hours cron: '0 0 */24 * * *' - ## 2 hours liveTimeout: 7200 batch: 100 + expiredUser: + ## 24 hours + cron: '0 0 */24 * * *' storage: project: ## 1 minute diff --git a/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java b/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java index d98cc33..6ee3940 100644 --- a/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java +++ b/src/test/java/com/epam/reportportal/jobs/storage/CalculateAllocatedStorageJobTest.java @@ -52,7 +52,7 @@ void shouldUpdateAllocatedStorageForAllProjects() { when(jdbcTemplate.queryForList(SELECT_PROJECT_IDS_QUERY, Long.class)).thenReturn(projectIds); when(jdbcTemplate.queryForObject(eq(SELECT_FILE_SIZE_SUM_BY_PROJECT_ID_QUERY), eq(Long.class), anyLong())).thenReturn(1000L); - calculateAllocatedStorageJob.calculate(); + calculateAllocatedStorageJob.execute(); verify(jdbcTemplate, times(2)).queryForObject(eq(SELECT_FILE_SIZE_SUM_BY_PROJECT_ID_QUERY), eq(Long.class), anyLong());