diff --git a/.github/workflows/check-md-link.yml b/.github/workflows/check-md-link.yml index 029f676c5..093496c85 100644 --- a/.github/workflows/check-md-link.yml +++ b/.github/workflows/check-md-link.yml @@ -41,5 +41,5 @@ jobs: with: use-quiet-mode: 'yes' config-file: '.github/workflows/check-md-link-config.json' - folder-path: 'regtests, .github, build-logic, polaris-core, service, dropwizard, extension, spec, k8, getting-started, helm' + folder-path: 'regtests, .github, build-logic, polaris-core, service, quarkus, extension, spec, k8, getting-started, helm' file-path: 'CHAT_BYLAWS.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, README.md, SECURITY.md' diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 830363adb..4cd317bf0 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -69,7 +69,7 @@ jobs: name: upload-test-artifacts path: | polaris-core/build/test-results/test - dropwizard/service/build/test-results/test + quarkus/service/build/test-results/test # NOTE: The Gradle Wrapper is the default and recommended way to run Gradle (https://docs.gradle.org/current/userguide/gradle_wrapper.html). # If your project does not have the Gradle Wrapper configured, you can use the following configuration to run Gradle with a specified version. diff --git a/Dockerfile b/Dockerfile index 9477cf07f..3e7c8c8ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,16 +32,36 @@ WORKDIR /app RUN rm -rf build # Build the rest catalog -RUN ./gradlew --no-daemon --info ${ECLIPSELINK_DEPS+"-PeclipseLinkDeps=$ECLIPSELINK_DEPS"} -PeclipseLink=$ECLIPSELINK clean prepareDockerDist +RUN ./gradlew --no-daemon --info ${ECLIPSELINK_DEPS+"-PeclipseLinkDeps=$ECLIPSELINK_DEPS"} -PeclipseLink=$ECLIPSELINK clean :polaris-quarkus-service:build -x test FROM registry.access.redhat.com/ubi9/openjdk-21-runtime:1.20-2.1729089285 -WORKDIR /app -COPY --from=build /app/dropwizard/service/build/docker-dist/bin /app/bin -COPY --from=build /app/dropwizard/service/build/docker-dist/lib /app/lib -COPY --from=build /app/polaris-server.yml /app + +LABEL org.opencontainers.image.source=https://github.com/apache/polaris +LABEL org.opencontainers.image.description="Apache Polaris (incubating)" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +ENV LANGUAGE='en_US:en' + +USER root +RUN groupadd --gid 10001 polaris \ + && useradd --uid 10000 --gid polaris polaris \ + && chown -R polaris:polaris /opt/jboss/container \ + && chown -R polaris:polaris /deployments + +USER polaris +WORKDIR /home/polaris +ENV USER=polaris +ENV UID=10000 +ENV HOME=/home/polaris + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/lib/ /deployments/lib/ +COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/*.jar /deployments/ +COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/app/ /deployments/app/ +COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/quarkus/ /deployments/quarkus/ EXPOSE 8181 +EXPOSE 8182 -# Run the resulting java binary -ENTRYPOINT ["/app/bin/polaris-service"] -CMD ["server", "polaris-server.yml"] +ENV AB_JOLOKIA_OFF="" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/LICENSE-BINARY-DIST b/LICENSE-BINARY-DIST index 6ab97ac6d..a50dabe28 100644 --- a/LICENSE-BINARY-DIST +++ b/LICENSE-BINARY-DIST @@ -208,6 +208,8 @@ Apache Polaris distributions contain some or all of the following dependencies, licensed under Apache License: +com.aayushatharva.brotli4j:brotli4j +com.aayushatharva.brotli4j:service com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.core:jackson-databind @@ -229,6 +231,7 @@ com.google.android:annotations com.google.api-client:google-api-client com.google.api.grpc:gapic-google-cloud-storage-v2 com.google.api.grpc:grpc-google-cloud-storage-v2 +com.google.api.grpc:proto-google-cloud-monitoring-v3 com.google.api.grpc:proto-google-cloud-storage-v2 com.google.api.grpc:proto-google-common-protos com.google.api.grpc:proto-google-iam-v1 @@ -236,20 +239,19 @@ com.google.api:api-common com.google.api:gax com.google.api:gax-grpc com.google.api:gax-httpjson -com.google.api.grpc:proto-google-cloud-monitoring-v3 com.google.apis:google-api-services-storage com.google.auth:google-auth-library-credentials com.google.auth:google-auth-library-oauth2-http com.google.auto.value:auto-value-annotations -com.google.cloud:google-cloud-core -com.google.cloud:google-cloud-core-grpc -com.google.cloud:google-cloud-core-http -com.google.cloud:google-cloud-storage -com.google.cloud:google-cloud-monitoring com.google.cloud.opentelemetry:detector-resources-support com.google.cloud.opentelemetry:exporter-metrics com.google.cloud.opentelemetry:shared-resourcemapping +com.google.cloud:google-cloud-core +com.google.cloud:google-cloud-core-grpc +com.google.cloud:google-cloud-core-http com.google.cloud:google-cloud-monitorin +com.google.cloud:google-cloud-monitoring +com.google.cloud:google-cloud-storage com.google.code.findbugs:jsr305 com.google.code.gson:gson com.google.errorprone:error_prone_annotations @@ -278,34 +280,8 @@ commons-io:commons-io commons-logging:commons-logging commons-net:commons-net dev.failsafe:failsafe +info.picocli:picocli io.airlift:aircompressor -io.dropwizard.logback:logback-throttling-appender -io.dropwizard.metrics:metrics-annotation -io.dropwizard.metrics:metrics-caffeine -io.dropwizard.metrics:metrics-core -io.dropwizard.metrics:metrics-healthchecks -io.dropwizard.metrics:metrics-jakarta-servlets -io.dropwizard.metrics:metrics-jersey3 -io.dropwizard.metrics:metrics-jetty11 -io.dropwizard.metrics:metrics-jmx -io.dropwizard.metrics:metrics-json -io.dropwizard.metrics:metrics-jvm -io.dropwizard.metrics:metrics-logback -io.dropwizard:dropwizard-auth -io.dropwizard:dropwizard-configuration -io.dropwizard:dropwizard-core -io.dropwizard:dropwizard-health -io.dropwizard:dropwizard-jackson -io.dropwizard:dropwizard-jersey -io.dropwizard:dropwizard-jetty -io.dropwizard:dropwizard-json-logging -io.dropwizard:dropwizard-lifecycle -io.dropwizard:dropwizard-logging -io.dropwizard:dropwizard-metrics -io.dropwizard:dropwizard-request-logging -io.dropwizard:dropwizard-servlets -io.dropwizard:dropwizard-util -io.dropwizard:dropwizard-validation io.grpc:grpc-alts io.grpc:grpc-api io.grpc:grpc-auth @@ -314,6 +290,7 @@ io.grpc:grpc-core io.grpc:grpc-googleapis io.grpc:grpc-grpclb io.grpc:grpc-inprocess +io.grpc:grpc-netty io.grpc:grpc-netty-shaded io.grpc:grpc-opentelemetry io.grpc:grpc-protobuf @@ -327,9 +304,11 @@ io.micrometer:micrometer-commons io.micrometer:micrometer-core io.micrometer:micrometer-observation io.micrometer:micrometer-registry-prometheus +io.micrometer:micrometer-registry-prometheus-simpleclient io.netty:netty-buffer io.netty:netty-codec io.netty:netty-codec-dns +io.netty:netty-codec-haproxy io.netty:netty-codec-http io.netty:netty-codec-http2 io.netty:netty-codec-socks @@ -351,18 +330,30 @@ io.netty:netty-transport-native-unix-common io.opencensus:opencensus-api io.opencensus:opencensus-contrib-http-util io.opencensus:opencensus-proto +io.opentelemetry.contrib:opentelemetry-gcp-resources +io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations +io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations-support +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17 +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8 io.opentelemetry.semconv:opentelemetry-semconv +io.opentelemetry.semconv:opentelemetry-semconv-incubating io.opentelemetry:opentelemetry-api io.opentelemetry:opentelemetry-api-incubator io.opentelemetry:opentelemetry-context +io.opentelemetry:opentelemetry-exporter-common io.opentelemetry:opentelemetry-exporter-logging +io.opentelemetry:opentelemetry-exporter-otlp +io.opentelemetry:opentelemetry-exporter-otlp-common +io.opentelemetry:opentelemetry-exporter-sender-okhttp io.opentelemetry:opentelemetry-sdk io.opentelemetry:opentelemetry-sdk-common +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi io.opentelemetry:opentelemetry-sdk-logs io.opentelemetry:opentelemetry-sdk-metrics io.opentelemetry:opentelemetry-sdk-trace -io.opentelemetry.contrib:opentelemetry-gcp-resources io.perfmark:perfmark-api io.projectreactor.netty:reactor-netty-core io.projectreactor.netty:reactor-netty-http @@ -375,18 +366,104 @@ io.prometheus:prometheus-metrics-exposition-formats io.prometheus:prometheus-metrics-model io.prometheus:prometheus-metrics-shaded-protobuf io.prometheus:prometheus-metrics-tracer-common +io.prometheus:simpleclient +io.prometheus:simpleclient_common +io.prometheus:simpleclient_tracer_common +io.prometheus:simpleclient_tracer_otel +io.prometheus:simpleclient_tracer_otel_agent +io.quarkus.arc:arc +io.quarkus.resteasy.reactive:resteasy-reactive +io.quarkus.resteasy.reactive:resteasy-reactive-common +io.quarkus.resteasy.reactive:resteasy-reactive-common-types +io.quarkus.resteasy.reactive:resteasy-reactive-jackson +io.quarkus.resteasy.reactive:resteasy-reactive-vertx +io.quarkus.security:quarkus-security +io.quarkus.vertx.utils:quarkus-vertx-utils +io.quarkus:quarkus-arc +io.quarkus:quarkus-bootstrap-runner +io.quarkus:quarkus-classloader-commons +io.quarkus:quarkus-container-image +io.quarkus:quarkus-container-image-docker +io.quarkus:quarkus-container-image-docker-common +io.quarkus:quarkus-core +io.quarkus:quarkus-credentials +io.quarkus:quarkus-development-mode-spi +io.quarkus:quarkus-fs-util +io.quarkus:quarkus-grpc-common +io.quarkus:quarkus-hibernate-validator +io.quarkus:quarkus-ide-launcher +io.quarkus:quarkus-jackson +io.quarkus:quarkus-jsonp +io.quarkus:quarkus-logging-json +io.quarkus:quarkus-micrometer +io.quarkus:quarkus-micrometer-registry-prometheus +io.quarkus:quarkus-mutiny +io.quarkus:quarkus-netty +io.quarkus:quarkus-opentelemetry +io.quarkus:quarkus-picocli +io.quarkus:quarkus-reactive-routes +io.quarkus:quarkus-rest +io.quarkus:quarkus-rest-common +io.quarkus:quarkus-rest-jackson +io.quarkus:quarkus-rest-jackson-common +io.quarkus:quarkus-security-runtime-spi +io.quarkus:quarkus-smallrye-context-propagation +io.quarkus:quarkus-smallrye-health +io.quarkus:quarkus-tls-registry +io.quarkus:quarkus-vertx +io.quarkus:quarkus-vertx-http +io.quarkus:quarkus-vertx-latebound-mdc-provider +io.quarkus:quarkus-virtual-threads io.smallrye.common:smallrye-common-annotation io.smallrye.common:smallrye-common-classloader io.smallrye.common:smallrye-common-constraint +io.smallrye.common:smallrye-common-cpu io.smallrye.common:smallrye-common-expression io.smallrye.common:smallrye-common-function +io.smallrye.common:smallrye-common-io +io.smallrye.common:smallrye-common-net +io.smallrye.common:smallrye-common-os +io.smallrye.common:smallrye-common-ref +io.smallrye.common:smallrye-common-vertx-context io.smallrye.config:smallrye-config io.smallrye.config:smallrye-config-common io.smallrye.config:smallrye-config-core +io.smallrye.config:smallrye-config-validator +io.smallrye.reactive:mutiny +io.smallrye.reactive:mutiny-smallrye-context-propagation +io.smallrye.reactive:mutiny-zero-flow-adapters +io.smallrye.reactive:smallrye-mutiny-vertx-auth-common +io.smallrye.reactive:smallrye-mutiny-vertx-bridge-common +io.smallrye.reactive:smallrye-mutiny-vertx-core +io.smallrye.reactive:smallrye-mutiny-vertx-runtime +io.smallrye.reactive:smallrye-mutiny-vertx-uri-template +io.smallrye.reactive:smallrye-mutiny-vertx-web +io.smallrye.reactive:smallrye-mutiny-vertx-web-common +io.smallrye.reactive:vertx-mutiny-generator +io.smallrye:smallrye-context-propagation +io.smallrye:smallrye-context-propagation-api +io.smallrye:smallrye-context-propagation-storage +io.smallrye:smallrye-fault-tolerance-vertx +io.smallrye:smallrye-health +io.smallrye:smallrye-health-api +io.smallrye:smallrye-health-provided-checks io.swagger:swagger-annotations io.swagger:swagger-core io.swagger:swagger-jaxrs io.swagger:swagger-models +io.vertx:vertx-auth-common +io.vertx:vertx-bridge-common +io.vertx:vertx-codegen +io.vertx:vertx-core +io.vertx:vertx-grpc +io.vertx:vertx-grpc-client +io.vertx:vertx-grpc-common +io.vertx:vertx-grpc-server +io.vertx:vertx-uri-template +io.vertx:vertx-web +io.vertx:vertx-web-common +jakarta.enterprise:jakarta.enterprise.cdi-api +jakarta.enterprise:jakarta.enterprise.lang-model jakarta.inject:jakarta.inject-api jakarta.validation:jakarta.validation-api javax.inject:javax.inject @@ -405,13 +482,14 @@ org.apache.curator:curator-client org.apache.curator:curator-framework org.apache.curator:curator-recipes org.apache.hadoop.thirdparty:hadoop-shaded-guava +org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_21 org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7 org.apache.hadoop:hadoop-annotations org.apache.hadoop:hadoop-auth org.apache.hadoop:hadoop-client-api +org.apache.hadoop:hadoop-client-runtime org.apache.hadoop:hadoop-common org.apache.hadoop:hadoop-hdfs-client -org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_21 org.apache.httpcomponents.client5:httpclient5 org.apache.httpcomponents.core5:httpcore5 org.apache.httpcomponents.core5:httpcore5-h2 @@ -452,9 +530,12 @@ org.eclipse.jetty:jetty-server org.eclipse.jetty:jetty-servlet org.eclipse.jetty:jetty-servlets org.eclipse.jetty:jetty-util +org.eclipse.jetty:jetty-util-ajax org.eclipse.jetty:jetty-webapp org.eclipse.jetty:jetty-xml org.eclipse.microprofile.config:microprofile-config-api +org.eclipse.microprofile.context-propagation:microprofile-context-propagation-api +org.eclipse.microprofile.health:microprofile-health-api org.glassfish.jersey.containers:jersey-container-servlet org.glassfish.jersey.containers:jersey-container-servlet-core org.glassfish.jersey.core:jersey-client @@ -465,7 +546,13 @@ org.glassfish.jersey.ext:jersey-metainf-services org.glassfish.jersey.inject:jersey-hk2 org.hibernate.validator:hibernate-validator org.javassist:javassist +org.jboss.logging:commons-logging-jboss-logging org.jboss.logging:jboss-logging +org.jboss.logging:jboss-logging-annotations +org.jboss.logmanager:jboss-logmanager +org.jboss.slf4j:slf4j-jboss-logmanager +org.jboss.threads:jboss-threads +org.jctools:jctools-core org.jetbrains.kotlin:kotlin-stdlib org.jetbrains.kotlin:kotlin-stdlib-common org.ow2.asm:asm @@ -473,6 +560,7 @@ org.reflections:reflections org.roaringbitmap:RoaringBitmap org.slf4j:jcl-over-slf4j org.slf4j:log4j-over-slf4j +org.wildfly.common:wildfly-common org.xerial.snappy:snappy-java org.yaml:snakeyaml software.amazon.awssdk:annotations @@ -6919,4 +7007,703 @@ org.threeten:threetenbp PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +--- +io.github.crac:org-crac +jakarta.interceptor:jakarta.interceptor-api +jakarta.json:jakarta.json-api +jakarta.transaction:jakarta.transaction-api +org.eclipse.parsson:jakarta.json +org.eclipse.parsson:parsson +org.glassfish.expressly:expressly + + +# Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + + "Contributor" means any person or entity that Distributes the Program. + + "Licensed Patents" mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions Distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement + or any Secondary License (as applicable), including Contributors. + + "Derivative Works" shall mean any work, whether in Source Code or other + form, that is based on (or derived from) the Program and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. + + "Modified Works" shall mean any work in Source Code or other form that + results from an addition to, deletion from, or modification of the + contents of the Program, including, for purposes of clarity any new file + in Source Code form that contains any contents of the Program. Modified + Works shall not include works that contain only declarations, + interfaces, types, classes, structures, or files of the Program solely + in each case in order to link to, bind by name, or subclass the Program + or Modified Works thereof. + + "Distribute" means the acts of a) distributing or b) making available + in any manner that enables the transfer of a copy. + + "Source Code" means the form of a Program preferred for making + modifications, including but not limited to software source code, + documentation source, and configuration files. + + "Secondary License" means either the GNU General Public License, + Version 2.0, or any later versions of that license, including any + exceptions or additional permissions as identified by the initial + Contributor. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + + 3. REQUIREMENTS + + 3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + + 3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + + 3.3 Contributors may not remove or alter any copyright, patent, + trademark, attribution notices, disclaimers of warranty, or limitations + of liability ("notices") contained within the Program from any copy of + the Program which they Distribute, provided that Contributors may add + their own appropriate notices. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product + offering should do so in a manner which does not create potential + liability for other Contributors. Therefore, if a Contributor includes + the Program in a commercial product offering, such Contributor + ("Commercial Contributor") hereby agrees to defend and indemnify every + other Contributor ("Indemnified Contributor") against any losses, + damages and costs (collectively "Losses") arising from claims, lawsuits + and other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the Program + in a commercial product offering. The obligations in this section do not + apply to any claims or Losses relating to any actual or alleged + intellectual property infringement. In order to qualify, an Indemnified + Contributor must: a) promptly notify the Commercial Contributor in + writing of such claim, and b) allow the Commercial Contributor to control, + and cooperate with the Commercial Contributor in, the defense and any + related settlement negotiations. The Indemnified Contributor may + participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those performance + claims and warranties, and if a court requires any other Contributor to + pay any damages as a result, the Commercial Contributor must pay + those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT + PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" + BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR + IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF + TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR + PURPOSE. Each Recipient is solely responsible for determining the + appropriateness of using and distributing the Program and assumes all + risks associated with its exercise of rights under this Agreement, + including but not limited to the risks and costs of program errors, + compliance with applicable laws, damage to or loss of data, programs + or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT + PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS + SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST + PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE + EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that the + Program itself (excluding combinations of the Program with other software + or hardware) infringes such Recipient's patent(s), then such Recipient's + rights granted under Section 2(b) shall terminate as of the date such + litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it + fails to comply with any of the material terms or conditions of this + Agreement and does not cure such failure in a reasonable period of + time after becoming aware of such noncompliance. If all Recipient's + rights under this Agreement terminate, Recipient agrees to cease use + and distribution of the Program as soon as reasonably practicable. + However, Recipient's obligations under this Agreement and any licenses + granted by Recipient relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and + may only be modified in the following manner. The Agreement Steward + reserves the right to publish new versions (including revisions) of + this Agreement from time to time. No one other than the Agreement + Steward has the right to modify this Agreement. The Eclipse Foundation + is the initial Agreement Steward. The Eclipse Foundation may assign the + responsibility to serve as the Agreement Steward to a suitable separate + entity. Each new version of the Agreement will be given a distinguishing + version number. The Program (including Contributions) may always be + Distributed subject to the version of the Agreement under which it was + received. In addition, after a new version of the Agreement is published, + Contributor may elect to Distribute the Program (including its + Contributions) under the new version. + + Except as expressly stated in Sections 2(a) and 2(b) above, Recipient + receives no rights or licenses to the intellectual property of any + Contributor under this Agreement, whether expressly, by implication, + estoppel or otherwise. All rights in the Program not expressly granted + under this Agreement are reserved. Nothing in this Agreement is intended + to be enforceable by any entity that is not a Contributor or Recipient. + No third-party beneficiary rights are created under this Agreement. + + Exhibit A - Form of Secondary Licenses Notice + + "This Source Code may also be made available under the following + Secondary Licenses when the conditions for such availability set forth + in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), + version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. + +--- + +## The GNU General Public License (GPL) Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor + Boston, MA 02110-1335 + USA + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your freedom to + share and change it. By contrast, the GNU General Public License is + intended to guarantee your freedom to share and change free software--to + make sure the software is free for all its users. This General Public + License applies to most of the Free Software Foundation's software and + to any other program whose authors commit to using it. (Some other Free + Software Foundation software is covered by the GNU Library General + Public License instead.) You can apply it to your programs, too. + + When we speak of free software, we are referring to freedom, not price. + Our General Public Licenses are designed to make sure that you have the + freedom to distribute copies of free software (and charge for this + service if you wish), that you receive source code or can get it if you + want it, that you can change the software or use pieces of it in new + free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid anyone + to deny you these rights or to ask you to surrender the rights. These + restrictions translate to certain responsibilities for you if you + distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether gratis + or for a fee, you must give the recipients all the rights that you have. + You must make sure that they, too, receive or can get the source code. + And you must show them these terms so they know their rights. + + We protect your rights with two steps: (1) copyright the software, and + (2) offer you this license which gives you legal permission to copy, + distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain + that everyone understands that there is no warranty for this free + software. If the software is modified by someone else and passed on, we + want its recipients to know that what they have is not the original, so + that any problems introduced by others will not reflect on the original + authors' reputations. + + Finally, any free program is threatened constantly by software patents. + We wish to avoid the danger that redistributors of a free program will + individually obtain patent licenses, in effect making the program + proprietary. To prevent this, we have made it clear that any patent must + be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and + modification follow. + + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains a + notice placed by the copyright holder saying it may be distributed under + the terms of this General Public License. The "Program", below, refers + to any such program or work, and a "work based on the Program" means + either the Program or any derivative work under copyright law: that is + to say, a work containing the Program or a portion of it, either + verbatim or with modifications and/or translated into another language. + (Hereinafter, translation is included without limitation in the term + "modification".) Each licensee is addressed as "you". + + Activities other than copying, distribution and modification are not + covered by this License; they are outside its scope. The act of running + the Program is not restricted, and the output from the Program is + covered only if its contents constitute a work based on the Program + (independent of having been made by running the Program). Whether that + is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's source + code as you receive it, in any medium, provided that you conspicuously + and appropriately publish on each copy an appropriate copyright notice + and disclaimer of warranty; keep intact all the notices that refer to + this License and to the absence of any warranty; and give any other + recipients of the Program a copy of this License along with the Program. + + You may charge a fee for the physical act of transferring a copy, and + you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion of + it, thus forming a work based on the Program, and copy and distribute + such modifications or work under the terms of Section 1 above, provided + that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any part + thereof, to be licensed as a whole at no charge to all third parties + under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a notice + that there is no warranty (or else, saying that you provide a + warranty) and that users may redistribute the program under these + conditions, and telling the user how to view a copy of this License. + (Exception: if the Program itself is interactive but does not + normally print such an announcement, your work based on the Program + is not required to print an announcement.) + + These requirements apply to the modified work as a whole. If + identifiable sections of that work are not derived from the Program, and + can be reasonably considered independent and separate works in + themselves, then this License, and its terms, do not apply to those + sections when you distribute them as separate works. But when you + distribute the same sections as part of a whole which is a work based on + the Program, the distribution of the whole must be on the terms of this + License, whose permissions for other licensees extend to the entire + whole, and thus to each and every part regardless of who wrote it. + + Thus, it is not the intent of this section to claim rights or contest + your rights to work written entirely by you; rather, the intent is to + exercise the right to control the distribution of derivative or + collective works based on the Program. + + In addition, mere aggregation of another work not based on the Program + with the Program (or with a work based on the Program) on a volume of a + storage or distribution medium does not bring the other work under the + scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, + under Section 2) in object code or executable form under the terms of + Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your cost + of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed + only for noncommercial distribution and only if you received the + program in object code or executable form with such an offer, in + accord with Subsection b above.) + + The source code for a work means the preferred form of the work for + making modifications to it. For an executable work, complete source code + means all the source code for all modules it contains, plus any + associated interface definition files, plus the scripts used to control + compilation and installation of the executable. However, as a special + exception, the source code distributed need not include anything that is + normally distributed (in either source or binary form) with the major + components (compiler, kernel, and so on) of the operating system on + which the executable runs, unless that component itself accompanies the + executable. + + If distribution of executable or object code is made by offering access + to copy from a designated place, then offering equivalent access to copy + the source code from the same place counts as distribution of the source + code, even though third parties are not compelled to copy the source + along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program + except as expressly provided under this License. Any attempt otherwise + to copy, modify, sublicense or distribute the Program is void, and will + automatically terminate your rights under this License. However, parties + who have received copies, or rights, from you under this License will + not have their licenses terminated so long as such parties remain in + full compliance. + + 5. You are not required to accept this License, since you have not + signed it. However, nothing else grants you permission to modify or + distribute the Program or its derivative works. These actions are + prohibited by law if you do not accept this License. Therefore, by + modifying or distributing the Program (or any work based on the + Program), you indicate your acceptance of this License to do so, and all + its terms and conditions for copying, distributing or modifying the + Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the + Program), the recipient automatically receives a license from the + original licensor to copy, distribute or modify the Program subject to + these terms and conditions. You may not impose any further restrictions + on the recipients' exercise of the rights granted herein. You are not + responsible for enforcing compliance by third parties to this License. + + 7. If, as a consequence of a court judgment or allegation of patent + infringement or for any other reason (not limited to patent issues), + conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot distribute + so as to satisfy simultaneously your obligations under this License and + any other pertinent obligations, then as a consequence you may not + distribute the Program at all. For example, if a patent license would + not permit royalty-free redistribution of the Program by all those who + receive copies directly or indirectly through you, then the only way you + could satisfy both it and this License would be to refrain entirely from + distribution of the Program. + + If any portion of this section is held invalid or unenforceable under + any particular circumstance, the balance of the section is intended to + apply and the section as a whole is intended to apply in other + circumstances. + + It is not the purpose of this section to induce you to infringe any + patents or other property right claims or to contest validity of any + such claims; this section has the sole purpose of protecting the + integrity of the free software distribution system, which is implemented + by public license practices. Many people have made generous + contributions to the wide range of software distributed through that + system in reliance on consistent application of that system; it is up to + the author/donor to decide if he or she is willing to distribute + software through any other system and a licensee cannot impose that choice. + + This section is intended to make thoroughly clear what is believed to be + a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in + certain countries either by patents or by copyrighted interfaces, the + original copyright holder who places the Program under this License may + add an explicit geographical distribution limitation excluding those + countries, so that distribution is permitted only in or among countries + not thus excluded. In such case, this License incorporates the + limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new + versions of the General Public License from time to time. Such new + versions will be similar in spirit to the present version, but may + differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies a version number of this License which applies to it and "any + later version", you have the option of following the terms and + conditions either of that version or of any later version published by + the Free Software Foundation. If the Program does not specify a version + number of this License, you may choose any version ever published by the + Free Software Foundation. + + 10. If you wish to incorporate parts of the Program into other free + programs whose distribution conditions are different, write to the + author to ask for permission. For software which is copyrighted by the + Free Software Foundation, write to the Free Software Foundation; we + sometimes make exceptions for this. Our decision will be guided by the + two goals of preserving the free status of all derivatives of our free + software and of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO + WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. + EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR + OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, + EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE + ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH + YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL + NECESSARY SERVICING, REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN + WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY + AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR + DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL + DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM + (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED + INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF + THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR + OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest + possible use to the public, the best way to achieve this is to make it + free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest to + attach them to the start of each source file to most effectively convey + the exclusion of warranty; and each file should have at least the + "copyright" line and a pointer to where the full notice is found. + + One line to give the program's name and a brief idea of what it does. + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + + Also add information on how to contact you by electronic and paper mail. + + If the program is interactive, make it output a short notice like this + when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type + `show w'. This is free software, and you are welcome to redistribute + it under certain conditions; type `show c' for details. + + The hypothetical commands `show w' and `show c' should show the + appropriate parts of the General Public License. Of course, the commands + you use may be called something other than `show w' and `show c'; they + could even be mouse-clicks or menu items--whatever suits your program. + + You should also get your employer (if you work as a programmer) or your + school, if any, to sign a "copyright disclaimer" for the program, if + necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + program `Gnomovision' (which makes passes at compilers) written by + James Hacker. + + signature of Ty Coon, 1 April 1989 + Ty Coon, President of Vice + + This General Public License does not permit incorporating your program + into proprietary programs. If your program is a subroutine library, you + may consider it more useful to permit linking proprietary applications + with the library. If this is what you want to do, use the GNU Library + General Public License instead of this License. + +--- + +## CLASSPATH EXCEPTION + + Linking this library statically or dynamically with other modules is + making a combined work based on this library. Thus, the terms and + conditions of the GNU General Public License version 2 cover the whole + combination. + + As a special exception, the copyright holders of this library give you + permission to link this library with independent modules to produce an + executable, regardless of the license terms of these independent + modules, and to copy and distribute the resulting executable under + terms of your choice, provided that you also meet, for each linked + independent module, the terms and conditions of the license of that + module. An independent module is a module which is not derived from or + based on this library. If you modify this library, you may extend this + exception to your version of the library, but you are not obligated to + do so. If you do not wish to do so, delete this exception statement + from your version. + + + +# Notices for Eclipse Project for Interceptors + +This content is produced and maintained by the Eclipse Project for Interceptors +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.interceptors + +## Trademarks + +Eclipse Project for Interceptors is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. + +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/interceptor-api + +## Third-party Content + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + diff --git a/aggregated-license-report/build.gradle.kts b/aggregated-license-report/build.gradle.kts index 5fe3de643..1e70825fb 100644 --- a/aggregated-license-report/build.gradle.kts +++ b/aggregated-license-report/build.gradle.kts @@ -22,7 +22,7 @@ import org.gradle.kotlin.dsl.support.unzipTo val licenseReports by configurations.creating { description = "Used to generate license reports" } dependencies { - licenseReports(project(":polaris-dropwizard-service", "licenseReports")) + licenseReports(project(":polaris-quarkus-service", "licenseReports")) } val collectLicenseReportJars by diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 0b05626e4..6d2043ebe 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -20,6 +20,7 @@ plugins { alias(libs.plugins.openapi.generator) id("polaris-client") + alias(libs.plugins.jandex) } dependencies { @@ -29,19 +30,21 @@ dependencies { implementation("org.apache.iceberg:iceberg-api") implementation("org.apache.iceberg:iceberg-core") - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.jakarta.validation.api) - compileOnly(libs.swagger.annotations) + implementation(libs.jakarta.annotation.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.validation.api) + implementation(libs.swagger.annotations) implementation(libs.jakarta.servlet.api) implementation(libs.jakarta.ws.rs.api) - compileOnly(platform(libs.micrometer.bom)) - compileOnly("io.micrometer:micrometer-core") + implementation(platform(libs.micrometer.bom)) + implementation("io.micrometer:micrometer-core") - compileOnly(platform(libs.jackson.bom)) - compileOnly("com.fasterxml.jackson.core:jackson-annotations") + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") } openApiGenerate { @@ -111,3 +114,5 @@ listOf("sourcesJar", "compileJava").forEach { task -> sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/api/iceberg-service/src/main/java/org/apache/polaris/service/types/TokenType.java b/api/iceberg-service/src/main/java/org/apache/polaris/service/types/TokenType.java index 1ef42c670..acf4ca8c3 100644 --- a/api/iceberg-service/src/main/java/org/apache/polaris/service/types/TokenType.java +++ b/api/iceberg-service/src/main/java/org/apache/polaris/service/types/TokenType.java @@ -54,8 +54,12 @@ public String toString() { return String.valueOf(value); } + // This method MUST be called 'fromString'; according to JAX-RS specs + // https://docs.oracle.com/javaee/7/api/javax/ws/rs/FormParam.html: + // "The type T of the annotated parameter must either [...] + // have a static method named valueOf or fromString". @JsonCreator - public static TokenType fromValue(String value) { + public static TokenType fromString(String value) { for (TokenType b : TokenType.values()) { if (b.value.equals(value)) { return b; diff --git a/api/management-model/build.gradle.kts b/api/management-model/build.gradle.kts index 313d28d65..488681410 100644 --- a/api/management-model/build.gradle.kts +++ b/api/management-model/build.gradle.kts @@ -20,6 +20,7 @@ plugins { alias(libs.plugins.openapi.generator) id("polaris-client") + alias(libs.plugins.jandex) } dependencies { @@ -60,3 +61,5 @@ listOf("sourcesJar", "compileJava").forEach { task -> sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/api/management-service/build.gradle.kts b/api/management-service/build.gradle.kts index 548c12596..a5cd11d63 100644 --- a/api/management-service/build.gradle.kts +++ b/api/management-service/build.gradle.kts @@ -20,6 +20,7 @@ plugins { alias(libs.plugins.openapi.generator) id("polaris-client") + alias(libs.plugins.jandex) } dependencies { @@ -74,3 +75,5 @@ listOf("sourcesJar", "compileJava").forEach { task -> sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/build.gradle.kts b/build.gradle.kts index e5fadafd4..902ee03d8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -24,6 +24,7 @@ buildscript { repositories { maven { url = java.net.URI("https://plugins.gradle.org/m2/") } } dependencies { classpath("com.diffplug.spotless:spotless-plugin-gradle:${libs.plugins.spotless.get().version}") + classpath("org.kordamp.gradle:jandex-gradle-plugin:${libs.plugins.jandex.get().version}") } } @@ -32,6 +33,8 @@ plugins { id("eclipse") id("polaris-root") alias(libs.plugins.rat) + // workaround for https://github.com/kordamp/jandex-gradle-plugin/issues/25 + alias(libs.plugins.jandex) apply false } val projectName = rootProject.file("ide-name.txt").readText().trim() @@ -88,7 +91,7 @@ tasks.named("rat").configure { excludes.add("logs/**") excludes.add("service/common/src/**/banner.txt") - excludes.add("dropwizard/service/logs") + excludes.add("quarkus/service/logs") excludes.add("site/node_modules/**") excludes.add("site/layouts/robots.txt") diff --git a/docker-compose.yml b/docker-compose.yml index 0757fa9cb..e2b3c352a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,17 +26,20 @@ services: - "8182" environment: AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS AZURE_TENANT_ID: $AZURE_TENANT_ID AZURE_CLIENT_ID: $AZURE_CLIENT_ID AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET - # add aws keys as dropwizard config - JAVA_OPTS: -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY + POLARIS_AUTHENTICATION_AUTHENTICATOR_TYPE: test + POLARIS_AUTHENTICATION_TOKEN_SERVICE_TYPE: test + QUARKUS_OTEL_SDK_DISABLED: "true" volumes: - ./regtests/credentials:/tmp/credentials/ healthcheck: - test: ["CMD", "curl", "http://localhost:8182/healthcheck"] + test: ["CMD", "curl", "http://localhost:8182/q/health"] interval: 10s timeout: 10s retries: 5 diff --git a/dropwizard/service/build.gradle.kts b/dropwizard/service/build.gradle.kts deleted file mode 100644 index 294279946..000000000 --- a/dropwizard/service/build.gradle.kts +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar - -plugins { - alias(libs.plugins.openapi.generator) - id("polaris-server") - id("polaris-license-report") - id("polaris-shadow-jar") - id("application") -} - -dependencies { - implementation(project(":polaris-core")) - implementation(project(":polaris-api-management-service")) - implementation(project(":polaris-api-iceberg-service")) - implementation(project(":polaris-service-common")) - - implementation(platform(libs.dropwizard.bom)) - implementation("io.dropwizard:dropwizard-core") - implementation("io.dropwizard:dropwizard-auth") - implementation("io.dropwizard:dropwizard-json-logging") - - implementation(platform(libs.iceberg.bom)) - implementation("org.apache.iceberg:iceberg-api") - implementation("org.apache.iceberg:iceberg-core") - implementation("org.apache.iceberg:iceberg-aws") - - // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329 - implementation(platform(libs.dnsjava)) - - implementation(libs.hadoop.common) { - exclude("org.slf4j", "slf4j-reload4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - exclude("org.apache.zookeeper", "zookeeper") - exclude("org.apache.hadoop.thirdparty", "hadoop-shaded-protobuf_3_25") - exclude("com.github.pjfanning", "jersey-json") - exclude("com.sun.jersey", "jersey-core") - exclude("com.sun.jersey", "jersey-server") - exclude("com.sun.jersey", "jersey-servlet") - } - - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.jakarta.servlet.api) - compileOnly(libs.jakarta.validation.api) - compileOnly(libs.jakarta.ws.rs.api) - - compileOnly(libs.smallrye.common.annotation) - - implementation(platform(libs.google.cloud.storage.bom)) - implementation("com.google.cloud:google-cloud-storage") - - implementation(platform(libs.awssdk.bom)) - implementation("software.amazon.awssdk:sts") - implementation("software.amazon.awssdk:iam-policy-builder") - implementation("software.amazon.awssdk:s3") - - implementation(platform(libs.azuresdk.bom)) - implementation("com.azure:azure-core") - - implementation(platform(libs.micrometer.bom)) - implementation("io.micrometer:micrometer-core") - implementation("io.micrometer:micrometer-registry-prometheus") - implementation(libs.prometheus.metrics.exporter.servlet.jakarta) - - implementation(platform(libs.opentelemetry.bom)) - implementation("io.opentelemetry:opentelemetry-api") - implementation("io.opentelemetry:opentelemetry-sdk-trace") - implementation("io.opentelemetry:opentelemetry-exporter-logging") - implementation(libs.opentelemetry.semconv) - - implementation(libs.smallrye.common.annotation) - - compileOnly(libs.swagger.annotations) - compileOnly(libs.spotbugs.annotations) - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.spotbugs.annotations) - - testImplementation(project(":polaris-api-management-model")) - - testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") - testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") - testImplementation("io.dropwizard:dropwizard-testing") - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - testImplementation("org.apache.iceberg:iceberg-spark-3.5_2.12") - testImplementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") - testImplementation("org.apache.spark:spark-sql_2.12:3.5.1") { - // exclude log4j dependencies - exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") - exclude("org.apache.logging.log4j", "log4j-api") - exclude("org.apache.logging.log4j", "log4j-1.2-api") - } - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(libs.auth0.jwt) - - testCompileOnly(libs.smallrye.common.annotation) - - testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) - testRuntimeOnly("org.junit.platform:junit-platform-launcher") - - testImplementation(project(":polaris-eclipselink")) -} - -if (project.properties.get("eclipseLink") == "true") { - dependencies { implementation(project(":polaris-eclipselink")) } -} - -tasks.named("test").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") - useJUnitPlatform() - maxParallelForks = 4 -} - -tasks.register("runApp").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - classpath = sourceSets["main"].runtimeClasspath - mainClass = "org.apache.polaris.service.dropwizard.PolarisApplication" - args("server", "$rootDir/polaris-server.yml") -} - -application { mainClass = "org.apache.polaris.service.dropwizard.PolarisApplication" } - -tasks.named("jar") { - manifest { attributes["Main-Class"] = "org.apache.polaris.service.dropwizard.PolarisApplication" } -} - -tasks.register("testJar") { - archiveClassifier.set("tests") - from(sourceSets.test.get().output) -} - -val shadowJar = - tasks.named("shadowJar") { - append("META-INF/hk2-locator/default") - finalizedBy("startShadowScripts") - } - -val startScripts = - tasks.named("startScripts") { applicationName = "polaris-service" } - -tasks.named("startShadowScripts") { applicationName = "polaris-service" } - -tasks.register("prepareDockerDist") { - into(project.layout.buildDirectory.dir("docker-dist")) - from(startScripts) { into("bin") } - from(configurations.runtimeClasspath) { into("lib") } - from(tasks.named("jar")) { into("lib") } -} - -tasks.named("build").configure { dependsOn("prepareDockerDist") } - -distributions { - main { - contents { - from("../../NOTICE") - from("../../LICENSE-BINARY-DIST").rename("LICENSE-BINARY-DIST", "LICENSE") - } - } - named("shadow") { - contents { - from("../../NOTICE") - from("../../LICENSE-BINARY-DIST").rename("LICENSE-BINARY-DIST", "LICENSE") - } - } -} - -tasks.named("assemble").configure { dependsOn("testJar") } diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java deleted file mode 100644 index b68d2d761..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import java.util.Map; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Command for bootstrapping root level service principals for each realm. This command will invoke - * a default implementation which generates random user id and secret. These credentials will be - * printed out to the log and standard output (stdout). - */ -public class BootstrapRealmsCommand extends ConfiguredCommand { - private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapRealmsCommand.class); - - public BootstrapRealmsCommand() { - super("bootstrap", "bootstraps principal credentials for all realms and prints them to log"); - } - - @Override - protected void run( - Bootstrap bootstrap, - Namespace namespace, - PolarisApplicationConfig configuration) { - MetaStoreManagerFactory metaStoreManagerFactory = - configuration.findService(MetaStoreManagerFactory.class); - - PolarisConfigurationStore configurationStore = - configuration.findService(PolarisConfigurationStore.class); - - // Execute the bootstrap - Map results = - metaStoreManagerFactory.bootstrapRealms(configuration.getDefaultRealms()); - - // Log any errors: - boolean success = true; - for (Map.Entry result : results.entrySet()) { - if (!result.getValue().isSuccess()) { - LOGGER.error( - "Bootstrapping `{}` failed: {}", - result.getKey(), - result.getValue().getReturnStatus().toString()); - success = false; - } - } - - if (success) { - LOGGER.info("Bootstrap completed successfully."); - } else { - LOGGER.error("Bootstrap encountered errors during operation."); - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java deleted file mode 100644 index 98c852a39..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java +++ /dev/null @@ -1,578 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig.REQUEST_BODY_BYTES_NO_LIMIT; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.DeserializationConfig; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.deser.std.StdValueInstantiator; -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.module.SimpleValueInstantiators; -import com.fasterxml.jackson.databind.type.TypeFactory; -import io.dropwizard.auth.AuthDynamicFeature; -import io.dropwizard.auth.AuthFilter; -import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter; -import io.dropwizard.configuration.EnvironmentVariableSubstitutor; -import io.dropwizard.configuration.SubstitutingSourceProvider; -import io.dropwizard.core.Application; -import io.dropwizard.core.setup.Bootstrap; -import io.dropwizard.core.setup.Environment; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.prometheusmetrics.PrometheusConfig; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.context.propagation.TextMapPropagator; -import io.opentelemetry.exporter.logging.LoggingSpanExporter; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import io.opentelemetry.semconv.ServiceAttributes; -import io.prometheus.metrics.exporter.servlet.jakarta.PrometheusMetricsServlet; -import jakarta.inject.Inject; -import jakarta.inject.Provider; -import jakarta.inject.Singleton; -import jakarta.servlet.DispatcherType; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.FilterRegistration; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.iceberg.rest.RESTSerializers; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.auth.PolarisAuthorizerImpl; -import org.apache.polaris.core.auth.PolarisGrantManager; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.context.RealmScoped; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache; -import org.apache.polaris.service.admin.PolarisServiceImpl; -import org.apache.polaris.service.admin.api.PolarisCatalogsApi; -import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; -import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; -import org.apache.polaris.service.auth.Authenticator; -import org.apache.polaris.service.catalog.IcebergCatalogAdapter; -import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; -import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; -import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApi; -import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2Api; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.config.Serializers; -import org.apache.polaris.service.config.TaskHandlerConfiguration; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.context.RealmScopeContext; -import org.apache.polaris.service.dropwizard.exception.JerseyViolationExceptionMapper; -import org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry; -import org.apache.polaris.service.dropwizard.persistence.cache.EntityCacheFactory; -import org.apache.polaris.service.dropwizard.throttling.StreamReadConstraintsExceptionMapper; -import org.apache.polaris.service.dropwizard.tracing.TracingFilter; -import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.exception.IcebergJsonProcessingExceptionMapper; -import org.apache.polaris.service.exception.PolarisExceptionMapper; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.apache.polaris.service.ratelimiter.RateLimiterFilter; -import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; -import org.apache.polaris.service.task.TableCleanupTaskHandler; -import org.apache.polaris.service.task.TaskExecutor; -import org.apache.polaris.service.task.TaskExecutorImpl; -import org.apache.polaris.service.task.TaskFileIOSupplier; -import org.eclipse.jetty.servlets.CrossOriginFilter; -import org.glassfish.hk2.api.Context; -import org.glassfish.hk2.api.Factory; -import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.hk2.api.TypeLiteral; -import org.glassfish.hk2.utilities.ServiceLocatorUtilities; -import org.glassfish.hk2.utilities.binding.AbstractBinder; -import org.glassfish.jersey.process.internal.RequestScoped; -import org.glassfish.jersey.servlet.ServletProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -public class PolarisApplication extends Application { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplication.class); - private ServiceLocator serviceLocator; - - public static void main(final String[] args) throws Exception { - new PolarisApplication().run(args); - printAsciiArt(); - } - - private static void printAsciiArt() throws IOException { - URL url = PolarisApplication.class.getResource("/org/apache/polaris/service/banner.txt"); - try (InputStream in = - requireNonNull(url, "banner.txt not found on classpath") - .openConnection() - .getInputStream()) { - System.out.println(new String(in.readAllBytes(), UTF_8)); - } - } - - @Override - public void initialize(Bootstrap bootstrap) { - // Enable variable substitution with environment variables - EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false); - SubstitutingSourceProvider provider = - new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), substitutor); - bootstrap.setConfigurationSourceProvider(provider); - - bootstrap.addCommand(new BootstrapRealmsCommand()); - bootstrap.addCommand(new PurgeRealmsCommand()); - serviceLocator = ServiceLocatorUtilities.createAndPopulateServiceLocator(); - ObjectMapper objectMapper = bootstrap.getObjectMapper(); - - // Register the PolarisApplicationConfig class with the ServiceLocator so that we can inject - // instances of the configuration or certain of the configuration fields into other classes. - // The configuration's postConstruct method registers all the configured fields as factories - // so that they are used for DI. - ServiceLocatorUtilities.addClasses(serviceLocator, PolarisApplicationConfig.class); - TypeFactory typeFactory = TypeFactory.defaultInstance(); - SimpleValueInstantiators instantiators = new SimpleValueInstantiators(); - instantiators.addValueInstantiator( - PolarisApplicationConfig.class, - new ServiceLocatorValueInstantiator( - objectMapper.getDeserializationConfig(), - typeFactory.constructType(PolarisApplicationConfig.class), - serviceLocator)); - - // Use the default ServiceLocator to discover the implementations of various contract providers - // and register them as subtypes with the ObjectMapper. This allows Jackson to discover the - // various implementations and use the type annotations to determine which instance to use when - // parsing the YAML configuration. - SimpleModule module = new SimpleModule(); - serviceLocator - .getDescriptors((c) -> true) - .forEach( - descriptor -> { - try { - Class klazz = - PolarisApplication.class - .getClassLoader() - .loadClass(descriptor.getImplementation()); - String name = descriptor.getName(); - if (name == null) { - module.registerSubtypes(klazz); - } else { - module.registerSubtypes(new NamedType(klazz, name)); - } - } catch (ClassNotFoundException e) { - LOGGER.error("Error loading class {}", descriptor.getImplementation(), e); - throw new RuntimeException("Unable to start Polaris application"); - } - }); - - ServiceLocatorUtilities.addClasses(serviceLocator, PolarisMetricRegistry.class); - ServiceLocatorUtilities.bind( - serviceLocator, - new AbstractBinder() { - @Override - protected void configure() { - bind(setupTracing()).to(OpenTelemetry.class); - bind(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)).to(MeterRegistry.class); - } - }); - module.setValueInstantiators(instantiators); - objectMapper.registerModule(module); - } - - /** - * Value instantiator that uses the ServiceLocator to create instances of the various service - * types - */ - private static class ServiceLocatorValueInstantiator extends StdValueInstantiator { - private final ServiceLocator serviceLocator; - - public ServiceLocatorValueInstantiator( - DeserializationConfig config, JavaType valueType, ServiceLocator serviceLocator) { - super(config, valueType); - this.serviceLocator = serviceLocator; - } - - @Override - public boolean canCreateUsingDefault() { - return true; - } - - @Override - public boolean canInstantiate() { - return true; - } - - @Override - public Object createUsingDefault(DeserializationContext ctxt) throws IOException { - return ServiceLocatorUtilities.findOrCreateService(serviceLocator, getValueClass()); - } - } - - @Override - public void run(PolarisApplicationConfig configuration, Environment environment) { - PolarisMetricRegistry polarisMetricRegistry = - configuration.findService(PolarisMetricRegistry.class); - - MetaStoreManagerFactory metaStoreManagerFactory = - configuration.findService(MetaStoreManagerFactory.class); - - // Use the PolarisApplicationConfig to register dependencies in the Jersey resource - // configuration. This uses a different ServiceLocator from the one in the bootstrap step - environment - .getApplicationContext() - .setAttribute(ServletProperties.SERVICE_LOCATOR, configuration.getServiceLocator()); - - environment - .jersey() - .register( - new AbstractBinder() { - @Override - protected void configure() { - bindFactory(CallContextFactory.class).to(CallContext.class).in(RequestScoped.class); - bindFactory(RealmContextFactory.class) - .to(RealmContext.class) - .in(RequestScoped.class); - bind(RealmScopeContext.class) - .in(Singleton.class) - .to(new TypeLiteral>() {}); - bindFactory(PolarisMetaStoreManagerFactory.class) - .to(PolarisMetaStoreManager.class) - .in(RealmScoped.class); - bindFactory(EntityCacheFactory.class).in(RealmScoped.class).to(EntityCache.class); - - bindFactory(PolarisRemoteCacheFactory.class) - .in(RealmScoped.class) - .to(PolarisRemoteCache.class); - - // factory to use a cache delegating grant cache - // currently depends explicitly on the metaStoreManager as the delegate grant - // manager - bindFactory(PolarisMetaStoreManagerFactory.class) - .in(RealmScoped.class) - .to(PolarisGrantManager.class); - polarisMetricRegistry.init( - IcebergRestCatalogApi.class, - IcebergRestConfigurationApi.class, - IcebergRestOAuth2Api.class, - PolarisCatalogsApi.class, - PolarisPrincipalsApi.class, - PolarisPrincipalRolesApi.class); - bindAsContract(RealmEntityManagerFactory.class).in(Singleton.class); - bind(PolarisCallContextCatalogFactory.class) - .to(CallContextCatalogFactory.class) - .in(Singleton.class); - bind(PolarisAuthorizerImpl.class).in(Singleton.class).to(PolarisAuthorizer.class); - bind(IcebergCatalogAdapter.class) - .in(Singleton.class) - .to(IcebergRestCatalogApiService.class) - .to(IcebergRestConfigurationApiService.class); - bind(PolarisServiceImpl.class) - .in(Singleton.class) - .to(PolarisCatalogsApiService.class) - .to(PolarisPrincipalsApiService.class) - .to(PolarisPrincipalRolesApiService.class); - FileIOFactory fileIOFactory = configuration.findService(FileIOFactory.class); - - TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); - TaskExecutorImpl taskExecutor = - new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); - TaskFileIOSupplier fileIOSupplier = - new TaskFileIOSupplier( - metaStoreManagerFactory, - fileIOFactory, - configuration.findService(PolarisConfigurationStore.class)); - taskExecutor.addTaskHandler( - new TableCleanupTaskHandler( - taskExecutor, metaStoreManagerFactory, fileIOSupplier)); - taskExecutor.addTaskHandler( - new ManifestFileCleanupTaskHandler( - fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); - - bind(taskExecutor).to(TaskExecutor.class); - } - }); - - // servlet filters don't use the underlying DI - environment - .servlets() - .addFilter( - "realmContext", - new ContextResolverFilter( - configuration.findService(RealmContextResolver.class), - configuration.findService(CallContextResolver.class))) - .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - LOGGER.info( - "Initializing PolarisCallContextCatalogFactory for metaStoreManagerType {}", - metaStoreManagerFactory); - - environment.jersey().register(IcebergRestCatalogApi.class); - environment.jersey().register(IcebergRestConfigurationApi.class); - - FilterRegistration.Dynamic corsRegistration = - environment.servlets().addFilter("CORS", CrossOriginFilter.class); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_ORIGINS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedOrigins())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_TIMING_ORIGINS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedTimingOrigins())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_METHODS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedMethods())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_HEADERS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedHeaders())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowCredentials())); - corsRegistration.setInitParameter( - CrossOriginFilter.PREFLIGHT_MAX_AGE_PARAM, - Objects.toString(configuration.getCorsConfiguration().getPreflightMaxAge())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, - configuration.getCorsConfiguration().getAllowCredentials()); - corsRegistration.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - OpenTelemetry openTelemetry = configuration.findService(OpenTelemetry.class); - environment - .servlets() - .addFilter("tracing", new TracingFilter(openTelemetry)) - .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - if (configuration.hasRateLimiter()) { - environment.jersey().register(RateLimiterFilter.class); - } - Authenticator authenticator = - configuration.findService(new TypeLiteral<>() {}); - AuthFilter oauthCredentialAuthFilter = - new OAuthCredentialAuthFilter.Builder() - .setAuthenticator(authenticator::authenticate) - .setPrefix("Bearer") - .buildAuthFilter(); - environment.jersey().register(new AuthDynamicFeature(oauthCredentialAuthFilter)); - environment.healthChecks().register("polaris", new PolarisHealthCheck()); - - environment.jersey().register(IcebergRestOAuth2Api.class); - environment.jersey().register(IcebergExceptionMapper.class); - environment.jersey().register(PolarisExceptionMapper.class); - - environment.jersey().register(PolarisCatalogsApi.class); - environment.jersey().register(PolarisPrincipalsApi.class); - environment.jersey().register(PolarisPrincipalRolesApi.class); - - ObjectMapper objectMapper = environment.getObjectMapper(); - objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - objectMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); - - long maxRequestBodyBytes = configuration.getMaxRequestBodyBytes(); - if (maxRequestBodyBytes != REQUEST_BODY_BYTES_NO_LIMIT) { - objectMapper - .getFactory() - .setStreamReadConstraints( - StreamReadConstraints.builder().maxDocumentLength(maxRequestBodyBytes).build()); - LOGGER.info("Limiting request body size to {} bytes", maxRequestBodyBytes); - } - - environment.jersey().register(StreamReadConstraintsExceptionMapper.class); - RESTSerializers.registerAll(objectMapper); - Serializers.registerSerializers(objectMapper); - environment.jersey().register(IcebergJsonProcessingExceptionMapper.class); - environment.jersey().register(JerseyViolationExceptionMapper.class); - - // for tests, we have to instantiate the TimedApplicationEventListener directly - environment.jersey().register(new TimedApplicationEventListener(polarisMetricRegistry)); - - environment - .admin() - .addServlet( - "metrics", - new PrometheusMetricsServlet( - ((PrometheusMeterRegistry) polarisMetricRegistry.getMeterRegistry()) - .getPrometheusRegistry())) - .addMapping("/metrics"); - - // For in-memory metastore we need to bootstrap Service and Service principal at startup (for - // default realm) - // We can not utilize dropwizard Bootstrap command as command and server will be running two - // different processes - // and in-memory state will be lost b/w invocation of bootstrap command and running a server - if (metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory) { - metaStoreManagerFactory.getOrCreateMetaStoreManager(configuration::getDefaultRealm); - } - - LOGGER.info("Server started successfully."); - } - - private static OpenTelemetry setupTracing() { - Resource resource = - Resource.getDefault().toBuilder() - .put(ServiceAttributes.SERVICE_NAME, "polaris") - .put(ServiceAttributes.SERVICE_VERSION, "0.1.0") - .build(); - SdkTracerProvider sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) - .setResource(resource) - .build(); - return OpenTelemetrySdk.builder() - .setTracerProvider(sdkTracerProvider) - .setPropagators( - ContextPropagators.create( - TextMapPropagator.composite( - W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance()))) - .build(); - } - - /** Resolves and sets ThreadLocal CallContext/RealmContext based on the request contents. */ - private static class ContextResolverFilter implements Filter { - private final RealmContextResolver realmContextResolver; - private final CallContextResolver callContextResolver; - - @Inject - public ContextResolverFilter( - RealmContextResolver realmContextResolver, CallContextResolver callContextResolver) { - this.realmContextResolver = realmContextResolver; - this.callContextResolver = callContextResolver; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - Stream headerNames = Collections.list(httpRequest.getHeaderNames()).stream(); - Map headers = - headerNames.collect(Collectors.toMap(Function.identity(), httpRequest::getHeader)); - RealmContext currentRealmContext = - realmContextResolver.resolveRealmContext( - httpRequest.getRequestURL().toString(), - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - headers); - CallContext currentCallContext = - callContextResolver.resolveCallContext( - currentRealmContext, - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - headers); - CallContext.setCurrentContext(currentCallContext); - try (MDC.MDCCloseable ignored1 = - MDC.putCloseable("realm", currentRealmContext.getRealmIdentifier()); - MDC.MDCCloseable ignored2 = - MDC.putCloseable("request_id", httpRequest.getHeader("request_id"))) { - chain.doFilter(request, response); - } finally { - currentCallContext.close(); - } - } - } - - /** Factory to create a CallContext based on the request contents. */ - @RequestScoped - private static class CallContextFactory implements Factory { - - @RequestScoped - @Override - public CallContext provide() { - return CallContext.getCurrentContext(); - } - - @Override - public void dispose(CallContext instance) {} - } - - @RequestScoped - private static class RealmContextFactory implements Factory { - @Inject Provider callContext; - - @RequestScoped - @Override - public RealmContext provide() { - return callContext.get().getRealmContext(); - } - - @Override - public void dispose(RealmContext instance) {} - } - - private static class PolarisMetaStoreManagerFactory implements Factory { - @Inject MetaStoreManagerFactory metaStoreManagerFactory; - - @RealmScoped - @Override - public PolarisMetaStoreManager provide() { - RealmContext realmContext = CallContext.getCurrentContext().getRealmContext(); - return metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - } - - @Override - public void dispose(PolarisMetaStoreManager instance) {} - } - - private static class PolarisRemoteCacheFactory implements Factory { - @Inject PolarisMetaStoreManager metaStoreManager; - - @RealmScoped - @Override - public PolarisRemoteCache provide() { - return metaStoreManager; - } - - @Override - public void dispose(PolarisRemoteCache instance) {} - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java deleted file mode 100644 index 259f1f07d..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Command for purging all metadata associated with a realm */ -public class PurgeRealmsCommand extends ConfiguredCommand { - private static final Logger LOGGER = LoggerFactory.getLogger(PurgeRealmsCommand.class); - - public PurgeRealmsCommand() { - super("purge", "purge principal credentials for all realms and prints them to log"); - } - - @Override - protected void run( - Bootstrap bootstrap, - Namespace namespace, - PolarisApplicationConfig configuration) - throws Exception { - MetaStoreManagerFactory metaStoreManagerFactory = - configuration.findService(MetaStoreManagerFactory.class); - - metaStoreManagerFactory.purgeRealms(configuration.getDefaultRealms()); - - LOGGER.info("Purge completed successfully."); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java deleted file mode 100644 index 88a41c957..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Tag; -import jakarta.inject.Inject; -import jakarta.ws.rs.ext.Provider; -import java.lang.reflect.Method; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry; -import org.glassfish.jersey.server.monitoring.ApplicationEvent; -import org.glassfish.jersey.server.monitoring.ApplicationEventListener; -import org.glassfish.jersey.server.monitoring.RequestEvent; -import org.glassfish.jersey.server.monitoring.RequestEventListener; - -/** - * An ApplicationEventListener that supports timing and error counting of Jersey resource methods - * annotated by {@link Timed}. It uses the {@link PolarisMetricRegistry} for metric collection and - * properly times the resource on success and increments the error counter on failure. - */ -@Provider -public class TimedApplicationEventListener implements ApplicationEventListener { - - /** - * Each API will increment a common counter (SINGLETON_METRIC_NAME) but have its API name tagged - * (TAG_API_NAME). - */ - public static final String SINGLETON_METRIC_NAME = "polaris.api"; - - public static final String TAG_API_NAME = "api_name"; - - // The PolarisMetricRegistry instance used for recording metrics and error counters. - private final PolarisMetricRegistry polarisMetricRegistry; - - @Inject - public TimedApplicationEventListener(PolarisMetricRegistry polarisMetricRegistry) { - this.polarisMetricRegistry = polarisMetricRegistry; - } - - @VisibleForTesting - public PolarisMetricRegistry getMetricRegistry() { - return polarisMetricRegistry; - } - - @Override - public void onEvent(ApplicationEvent event) {} - - @Override - public RequestEventListener onRequest(RequestEvent event) { - return new TimedRequestEventListener(); - } - - /** - * A RequestEventListener implementation that handles timing of resource method execution and - * increments error counters on failures. The lifetime of the listener is tied to a single HTTP - * request. - */ - private class TimedRequestEventListener implements RequestEventListener { - private String metric; - private Stopwatch sw; - - /** Handles various types of RequestEvents to start timing, stop timing, and record metrics. */ - @Override - public void onEvent(RequestEvent event) { - String realmId = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); - if (event.getType() == RequestEvent.Type.REQUEST_MATCHED) { - Method method = - event.getUriInfo().getMatchedResourceMethod().getInvocable().getHandlingMethod(); - if (method.isAnnotationPresent(Timed.class)) { - Timed timedApi = method.getAnnotation(Timed.class); - metric = timedApi.value(); - - // Increment both the counter with the API name in the metric name and a common metric - polarisMetricRegistry.incrementCounter(metric, realmId); - polarisMetricRegistry.incrementCounter( - SINGLETON_METRIC_NAME, realmId, Tag.of(TAG_API_NAME, metric)); - } - } else if (event.getType() == RequestEvent.Type.RESOURCE_METHOD_START && metric != null) { - sw = Stopwatch.createStarted(); - } else if (event.getType() == RequestEvent.Type.FINISHED && metric != null) { - if (event.isSuccess()) { - sw.stop(); - polarisMetricRegistry.recordTimer(metric, sw.elapsed(TimeUnit.MILLISECONDS), realmId); - } else { - int statusCode = event.getContainerResponse().getStatus(); - - // Increment both the counter with the API name in the metric name and a common metric - polarisMetricRegistry.incrementErrorCounter(metric, statusCode, realmId); - polarisMetricRegistry.incrementErrorCounter( - SINGLETON_METRIC_NAME, - realmId, - List.of( - Tag.of(TAG_API_NAME, metric), Tag.of(TAG_RESP_CODE, String.valueOf(statusCode)))); - } - } - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java deleted file mode 100644 index 9055d73a7..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; - -public class CorsConfiguration { - private List allowedOrigins = List.of("*"); - private List allowedTimingOrigins = List.of("*"); - private List allowedMethods = List.of("*"); - private List allowedHeaders = List.of("*"); - private List exposedHeaders = List.of("*"); - private Integer preflightMaxAge = 600; - private String allowCredentials = "true"; - - public List getAllowedOrigins() { - return allowedOrigins; - } - - @JsonProperty("allowed-origins") - public void setAllowedOrigins(List allowedOrigins) { - this.allowedOrigins = allowedOrigins; - } - - public void setAllowedTimingOrigins(List allowedTimingOrigins) { - this.allowedTimingOrigins = allowedTimingOrigins; - } - - @JsonProperty("allowed-timing-origins") - public List getAllowedTimingOrigins() { - return allowedTimingOrigins; - } - - public List getAllowedMethods() { - return allowedMethods; - } - - @JsonProperty("allowed-methods") - public void setAllowedMethods(List allowedMethods) { - this.allowedMethods = allowedMethods; - } - - public List getAllowedHeaders() { - return allowedHeaders; - } - - @JsonProperty("allowed-headers") - public void setAllowedHeaders(List allowedHeaders) { - this.allowedHeaders = allowedHeaders; - } - - public List getExposedHeaders() { - return exposedHeaders; - } - - @JsonProperty("exposed-headers") - public void setExposedHeaders(List exposedHeaders) { - this.exposedHeaders = exposedHeaders; - } - - public Integer getPreflightMaxAge() { - return preflightMaxAge; - } - - @JsonProperty("preflight-max-age") - public void setPreflightMaxAge(Integer preflightMaxAge) { - this.preflightMaxAge = preflightMaxAge; - } - - public String getAllowCredentials() { - return allowCredentials; - } - - @JsonProperty("allowed-credentials") - public void setAllowCredentials(String allowCredentials) { - this.allowCredentials = allowCredentials; - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java deleted file mode 100644 index 2c351d0f0..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java +++ /dev/null @@ -1,458 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.base.Preconditions; -import io.dropwizard.core.Configuration; -import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Inject; -import java.io.IOException; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; -import org.apache.commons.lang3.StringUtils; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.apache.polaris.service.auth.Authenticator; -import org.apache.polaris.service.auth.TokenBrokerFactory; -import org.apache.polaris.service.auth.TokenBrokerFactoryConfig; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.DefaultConfigurationStore; -import org.apache.polaris.service.config.TaskHandlerConfiguration; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.ratelimiter.RateLimiter; -import org.apache.polaris.service.ratelimiter.TokenBucketFactory; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.glassfish.hk2.api.Factory; -import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.hk2.api.TypeLiteral; -import org.glassfish.hk2.utilities.ServiceLocatorUtilities; -import org.glassfish.hk2.utilities.binding.AbstractBinder; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; - -/** - * Configuration specific to a Polaris REST Service. Place these entries in a YML file for them to - * be picked up, i.e. `iceberg-rest-server.yml` - */ -public class PolarisApplicationConfig extends Configuration { - /** - * Override the default binding of registered services so that the configured instances are used. - */ - private static final int OVERRIDE_BINDING_RANK = 10; - - private MetaStoreManagerFactory metaStoreManagerFactory; - private String defaultRealm = "default-realm"; - private RealmContextResolver realmContextResolver; - private CallContextResolver callContextResolver; - private Authenticator polarisAuthenticator; - private CorsConfiguration corsConfiguration = new CorsConfiguration(); - private TaskHandlerConfiguration taskHandler = new TaskHandlerConfiguration(); - private Map globalFeatureConfiguration = Map.of(); - private Map> realmConfiguration = Map.of(); - private List defaultRealms; - private String awsAccessKey; - private String awsSecretKey; - private FileIOFactory fileIOFactory; - private RateLimiter rateLimiter; - private TokenBucketFactory tokenBucketFactory; - private TokenBrokerConfig tokenBroker = new TokenBrokerConfig(); - - private AccessToken gcpAccessToken; - - public static final long REQUEST_BODY_BYTES_NO_LIMIT = -1; - private long maxRequestBodyBytes = REQUEST_BODY_BYTES_NO_LIMIT; - - @Inject ServiceLocator serviceLocator; - - @PostConstruct - public void bindToServiceLocator() { - ServiceLocatorUtilities.bind(serviceLocator, binder()); - } - - public ServiceLocator getServiceLocator() { - return serviceLocator; - } - - @Nonnull - public AbstractBinder binder() { - PolarisApplicationConfig config = this; - return new AbstractBinder() { - @Override - protected void configure() { - bindFactory(SupplierFactory.create(serviceLocator, config::getStorageIntegrationProvider)) - .to(PolarisStorageIntegrationProvider.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getMetaStoreManagerFactory)) - .to(MetaStoreManagerFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::createConfigurationStore)) - .to(PolarisConfigurationStore.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getFileIOFactory)) - .to(FileIOFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getPolarisAuthenticator)) - .to(Authenticator.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, () -> tokenBroker)) - .to(TokenBrokerFactoryConfig.class); - bindFactory( - SupplierFactory.create( - serviceLocator, - () -> - serviceLocator.getService(TokenBrokerFactory.class, tokenBroker.getType()))) - .to(TokenBrokerFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getOauth2Service)) - .to(IcebergRestOAuth2ApiService.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getCallContextResolver)) - .to(CallContextResolver.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getRealmContextResolver)) - .to(RealmContextResolver.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getRateLimiter)) - .to(RateLimiter.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getTokenBucketFactory)) - .to(TokenBucketFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - } - }; - } - - /** - * Factory implementation that uses the provided supplier method to retrieve the instance and then - * uses the {@link #serviceLocator} to inject dependencies into the instance. This is necessary - * since the DI framework doesn't automatically inject dependencies into the instances created. - * - * @param - */ - private static final class SupplierFactory implements Factory { - private final ServiceLocator serviceLocator; - private final Supplier supplier; - - private static SupplierFactory create( - ServiceLocator serviceLocator, Supplier supplier) { - return new SupplierFactory<>(serviceLocator, supplier); - } - - private SupplierFactory(ServiceLocator serviceLocator, Supplier supplier) { - this.serviceLocator = serviceLocator; - this.supplier = supplier; - } - - @Override - public T provide() { - T obj = supplier.get(); - serviceLocator.inject(obj); - return obj; - } - - @Override - public void dispose(T instance) {} - } - - public T findService(Class serviceClass) { - return serviceLocator.getService(serviceClass); - } - - public T findService(TypeLiteral serviceClass) { - return serviceLocator.getService(serviceClass.getRawType()); - } - - @JsonProperty("metaStoreManager") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - private MetaStoreManagerFactory getMetaStoreManagerFactory() { - return metaStoreManagerFactory; - } - - @JsonProperty("io") - @JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "factoryType") - public void setFileIOFactory(FileIOFactory fileIOFactory) { - this.fileIOFactory = fileIOFactory; - } - - private FileIOFactory getFileIOFactory() { - return fileIOFactory; - } - - @JsonProperty("authenticator") - @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "class") - public void setPolarisAuthenticator( - Authenticator polarisAuthenticator) { - this.polarisAuthenticator = polarisAuthenticator; - } - - private Authenticator getPolarisAuthenticator() { - return polarisAuthenticator; - } - - @JsonProperty("tokenBroker") - public void setTokenBroker(TokenBrokerConfig tokenBroker) { - this.tokenBroker = tokenBroker; - } - - private RealmContextResolver getRealmContextResolver() { - realmContextResolver.setDefaultRealm(this.defaultRealm); - return realmContextResolver; - } - - @JsonProperty("realmContextResolver") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setRealmContextResolver(RealmContextResolver realmContextResolver) { - this.realmContextResolver = realmContextResolver; - } - - private CallContextResolver getCallContextResolver() { - return callContextResolver; - } - - @JsonProperty("callContextResolver") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setCallContextResolver(CallContextResolver callContextResolver) { - this.callContextResolver = callContextResolver; - } - - private IcebergRestOAuth2ApiService oauth2Service; - - @JsonProperty("oauth2") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setOauth2Service(IcebergRestOAuth2ApiService oauth2Service) { - this.oauth2Service = oauth2Service; - } - - private IcebergRestOAuth2ApiService getOauth2Service() { - return oauth2Service; - } - - public String getDefaultRealm() { - return defaultRealm; - } - - @JsonProperty("defaultRealm") - public void setDefaultRealm(String defaultRealm) { - this.defaultRealm = defaultRealm; - } - - @JsonProperty("cors") - public CorsConfiguration getCorsConfiguration() { - return corsConfiguration; - } - - @JsonProperty("cors") - public void setCorsConfiguration(CorsConfiguration corsConfiguration) { - this.corsConfiguration = corsConfiguration; - } - - @JsonProperty("rateLimiter") - private RateLimiter getRateLimiter() { - return rateLimiter; - } - - public boolean hasRateLimiter() { - return rateLimiter != null; - } - - @JsonProperty("rateLimiter") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setRateLimiter(@Nullable RateLimiter rateLimiter) { - this.rateLimiter = rateLimiter; - } - - @JsonProperty("tokenBucketFactory") - private TokenBucketFactory getTokenBucketFactory() { - return tokenBucketFactory; - } - - @JsonProperty("tokenBucketFactory") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setTokenBucketFactory(@Nullable TokenBucketFactory tokenBucketFactory) { - this.tokenBucketFactory = tokenBucketFactory; - } - - public void setTaskHandler(TaskHandlerConfiguration taskHandler) { - this.taskHandler = taskHandler; - } - - public TaskHandlerConfiguration getTaskHandler() { - return taskHandler; - } - - @JsonProperty("featureConfiguration") - public void setFeatureConfiguration(Map featureConfiguration) { - this.globalFeatureConfiguration = featureConfiguration; - } - - @JsonProperty("realmFeatureConfiguration") - public void setRealmFeatureConfiguration(Map> realmConfiguration) { - this.realmConfiguration = realmConfiguration; - } - - @JsonProperty("maxRequestBodyBytes") - public void setMaxRequestBodyBytes(long maxRequestBodyBytes) { - // The underlying library that we use to implement the limit treats all values <= 0 as the - // same, so we block all but -1 to prevent ambiguity. - Preconditions.checkArgument( - maxRequestBodyBytes == -1 || maxRequestBodyBytes > 0, - "maxRequestBodyBytes must be a positive integer or %s to specify no limit.", - REQUEST_BODY_BYTES_NO_LIMIT); - - this.maxRequestBodyBytes = maxRequestBodyBytes; - } - - public long getMaxRequestBodyBytes() { - return maxRequestBodyBytes; - } - - private PolarisConfigurationStore createConfigurationStore() { - return new DefaultConfigurationStore(globalFeatureConfiguration, realmConfiguration); - } - - public List getDefaultRealms() { - return defaultRealms; - } - - private AwsCredentialsProvider credentialsProvider() { - if (StringUtils.isNotBlank(awsAccessKey) && StringUtils.isNotBlank(awsSecretKey)) { - LoggerFactory.getLogger(PolarisApplicationConfig.class) - .warn("Using hard-coded AWS credentials - this is not recommended for production"); - return StaticCredentialsProvider.create( - AwsBasicCredentials.create(awsAccessKey, awsSecretKey)); - } - return null; - } - - public void setAwsAccessKey(String awsAccessKey) { - this.awsAccessKey = awsAccessKey; - } - - public void setAwsSecretKey(String awsSecretKey) { - this.awsSecretKey = awsSecretKey; - } - - public void setDefaultRealms(List defaultRealms) { - this.defaultRealms = defaultRealms; - } - - private PolarisStorageIntegrationProvider storageIntegrationProvider; - - public void setStorageIntegrationProvider( - PolarisStorageIntegrationProvider storageIntegrationProvider) { - this.storageIntegrationProvider = storageIntegrationProvider; - } - - private PolarisStorageIntegrationProvider getStorageIntegrationProvider() { - if (storageIntegrationProvider == null) { - storageIntegrationProvider = - new PolarisStorageIntegrationProviderImpl( - () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - AwsCredentialsProvider awsCredentialsProvider = credentialsProvider(); - if (awsCredentialsProvider != null) { - stsClientBuilder.credentialsProvider(awsCredentialsProvider); - } - return stsClientBuilder.build(); - }, - getGcpCredentialsProvider()); - } - return storageIntegrationProvider; - } - - private Supplier getGcpCredentialsProvider() { - return () -> - Optional.ofNullable(gcpAccessToken) - .map(GoogleCredentials::create) - .orElseGet( - () -> { - try { - return GoogleCredentials.getApplicationDefault(); - } catch (IOException e) { - throw new RuntimeException("Failed to get GCP credentials", e); - } - }); - } - - @JsonProperty("gcp_credentials") - void setGcpCredentials(GcpAccessToken token) { - this.gcpAccessToken = - new AccessToken( - token.getAccessToken(), - new Date(System.currentTimeMillis() + token.getExpiresIn() * 1000)); - } - - /** - * A static AccessToken representation used to store a static token and expiration date. This - * should strictly be used for testing. - */ - static class GcpAccessToken { - private String accessToken; - private long expiresIn; - - public GcpAccessToken() {} - - public GcpAccessToken(String accessToken, long expiresIn) { - this.accessToken = accessToken; - this.expiresIn = expiresIn; - } - - public String getAccessToken() { - return accessToken; - } - - @JsonProperty("access_token") - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } - - public long getExpiresIn() { - return expiresIn; - } - - @JsonProperty("expires_in") - public void setExpiresIn(long expiresIn) { - this.expiresIn = expiresIn; - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/TokenBrokerConfig.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/TokenBrokerConfig.java deleted file mode 100644 index 3e349fcfa..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/TokenBrokerConfig.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.annotation.Nullable; -import org.apache.polaris.service.auth.TokenBrokerFactoryConfig; - -/** - * This object receives the subsection of the server config YAML (`polaris-server.yml`) that is - * related to JWT brokers. - */ -public class TokenBrokerConfig implements TokenBrokerFactoryConfig { - private String type = "none"; - private int maxTokenGenerationInSeconds = 3600; - private String file; - private String secret; - - @JsonProperty("type") - public void setType(String type) { - this.type = type; - } - - @JsonProperty("file") - public void setFile(String file) { - this.file = file; - } - - @JsonProperty("secret") - public void setSecret(String secret) { - this.secret = secret; - } - - @JsonProperty("maxTokenGenerationInSeconds") - public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; - } - - public String getType() { - return type; - } - - @Override - public int maxTokenGenerationInSeconds() { - return maxTokenGenerationInSeconds; - } - - @Nullable - @Override - public String file() { - return file; - } - - @Nullable - @Override - public String secret() { - return secret; - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java deleted file mode 100644 index f89ea64e7..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.context; - -import jakarta.inject.Inject; -import jakarta.inject.Singleton; -import java.lang.annotation.Annotation; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.context.RealmScoped; -import org.glassfish.hk2.api.ActiveDescriptor; -import org.glassfish.hk2.api.Context; -import org.glassfish.hk2.api.IterableProvider; -import org.glassfish.hk2.api.ServiceHandle; -import org.glassfish.hk2.api.ServiceLocator; - -@Singleton -public class RealmScopeContext implements Context { - private final Map, Object>> contexts = new ConcurrentHashMap<>(); - - @Inject private ServiceLocator locator; - @Inject private IterableProvider realmContextProvider; - - @Override - public Class getScope() { - return RealmScoped.class; - } - - @SuppressWarnings("unchecked") - @Override - public U findOrCreate(ActiveDescriptor activeDescriptor, ServiceHandle root) { - RealmContext realmContext = realmContextProvider.iterator().next(); - Map, Object> contextMap = - contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new ConcurrentHashMap<>()); - return (U) contextMap.computeIfAbsent(activeDescriptor, k -> activeDescriptor.create(root)); - } - - @Override - public boolean containsKey(ActiveDescriptor descriptor) { - RealmContext realmContext = realmContextProvider.iterator().next(); - Map, Object> contextMap = - contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new HashMap<>()); - return contextMap.containsKey(descriptor); - } - - @Override - public void destroyOne(ActiveDescriptor descriptor) { - RealmContext realmContext = realmContextProvider.iterator().next(); - Map, Object> contextMap = - contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new HashMap<>()); - contextMap.remove(descriptor); - } - - @Override - public boolean supportsNullCreation() { - return false; - } - - @Override - public boolean isActive() { - Optional first = - locator.getAllServices(Context.class).stream() - .filter( - context -> - context - .getScope() - .equals( - realmContextProvider - .getHandle() - .getActiveDescriptor() - .getScopeAnnotation())) - .findFirst(); - return first.map(Context::isActive).orElse(false); - } - - @Override - public void shutdown() { - contexts.clear(); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java deleted file mode 100644 index 4885b3a54..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.exception; - -import io.dropwizard.jersey.validation.JerseyViolationException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; -import org.apache.polaris.service.exception.IcebergConstraintViolationExceptionMapper; - -/** - * Override of the default JerseyViolationExceptionMapper to provide an Iceberg ErrorResponse with - * the exception details. - */ -@Provider -public class JerseyViolationExceptionMapper implements ExceptionMapper { - - private IcebergConstraintViolationExceptionMapper icebergMapper = - new IcebergConstraintViolationExceptionMapper(); - - @Override - public Response toResponse(JerseyViolationException exception) { - return icebergMapper.toResponse(exception); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java deleted file mode 100644 index 26aec8bcc..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.logging; - -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.pattern.ExtendedThrowableProxyConverter; -import ch.qos.logback.classic.pattern.RootCauseFirstThrowableProxyConverter; -import ch.qos.logback.classic.pattern.ThrowableHandlingConverter; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.LayoutBase; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.collect.ImmutableMap; -import io.dropwizard.logging.json.AbstractJsonLayoutBaseFactory; -import io.dropwizard.logging.json.EventAttribute; -import io.dropwizard.logging.json.layout.EventJsonLayout; -import io.dropwizard.logging.json.layout.ExceptionFormat; -import io.dropwizard.logging.json.layout.JsonFormatter; -import io.dropwizard.logging.json.layout.TimestampFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.stream.Collectors; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * Basically a direct copy of {@link io.dropwizard.logging.json.EventJsonLayoutBaseFactory} that - * adds support for {@link ILoggingEvent#getKeyValuePairs()} in the output. By default, additional - * key/value pairs are included as the `params` field of the json output, but they can optionally be - * flattened into the log event output. - * - *

To use this appender, change the appender type to `polaris` - * loggers: - * org.apache.iceberg.rest: DEBUG - * org.apache.iceberg.polaris: DEBUG - * appenders: - * - type: console - * threshold: ALL - * layout: - * type: polaris - * flattenKeyValues: false - * includeKeyValues: true - * - */ -@JsonTypeName("polaris") -public class PolarisJsonLayoutFactory extends AbstractJsonLayoutBaseFactory { - private EnumSet includes = - EnumSet.of( - EventAttribute.LEVEL, - EventAttribute.THREAD_NAME, - EventAttribute.MDC, - EventAttribute.MARKER, - EventAttribute.LOGGER_NAME, - EventAttribute.MESSAGE, - EventAttribute.EXCEPTION, - EventAttribute.TIMESTAMP); - - private Set includesMdcKeys = Collections.emptySet(); - private boolean flattenMdc = false; - private boolean includeKeyValues = true; - private boolean flattenKeyValues = false; - - @Nullable private ExceptionFormat exceptionFormat; - - @JsonProperty - public EnumSet getIncludes() { - return includes; - } - - @JsonProperty - public void setIncludes(EnumSet includes) { - this.includes = includes; - } - - @JsonProperty - public Set getIncludesMdcKeys() { - return includesMdcKeys; - } - - @JsonProperty - public void setIncludesMdcKeys(Set includesMdcKeys) { - this.includesMdcKeys = includesMdcKeys; - } - - @JsonProperty - public boolean isFlattenMdc() { - return flattenMdc; - } - - @JsonProperty - public void setFlattenMdc(boolean flattenMdc) { - this.flattenMdc = flattenMdc; - } - - @JsonProperty - public boolean isIncludeKeyValues() { - return includeKeyValues; - } - - @JsonProperty - public void setIncludeKeyValues(boolean includeKeyValues) { - this.includeKeyValues = includeKeyValues; - } - - @JsonProperty - public boolean isFlattenKeyValues() { - return flattenKeyValues; - } - - @JsonProperty - public void setFlattenKeyValues(boolean flattenKeyValues) { - this.flattenKeyValues = flattenKeyValues; - } - - /** - * @since 2.0 - */ - @JsonProperty("exception") - public void setExceptionFormat(ExceptionFormat exceptionFormat) { - this.exceptionFormat = exceptionFormat; - } - - /** - * @since 2.0 - */ - @JsonProperty("exception") - @Nullable - public ExceptionFormat getExceptionFormat() { - return exceptionFormat; - } - - @Override - public LayoutBase build(LoggerContext context, TimeZone timeZone) { - final PolarisJsonLayout jsonLayout = - new PolarisJsonLayout( - createDropwizardJsonFormatter(), - createTimestampFormatter(timeZone), - createThrowableProxyConverter(context), - includes, - getCustomFieldNames(), - getAdditionalFields(), - includesMdcKeys, - flattenMdc, - includeKeyValues, - flattenKeyValues); - jsonLayout.setContext(context); - return jsonLayout; - } - - public static class PolarisJsonLayout extends EventJsonLayout { - private final boolean includeKeyValues; - private final boolean flattenKeyValues; - - public PolarisJsonLayout( - JsonFormatter jsonFormatter, - TimestampFormatter timestampFormatter, - ThrowableHandlingConverter throwableProxyConverter, - Set includes, - Map customFieldNames, - Map additionalFields, - Set includesMdcKeys, - boolean flattenMdc, - boolean includeKeyValues, - boolean flattenKeyValues) { - super( - jsonFormatter, - timestampFormatter, - throwableProxyConverter, - includes, - customFieldNames, - additionalFields, - includesMdcKeys, - flattenMdc); - this.includeKeyValues = includeKeyValues; - this.flattenKeyValues = flattenKeyValues; - } - - @Override - protected Map toJsonMap(ILoggingEvent event) { - Map jsonMap = super.toJsonMap(event); - if (!includeKeyValues) { - return jsonMap; - } - Map keyValueMap = - event.getKeyValuePairs() == null - ? Map.of() - : event.getKeyValuePairs().stream() - .collect(Collectors.toMap(kv -> kv.key, kv -> kv.value)); - ImmutableMap.Builder builder = - ImmutableMap.builder().putAll(jsonMap); - if (flattenKeyValues) { - builder.putAll(keyValueMap); - } else { - builder.put("params", keyValueMap); - } - return builder.build(); - } - } - - protected ThrowableHandlingConverter createThrowableProxyConverter(LoggerContext context) { - if (exceptionFormat == null) { - return new RootCauseFirstThrowableProxyConverter(); - } - - ThrowableHandlingConverter throwableHandlingConverter; - if (exceptionFormat.isRootFirst()) { - throwableHandlingConverter = new RootCauseFirstThrowableProxyConverter(); - } else { - throwableHandlingConverter = new ExtendedThrowableProxyConverter(); - } - - List options = new ArrayList<>(); - // depth must be added first - options.add(exceptionFormat.getDepth()); - options.addAll(exceptionFormat.getEvaluators()); - - throwableHandlingConverter.setOptionList(options); - throwableHandlingConverter.setContext(context); - - return throwableHandlingConverter; - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java deleted file mode 100644 index c34249a85..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.monitor; - -import com.google.common.annotations.VisibleForTesting; -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import io.micrometer.core.instrument.Timer; -import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; -import io.micrometer.core.instrument.binder.system.ProcessorMetrics; -import jakarta.inject.Inject; -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -/** - * Wrapper around the Micrometer {@link MeterRegistry} providing additional metric management - * functions for the Polaris application. Implements in-memory caching of timers and counters. - * Records two metrics for each instrument with one tagged by the realm ID (realm-specific metric) - * and one without. The realm-specific metric is suffixed with ".realm". - * - *

Uppercase tag names are now deprecated. Prefer snake_casing instead. Old metrics are emitted - * with both variations but the uppercase versions may eventually be removed. New methods for tag - * emission (those that allow you to pass in an Iterable) only emit the snake_case version. - */ -public class PolarisMetricRegistry { - private final MeterRegistry meterRegistry; - private final ConcurrentMap timers = new ConcurrentHashMap<>(); - private final ConcurrentMap counters = new ConcurrentHashMap<>(); - - /** - * @deprecated See class Javadoc. - */ - @Deprecated public static final String TAG_REALM_DEPRECATED = "REALM_ID"; - - public static final String TAG_REALM = "realm_id"; - - /** - * @deprecated See class Javadoc. - */ - @Deprecated public static final String TAG_RESP_CODE_DEPRECATED = "HTTP_RESPONSE_CODE"; - - public static final String TAG_RESP_CODE = "http_response_code"; - - public static final String SUFFIX_COUNTER = ".count"; - public static final String SUFFIX_ERROR = ".error"; - public static final String SUFFIX_REALM = ".realm"; - - @Inject - public PolarisMetricRegistry(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - new ClassLoaderMetrics().bindTo(meterRegistry); - new JvmMemoryMetrics().bindTo(meterRegistry); - new JvmGcMetrics().bindTo(meterRegistry); - new ProcessorMetrics().bindTo(meterRegistry); - new JvmThreadMetrics().bindTo(meterRegistry); - } - - public MeterRegistry getMeterRegistry() { - return meterRegistry; - } - - @VisibleForTesting - public void clear() { - meterRegistry.clear(); - counters.clear(); - } - - public void init(Class... classes) { - for (Class clazz : classes) { - Method[] methods = clazz.getDeclaredMethods(); - for (Method method : methods) { - if (method.isAnnotationPresent(Timed.class)) { - Timed timedApi = method.getAnnotation(Timed.class); - String metric = timedApi.value(); - timers.put(metric, Timer.builder(metric).register(meterRegistry)); - counters.put( - metric + SUFFIX_COUNTER, - Counter.builder(metric + SUFFIX_COUNTER).register(meterRegistry)); - - // Error counters contain the HTTP response code in a tag, thus caching them would not be - // meaningful. - Counter.builder(metric + SUFFIX_ERROR) - .tags(TAG_RESP_CODE, "400", TAG_RESP_CODE_DEPRECATED, "400") - .register(meterRegistry); - Counter.builder(metric + SUFFIX_ERROR) - .tags(TAG_RESP_CODE, "500", TAG_RESP_CODE_DEPRECATED, "500") - .register(meterRegistry); - } - } - } - } - - public void recordTimer(String metric, long elapsedTimeMs, String realmId) { - Timer timer = - timers.computeIfAbsent(metric, m -> Timer.builder(metric).register(meterRegistry)); - timer.record(elapsedTimeMs, TimeUnit.MILLISECONDS); - - Timer timerRealm = - timers.computeIfAbsent( - metric + SUFFIX_REALM, - m -> - Timer.builder(metric + SUFFIX_REALM) - .tag(TAG_REALM, realmId) - .tag(TAG_REALM_DEPRECATED, realmId) - .register(meterRegistry)); - timerRealm.record(elapsedTimeMs, TimeUnit.MILLISECONDS); - } - - /** - * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric. - * Counters are cached. - */ - public void incrementCounter(String metric, String realmId) { - String counterMetric = metric + SUFFIX_COUNTER; - Counter counter = - counters.computeIfAbsent( - counterMetric, m -> Counter.builder(counterMetric).register(meterRegistry)); - counter.increment(); - - Counter counterRealm = - counters.computeIfAbsent( - counterMetric + SUFFIX_REALM, - m -> - Counter.builder(counterMetric + SUFFIX_REALM) - .tag(TAG_REALM, realmId) - .tag(TAG_REALM_DEPRECATED, realmId) - .register(meterRegistry)); - counterRealm.increment(); - } - - /** - * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric. - * Both metrics are tagged with the statusCode and counters are not cached. - */ - public void incrementErrorCounter(String metric, int statusCode, String realmId) { - String errorMetric = metric + SUFFIX_ERROR; - Counter.builder(errorMetric) - .tag(TAG_RESP_CODE, String.valueOf(statusCode)) - .tag(TAG_RESP_CODE_DEPRECATED, String.valueOf(statusCode)) - .register(meterRegistry) - .increment(); - - Counter.builder(errorMetric + SUFFIX_REALM) - .tag(TAG_RESP_CODE, String.valueOf(statusCode)) - .tag(TAG_RESP_CODE_DEPRECATED, String.valueOf(statusCode)) - .tag(TAG_REALM, realmId) - .tag(TAG_REALM_DEPRECATED, realmId) - .register(meterRegistry) - .increment(); - } - - /** - * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementCounter(String metric, String realmId, Iterable tags) { - Counter.builder(metric + SUFFIX_COUNTER).tags(tags).register(meterRegistry).increment(); - - Counter.builder(metric + SUFFIX_COUNTER + SUFFIX_REALM) - .tags(tags) - .tag(TAG_REALM, realmId) - .register(meterRegistry) - .increment(); - } - - /** - * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementCounter(String metric, String realmId, Tag tag) { - incrementCounter(metric, realmId, Collections.singleton(tag)); - } - - /** - * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementErrorCounter(String metric, String realmId, Iterable tags) { - Counter.builder(metric + SUFFIX_ERROR).tags(tags).register(meterRegistry).increment(); - - Counter.builder(metric + SUFFIX_ERROR + SUFFIX_REALM) - .tags(tags) - .tag(TAG_REALM, realmId) - .register(meterRegistry) - .increment(); - } - - /** - * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementErrorCounter(String metric, String realmId, Tag tag) { - incrementErrorCounter(metric, realmId, Collections.singleton(tag)); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java deleted file mode 100644 index 0313fd85c..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.persistence.cache; - -import jakarta.inject.Inject; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmScoped; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.glassfish.hk2.api.Factory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EntityCacheFactory implements Factory { - private static Logger LOGGER = LoggerFactory.getLogger(EntityCacheFactory.class); - @Inject PolarisMetaStoreManager metaStoreManager; - - @RealmScoped - @Override - public EntityCache provide() { - LOGGER.debug( - "Creating new EntityCache instance for realm {}", - CallContext.getCurrentContext().getRealmContext().getRealmIdentifier()); - return new EntityCache(metaStoreManager); - } - - @Override - public void dispose(EntityCache instance) { - // no-op - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java deleted file mode 100644 index 5ec8fd79a..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.throttling; - -import static org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; - -import com.fasterxml.jackson.core.exc.StreamConstraintsException; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; - -/** - * Handles exceptions during the request that are a result of stream constraints such as the request - * being too large - */ -public class StreamReadConstraintsExceptionMapper - implements ExceptionMapper { - - @Override - public Response toResponse(StreamConstraintsException exception) { - return Response.status(Response.Status.REQUEST_ENTITY_TOO_LARGE) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new RequestThrottlingErrorResponse(REQUEST_TOO_LARGE)) - .build(); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java deleted file mode 100644 index f6e52ac55..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.tracing; - -import io.opentelemetry.context.propagation.TextMapGetter; -import io.opentelemetry.context.propagation.TextMapSetter; -import jakarta.annotation.Nullable; -import jakarta.servlet.http.HttpServletRequest; -import java.net.http.HttpRequest; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.StreamSupport; - -/** - * Implementation of {@link TextMapSetter} and {@link TextMapGetter} that can handle an {@link - * HttpServletRequest} for extracting headers and sets headers on a {@link HttpRequest.Builder}. - */ -public class HeadersMapAccessor - implements TextMapSetter, TextMapGetter { - @Override - public Iterable keys(HttpServletRequest carrier) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize( - carrier.getHeaderNames().asIterator(), Spliterator.IMMUTABLE), - false) - .toList(); - } - - @Nullable - @Override - public String get(@Nullable HttpServletRequest carrier, String key) { - return carrier == null ? null : carrier.getHeader(key); - } - - @Override - public void set(@Nullable HttpRequest.Builder carrier, String key, String value) { - if (carrier != null) { - carrier.setHeader(key, value); - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java deleted file mode 100644 index 30f5919a5..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.tracing; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import io.opentelemetry.semconv.HttpAttributes; -import io.opentelemetry.semconv.ServerAttributes; -import io.opentelemetry.semconv.UrlAttributes; -import jakarta.annotation.Priority; -import jakarta.inject.Inject; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.Priorities; -import java.io.IOException; -import org.apache.polaris.core.context.CallContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -/** - * Servlet {@link Filter} that starts an OpenTracing {@link Span}, propagating the calling context - * from HTTP headers, if present. "spanId" and "traceId" are added to the logging MDC so that all - * logs recorded in the request will contain the current span and trace id. Downstream HTTP calls - * should use the OpenTelemetry {@link io.opentelemetry.context.propagation.ContextPropagators} to - * include the current trace id in the request headers. - */ -@Priority(Priorities.AUTHENTICATION - 1) -public class TracingFilter implements Filter { - private static final Logger LOGGER = LoggerFactory.getLogger(TracingFilter.class); - private final OpenTelemetry openTelemetry; - - @Inject - public TracingFilter(OpenTelemetry openTelemetry) { - this.openTelemetry = openTelemetry; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - Context extractedContext = - openTelemetry - .getPropagators() - .getTextMapPropagator() - .extract(Context.current(), httpRequest, new HeadersMapAccessor()); - try (Scope scope = extractedContext.makeCurrent()) { - Tracer tracer = openTelemetry.getTracer(httpRequest.getPathInfo()); - Span span = - tracer - .spanBuilder(httpRequest.getMethod() + " " + httpRequest.getPathInfo()) - .setSpanKind(SpanKind.SERVER) - .setAttribute( - "realm", CallContext.getCurrentContext().getRealmContext().getRealmIdentifier()) - .startSpan(); - - try (Scope ignored = span.makeCurrent(); - MDC.MDCCloseable spanId = MDC.putCloseable("spanId", span.getSpanContext().getSpanId()); - MDC.MDCCloseable traceId = - MDC.putCloseable("traceId", span.getSpanContext().getTraceId())) { - LOGGER - .atInfo() - .addKeyValue("spanId", span.getSpanContext().getSpanId()) - .addKeyValue("traceId", span.getSpanContext().getTraceId()) - .addKeyValue("parentContext", extractedContext) - .log("Started span with parent"); - span.setAttribute(HttpAttributes.HTTP_REQUEST_METHOD, httpRequest.getMethod()); - span.setAttribute(ServerAttributes.SERVER_ADDRESS, httpRequest.getServerName()); - span.setAttribute(UrlAttributes.URL_SCHEME, httpRequest.getScheme()); - span.setAttribute(UrlAttributes.URL_PATH, httpRequest.getPathInfo()); - - chain.doFilter(request, response); - } finally { - span.end(); - } - } - } -} diff --git a/dropwizard/service/src/main/resources/META-INF/hk2-locator/default b/dropwizard/service/src/main/resources/META-INF/hk2-locator/default deleted file mode 100644 index b5eacf9ce..000000000 --- a/dropwizard/service/src/main/resources/META-INF/hk2-locator/default +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -[org.apache.polaris.service.auth.DefaultPolarisAuthenticator]S -contract={org.apache.polaris.service.auth.Authenticator} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator]S -contract={org.apache.polaris.service.auth.Authenticator} -name=test -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory]S -contract={org.apache.polaris.core.persistence.MetaStoreManagerFactory} -name=in-memory -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.DefaultOAuth2ApiService]S -contract={org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.TestOAuth2ApiService]S -contract={org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} -name=test -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.JWTSymmetricKeyFactory]S -contract={org.apache.polaris.service.auth.TokenBrokerFactory} -name=symmetric-key -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.JWTRSAKeyPairFactory]S -contract={org.apache.polaris.service.auth.TokenBrokerFactory} -name=rsa-key-pair -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.NoneTokenBrokerFactory]S -contract={org.apache.polaris.service.auth.TokenBrokerFactory} -name=none -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.context.DefaultRealmContextResolver]S -contract={org.apache.polaris.service.context.RealmContextResolver} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.context.DefaultCallContextResolver]S -contract={org.apache.polaris.service.context.CallContextResolver} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.core.storage.PolarisStorageIntegrationProvider]S -contract={org.apache.polaris.core.storage.PolarisStorageIntegrationProvider} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.catalog.io.DefaultFileIOFactory]S -contract={org.apache.polaris.service.catalog.io.FileIOFactory} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.catalog.io.WasbTranslatingFileIOFactory]S -contract={org.apache.polaris.service.catalog.io.FileIOFactory} -name=wasb -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.ratelimiter.NoOpRateLimiter]S -contract={org.apache.polaris.service.ratelimiter.RateLimiter} -name=no-op -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter]S -contract={org.apache.polaris.service.ratelimiter.RateLimiter} -name=realm-token-bucket -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.ratelimiter.DefaultTokenBucketFactory]S -contract={org.apache.polaris.service.ratelimiter.TokenBucketFactory} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java deleted file mode 100644 index 112b121c3..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(DropwizardExtensionsSupport.class) -public class PolarisApplicationConfigurationTest { - - public static final String CONFIG_PATH = - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"); - // Bind to random ports to support parallelism - public static final ConfigOverride RANDOM_APP_PORT = - ConfigOverride.config("server.applicationConnectors[0].port", "0"); - public static final ConfigOverride RANDOM_ADMIN_PORT = - ConfigOverride.config("server.adminConnectors[0].port", "0"); - - @Nested - class DefaultMetastore { - private final DropwizardAppExtension app = - new DropwizardAppExtension<>( - PolarisApplication.class, CONFIG_PATH, RANDOM_APP_PORT, RANDOM_ADMIN_PORT); - - @Test - void testMetastoreType() { - assertThat(app.getConfiguration().findService(MetaStoreManagerFactory.class)) - .isInstanceOf(InMemoryPolarisMetaStoreManagerFactory.class); - } - } - - @Nested - class EclipseLinkMetastore { - private final DropwizardAppExtension app = - new DropwizardAppExtension<>( - PolarisApplication.class, - CONFIG_PATH, - RANDOM_APP_PORT, - RANDOM_ADMIN_PORT, - ConfigOverride.config("metaStoreManager.type", "eclipse-link"), - ConfigOverride.config("metaStoreManager.persistence-unit", "test-unit"), - ConfigOverride.config("metaStoreManager.conf-file", "/test-conf-file")); - - @Test - void testMetastoreType() { - assertThat(app.getConfiguration().findService(MetaStoreManagerFactory.class)) - .isInstanceOf(EclipseLinkPolarisMetaStoreManagerFactory.class) - .extracting("persistenceUnitName", "confFile") - .containsExactly("test-unit", "/test-conf-file"); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java deleted file mode 100644 index 76c40cfe2..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.SINGLETON_METRIC_NAME; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.TAG_API_NAME; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_COUNTER; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_ERROR; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_REALM; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_REALM; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_REALM_DEPRECATED; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE_DEPRECATED; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Tag; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.dropwizard.test.TestMetricsUtil; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public class TimedApplicationEventListenerTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - private static final int ERROR_CODE = Response.Status.NOT_FOUND.getStatusCode(); - private static final String ENDPOINT = "api/management/v1/principals"; - private static final String API_ANNOTATION = - Arrays.stream(PolarisPrincipalsApi.class.getMethods()) - .filter(m -> m.getName().contains("getPrincipal")) - .findFirst() - .orElseThrow() - .getAnnotation(Timed.class) - .value(); - - private static PolarisConnectionExtension.PolarisToken userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; - private static String realm; - - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) - throws IOException { - TimedApplicationEventListenerTest.userToken = userToken; - TimedApplicationEventListenerTest.snowmanCredentials = snowmanCredentials; - TimedApplicationEventListenerTest.realm = realm; - } - - @BeforeEach - public void clearMetrics() { - getPolarisMetricRegistry().clear(); - } - - @Test - public void testMetricsEmittedOnSuccessfulRequest() { - sendSuccessfulRequest(); - Assertions.assertTrue(getPerApiMetricCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricCount() > 0); - Assertions.assertTrue(getCommonMetricCount() > 0); - Assertions.assertTrue(getCommonRealmMetricCount() > 0); - Assertions.assertEquals(0, getPerApiMetricErrorCount()); - Assertions.assertEquals(0, getPerApiRealmMetricErrorCount()); - Assertions.assertEquals(0, getCommonMetricErrorCount()); - Assertions.assertEquals(0, getCommonRealmMetricErrorCount()); - } - - @Test - public void testMetricsEmittedOnFailedRequest() { - sendFailingRequest(); - Assertions.assertTrue(getPerApiMetricCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricCount() > 0); - Assertions.assertTrue(getCommonMetricCount() > 0); - Assertions.assertTrue(getCommonRealmMetricCount() > 0); - Assertions.assertTrue(getPerApiMetricErrorCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricErrorCount() > 0); - Assertions.assertTrue(getCommonMetricErrorCount() > 0); - Assertions.assertTrue(getCommonRealmMetricErrorCount() > 0); - } - - private PolarisMetricRegistry getPolarisMetricRegistry() { - TimedApplicationEventListener listener = - (TimedApplicationEventListener) - EXT.getEnvironment().jersey().getResourceConfig().getSingletons().stream() - .filter( - s -> - TimedApplicationEventListener.class - .getName() - .equals(s.getClass().getName())) - .findAny() - .orElseThrow(); - return listener.getMetricRegistry(); - } - - private double getPerApiMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, API_ANNOTATION + SUFFIX_COUNTER, Collections.emptyList()); - } - - @SuppressWarnings("deprecation") - private double getPerApiRealmMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_COUNTER + SUFFIX_REALM, - List.of(Tag.of(TAG_REALM, realm), Tag.of(TAG_REALM_DEPRECATED, realm))); - } - - @SuppressWarnings("deprecation") - private double getPerApiMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_ERROR, - List.of( - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)), - Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE)))); - } - - @SuppressWarnings("deprecation") - private double getPerApiRealmMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_ERROR + SUFFIX_REALM, - List.of( - Tag.of(TAG_REALM, realm), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)), - Tag.of(TAG_REALM_DEPRECATED, realm), - Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE)))); - } - - private double getCommonMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_COUNTER, - Collections.singleton(Tag.of(TAG_API_NAME, API_ANNOTATION))); - } - - private double getCommonRealmMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_COUNTER + SUFFIX_REALM, - List.of(Tag.of(TAG_API_NAME, API_ANNOTATION), Tag.of(TAG_REALM, realm))); - } - - private double getCommonMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR, - List.of( - Tag.of(TAG_API_NAME, API_ANNOTATION), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)))); - } - - private double getCommonRealmMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR + SUFFIX_REALM, - List.of( - Tag.of(TAG_API_NAME, API_ANNOTATION), - Tag.of(TAG_REALM, realm), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)))); - } - - private int sendRequest(String principalName) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/%s/%s", EXT.getLocalPort(), ENDPOINT, principalName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - return response.getStatus(); - } - } - - private void sendSuccessfulRequest() { - Assertions.assertEquals( - Response.Status.OK.getStatusCode(), - sendRequest(snowmanCredentials.identifier().principalName())); - } - - private void sendFailingRequest() { - Assertions.assertEquals(ERROR_CODE, sendRequest("notarealprincipal")); - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java deleted file mode 100644 index c1a082533..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.admin; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import io.dropwizard.core.setup.Environment; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.Feature; -import jakarta.ws.rs.core.FeatureContext; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.persistence.cache.EntityCacheFactory; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.jersey.process.internal.RequestScope; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * tests around the {@link EntityCacheFactory} and ensuring that the {@link - * org.apache.polaris.core.persistence.cache.EntityCache} is managed per realm. - */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) -public class PolarisRealmEntityCacheTest { - private static ServiceLocatorAccessor accessor = new ServiceLocatorAccessor(); - - /** - * Injectable {@link Feature} that allows us to access the {@link ServiceLocator} from the jersey - * resource configuration. - */ - private static final class ServiceLocatorAccessor implements Feature { - @Inject ServiceLocator serviceLocator; - - @Override - public boolean configure(FeatureContext context) { - return true; - } - - public ServiceLocator getServiceLocator() { - return serviceLocator; - } - } - - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0")) - .addListener( - new DropwizardAppExtension.ServiceListener() { - @Override - public void onRun( - PolarisApplicationConfig configuration, - Environment environment, - DropwizardAppExtension rule) - throws Exception { - environment.jersey().register(accessor); - } - }); - private static String userToken; - private static String realm; - - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); - } - - @Test - public void testRealmEntityCacheEquality() { - ServiceLocator serviceLocator = accessor.getServiceLocator(); - RequestScope requestScope = serviceLocator.getService(RequestScope.class); - EntityCache cache1; - // check that multiple calls to the serviceLocator return the same instance of the EntityCache - // within the same call context - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - cache1 = requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - EntityCache cache2 = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache1).isSameAs(cache2); - } - - // in a new call context with a different realm, the EntityCache should be different - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> "anotherrealm", new PolarisCallContext(null, null)))) { - EntityCache cache2 = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache1).isNotSameAs(cache2); - } - - // but if we start a new call context with the original realm, we'll get the same EntityCache - // instance - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - EntityCache cache2 = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache1).isSameAs(cache2); - } - } - - @Test - public void testCacheForRealm() { - // create a catalog - String catalogName = "mycachecatalog"; - ServiceLocator serviceLocator = accessor.getServiceLocator(); - RequestScope requestScope = serviceLocator.getService(RequestScope.class); - listCatalogs(); - - // check for the catalog - it should not exist - // the service_admin role, however, should exist - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - EntityCache cache = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache).isNotNull(); - EntityCacheEntry cachedCatalog = - cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); - assertThat(cachedCatalog).isNull(); - EntityCacheEntry serviceAdmin = - cache.getEntityByName( - new EntityCacheByNameKey( - PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())); - assertThat(serviceAdmin) - .isNotNull() - .extracting(EntityCacheEntry::getEntity) - .returns(PolarisEntityType.PRINCIPAL_ROLE, PolarisBaseEntity::getType) - .returns( - PolarisEntityConstants.getNameOfPrincipalServiceAdminRole(), - PolarisBaseEntity::getName); - } - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - createCatalogRole(catalogName, "my_cr", userToken); - - // now check again for the catalog - it should exist - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - EntityCache cache = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache).isNotNull(); - EntityCacheEntry cachedCatalog = - cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); - assertThat(cachedCatalog) - .isNotNull() - .extracting(EntityCacheEntry::getEntity) - .returns(catalogName, PolarisBaseEntity::getName); - } - - // but if we check a different realm, the catalog should not exist in the cache - // the service_admin role also does not exist, since it's never been used - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> "another-realm", new PolarisCallContext(null, null)))) { - EntityCache cache = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache).isNotNull(); - EntityCacheEntry cachedCatalog = - cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); - assertThat(cachedCatalog).isNull(); - EntityCacheEntry serviceAdmin = - cache.getEntityByName( - new EntityCacheByNameKey( - PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())); - assertThat(serviceAdmin).isNull(); - } - } - - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); - } - - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); - } - - private static void listCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalog(Catalog catalog) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalogRole( - String catalogName, String catalogRoleName, String catalogAdminToken) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", - catalogAdminToken) - .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java deleted file mode 100644 index a2dc8370a..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.ratelimiter; - -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.SINGLETON_METRIC_NAME; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.TAG_API_NAME; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_ERROR; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import io.micrometer.core.instrument.Tag; -import jakarta.ws.rs.core.Response; -import java.time.Duration; -import java.util.List; -import java.util.function.Consumer; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.dropwizard.test.TestMetricsUtil; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.threeten.extra.MutableClock; - -/** Main integration tests for rate limiting */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public class RateLimiterFilterTest { - private static final long REQUESTS_PER_SECOND = 5; - private static final long WINDOW_SECONDS = 10; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("tokenBucketFactory.type", "mock"), - ConfigOverride.config( - "tokenBucketFactory.requestsPerSecond", String.valueOf(REQUESTS_PER_SECOND)), - ConfigOverride.config( - "tokenBucketFactory.windowSeconds", String.valueOf(WINDOW_SECONDS))); - - private static String userToken; - private static String realm; - private static MutableClock clock = MockTokenBucketFactory.CLOCK; - - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, @PolarisRealm String polarisRealm) { - realm = polarisRealm; - RateLimiterFilterTest.userToken = userToken.token(); - } - - @BeforeEach - @AfterEach - public void resetRateLimiter() { - clock.add( - Duration.ofSeconds(2 * WINDOW_SECONDS)); // Clear any counters from before/after this test - } - - @Test - public void testRateLimiter() { - Consumer requestAsserter = - TestUtil.constructRequestAsserter(EXT, userToken, realm); - - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW_SECONDS; i++) { - requestAsserter.accept(Response.Status.OK); - } - requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); - - // Ensure that a different realm identifier gets a separate limit - Consumer requestAsserter2 = - TestUtil.constructRequestAsserter(EXT, userToken, realm + "2"); - requestAsserter2.accept(Response.Status.OK); - } - - @Test - public void testMetricsAreEmittedWhenRateLimiting() { - Consumer requestAsserter = - TestUtil.constructRequestAsserter(EXT, userToken, realm); - - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW_SECONDS; i++) { - requestAsserter.accept(Response.Status.OK); - } - requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); - - assertTrue( - TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR, - List.of( - Tag.of(TAG_API_NAME, "polaris.principal-roles.listPrincipalRoles"), - Tag.of( - TAG_RESP_CODE, - String.valueOf(Response.Status.TOO_MANY_REQUESTS.getStatusCode())))) - > 0); - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java deleted file mode 100644 index efeee130c..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.ratelimiter; - -import static org.apache.polaris.service.dropwizard.ratelimiter.MockTokenBucketFactory.CLOCK; - -import java.time.Duration; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.ratelimiter.DefaultTokenBucketFactory; -import org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** Main unit test class for TokenBucketRateLimiter */ -public class RealmTokenBucketRateLimiterTest { - @Test - void testDifferentBucketsDontTouch() { - RealmTokenBucketRateLimiter rateLimiter = new RealmTokenBucketRateLimiter(); - rateLimiter.setTokenBucketFactory(new DefaultTokenBucketFactory(10, 10, CLOCK)); - - for (int i = 0; i < 202; i++) { - String realm = (i % 2 == 0) ? "realm1" : "realm2"; - CallContext.setCurrentContext(CallContext.of(() -> realm, null)); - - if (i < 200) { - Assertions.assertTrue(rateLimiter.canProceed()); - } else { - assertCannotProceed(rateLimiter); - } - } - - CLOCK.add(Duration.ofSeconds(1)); - for (int i = 0; i < 22; i++) { - String realm = (i % 2 == 0) ? "realm1" : "realm2"; - CallContext.setCurrentContext(CallContext.of(() -> realm, null)); - - if (i < 20) { - Assertions.assertTrue(rateLimiter.canProceed()); - } else { - assertCannotProceed(rateLimiter); - } - } - } - - private void assertCannotProceed(RealmTokenBucketRateLimiter rateLimiter) { - for (int i = 0; i < 5; i++) { - Assertions.assertFalse(rateLimiter.canProceed()); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java deleted file mode 100644 index 0676fb20a..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import jakarta.annotation.Nullable; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.Arrays; -import java.util.Optional; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.platform.commons.util.ReflectionUtils; -import org.slf4j.LoggerFactory; - -public class DropwizardTestEnvironmentResolver implements TestEnvironmentResolver { - /** - * Resolves the TestEnvironment to point to the local Dropwizard instance. - * - * @param extensionContext - * @return - */ - @Override - public TestEnvironment resolveTestEnvironment(ExtensionContext extensionContext) { - try { - DropwizardAppExtension dropwizardAppExtension = findDropwizardExtension(extensionContext); - if (dropwizardAppExtension == null) { - throw new ParameterResolutionException("Could not find DropwizardAppExtension."); - } - return new TestEnvironment( - dropwizardAppExtension.client(), - String.format("http://localhost:%d", dropwizardAppExtension.getLocalPort())); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - public static @Nullable DropwizardAppExtension findDropwizardExtension( - ExtensionContext extensionContext) throws IllegalAccessException { - Field dropwizardExtensionField = - findAnnotatedFields(extensionContext.getRequiredTestClass(), true); - if (dropwizardExtensionField == null) { - LoggerFactory.getLogger(PolarisGrantRecord.class) - .warn( - "Unable to find dropwizard extension field in test class {}", - extensionContext.getRequiredTestClass()); - return null; - } - DropwizardAppExtension appExtension = - (DropwizardAppExtension) ReflectionUtils.makeAccessible(dropwizardExtensionField).get(null); - return appExtension; - } - - private static Field findAnnotatedFields(Class testClass, boolean isStaticMember) { - final Optional set = - Arrays.stream(testClass.getDeclaredFields()) - .filter(m -> isStaticMember == Modifier.isStatic(m.getModifiers())) - .filter(m -> DropwizardAppExtension.class.isAssignableFrom(m.getType())) - .findFirst(); - if (set.isPresent()) { - return set.get(); - } - if (!testClass.getSuperclass().equals(Object.class)) { - return findAnnotatedFields(testClass.getSuperclass(), isStaticMember); - } - return null; - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java deleted file mode 100644 index a7d08d1ce..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.test.DropwizardTestEnvironmentResolver.findDropwizardExtension; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; - -public class PolarisConnectionExtension - implements BeforeAllCallback, AfterAllCallback, ParameterResolver { - - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private MetaStoreManagerFactory metaStoreManagerFactory; - private DropwizardAppExtension dropwizardAppExtension; - - public record PolarisToken(String token) {} - - private static PolarisPrincipalSecrets adminSecrets; - private static String realm; - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - dropwizardAppExtension = findDropwizardExtension(extensionContext); - if (dropwizardAppExtension == null) { - return; - } - - // Generate unique realm using test name for each test since the tests can run in parallel - realm = extensionContext.getRequiredTestClass().getName().replace('.', '_'); - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .put(REALM_PROPERTY_KEY, realm); - - try { - PolarisApplicationConfig config = - (PolarisApplicationConfig) dropwizardAppExtension.getConfiguration(); - metaStoreManagerFactory = config.findService(MetaStoreManagerFactory.class); - if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - metaStoreManagerFactory.bootstrapRealms(List.of(realm)); - } - - URI testEnvUri = TestEnvironmentExtension.getEnv(extensionContext).baseUri(); - String path = testEnvUri.getPath(); - if (path.isEmpty()) { - path = "/"; - } - - RealmContext realmContext = - config - .findService(RealmContextResolver.class) - .resolveRealmContext( - String.format("%s://%s", testEnvUri.getScheme(), testEnvUri.getHost()), - "GET", - path, - Map.of(REALM_PROPERTY_KEY, realm)); - CallContext ctx = - config - .findService(CallContextResolver.class) - .resolveCallContext(realmContext, "GET", path, Map.of()); - CallContext.setCurrentContext(ctx); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); - PolarisMetaStoreManager.EntityResult principal = - metaStoreManager.readEntityByName( - ctx.getPolarisCallContext(), - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - PolarisEntityConstants.getRootPrincipalName()); - - Map propertiesMap = readInternalProperties(principal); - adminSecrets = - metaStoreManager - .loadPrincipalSecrets(ctx.getPolarisCallContext(), propertiesMap.get("client_id")) - .getPrincipalSecrets(); - } finally { - CallContext.unsetCurrentContext(); - } - } - - @Override - public void afterAll(ExtensionContext context) { - if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - metaStoreManagerFactory.purgeRealms(List.of(realm)); - } - } - - public static void createTestDir(String realm) throws IOException { - // Set up the database location - Path testDir = Path.of("build/test_data/polaris/" + realm); - if (Files.exists(testDir)) { - if (Files.isDirectory(testDir)) { - Files.walk(testDir) - .sorted(Comparator.reverseOrder()) - .forEach( - path -> { - try { - Files.delete(path); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - } else { - Files.delete(testDir); - } - } - Files.createDirectories(testDir); - } - - static PolarisPrincipalSecrets getAdminSecrets() { - return adminSecrets; - } - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameterContext - .getParameter() - .getType() - .equals(PolarisConnectionExtension.PolarisToken.class) - || parameterContext.getParameter().getType().equals(MetaStoreManagerFactory.class) - || parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class) - || (parameterContext.getParameter().getType().equals(String.class) - && parameterContext.getParameter().isAnnotationPresent(PolarisRealm.class)); - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - if (parameterContext.getParameter().getType().equals(PolarisToken.class)) { - try { - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String token = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - return new PolarisToken(token); - } catch (IllegalAccessException e) { - throw new ParameterResolutionException(e.getMessage()); - } - } else if (parameterContext.getParameter().getType().equals(String.class) - && parameterContext.getParameter().isAnnotationPresent(PolarisRealm.class)) { - return realm; - } else if (parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class)) { - return adminSecrets; - } else { - return metaStoreManagerFactory; - } - } - - private static Map readInternalProperties( - PolarisMetaStoreManager.EntityResult principal) { - try { - return OBJECT_MAPPER.readValue( - principal.getEntity().getInternalProperties(), - new TypeReference>() {}); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java deleted file mode 100644 index cbf8b0f51..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowmanCredentialsExtension - implements BeforeAllCallback, AfterAllCallback, ParameterResolver { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowmanCredentialsExtension.class); - private SnowmanCredentials snowmanCredentials; - - public record SnowmanIdentifier(String principalName, String principalRoleName) {} - - public record SnowmanCredentials( - String clientId, String clientSecret, SnowmanIdentifier identifier) {} - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - PolarisPrincipalSecrets adminSecrets = PolarisConnectionExtension.getAdminSecrets(); - String realm = - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .get(REALM_PROPERTY_KEY, String.class); - - if (adminSecrets == null) { - LOGGER - .atError() - .log( - "No admin secrets configured - you must also configure your test with PolarisConnectionExtension"); - return; - } - - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String userToken = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - - SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); - PrincipalRole principalRole = new PrincipalRole(snowmanIdentifier.principalRoleName()); - try (Response createPrResponse = - testEnv - .apiClient() - .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principalRole))) { - assertThat(createPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - Principal principal = new Principal(snowmanIdentifier.principalName()); - - try (Response createPResponse = - testEnv - .apiClient() - .target(String.format("%s/api/management/v1/principals", testEnv.baseUri())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) // how is token getting used? - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principal))) { - assertThat(createPResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials snowmanWithCredentials = - createPResponse.readEntity(PrincipalWithCredentials.class); - try (Response rotateResp = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s/rotate", - testEnv.baseUri(), principal.getName())) - .request(MediaType.APPLICATION_JSON) - .header( - "Authorization", - "Bearer " - + TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret(), - realm)) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(snowmanWithCredentials))) { - - assertThat(rotateResp).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - - // Use the rotated credentials. - snowmanWithCredentials = rotateResp.readEntity(PrincipalWithCredentials.class); - } - snowmanCredentials = - new SnowmanCredentials( - snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret(), - snowmanIdentifier); - } - try (Response assignPrResponse = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s/principal-roles", - testEnv.baseUri(), principal.getName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) // how is token getting used? - .header(REALM_PROPERTY_KEY, realm) - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(assignPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - @Override - public void afterAll(ExtensionContext extensionContext) throws Exception { - PolarisPrincipalSecrets adminSecrets = PolarisConnectionExtension.getAdminSecrets(); - String realm = - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .get(REALM_PROPERTY_KEY, String.class); - - if (adminSecrets == null) { - LOGGER - .atError() - .log( - "No admin secrets configured - you must also configure your test with PolarisConnectionExtension"); - return; - } - - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String userToken = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - - SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principal-roles/%s", - testEnv.baseUri(), snowmanIdentifier.principalRoleName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s", - testEnv.baseUri(), snowmanIdentifier.principalName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - } - - // FIXME - this would be better done with a Credentials-specific annotation processor so - // tests could declare which credentials they want (e.g., @TestCredentials("root") ) - // For now, snowman comes from here and root comes from PolarisConnectionExtension - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - - return parameterContext.getParameter().getType() == SnowmanCredentials.class; - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return snowmanCredentials; - } - - private static SnowmanIdentifier getSnowmanIdentifier(TestEnvironment testEnv) { - return new SnowmanIdentifier("snowman" + testEnv.testId(), "catalog-admin" + testEnv.testId()); - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java deleted file mode 100644 index 52257c4ca..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.micrometer.core.instrument.Tag; -import jakarta.ws.rs.core.Response; -import java.util.Collection; -import java.util.List; -import org.apache.commons.lang3.StringUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; - -/** Utils for working with metrics in tests */ -public class TestMetricsUtil { - private static final String SUFFIX_TOTAL = ".total"; - - /** Gets a total counter by calling the Prometheus metrics endpoint */ - public static double getTotalCounter( - DropwizardAppExtension dropwizardAppExtension, - String metricName, - Collection tags) { - - metricName += SUFFIX_TOTAL; - metricName = metricName.replace('.', '_').replace('-', '_'); - - // Example of a line from the metrics endpoint: - // polaris_TimedApi_count_realm_total{API_NAME="polaris.principals.getPrincipal",REALM_ID="org_apache_polaris_service_TimedApplicationEventListenerTest"} 1.0 - // This method assumes that tag ordering isn't guaranteed - List tagFilters = - tags.stream().map(tag -> String.format("%s=\"%s\"", tag.getKey(), tag.getValue())).toList(); - - Response response = - dropwizardAppExtension - .client() - .target( - String.format("http://localhost:%d/metrics", dropwizardAppExtension.getAdminPort())) - .request() - .get(); - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - String[] responseLines = response.readEntity(String.class).split("\n"); - for (String line : responseLines) { - int numTags = - StringUtils.countMatches(line, '='); // Assumes the tag values don't contain an '=' - if (line.startsWith(metricName) - && tagFilters.stream().allMatch(line::contains) - && numTags == tagFilters.size()) { - return Double.parseDouble(line.split(" ")[1]); - } - } - return 0; - } -} diff --git a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml deleted file mode 100644 index 9d8f770e5..000000000 --- a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml +++ /dev/null @@ -1,164 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -server: - # Maximum number of threads. - maxThreads: 200 - - # Minimum number of thread to keep alive. - minThreads: 10 - applicationConnectors: - # HTTP-specific options. - - type: http - - # The port on which the HTTP server listens for service requests. - port: 8181 - - adminConnectors: - - type: http - port: 8182 - - # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the - # socket will listen on all interfaces. - #bindHost: localhost - - # ssl: - # keyStore: ./example.keystore - # keyStorePassword: example - # - # keyStoreType: JKS # (optional, JKS is default) - - # HTTP request log settings - requestLog: - appenders: - # Settings for logging to stdout. - - type: console - - # Settings for logging to a file. - - type: file - - # The file to which statements will be logged. - currentLogFilename: ./logs/request.log - - # When the log file rolls over, the file will be archived to requests-2012-03-15.log.gz, - # requests.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/requests-%d.log.gz - - # The maximum number of log files to archive. - archivedFileCount: 14 - - # Enable archiving if the request log entries go to the their own file - archive: true - -featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: true - ALLOW_WILDCARD_LOCATION: true - ALLOW_SPECIFYING_FILE_IO_IMPL: true - SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION: true - ALLOW_OVERLAPPING_CATALOG_URLS: true - ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - FILE - - S3 - - GCS - - AZURE - -metaStoreManager: - type: in-memory - -io: - factoryType: default - -oauth2: - type: default - -authenticator: - class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - -tokenBroker: - type: symmetric-key - secret: polaris - -callContextResolver: - type: default - -realmContextResolver: - type: default - -defaultRealm: POLARIS - -cors: - allowed-origins: - - localhost - - # Logging settings. -logging: - - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. - level: INFO - - # Logger-specific levels. - loggers: - org.apache.polaris: DEBUG - # only used to warn about the tokens endpoint being deprecated - org.apache.iceberg.rest.RESTSessionCatalog: ERROR - - appenders: - - - type: console - # If true, write log statements to stdout. - # enabled: true - # Do not display log statements below this threshold to stdout. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" - - # Settings for logging to a file. - - type: file - # If true, write log statements to a file. - # enabled: true - # Do not write log statements below this threshold to the file. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c: %m %kvp%n%ex" - - # when using json logging, you must use a format like this, else the - # mdc section of the json log will be incorrect - # logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X] %c: %m%n%ex" - - # The file to which statements will be logged. - currentLogFilename: ./logs/iceberg-rest.log - # When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz, - # polaris.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/iceberg-rest-%d.log.gz - # The maximum number of log files to archive. - archivedFileCount: 14 - -# Limits the size of request bodies sent to Polaris. -1 means no limit. -maxRequestBodyBytes: 1000000 - -# Limits the request rate per realm. -rateLimiter: - type: realm-token-bucket - -# The token bucket factory to use when using the realm-token-bucket rate limiter. -tokenBucketFactory: - type: default - requestsPerSecond: 9999 - windowSeconds: 10 diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 20802ba65..ea2c436d9 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -23,18 +23,23 @@ fun isValidDep(dep: String): Boolean { } plugins { + alias(libs.plugins.quarkus) id("polaris-server") `java-library` + alias(libs.plugins.jandex) } dependencies { implementation(project(":polaris-core")) implementation(project(":polaris-jpa-model")) + implementation(libs.eclipselink) - implementation(platform(libs.dropwizard.bom)) - implementation(libs.jakarta.inject.api) - implementation(libs.smallrye.common.annotation) - implementation("io.dropwizard:dropwizard-jackson") + + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-core") + + implementation(libs.slf4j.api) + val eclipseLinkDeps: String? = project.findProperty("eclipseLinkDeps") as String? eclipseLinkDeps?.let { val dependenciesList = it.split(",") @@ -48,15 +53,25 @@ dependencies { } } + // only for @VisibleForTesting + compileOnly(libs.guava) + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly("io.smallrye.common:smallrye-common-annotation") // @Identifier + compileOnly("io.smallrye.config:smallrye-config-core") // @ConfigMapping + + compileOnly(platform(libs.jackson.bom)) + compileOnly("com.fasterxml.jackson.core:jackson-annotations") + compileOnly("com.fasterxml.jackson.core:jackson-core") testImplementation(libs.h2) testImplementation(testFixtures(project(":polaris-core"))) testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) + testImplementation(libs.bundles.junit.testing) + testRuntimeOnly("org.junit.jupiter:junit-jupiter") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } @@ -69,3 +84,7 @@ tasks.register("createTestConfJar") { sourceSets { test { resources.srcDir(layout.buildDirectory.dir("conf")) } } tasks.named("processTestResources") { dependsOn("createTestConfJar") } + +tasks.named("javadoc") { dependsOn("jandex") } + +tasks.named("quarkusDependenciesBuild") { dependsOn("jandex") } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java new file mode 100644 index 000000000..cfa7b6679 --- /dev/null +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.persistence.impl.eclipselink; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; +import java.nio.file.Path; +import java.util.Optional; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.persistence.eclipselink") +public interface EclipseLinkConfiguration { + + /** + * The path to the EclipseLink configuration file. If not provided, the default (built-in) + * configuration file will be used. + */ + Optional configurationFile(); + + /** + * The name of the persistence unit to use. If not provided, or if {@link #configurationFile()} is + * not provided, the default persistence unit name ({@code polaris}) will be used. + */ + @WithDefault("polaris") + String persistenceUnit(); +} diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 21e466b78..3767a0645 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -18,13 +18,16 @@ */ package org.apache.polaris.extension.persistence.impl.eclipselink; -import com.fasterxml.jackson.annotation.JsonProperty; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.nio.file.Path; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -34,16 +37,13 @@ * using an EclipseLink based meta store to store and retrieve all Polaris metadata. It can be * configured through persistence.xml to use supported RDBMS as the meta store. */ +@ApplicationScoped @Identifier("eclipse-link") public class EclipseLinkPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - @JsonProperty("conf-file") - private String confFile; - @JsonProperty("persistence-unit") - private String persistenceUnitName; - - @Inject protected PolarisStorageIntegrationProvider storageIntegration; + @Inject EclipseLinkConfiguration eclipseLinkConfiguration; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Override protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { @@ -52,13 +52,25 @@ protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics @Override protected PolarisMetaStoreSession createMetaStoreSession( - @Nonnull PolarisEclipseLinkStore store, @Nonnull RealmContext realmContext) { + @Nonnull PolarisEclipseLinkStore store, + @Nonnull RealmContext realmContext, + @Nullable PolarisCredentialsBootstrap credentialsBootstrap) { return new PolarisEclipseLinkMetaStoreSessionImpl( store, - storageIntegration, + storageIntegrationProvider, realmContext, - confFile, - persistenceUnitName, - secretsGenerator(realmContext)); + configurationFile(), + persistenceUnitName(), + secretsGenerator(realmContext, credentialsBootstrap)); + } + + private String configurationFile() { + return eclipseLinkConfiguration.configurationFile().map(Path::toString).orElse(null); + } + + private String persistenceUnitName() { + return eclipseLinkConfiguration.configurationFile().isPresent() + ? eclipseLinkConfiguration.persistenceUnit() + : null; } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e1d8ae56a..6e489ddfc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -20,11 +20,13 @@ [versions] hadoop = "3.4.1" iceberg = "1.7.1" -dropwizard = "4.0.11" +junit = "5.11.4" +quarkus = "3.17.6" slf4j = "2.0.16" swagger = "1.6.14" [bundles] +junit-testing = ["assertj-core", "mockito-core", "mockito-junit-jupiter", "junit-jupiter-api", "junit-jupiter-params", "junit-platform-reporting"] [libraries] @@ -40,7 +42,6 @@ bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.1.8" } commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.2" } commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" } -dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version.ref = "dropwizard" } eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.5" } errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.29.2" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.42.0" } @@ -49,25 +50,32 @@ h2 = { module = "com.h2database:h2", version = "2.3.232" } # Strict dnsjava downgrade due to https://github.com/dnsjava/dnsjava/issues/329 dnsjava = { module = "dnsjava:dnsjava", version = "3.5.3!!" } hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "hadoop" } +hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "hadoop" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client", version.ref = "hadoop" } iceberg-bom = { module = "org.apache.iceberg:iceberg-bom", version.ref = "iceberg" } jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version = "2.17.2" } jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version = "3.0.0" } +jakarta-enterprise-cdi-api = { module = "jakarta.enterprise:jakarta.enterprise.cdi-api", version = "4.1.0" } jakarta-inject-api = { module = "jakarta.inject:jakarta.inject-api", version = "2.0.1" } jakarta-persistence-api = { module = "jakarta.persistence:jakarta.persistence-api", version = "3.1.0" } jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version = "6.1.0" } jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.0.2" } jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "3.1.0" } -javax-annotation-api = { module = "javax.annotation:javax.annotation-api", version = "1.3.2" } +javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } jetbrains-annotations = { module = "org.jetbrains:annotations", version = "24.1.0" } -junit-bom = { module = "org.junit:junit-bom", version = "5.10.3" } +junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" } +junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit" } +junit-platform-reporting = { module = "org.junit.platform:junit-platform-reporting", version = "1.10.3" } logback-core = { module = "ch.qos.logback:logback-core", version = "1.4.14" } micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.13.6" } mockito-core = { module = "org.mockito:mockito-core", version = "5.11.0" } +mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.11.0" } opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.38.0" } opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.25.0-alpha" } prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:prometheus-metrics-exporter-servlet-jakarta", version = "1.3.0" } +quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" } s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version = "3.11.0" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } smallrye-common-annotation = { module = "io.smallrye.common:smallrye-common-annotation", version = "2.9.0" } @@ -78,6 +86,8 @@ testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" } [plugins] +jandex = { id = "org.kordamp.gradle.jandex", version = "2.1.0" } openapi-generator = { id = "org.openapi.generator", version = "7.6.0" } +quarkus = { id = "io.quarkus", version.ref = "quarkus" } rat = { id = "org.nosphere.apache.rat", version = "0.8.1" } spotless = { id = "com.diffplug.spotless", version = "6.25.0" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index bf7feec16..32705debf 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -23,7 +23,9 @@ polaris-api-iceberg-service=api/iceberg-service polaris-api-management-model=api/management-model polaris-api-management-service=api/management-service polaris-service-common=service/common -polaris-dropwizard-service=dropwizard/service +polaris-quarkus-service=quarkus/service +polaris-quarkus-server=quarkus/server +polaris-quarkus-admin=quarkus/admin polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model aggregated-license-report=aggregated-license-report diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts index d07b12385..d51bba6cf 100644 --- a/polaris-core/build.gradle.kts +++ b/polaris-core/build.gradle.kts @@ -21,6 +21,7 @@ plugins { id("polaris-client") id("java-library") id("java-test-fixtures") + alias(libs.plugins.jandex) } dependencies { @@ -108,3 +109,5 @@ dependencies { compileOnly(libs.jakarta.annotation.api) } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java index 2b64fe664..abe598714 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java @@ -58,6 +58,14 @@ public PolarisCallContext( this.clock = Clock.system(ZoneId.systemDefault()); } + public static PolarisCallContext copyOf(PolarisCallContext original) { + return new PolarisCallContext( + original.getMetaStore(), + original.getDiagServices(), + original.getConfigurationStore(), + original.getClock()); + } + public PolarisMetaStoreSession getMetaStore() { return metaStore; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java index d9962f9c3..71c457720 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java @@ -100,8 +100,9 @@ public Map contextVariables() { * CallContext}. */ static CallContext copyOf(CallContext base) { - RealmContext realmContext = base.getRealmContext(); - PolarisCallContext polarisCallContext = base.getPolarisCallContext(); + String realmId = base.getRealmContext().getRealmIdentifier(); + RealmContext realmContext = () -> realmId; + PolarisCallContext polarisCallContext = PolarisCallContext.copyOf(base.getPolarisCallContext()); Map contextVariables = base.contextVariables().entrySet().stream() .filter(e -> !e.getKey().equals(CLOSEABLES)) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 1e2ca3006..6c2d6565a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.persistence; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,6 +35,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +50,7 @@ public abstract class LocalPolarisMetaStoreManagerFactory final Map metaStoreManagerMap = new HashMap<>(); final Map storageCredentialCacheMap = new HashMap<>(); + final Map entityCacheMap = new HashMap<>(); final Map backingStoreMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); @@ -55,50 +58,49 @@ public abstract class LocalPolarisMetaStoreManagerFactory private static final Logger LOGGER = LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class); - private boolean bootstrap; - protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); protected abstract PolarisMetaStoreSession createMetaStoreSession( - @Nonnull StoreType store, @Nonnull RealmContext realmContext); + @Nonnull StoreType store, + @Nonnull RealmContext realmContext, + @Nullable PolarisCredentialsBootstrap credentialsBootstrap); - protected PrincipalSecretsGenerator secretsGenerator(RealmContext realmContext) { - if (bootstrap) { - return PrincipalSecretsGenerator.bootstrap(realmContext.getRealmIdentifier()); + protected PrincipalSecretsGenerator secretsGenerator( + RealmContext realmContext, @Nullable PolarisCredentialsBootstrap credentialsBootstrap) { + if (credentialsBootstrap != null) { + return PrincipalSecretsGenerator.bootstrap( + realmContext.getRealmIdentifier(), credentialsBootstrap); } else { return PrincipalSecretsGenerator.RANDOM_SECRETS; } } - private void initializeForRealm(RealmContext realmContext) { + private void initializeForRealm( + RealmContext realmContext, PolarisCredentialsBootstrap credentialsBootstrap) { final StoreType backingStore = createBackingStore(diagServices); backingStoreMap.put(realmContext.getRealmIdentifier(), backingStore); sessionSupplierMap.put( realmContext.getRealmIdentifier(), - () -> createMetaStoreSession(backingStore, realmContext)); + () -> createMetaStoreSession(backingStore, realmContext, credentialsBootstrap)); PolarisMetaStoreManager metaStoreManager = new PolarisMetaStoreManagerImpl(); metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); } @Override - public synchronized Map bootstrapRealms(List realms) { + public synchronized Map bootstrapRealms( + List realms, PolarisCredentialsBootstrap credentialsBootstrap) { Map results = new HashMap<>(); - bootstrap = true; - try { - for (String realm : realms) { - RealmContext realmContext = () -> realm; - if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext); - PrincipalSecretsResult secretsResult = - bootstrapServiceAndCreatePolarisPrincipalForRealm( - realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); - results.put(realmContext.getRealmIdentifier(), secretsResult); - } + for (String realm : realms) { + RealmContext realmContext = () -> realm; + if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { + initializeForRealm(realmContext, credentialsBootstrap); + PrincipalSecretsResult secretsResult = + bootstrapServiceAndCreatePolarisPrincipalForRealm( + realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); + results.put(realmContext.getRealmIdentifier(), secretsResult); } - } finally { - bootstrap = false; } return results; @@ -124,7 +126,7 @@ public void purgeRealms(List realms) { public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( RealmContext realmContext) { if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext); + initializeForRealm(realmContext, null); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } @@ -135,7 +137,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext); + initializeForRealm(realmContext, null); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } else { @@ -156,6 +158,16 @@ public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); } + @Override + public synchronized EntityCache getOrCreateEntityCache(RealmContext realmContext) { + if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { + PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); + entityCacheMap.put(realmContext.getRealmIdentifier(), new EntityCache(metaStoreManager)); + } + + return entityCacheMap.get(realmContext.getRealmIdentifier()); + } + /** * This method bootstraps service for a given realm: i.e. creates all the needed entities in the * metastore and creates a root service principal. After that we rotate the root principal diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index 7cdb36c5a..4e37f034c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -23,12 +23,10 @@ import java.util.function.Supplier; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.cache.StorageCredentialCache; -/** - * Configuration interface for configuring the {@link PolarisMetaStoreManager} via Dropwizard - * configuration - */ +/** Configuration interface for configuring the {@link PolarisMetaStoreManager}. */ public interface MetaStoreManagerFactory { PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext); @@ -37,7 +35,10 @@ public interface MetaStoreManagerFactory { StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext); - Map bootstrapRealms(List realms); + EntityCache getOrCreateEntityCache(RealmContext realmContext); + + Map bootstrapRealms( + List realms, PolarisCredentialsBootstrap credentialsBootstrap); /** Purge all metadata for the realms provided */ void purgeRealms(List realms); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 8836a171c..05efba7fb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -23,20 +23,17 @@ import com.github.benmanes.caffeine.cache.RemovalListener; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; -import jakarta.inject.Inject; import java.util.AbstractMap; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.RealmScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.CachedEntryResult; /** The entity cache, can be private or shared */ -@RealmScoped public class EntityCache { // cache mode @@ -56,7 +53,6 @@ public class EntityCache { * * @param polarisRemoteCache the meta store manager implementation */ - @Inject public EntityCache(@Nonnull PolarisRemoteCache polarisRemoteCache) { // by name cache diff --git a/polaris-server.yml b/polaris-server.yml deleted file mode 100644 index 6d2fee83c..000000000 --- a/polaris-server.yml +++ /dev/null @@ -1,184 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -server: - # Maximum number of threads. - maxThreads: 200 - - # Minimum number of thread to keep alive. - minThreads: 10 - applicationConnectors: - # HTTP-specific options. - - type: http - - # The port on which the HTTP server listens for service requests. - port: 8181 - - adminConnectors: - - type: http - port: 8182 - - # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the - # socket will listen on all interfaces. - #bindHost: localhost - - # ssl: - # keyStore: ./example.keystore - # keyStorePassword: example - # - # keyStoreType: JKS # (optional, JKS is default) - - # HTTP request log settings - requestLog: - appenders: - # Settings for logging to stdout. - - type: console - - # Settings for logging to a file. - - type: file - - # The file to which statements will be logged. - currentLogFilename: ./logs/request.log - - # When the log file rolls over, the file will be archived to requests-2012-03-15.log.gz, - # requests.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/requests-%d.log.gz - - # The maximum number of log files to archive. - archivedFileCount: 14 - - # Enable archiving if the request log entries go to the their own file - archive: true - -featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - S3 - - GCS - - AZURE - - FILE - -callContextResolver: - type: default - -defaultRealm: default-realm - -realmContextResolver: - type: default - -defaultRealms: - - default-realm - -metaStoreManager: - type: in-memory - # type: eclipse-link # uncomment to use eclipse-link as metastore - # persistence-unit: polaris - -io: - factoryType: wasb - -# TODO - avoid duplicating token broker config -oauth2: - type: test - -authenticator: - class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator - -# - uncomment to support Auth0 JWT tokens -#oauth2: -# type: default -#authenticator: -# class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator # - uncomment to support Auth0 JWT tokens -# -#tokenBroker: -# type: symmetric-key -# secret: polaris - -cors: - allowed-origins: - - http://localhost:8080 - allowed-timing-origins: - - http://localhost:8080 - allowed-methods: - - PATCH - - POST - - DELETE - - GET - - PUT - allowed-headers: - - "*" - exposed-headers: - - "*" - preflight-max-age: 600 - allowed-credentials: true - -# Logging settings. - -logging: - - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. - level: INFO - - # Logger-specific levels. - loggers: - org.apache.iceberg.rest: DEBUG - org.apache.polaris: INFO - - appenders: - - - type: console - # If true, write log statements to stdout. - # enabled: true - # Do not display log statements below this threshold to stdout. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" - - # Settings for logging to a file. - - type: file - # If true, write log statements to a file. - # enabled: true - # Do not write log statements below this threshold to the file. - threshold: ALL - layout: - type: polaris - flattenKeyValues: false - includeKeyValues: true - - # The file to which statements will be logged. - currentLogFilename: ./logs/polaris.log - # When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz, - # polaris.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/polaris-%d.log.gz - # The maximum number of log files to archive. - archivedFileCount: 14 - -# Limits the size of request bodies sent to Polaris. -1 means no limit. -maxRequestBodyBytes: -1 - -# Optional, not specifying a "rateLimiter" section also means no rate limiter -rateLimiter: - type: no-op - # Uncomment to use the realm-token-bucket rate limiter - # type: realm-token-bucket - -# The token bucket factory to use when using the realm-token-bucket rate limiter. -tokenBucketFactory: - type: default - requestsPerSecond: 9999 - windowSeconds: 10 diff --git a/quarkus/admin/README.md b/quarkus/admin/README.md new file mode 100644 index 000000000..9720695a4 --- /dev/null +++ b/quarkus/admin/README.md @@ -0,0 +1,26 @@ +# Polaris Server Admin Tool + +This module contains a maintenance tool for the Polaris server. It is a Quarkus application that can +be used to perform various maintenance tasks targeting Polaris database directly. + +Building this module will create a runnable uber-jar that can be executed from the command line. + +To also build the Docker image, you can use the following command: + +```shell +./gradlew :polaris-quarkus-admin:build -Dquarkus.container-image.build=true +``` + +## Running the Admin Tool + +The admin tool can be run from the command line using the following command: + +```shell +java -jar polaris-quarkus-admin--runner.jar --help +``` + +Using the Docker image, you can run the admin tool with the following command: + +```shell +docker run --rm -it polaris-server-admin-tool: --help +``` \ No newline at end of file diff --git a/quarkus/admin/build.gradle.kts b/quarkus/admin/build.gradle.kts new file mode 100644 index 000000000..9cf4c9400 --- /dev/null +++ b/quarkus/admin/build.gradle.kts @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import io.quarkus.gradle.tasks.QuarkusBuild + +plugins { + alias(libs.plugins.quarkus) + alias(libs.plugins.openapi.generator) + id("polaris-server") + id("polaris-license-report") +} + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-service")) + implementation(project(":polaris-api-iceberg-service")) + implementation(project(":polaris-service-common")) + implementation(project(":polaris-quarkus-service")) + + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-picocli") + implementation("io.quarkus:quarkus-container-image-docker") + + implementation("org.jboss.slf4j:slf4j-jboss-logmanager") + + // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329 + implementation(platform(libs.dnsjava)) + + testImplementation(enforcedPlatform(libs.quarkus.bom)) + testImplementation("io.quarkus:quarkus-junit5") + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.bundles.junit.testing) +} + +tasks.withType().configureEach { + from("src/main/resources") { + expand("polarisVersion" to version) + duplicatesStrategy = DuplicatesStrategy.INCLUDE + } +} + +quarkus { + quarkusBuildProperties.put("quarkus.package.type", "uber-jar") + // Pull manifest attributes from the "main" `jar` task to get the + // release-information into the jars generated by Quarkus. + quarkusBuildProperties.putAll( + provider { + tasks + .named("jar", Jar::class.java) + .get() + .manifest + .attributes + .map { e -> "quarkus.package.jar.manifest.attributes.\"${e.key}\"" to e.value.toString() } + .toMap() + } + ) +} + +publishing { + publications { + named("maven") { + val quarkusBuild = tasks.getByName("quarkusBuild") + artifact(quarkusBuild.runnerJar) { + classifier = "runner" + builtBy(quarkusBuild) + } + } + } +} diff --git a/quarkus/admin/src/main/docker/Dockerfile.jvm b/quarkus/admin/src/main/docker/Dockerfile.jvm new file mode 100644 index 000000000..9f992a30a --- /dev/null +++ b/quarkus/admin/src/main/docker/Dockerfile.jvm @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2.1726695192 + +LABEL org.opencontainers.image.source=https://github.com/apache/polaris +LABEL org.opencontainers.image.description="Apache Polaris (incubating) Admin Tool" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +ENV LANGUAGE='en_US:en' + +USER root +RUN groupadd --gid 10001 polaris \ + && useradd --uid 10000 --gid polaris polaris \ + && chown -R polaris:polaris /opt/jboss/container \ + && chown -R polaris:polaris /deployments + +USER polaris +WORKDIR /deployments +ENV USER=polaris +ENV UID=10000 +ENV HOME=/home/polaris +ENV PWD=/deployments + +COPY --chown=polaris:polaris build/*-runner.jar /deployments/polaris-server-admin-tool.jar + +ENTRYPOINT [ "java", "-jar", "/deployments/polaris-server-admin-tool.jar" ] \ No newline at end of file diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java new file mode 100644 index 000000000..ce9957242 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import jakarta.inject.Inject; +import java.util.concurrent.Callable; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +public abstract class BaseCommand implements Callable { + + public static final Integer EXIT_CODE_GENERIC_ERROR = 1; + public static final Integer EXIT_CODE_BOOTSTRAP_ERROR = 2; + public static final Integer EXIT_CODE_PURGE_ERROR = 3; + + @Inject QuarkusPersistenceConfiguration persistenceConfiguration; + + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + + @Spec CommandSpec spec; + + protected void warnOnInMemory() { + if (persistenceConfiguration.type().equalsIgnoreCase("in-memory")) { + spec.commandLine() + .getErr() + .println( + spec.commandLine() + .getColorScheme() + .errorText( + """ + ********************************************************************************************* + ** Running the Admin Tool on a Polaris instance with in-memory persistence is meaningless! ** + ********************************************************************************************* + """)); + } + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java new file mode 100644 index 000000000..fe82a1b40 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; +import picocli.CommandLine; + +@CommandLine.Command( + name = "bootstrap", + mixinStandardHelpOptions = true, + description = "Bootstraps principal credentials for all realms and prints them to log.") +public class BootstrapCommand extends BaseCommand { + + @CommandLine.Option( + names = {"-r", "--realm"}, + required = true, + description = "The name of the realm to bootstrap.") + List realms; + + @CommandLine.Option( + names = {"-c", "--credential"}, + required = true, + description = + "Principal credentials to bootstrap. Must be of the form 'realm,userName,clientId,clientSecret'.") + List credentials; + + @Override + public Integer call() { + warnOnInMemory(); + + // Execute the bootstrap + PolarisCredentialsBootstrap credentialsBootstrap = + PolarisCredentialsBootstrap.fromString(String.join(";", credentials)); + Map results = + metaStoreManagerFactory.bootstrapRealms(realms, credentialsBootstrap); + + // Log any errors: + boolean success = true; + for (Map.Entry result : results.entrySet()) { + if (!result.getValue().isSuccess()) { + String realm = result.getKey(); + spec.commandLine() + .getErr() + .printf( + "Bootstrapping '%s' failed: %s%n", + realm, result.getValue().getReturnStatus().toString()); + success = false; + } + } + + if (success) { + spec.commandLine().getOut().println("Bootstrap completed successfully."); + return 0; + } else { + spec.commandLine().getErr().println("Bootstrap encountered errors during operation."); + return EXIT_CODE_BOOTSTRAP_ERROR; + } + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java new file mode 100644 index 000000000..c81d9273b --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import io.quarkus.picocli.runtime.annotations.TopCommand; +import java.io.PrintWriter; +import picocli.CommandLine.Command; +import picocli.CommandLine.HelpCommand; + +@TopCommand +@Command( + name = "polaris-admin-tool-runner.jar", + mixinStandardHelpOptions = true, + versionProvider = PolarisVersionProvider.class, + description = "Nessie Server Admin Tool", + subcommands = { + HelpCommand.class, + BootstrapCommand.class, + PurgeCommand.class, + }) +public class PolarisAdminTool extends BaseCommand { + + @Override + public Integer call() { + return info(); + } + + private int info() { + warnOnInMemory(); + + PrintWriter out = spec.commandLine().getOut(); + + out.println("Polaris server information & maintenance tool."); + out.println("Use the 'help' command."); + out.println(); + return 0; + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java new file mode 100644 index 000000000..090948b18 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import java.io.InputStream; +import java.net.URL; +import java.util.Objects; +import java.util.Properties; +import picocli.CommandLine.IVersionProvider; + +public class PolarisVersionProvider implements IVersionProvider { + + @Override + public String[] getVersion() throws Exception { + URL resource = + Objects.requireNonNull(PolarisVersionProvider.class.getResource("version.properties")); + try (InputStream input = resource.openConnection().getInputStream()) { + Properties props = new Properties(); + props.load(input); + return new String[] {props.getProperty("polaris.version")}; + } + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java new file mode 100644 index 000000000..292ffcab8 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import java.util.List; +import picocli.CommandLine; + +@CommandLine.Command( + name = "purge", + mixinStandardHelpOptions = true, + description = "Purge principal credentials for all realms and prints them to log.") +public class PurgeCommand extends BaseCommand { + + @CommandLine.Option( + names = {"-r", "--realm"}, + required = true, + description = "The name of the realm to purge.") + List realms; + + @Override + public Integer call() { + warnOnInMemory(); + + try { + metaStoreManagerFactory.purgeRealms(realms); + spec.commandLine().getOut().println("Purge completed successfully."); + return 0; + } catch (Exception e) { + spec.commandLine().getErr().println("Purge encountered errors during operation."); + return EXIT_CODE_PURGE_ERROR; + } + } +} diff --git a/dropwizard/service/src/main/resources/log4j.properties b/quarkus/admin/src/main/resources/application.properties similarity index 73% rename from dropwizard/service/src/main/resources/log4j.properties rename to quarkus/admin/src/main/resources/application.properties index 37c1696bd..380fbd557 100644 --- a/dropwizard/service/src/main/resources/log4j.properties +++ b/quarkus/admin/src/main/resources/application.properties @@ -17,8 +17,10 @@ # under the License. # -log4j.rootLogger=INFO, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n +quarkus.application.name=Apache Polaris Server Admin Tool (incubating) + +quarkus.container-image.build=false +quarkus.container-image.push=false +quarkus.container-image.registry=docker.io +quarkus.container-image.group=apache +quarkus.container-image.name=polaris-server-admin-tool \ No newline at end of file diff --git a/dropwizard/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory b/quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties similarity index 91% rename from dropwizard/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory rename to quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties index 2e1cd0210..b36811eef 100644 --- a/dropwizard/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory +++ b/quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.dropwizard.logging.PolarisJsonLayoutFactory \ No newline at end of file +polaris.version=${polarisVersion} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java new file mode 100644 index 000000000..0e964c07f --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.main.Launch; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainTest; +import org.junit.jupiter.api.Test; + +@QuarkusMainTest +class BootstrapCommandTest { + + @Test + @Launch( + value = { + "bootstrap", + "-r", + "realm1", + "-r", + "realm2", + "-c", + "realm1,root,root,s3cr3t", + "-c", + "realm2,root,root,s3cr3t" + }) + public void testBootstrap(LaunchResult result) { + assertThat(result.getOutput()).contains("Bootstrap completed successfully."); + } +} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java new file mode 100644 index 000000000..902cf8a10 --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.main.Launch; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainTest; +import org.junit.jupiter.api.Test; + +@QuarkusMainTest +class PurgeCommandTest { + + @Test + @Launch(value = {"purge", "-r", "realm1", "-r", "realm2"}) + public void testPurge(LaunchResult result) { + assertThat(result.getOutput()).contains("Purge completed successfully."); + } +} diff --git a/quarkus/server/README.md b/quarkus/server/README.md new file mode 100644 index 000000000..7fb773ece --- /dev/null +++ b/quarkus/server/README.md @@ -0,0 +1,12 @@ +# Polaris Quarkus Server + +This module contains the Quarkus-based Polaris server main artifact. + +Building this module will create a zip/tar distribution with the Polaris server. + +To also build the Docker image, you can use the following command (a running Docker daemon is +required): + +```shell +./gradlew :polaris-quarkus-server:build -Dquarkus.container-image.build=true +``` diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts new file mode 100644 index 000000000..e4b0f9e00 --- /dev/null +++ b/quarkus/server/build.gradle.kts @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + alias(libs.plugins.quarkus) + alias(libs.plugins.openapi.generator) + id("polaris-server") + id("polaris-license-report") + id("distribution") +} + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-service")) + implementation(project(":polaris-api-iceberg-service")) + implementation(project(":polaris-service-common")) + implementation(project(":polaris-quarkus-service")) + + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-container-image-docker") + + // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329 + implementation(platform(libs.dnsjava)) +} + +quarkus { + quarkusBuildProperties.put("quarkus.package.type", "fast-jar") + // Pull manifest attributes from the "main" `jar` task to get the + // release-information into the jars generated by Quarkus. + quarkusBuildProperties.putAll( + provider { + tasks + .named("jar", Jar::class.java) + .get() + .manifest + .attributes + .map { e -> "quarkus.package.jar.manifest.attributes.\"${e.key}\"" to e.value.toString() } + .toMap() + } + ) +} + +tasks.named("distZip") { dependsOn("quarkusBuild") } + +tasks.named("distTar") { dependsOn("quarkusBuild") } + +distributions { + main { + contents { + from(project.layout.buildDirectory.dir("quarkus-app")) + from("../../NOTICE") + from("../../LICENSE-BINARY-DIST").rename("LICENSE-BINARY-DIST", "LICENSE") + exclude("lib/main/io.quarkus.quarkus-container-image*") + } + } +} diff --git a/quarkus/server/src/main/docker/Dockerfile.jvm b/quarkus/server/src/main/docker/Dockerfile.jvm new file mode 100644 index 000000000..9a1cced35 --- /dev/null +++ b/quarkus/server/src/main/docker/Dockerfile.jvm @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2.1726695192 + +LABEL org.opencontainers.image.source=https://github.com/apache/polaris +LABEL org.opencontainers.image.description="Apache Polaris (incubating)" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +ENV LANGUAGE='en_US:en' + +USER root +RUN groupadd --gid 10001 polaris \ + && useradd --uid 10000 --gid polaris polaris \ + && chown -R polaris:polaris /opt/jboss/container \ + && chown -R polaris:polaris /deployments + +USER polaris +WORKDIR /home/polaris +ENV USER=polaris +ENV UID=10000 +ENV HOME=/home/polaris + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --chown=polaris:polaris build/quarkus-app/lib/ /deployments/lib/ +COPY --chown=polaris:polaris build/quarkus-app/*.jar /deployments/ +COPY --chown=polaris:polaris build/quarkus-app/app/ /deployments/app/ +COPY --chown=polaris:polaris build/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8181 +EXPOSE 8182 + +ENV AB_JOLOKIA_OFF="" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/quarkus/server/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor b/quarkus/server/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor new file mode 100644 index 000000000..b92210f72 --- /dev/null +++ b/quarkus/server/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +io.smallrye.config.LoggingConfigSourceInterceptor \ No newline at end of file diff --git a/quarkus/server/src/main/resources/application.properties b/quarkus/server/src/main/resources/application.properties new file mode 100644 index 000000000..3f5018a77 --- /dev/null +++ b/quarkus/server/src/main/resources/application.properties @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +quarkus.application.name=Apache Polaris Server (incubating) + +quarkus.container-image.build=false +quarkus.container-image.push=false +quarkus.container-image.registry=docker.io +quarkus.container-image.group=apache +quarkus.container-image.name=polaris diff --git a/quarkus/service/README-quarkus.md b/quarkus/service/README-quarkus.md new file mode 100644 index 000000000..1add64333 --- /dev/null +++ b/quarkus/service/README-quarkus.md @@ -0,0 +1,97 @@ + + +This module contains the Polaris Service powered by Quarkus (instead of Dropwizard). + +# Main differences + +* Bean injection (CDI) is made using `@ApplicationScoped` annotation on class and injected in other classes using `@Inject` annotation (https://quarkus.io/guides/cdi-reference). +* Codehale metrics registry and opentelemetry boilerplate (prometheus exporter included) are not needed anymore: Quarkus provides it "out of the box" (https://quarkus.io/guides/opentelemetry) +* `PolarisHealthCheck` is not needed anymore: Quarkus provides it "out of the box" (https://quarkus.io/guides/smallrye-health) +* `TimedApplicationEventListener` and the `@TimedApi` annotation are replaced by Quarkus (micrometer) `@Timed` annotation (https://quarkus.io/guides/telemetry-micrometer) +* `PolarisJsonLayoutFactory` is not needed anymore: Quarkus provides it by configuration (using `quarkus.log.*` configuration) +* `PolarisApplication` is not needed, Quarkus provide a "main" application out of the box (it's possible to provide `QuarkusApplication` for control the startup and also using `@Startup` annotation) +* CORS boilerplate is not needed anymore: Quarkus supports it via configuration (using `quarkus.http.cors.*` configuration) +* CLI is not part of `polaris-service` anymore, we have (will have) a dedicated module (`polaris-cli`) + +# Build and run + +To build `polaris-service` you simply do: + +``` +./gradlew :polaris-service:build +``` + +The build creates ready to run package: +* in the `build/quarkus-app` folder, you can run with `java -jar quarkus-run.jar` +* the `build/distributions` folder contains tar/zip distributions you can extract + +You can directly run Polaris service (in the build scope) using: + +``` +./gradlew :polaris-quarkus-service:quarkusRun +``` + +You can run in Dev mode as well: + +``` +./gradlew --console=plain :polaris-quarkus-service:quarkusDev +``` + +You can directly build a Docker image using: + +``` +./gradlew :polaris-quarkus-service:imageBuild +``` + +# Configuration + +The main configuration file is not the `application.properties`. The default configuration is +packaged as part of the `polaris-quarkus-service`. `polaris-quarkus-service` uses several +configuration sources (in this order): +* system properties +* environment variables +* `.env` file in the current working directory +* `$PWD/config/application.properties` file +* the `application.properties` packaged in the `polaris-quarkus-service` application + +It means you can override some configuration property using environment variables for example. + +By default, `polaris-quarkus-service` uses 8181 as the HTTP port (defined in the `quarkus.http.port` +configuration property) and 8182 as the management port (defined in the `quarkus.management.port` +configuration property). + +You can find more details here: https://quarkus.io/guides/config + +# TODO + +* Modify `CallContext` and remove all usages of `ThreadLocal`, replace with proper context propagation. +* Remove `PolarisCallContext` – it's just an aggregation of CDI beans +* Use `@QuarkustIntegrationTest` for integration tests +* Remove `OAuthCredentialAuthFilter` and replace with Quarkus OIDC security +* Create `polaris-cli` module, add Bootstrap and Purge commands +* Adapt Helm charts, Dockerfiles, K8s examples +* Fix intermittent Gradle build error : SRCFG00011: Could not expand value + platform.quarkus.native.builder-image in property quarkus.native.builder-image + (https://github.com/quarkusio/quarkus/issues/19139). + The Quarkus issue says that using the Quarkus-platform bom helps (it's really what we should depend on). But IIRC there were some dependency issues with Spark/Scala, which prevents us from using the Quarkus-platform bom. + +* Update documentation/README/... + +* Do we want to support existing json configuration file as configuration source ? diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts new file mode 100644 index 000000000..0062fe8b0 --- /dev/null +++ b/quarkus/service/build.gradle.kts @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + alias(libs.plugins.quarkus) + alias(libs.plugins.jandex) + id("polaris-server") +} + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-service")) + implementation(project(":polaris-api-iceberg-service")) + implementation(project(":polaris-service-common")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + implementation("org.apache.iceberg:iceberg-aws") + + // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329 + implementation(platform(libs.dnsjava)) + + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-logging-json") + implementation("io.quarkus:quarkus-rest-jackson") + implementation("io.quarkus:quarkus-reactive-routes") + implementation("io.quarkus:quarkus-hibernate-validator") + implementation("io.quarkus:quarkus-smallrye-health") + implementation("io.quarkus:quarkus-micrometer") + implementation("io.quarkus:quarkus-micrometer-registry-prometheus") + implementation("io.quarkus:quarkus-opentelemetry") + implementation("io.quarkus:quarkus-smallrye-context-propagation") + + implementation(libs.jakarta.enterprise.cdi.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.validation.api) + implementation(libs.jakarta.ws.rs.api) + + implementation(libs.caffeine) + implementation(libs.guava) + implementation(libs.slf4j.api) + + implementation("org.jboss.slf4j:slf4j-jboss-logmanager") + + implementation(libs.hadoop.client.api) + implementation(libs.hadoop.client.runtime) + + implementation(libs.auth0.jwt) + + implementation(libs.bouncycastle.bcprov) + + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.spotbugs.annotations) + + implementation(platform(libs.google.cloud.storage.bom)) + implementation("com.google.cloud:google-cloud-storage") + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:sts") + implementation("software.amazon.awssdk:iam-policy-builder") + implementation("software.amazon.awssdk:s3") + implementation(platform(libs.azuresdk.bom)) + implementation("com.azure:azure-core") + + compileOnly(libs.swagger.annotations) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") + + implementation(libs.jakarta.servlet.api) + + testImplementation(project(":polaris-api-management-model")) + + testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") + testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") + + testImplementation("org.apache.iceberg:iceberg-spark-3.5_2.12") + testImplementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") + testImplementation("org.apache.spark:spark-sql_2.12:3.5.1") { + // exclude log4j dependencies + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-api") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.slf4j", "jul-to-slf4j") + } + + testImplementation("software.amazon.awssdk:glue") + testImplementation("software.amazon.awssdk:kms") + testImplementation("software.amazon.awssdk:dynamodb") + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.bundles.junit.testing) + + testImplementation(platform(libs.quarkus.bom)) + testImplementation("io.quarkus:quarkus-junit5") + testImplementation("io.quarkus:quarkus-junit5-mockito") + testImplementation("io.quarkus:quarkus-rest-client") + testImplementation("io.quarkus:quarkus-rest-client-jackson") + testImplementation("io.rest-assured:rest-assured") + + testImplementation(platform(libs.testcontainers.bom)) + testImplementation("org.testcontainers:testcontainers") + testImplementation(libs.s3mock.testcontainers) + + // required for PolarisSparkIntegrationTest + testImplementation(enforcedPlatform("org.scala-lang:scala-library:2.12.18")) + testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:2.12.18")) + testImplementation(libs.javax.servlet.api) + testImplementation( + enforcedPlatform("org.antlr:antlr4-runtime:4.9.3") + ) // cannot be higher than 4.9.3 + + testImplementation("org.hawkular.agent:prometheus-scraper:0.23.0.Final") +} + +tasks.withType(Test::class.java).configureEach { + systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") + addSparkJvmOptions() +} + +tasks.named("test").configure { + if (System.getenv("AWS_REGION") == null) { + environment("AWS_REGION", "us-west-2") + } + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + useJUnitPlatform() + maxParallelForks = 4 +} + +/** + * Adds the JPMS options required for Spark to run on Java 17, taken from the + * `DEFAULT_MODULE_OPTIONS` constant in `org.apache.spark.launcher.JavaModuleOptions`. + */ +fun JavaForkOptions.addSparkJvmOptions() { + jvmArgs = + (jvmArgs ?: emptyList()) + + listOf( + // Spark 3.3+ + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED", + // Spark 3.4+ + "-Djdk.reflect.useDirectMethodHandle=false" + ) +} + +tasks.named("compileJava") { dependsOn("compileQuarkusGeneratedSourcesJava") } + +tasks.named("sourcesJar") { dependsOn("compileQuarkusGeneratedSourcesJava") } + +tasks.named("javadoc") { dependsOn("jandex") } + +tasks.named("quarkusDependenciesBuild") { dependsOn("jandex") } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/auth/QuarkusAuthenticationConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/auth/QuarkusAuthenticationConfiguration.java new file mode 100644 index 000000000..9ae77291d --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/auth/QuarkusAuthenticationConfiguration.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.auth; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import org.apache.polaris.service.auth.AuthenticationConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.authentication") +public interface QuarkusAuthenticationConfiguration extends AuthenticationConfiguration { + + @Override + QuarkusAuthenticatorConfiguration authenticator(); + + @Override + QuarkusTokenServiceConfiguration tokenService(); + + @Override + QuarkusTokenBrokerConfiguration tokenBroker(); + + interface QuarkusAuthenticatorConfiguration extends AuthenticatorConfiguration { + + /** + * The type of the authenticator. Must be a registered {@link + * org.apache.polaris.service.auth.Authenticator} identifier. + */ + String type(); + } + + interface QuarkusTokenServiceConfiguration extends TokenServiceConfiguration { + + /** + * The type of the OAuth2 service. Must be a registered {@link + * org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} identifier. + */ + String type(); + } + + interface QuarkusTokenBrokerConfiguration extends TokenBrokerConfiguration { + + /** + * The type of the token broker factory. Must be a registered {@link + * org.apache.polaris.service.auth.TokenBrokerFactory} identifier. + */ + String type(); + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/auth/QuarkusOAuthFilter.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/auth/QuarkusOAuthFilter.java new file mode 100644 index 000000000..57ffc82c2 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/auth/QuarkusOAuthFilter.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.auth; + +import jakarta.inject.Inject; +import jakarta.ws.rs.NotAuthorizedException; +import jakarta.ws.rs.Priorities; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.SecurityContext; +import java.security.Principal; +import java.util.Optional; +import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.service.auth.Authenticator; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.jboss.resteasy.reactive.server.ServerRequestFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO replace with all authN with Quarkus native authN +public class QuarkusOAuthFilter { + + private static final Logger LOGGER = LoggerFactory.getLogger(QuarkusOAuthFilter.class); + + private static final String CHALLENGE_FORMAT = "Bearer realm=\"%s\""; + + /** + * Query parameter used to pass Bearer token + * + * @see The OAuth 2.0 Authorization + * Framework: Bearer Token Usage + */ + public static final String OAUTH_ACCESS_TOKEN_PARAM = "access_token"; + + @Inject Authenticator authenticator; + @Inject CallContext callContext; + + @ServerRequestFilter(priority = Priorities.AUTHENTICATION) + public void authenticate(ContainerRequestContext requestContext) { + + if (requestContext.getUriInfo().getPath().equals("/api/catalog/v1/oauth/tokens")) { + return; + } + + String credentials = + getCredentials(requestContext.getHeaders().getFirst(HttpHeaders.AUTHORIZATION)); + + // If Authorization header is not used, check query parameter where token can be passed as well + if (credentials == null) { + credentials = + requestContext.getUriInfo().getQueryParameters().getFirst(OAUTH_ACCESS_TOKEN_PARAM); + } + + if (!authenticate(requestContext, credentials, SecurityContext.BASIC_AUTH)) { + throw new NotAuthorizedException( + "Credentials are required to access this resource.", + String.format(CHALLENGE_FORMAT, callContext.getRealmContext().getRealmIdentifier())); + } + } + + /** + * Parses a value of the `Authorization` header in the form of `Bearer a892bf3e284da9bb40648ab10`. + * + * @param header the value of the `Authorization` header + * @return a token + */ + @Nullable + private String getCredentials(String header) { + if (header == null) { + return null; + } + + final int space = header.indexOf(' '); + if (space <= 0) { + return null; + } + + final String method = header.substring(0, space); + if (!"Bearer".equalsIgnoreCase(method)) { + return null; + } + + return header.substring(space + 1); + } + + /** + * Authenticates a request with user credentials and setup the security context. + * + * @param requestContext the context of the request + * @param credentials the user credentials + * @param scheme the authentication scheme; one of {@code BASIC_AUTH, FORM_AUTH, CLIENT_CERT_AUTH, + * DIGEST_AUTH}. See {@link SecurityContext} + * @return {@code true}, if the request is authenticated, otherwise {@code false} + */ + protected boolean authenticate( + ContainerRequestContext requestContext, @Nullable String credentials, String scheme) { + if (credentials == null) { + return false; + } + + CallContext.setCurrentContext(callContext); + + Optional principal = authenticator.authenticate(credentials); + if (principal.isEmpty()) { + return false; + } + + LOGGER.debug("Authenticated user: {}", principal.get().getName()); + + AuthenticatedPolarisPrincipal prince = principal.get(); + SecurityContext securityContext = augmentSecurityContext(requestContext, scheme, prince); + requestContext.setSecurityContext(securityContext); + return true; + } + + private static SecurityContext augmentSecurityContext( + ContainerRequestContext requestContext, String scheme, AuthenticatedPolarisPrincipal prince) { + SecurityContext securityContext = requestContext.getSecurityContext(); + boolean secure = securityContext != null && securityContext.isSecure(); + return new SecurityContext() { + @Override + public Principal getUserPrincipal() { + return prince; + } + + @Override + public boolean isUserInRole(String role) { + return true; // TODO: implement role-based access control + } + + @Override + public boolean isSecure() { + return secure; + } + + @Override + public String getAuthenticationScheme() { + return scheme; + } + }; + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/RequestThrottlingErrorResponse.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/catalog/io/QuarkusFileIOConfiguration.java similarity index 65% rename from dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/RequestThrottlingErrorResponse.java rename to quarkus/service/src/main/java/org/apache/polaris/service/quarkus/catalog/io/QuarkusFileIOConfiguration.java index fa07647c1..cebbe7673 100644 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/RequestThrottlingErrorResponse.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/catalog/io/QuarkusFileIOConfiguration.java @@ -16,18 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.throttling; +package org.apache.polaris.service.quarkus.catalog.io; -import com.fasterxml.jackson.annotation.JsonProperty; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; -/** - * Response object for errors caused by DoS-prevention throttling mechanisms, such as request size - * limits - */ -public record RequestThrottlingErrorResponse( - @JsonProperty("error_type") RequestThrottlingErrorType errorType) { - public enum RequestThrottlingErrorType { - REQUEST_TOO_LARGE, - ; - } +@StaticInitSafe +@ConfigMapping(prefix = "polaris.file-io") +public interface QuarkusFileIOConfiguration { + + /** + * The type of the catalog IO to use. Must be a registered {@link + * org.apache.polaris.service.catalog.io.FileIOFactory} identifier. + */ + String type(); } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java new file mode 100644 index 000000000..76ba04968 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusFeaturesConfiguration.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.config; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithParentName; +import java.util.Map; +import org.apache.polaris.service.config.FeaturesConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.features") +public interface QuarkusFeaturesConfiguration extends FeaturesConfiguration { + + @Override + Map defaults(); + + @Override + Map realmOverrides(); + + interface QuarkusRealmOverrides extends RealmOverrides { + @WithParentName + @Override + Map overrides(); + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusJacksonConfig.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusJacksonConfig.java new file mode 100644 index 000000000..4550fc65e --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusJacksonConfig.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.config; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import io.quarkus.jackson.ObjectMapperCustomizer; +import io.quarkus.runtime.configuration.MemorySize; +import io.quarkus.runtime.configuration.MemorySizeConverter; +import io.smallrye.config.WithConverter; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.service.config.Serializers; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class QuarkusJacksonConfig implements ObjectMapperCustomizer { + + private static final Logger LOGGER = LoggerFactory.getLogger(QuarkusJacksonConfig.class); + + private final long maxBodySize; + + @Inject + public QuarkusJacksonConfig( + @ConfigProperty(name = "quarkus.http.limits.max-body-size") + @WithConverter(MemorySizeConverter.class) + MemorySize maxBodySize) { + this.maxBodySize = maxBodySize.asLongValue(); + } + + @Override + public void customize(ObjectMapper objectMapper) { + objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(objectMapper); + Serializers.registerSerializers(objectMapper); + objectMapper + .getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder().maxDocumentLength(maxBodySize).build()); + LOGGER.info("Limiting request body size to {} bytes", maxBodySize); + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java new file mode 100644 index 000000000..647f309f1 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.config; + +import io.smallrye.common.annotation.Identifier; +import io.smallrye.context.SmallRyeManagedExecutor; +import io.vertx.core.http.HttpServerRequest; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.RequestScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Disposes; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; +import jakarta.ws.rs.core.Context; +import java.time.Clock; +import java.util.HashMap; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisConfigurationStore; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.apache.polaris.service.auth.Authenticator; +import org.apache.polaris.service.auth.TokenBrokerFactory; +import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; +import org.apache.polaris.service.catalog.io.FileIOFactory; +import org.apache.polaris.service.context.RealmContextResolver; +import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationConfiguration; +import org.apache.polaris.service.quarkus.catalog.io.QuarkusFileIOConfiguration; +import org.apache.polaris.service.quarkus.context.QuarkusContextConfiguration; +import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; +import org.apache.polaris.service.quarkus.ratelimiter.QuarkusRateLimiterConfiguration; +import org.apache.polaris.service.ratelimiter.RateLimiter; +import org.apache.polaris.service.ratelimiter.TokenBucketFactory; +import org.apache.polaris.service.task.TaskHandlerConfiguration; +import org.eclipse.microprofile.context.ManagedExecutor; +import org.eclipse.microprofile.context.ThreadContext; + +public class QuarkusProducers { + + @Produces + @ApplicationScoped // cannot be singleton because it is mocked in tests + public Clock clock() { + return Clock.systemDefaultZone(); + } + + // Polaris core beans - application scope + + @Produces + @ApplicationScoped + public StorageCredentialCache storageCredentialCache() { + return new StorageCredentialCache(); + } + + @Produces + @ApplicationScoped + public PolarisAuthorizer polarisAuthorizer(PolarisConfigurationStore configurationStore) { + return new PolarisAuthorizerImpl(configurationStore); + } + + @Produces + @ApplicationScoped + public PolarisDiagnostics polarisDiagnostics() { + return new PolarisDefaultDiagServiceImpl(); + } + + // Polaris core beans - request scope + + @Produces + @RequestScoped + public RealmContext realmContext( + @Context HttpServerRequest request, RealmContextResolver realmContextResolver) { + return realmContextResolver.resolveRealmContext( + request.absoluteURI(), + request.method().name(), + request.path(), + request.headers().entries().stream() + .collect(HashMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), HashMap::putAll)); + } + + @Produces + @RequestScoped + public PolarisCallContext polarisCallContext( + RealmContext realmContext, + PolarisDiagnostics diagServices, + PolarisConfigurationStore configurationStore, + MetaStoreManagerFactory metaStoreManagerFactory, + Clock clock) { + PolarisMetaStoreSession metaStoreSession = + metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); + return new PolarisCallContext(metaStoreSession, diagServices, configurationStore, clock); + } + + @Produces + @RequestScoped + public CallContext callContext(RealmContext realmContext, PolarisCallContext polarisCallContext) { + return CallContext.of(realmContext, polarisCallContext); + } + + public void closeCallContext(@Disposes CallContext callContext) { + callContext.close(); + } + + // Polaris service beans - selected from @Identifier-annotated beans + + @Produces + public RealmContextResolver realmContextResolver( + QuarkusContextConfiguration config, + @Any Instance realmContextResolvers) { + return realmContextResolvers + .select(Identifier.Literal.of(config.realmContextResolver().type())) + .get(); + } + + @Produces + public FileIOFactory fileIOFactory( + QuarkusFileIOConfiguration config, @Any Instance fileIOFactories) { + return fileIOFactories.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public MetaStoreManagerFactory metaStoreManagerFactory( + QuarkusPersistenceConfiguration config, + @Any Instance metaStoreManagerFactories) { + return metaStoreManagerFactories.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public RateLimiter rateLimiter( + QuarkusRateLimiterConfiguration config, @Any Instance rateLimiters) { + return rateLimiters.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public TokenBucketFactory tokenBucketFactory( + QuarkusRateLimiterConfiguration config, + @Any Instance tokenBucketFactories) { + return tokenBucketFactories.select(Identifier.Literal.of(config.tokenBucket().type())).get(); + } + + @Produces + public Authenticator authenticator( + QuarkusAuthenticationConfiguration config, + @Any Instance> authenticators) { + return authenticators.select(Identifier.Literal.of(config.authenticator().type())).get(); + } + + @Produces + public IcebergRestOAuth2ApiService icebergRestOAuth2ApiService( + QuarkusAuthenticationConfiguration config, + @Any Instance services) { + return services.select(Identifier.Literal.of(config.tokenService().type())).get(); + } + + @Produces + public TokenBrokerFactory tokenBrokerFactory( + QuarkusAuthenticationConfiguration config, + @Any Instance tokenBrokerFactories) { + return tokenBrokerFactories.select(Identifier.Literal.of(config.tokenBroker().type())).get(); + } + + // other beans + + @Produces + @Singleton + @Identifier("task-executor") + public ManagedExecutor taskExecutor(TaskHandlerConfiguration config) { + return SmallRyeManagedExecutor.builder() + .injectionPointName("task-executor") + .propagated(ThreadContext.ALL_REMAINING) + .maxAsync(config.maxConcurrentTasks()) + .maxQueued(config.maxQueuedTasks()) + .build(); + } + + public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { + executor.close(); + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/context/QuarkusContextConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/context/QuarkusContextConfiguration.java new file mode 100644 index 000000000..9cd617b91 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/context/QuarkusContextConfiguration.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.context; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import org.apache.polaris.service.context.ContextConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.context") +public interface QuarkusContextConfiguration extends ContextConfiguration { + + @Override + QuarkusRealmContextResolverConfiguration realmContextResolver(); + + interface QuarkusRealmContextResolverConfiguration extends RealmContextResolverConfiguration { + + /** + * The type of the realm context resolver. Must be a registered {@link + * org.apache.polaris.service.context.RealmContextResolver} identifier. + */ + String type(); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/logging/QuarkusLoggingConfiguration.java similarity index 63% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java rename to quarkus/service/src/main/java/org/apache/polaris/service/quarkus/logging/QuarkusLoggingConfiguration.java index 88d5696b3..145bd740e 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/logging/QuarkusLoggingConfiguration.java @@ -16,17 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.test; +package org.apache.polaris.service.quarkus.logging; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import java.util.Map; -/** - * Annotation used to specify where to inject the Polaris test realm identifier. This is provided by - * PolarisConnectionExtension. - */ -@Target({ElementType.PARAMETER}) -@Retention(RetentionPolicy.RUNTIME) -public @interface PolarisRealm {} +@StaticInitSafe +@ConfigMapping(prefix = "polaris.log") +public interface QuarkusLoggingConfiguration { + + /** The name of the header that contains the request ID. */ + String requestIdHeaderName(); + + /** Additional MDC values to include in the log context. */ + Map mdc(); +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/logging/QuarkusLoggingMDCFilter.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/logging/QuarkusLoggingMDCFilter.java new file mode 100644 index 000000000..98fb2888d --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/logging/QuarkusLoggingMDCFilter.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.logging; + +import io.quarkus.vertx.web.RouteFilter; +import io.vertx.ext.web.RoutingContext; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.polaris.core.context.RealmContext; +import org.slf4j.MDC; + +@ApplicationScoped +public class QuarkusLoggingMDCFilter { + + public static final int PRIORITY = RouteFilter.DEFAULT_PRIORITY + 100; + + private static final String REQUEST_ID_KEY = "requestId"; + private static final String REALM_ID_KEY = "realmId"; + + @Inject RealmContext realmContext; + + @Inject QuarkusLoggingConfiguration loggingConfiguration; + + public static String requestId(RoutingContext rc) { + return rc.get(REQUEST_ID_KEY); + } + + public static String realmId(RoutingContext rc) { + return rc.get(REALM_ID_KEY); + } + + @RouteFilter(value = PRIORITY) + public void applyMDCContext(RoutingContext rc) { + // The request scope is active here, so any MDC values set here will be propagated to + // threads handling the request. + // Also put the MDC values in the request context for use by other filters and handlers + loggingConfiguration.mdc().forEach(MDC::put); + loggingConfiguration.mdc().forEach(rc::put); + var requestId = rc.request().getHeader(loggingConfiguration.requestIdHeaderName()); + if (requestId != null) { + MDC.put(REQUEST_ID_KEY, requestId); + rc.put(REQUEST_ID_KEY, requestId); + } + MDC.put(REALM_ID_KEY, realmContext.getRealmIdentifier()); + rc.put(REALM_ID_KEY, realmContext.getRealmIdentifier()); + // Do not explicitly remove the MDC values from the request context with an end handler, + // as this could remove MDC context still in use in TaskExecutor threads + // rc.addEndHandler( + // (v) -> { + // MDC.remove(REQUEST_ID_MDC_KEY); + // MDC.remove(REALM_ID_MDC_KEY); + // loggingConfiguration.mdc().keySet().forEach(MDC::remove); + // }); + rc.next(); + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMeterFilterProducer.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMeterFilterProducer.java new file mode 100644 index 000000000..28f77e913 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMeterFilterProducer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.metrics; + +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.config.MeterFilter; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.util.stream.Collectors; + +public class QuarkusMeterFilterProducer { + + @Inject QuarkusMetricsConfiguration configuration; + + @Produces + @Singleton + public MeterFilter produceGlobalMeterFilter() { + return MeterFilter.commonTags( + this.configuration.tags().entrySet().stream() + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toSet())); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMetricsConfiguration.java similarity index 67% rename from polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java rename to quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMetricsConfiguration.java index 9bf456f58..c599e85e2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMetricsConfiguration.java @@ -16,16 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.context; +package org.apache.polaris.service.quarkus.metrics; -import jakarta.inject.Scope; -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import java.util.Map; -@Scope -@Documented -@Retention(java.lang.annotation.RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE, ElementType.METHOD}) -public @interface RealmScoped {} +@StaticInitSafe +@ConfigMapping(prefix = "polaris.metrics") +public interface QuarkusMetricsConfiguration { + + /** Additional tags to include in the metrics. */ + Map tags(); +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusValueExpressionResolver.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusValueExpressionResolver.java new file mode 100644 index 000000000..98783de60 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusValueExpressionResolver.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.metrics; + +import io.micrometer.common.annotation.ValueExpressionResolver; +import io.micrometer.common.lang.Nullable; +import jakarta.annotation.Nonnull; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.context.RealmContext; + +@ApplicationScoped +public class QuarkusValueExpressionResolver implements ValueExpressionResolver { + + @Override + public String resolve(@Nonnull String expression, @Nullable Object parameter) { + // TODO maybe replace with CEL of some expression engine and make this more generic + if (parameter instanceof RealmContext realmContext && expression.equals("realmIdentifier")) { + return realmContext.getRealmIdentifier(); + } + return null; + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/RealmIdTagContributor.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/RealmIdTagContributor.java new file mode 100644 index 000000000..147ebd7fd --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/RealmIdTagContributor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.metrics; + +import io.micrometer.core.instrument.Tags; +import io.quarkus.micrometer.runtime.HttpServerMetricsTagsContributor; +import io.vertx.core.http.HttpServerRequest; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.HashMap; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.service.context.RealmContextResolver; + +@ApplicationScoped +public class RealmIdTagContributor implements HttpServerMetricsTagsContributor { + + public static final String TAG_REALM = "realm_id"; + + @Inject RealmContextResolver realmContextResolver; + + @Override + public Tags contribute(Context context) { + // FIXME request scope does not work here, so we have to resolve the realm context manually + HttpServerRequest request = context.request(); + RealmContext realmContext = resolveRealmContext(request); + return Tags.of(TAG_REALM, realmContext.getRealmIdentifier()); + } + + private RealmContext resolveRealmContext(HttpServerRequest request) { + return realmContextResolver.resolveRealmContext( + request.absoluteURI(), + request.method().name(), + request.path(), + request.headers().entries().stream() + .collect(HashMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), HashMap::putAll)); + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/persistence/QuarkusPersistenceConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/persistence/QuarkusPersistenceConfiguration.java new file mode 100644 index 000000000..5c3223f69 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/persistence/QuarkusPersistenceConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.persistence; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.persistence") +public interface QuarkusPersistenceConfiguration { + + /** + * The type of the persistence to use. Must be a registered {@link + * org.apache.polaris.core.persistence.MetaStoreManagerFactory} identifier. + */ + String type(); +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/ratelimiter/QuarkusRateLimiterConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/ratelimiter/QuarkusRateLimiterConfiguration.java new file mode 100644 index 000000000..0a1e72de8 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/ratelimiter/QuarkusRateLimiterConfiguration.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.ratelimiter; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import org.apache.polaris.service.ratelimiter.RateLimiterConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.rate-limiter") +public interface QuarkusRateLimiterConfiguration extends RateLimiterConfiguration { + + /** + * The type of the rate limiter. Must be a registered {@link + * org.apache.polaris.service.ratelimiter.RateLimiter} identifier. + */ + String type(); + + /** The configuration for the token bucket rate limiter. */ + @Override + QuarkusTokenBucketConfiguration tokenBucket(); + + interface QuarkusTokenBucketConfiguration extends TokenBucketConfiguration { + + /** + * The type of the token bucket factory. Must be a registered {@link + * org.apache.polaris.service.ratelimiter.TokenBucketFactory} identifier. + */ + String type(); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/storage/QuarkusStorageConfiguration.java similarity index 52% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java rename to quarkus/service/src/main/java/org/apache/polaris/service/quarkus/storage/QuarkusStorageConfiguration.java index 232a57b7e..cf2c716bc 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/storage/QuarkusStorageConfiguration.java @@ -16,22 +16,32 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.test; +package org.apache.polaris.service.quarkus.storage; -import jakarta.ws.rs.client.Client; -import java.net.URI; -import java.util.UUID; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; +import java.time.Duration; +import java.util.*; +import org.apache.polaris.service.storage.StorageConfiguration; -/** - * Defines the test environment that a test should run in. - * - * @param apiClient The HTTP client to use when making requests - * @param baseUri The base URL that requests should target, for example http://localhost:1234 - * @param testId An ID unique to this test. This can be used to prefix resource names, such as - * catalog names, to prevent collision. - */ -public record TestEnvironment(Client apiClient, URI baseUri, String testId) { - public TestEnvironment(Client apiClient, String baseUri) { - this(apiClient, URI.create(baseUri), UUID.randomUUID().toString().replace("-", "")); - } +@StaticInitSafe +@ConfigMapping(prefix = "polaris.storage") +public interface QuarkusStorageConfiguration extends StorageConfiguration { + + @WithName("aws.access-key") + @Override + Optional awsAccessKey(); + + @WithName("aws.secret-key") + @Override + Optional awsSecretKey(); + + @WithName("gcp.token") + @Override + Optional gcpAccessToken(); + + @WithName("gcp.lifespan") + @Override + Optional gcpAccessTokenLifespan(); } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/task/QuarkusTaskExecutorImpl.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/task/QuarkusTaskExecutorImpl.java new file mode 100644 index 000000000..8f38de648 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/task/QuarkusTaskExecutorImpl.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.task; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.quarkus.runtime.Startup; +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.concurrent.ExecutorService; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.tracing.QuarkusTracingFilter; +import org.apache.polaris.service.task.TaskExecutorImpl; +import org.apache.polaris.service.task.TaskFileIOSupplier; + +@ApplicationScoped +public class QuarkusTaskExecutorImpl extends TaskExecutorImpl { + + private final Tracer tracer; + + public QuarkusTaskExecutorImpl() { + this(null, null, null, null); + } + + @Inject + public QuarkusTaskExecutorImpl( + @Identifier("task-executor") ExecutorService executorService, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskFileIOSupplier fileIOSupplier, + Tracer tracer) { + super(executorService, metaStoreManagerFactory, fileIOSupplier); + this.tracer = tracer; + } + + @Startup + @Override + public void init() { + super.init(); + } + + @Override + protected void handleTask(long taskEntityId, CallContext callContext, int attempt) { + Span span = + tracer + .spanBuilder("polaris.task") + .setParent(Context.current()) + .setAttribute( + QuarkusTracingFilter.REALM_ID_ATTRIBUTE, + callContext.getRealmContext().getRealmIdentifier()) + .setAttribute("polaris.task.entity.id", taskEntityId) + .setAttribute("polaris.task.attempt", attempt) + .startSpan(); + try (Scope ignored = span.makeCurrent()) { + super.handleTask(taskEntityId, callContext, attempt); + } finally { + span.end(); + } + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/task/QuarkusTaskHandlerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/task/QuarkusTaskHandlerConfiguration.java new file mode 100644 index 000000000..ff9416f97 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/task/QuarkusTaskHandlerConfiguration.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.task; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; +import org.apache.polaris.service.task.TaskHandlerConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.tasks") +public interface QuarkusTaskHandlerConfiguration extends TaskHandlerConfiguration { + + @WithDefault("-1") + @Override + int maxConcurrentTasks(); + + @WithDefault("-1") + @Override + int maxQueuedTasks(); +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/tracing/QuarkusTracingFilter.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/tracing/QuarkusTracingFilter.java new file mode 100644 index 000000000..28811ad91 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/tracing/QuarkusTracingFilter.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.tracing; + +import io.opentelemetry.api.trace.Span; +import io.quarkus.vertx.web.RouteFilter; +import io.vertx.ext.web.RoutingContext; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.service.quarkus.logging.QuarkusLoggingMDCFilter; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +@ApplicationScoped +public class QuarkusTracingFilter { + + public static final String REQUEST_ID_ATTRIBUTE = "polaris.request.id"; + public static final String REALM_ID_ATTRIBUTE = "polaris.realm"; + + @ConfigProperty(name = "quarkus.otel.sdk.disabled") + boolean sdkDisabled; + + @RouteFilter(QuarkusLoggingMDCFilter.PRIORITY - 1) + public void applySpanAttributes(RoutingContext rc) { + if (!sdkDisabled) { + Span span = Span.current(); + String requestId = QuarkusLoggingMDCFilter.requestId(rc); + String realmId = QuarkusLoggingMDCFilter.realmId(rc); + if (requestId != null) { + span.setAttribute(REQUEST_ID_ATTRIBUTE, requestId); + } + span.setAttribute(REALM_ID_ATTRIBUTE, realmId); + } + rc.next(); + } +} diff --git a/quarkus/service/src/main/resources/application.properties b/quarkus/service/src/main/resources/application.properties new file mode 100644 index 000000000..ad270c455 --- /dev/null +++ b/quarkus/service/src/main/resources/application.properties @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +quarkus.banner.path=/org/apache/polaris/service/banner.txt + +quarkus.config.mapping.validate-unknown=true + +quarkus.http.auth.basic=false +quarkus.http.access-log.enabled=true +# quarkus.http.access-log.pattern=common +quarkus.http.enable-compression=true +quarkus.http.enable-decompression=true +quarkus.http.body.handle-file-uploads=false +quarkus.http.limits.max-body-size=10240K +quarkus.http.compress-media-types=application/json,text/html,text/plain + +quarkus.http.cors.origins=http://localhost:8080 +quarkus.http.cors.methods=PATCH, POST, DELETE, GET, PUT +quarkus.http.cors.headers=* +quarkus.http.cors.exposed-headers=* +quarkus.http.cors.access-control-max-age=PT10M +quarkus.http.cors.access-control-allow-credentials=true + +quarkus.http.port=8181 +quarkus.http.test-port=0 + +quarkus.log.level=INFO +quarkus.log.console.enable=true +quarkus.log.console.level=ALL +quarkus.log.console.json=false +quarkus.log.console.format=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n +quarkus.log.file.enable=true +quarkus.log.file.level=ALL +quarkus.log.file.json=false +quarkus.log.file.format=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n +quarkus.log.file.path=./logs/polaris.log +quarkus.log.file.rotation.file-suffix=.yyyy-MM-dd.gz +quarkus.log.file.rotation.max-file-size=10M +quarkus.log.file.rotation.max-backup-index=14 +quarkus.log.category."org.apache.polaris".level=INFO +quarkus.log.category."org.apache.iceberg.rest".level=INFO +quarkus.log.category."io.smallrye.config".level=INFO + +quarkus.management.enabled=true +quarkus.management.port=8182 +quarkus.management.test-port=0 + +quarkus.micrometer.enabled=true +quarkus.micrometer.export.prometheus.enabled=true + +quarkus.otel.enabled=true +quarkus.otel.sdk.disabled=false +# quarkus.otel.exporter.otlp.endpoint=http://otlp-collector:4317 +# quarkus.otel.resource.attributes=service.name=polaris,deployment.env=prod,region=us-west-2 +# quarkus.otel.service.name=polaris +# quarkus.otel.traces.sampler=parentbased_always_on +# quarkus.otel.traces.sampler.arg=1.0d + +polaris.context.realm-context-resolver.default-realm=default-realm +polaris.context.realm-context-resolver.type=default + +polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false +polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +# realm overrides +# polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true +# polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true + +polaris.persistence.type=in-memory + +polaris.file-io.type=default + +polaris.log.request-id-header-name=request_id +# polaris.log.mdc.aid=polaris +# polaris.log.mdc.sid=polaris-service + +polaris.metrics.tags.application=Polaris +# polaris.metrics.tags.service=polaris +# polaris.metrics.tags.environment=prod +# polaris.metrics.tags.region=us-west-2 + +# polaris.tasks.max-concurrent-tasks=100 +# polaris.tasks.max-queued-tasks=1000 + +polaris.rate-limiter.type=default +polaris.rate-limiter.token-bucket.type=default +polaris.rate-limiter.token-bucket.requests-per-second=9999 +polaris.rate-limiter.token-bucket.window=PT10S + +polaris.authentication.authenticator.type=default +polaris.authentication.token-service.type=default +polaris.authentication.token-broker.type=rsa-key-pair +polaris.authentication.token-broker.max-token-generation=PT1H +# polaris.authentication.token-broker.rsa-key-pair.public-key-file=/tmp/public.key +# polaris.authentication.token-broker.rsa-key-pair.private-key-file=/tmp/private.key +# polaris.authentication.token-broker.symmetric-key.secret=secret +# polaris.authentication.token-broker.symmetric-key.file=/tmp/symmetric.key + +# If the following properties are unset, the default credential provider chain will be used +# polaris.storage.aws.access-key=accessKey +# polaris.storage.aws.secret-key=secretKey +# polaris.storage.gcp.token=token +# polaris.storage.gcp.lifespan=PT1H + +%test.quarkus.log.file.enable=false +%test.quarkus.log.category."org.apache.polaris".level=INFO +%test.quarkus.log.category."org.apache.iceberg.rest".level=INFO +%test.quarkus.log.category."org.apache.iceberg.rest.RESTSessionCatalog".level=ERROR +%test.quarkus.log.category."org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl".level=ERROR +%test.quarkus.log.category."org.apache.polaris.service.context.DefaultRealmContextResolver".level=ERROR +%test.quarkus.log.category."org.apache.polaris.service.catalog.PolarisCatalogHandlerWrapper".level=ERROR +%test.quarkus.log.category."org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl".level=ERROR +%test.quarkus.http.limits.max-body-size=1000000 +%test.quarkus.otel.sdk.disabled=true +%test.polaris.authentication.token-broker.type=symmetric-key +%test.polaris.authentication.token-broker.symmetric-key.secret=polaris +%test.polaris.features.defaults."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false +%test.polaris.features.defaults."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false +%test.polaris.features.defaults."ALLOW_OVERLAPPING_CATALOG_URLS"=true +%test.polaris.features.defaults."ALLOW_SPECIFYING_FILE_IO_IMPL"=true +%test.polaris.features.defaults."ALLOW_WILDCARD_LOCATION"=true +%test.polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true +%test.polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true +%test.polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true +%test.polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +%test.polaris.context.realm-context-resolver.default-realm=POLARIS +%test.polaris.storage.aws.access-key=accessKey +%test.polaris.storage.aws.secret-key=secretKey +%test.polaris.storage.gcp.token=token +%test.polaris.storage.gcp.lifespan=PT1H diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/IcebergExceptionMapperTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/IcebergExceptionMapperTest.java similarity index 100% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/IcebergExceptionMapperTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/IcebergExceptionMapperTest.java diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/PolarisApplicationIntegrationTest.java similarity index 75% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/PolarisApplicationIntegrationTest.java index e7f7cd4a2..0b76d0f13 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/PolarisApplicationIntegrationTest.java @@ -16,31 +16,28 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.quarkus; +import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.ProcessingException; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.function.Supplier; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionData; @@ -80,127 +77,91 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) + +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisApplicationIntegrationTest { - @TempDir private static Path tempDir; - private static final Supplier CURRENT_LOG = - () -> tempDir.resolve("application.log").toString(); private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); public static final String PRINCIPAL_ROLE_NAME = "admin"; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0"), // Bind to random port to support parallelism - ConfigOverride.config("logging.appenders[1].type", "file"), - ConfigOverride.config("logging.appenders[1].currentLogFilename", CURRENT_LOG)); - - private static String userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; - private static Path testDir; - private static String realm; - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String polarisRealm) - throws IOException { - realm = polarisRealm; + @Inject PolarisIntegrationTestHelper helper; - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .contains("PolarisApplication: Server started successfully"); + @ConfigProperty(name = "quarkus.http.limits.max-body-size") + long maxBodySize; - testDir = Path.of("build/test_data/iceberg/" + realm); - FileUtils.deleteQuietly(testDir.toFile()); - Files.createDirectories(testDir); - PolarisApplicationIntegrationTest.userToken = userToken.token(); - PolarisApplicationIntegrationTest.snowmanCredentials = snowmanCredentials; + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; + @BeforeAll + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); try (Response createPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(principalRole))) { assertThat(createPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response assignPrResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/principals/%s/principal-roles", - EXT.getLocalPort(), snowmanCredentials.identifier().principalName())) + "%s/api/management/v1/principals/%s/principal-roles", + testEnv.baseUri(), fixture.snowmanCredentials.identifier().principalName())) .request("application/json") - .header("Authorization", "Bearer " + PolarisApplicationIntegrationTest.userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put(Entity.json(principalRole))) { assertThat(assignPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - - assertZeroErrorsInApplicationLog(); } @AfterAll - public static void deletePrincipalRole() { - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s", - EXT.getLocalPort(), PRINCIPAL_ROLE_NAME)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - } - - private static void assertZeroErrorsInApplicationLog() { - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .hasSizeGreaterThan(0) - .doesNotContain("ERROR", "FATAL"); + public void destroyFixture() { + if (fixture != null) { + fixture + .client + .target( + String.format( + "%s/api/management/v1/principal-roles/%s", + testEnv.baseUri(), PRINCIPAL_ROLE_NAME)) + .request("application/json") + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .delete() + .close(); + fixture.destroy(); + } } /** @@ -210,7 +171,7 @@ private static void assertZeroErrorsInApplicationLog() { * @param testInfo */ @BeforeEach - public void before(TestInfo testInfo) { + public void createTestCatalog(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( @@ -221,7 +182,7 @@ public void before(TestInfo testInfo) { }); } - private static void createCatalog( + private void createCatalog( String catalogName, Catalog.TypeEnum catalogType, String principalRoleName) { createCatalog( catalogName, @@ -236,7 +197,7 @@ private static void createCatalog( "s3://my-bucket/path/to/data"); } - private static void createCatalog( + private void createCatalog( String catalogName, Catalog.TypeEnum catalogType, String principalRoleName, @@ -263,39 +224,41 @@ private static void createCatalog( .setStorageConfigInfo(storageConfig) .build(); try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", testEnv.baseUri())) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - EXT.getLocalPort(), + "%s/api/management/v1/catalogs/%s/catalog-roles/%s", + testEnv.baseUri(), catalogName, PolarisEntityConstants.getNameOfCatalogAdminRole())) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - EXT.getLocalPort(), principalRoleName, catalogName)) + "%s/api/management/v1/principal-roles/%s/catalog-roles/%s", + testEnv.baseUri(), principalRoleName, catalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put(Entity.json(catalogRole))) { assertThat(assignResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -303,21 +266,21 @@ private static void createCatalog( } } - private static RESTSessionCatalog newSessionCatalog(String catalog) { + private RESTSessionCatalog newSessionCatalog(String catalog) { RESTSessionCatalog sessionCatalog = new RESTSessionCatalog(); sessionCatalog.initialize( "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + testEnv.baseUri() + "/api/catalog", OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + fixture.snowmanCredentials.clientId() + ":" + fixture.snowmanCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", catalog, "header." + REALM_PROPERTY_KEY, - realm)); + fixture.realm)); return sessionCatalog; } @@ -506,16 +469,17 @@ public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throw } @Test - public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -524,7 +488,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -553,16 +517,17 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws } @Test - public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -571,7 +536,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -606,16 +571,17 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO } @Test - public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergDropTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -624,7 +590,7 @@ public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOEx TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -661,15 +627,17 @@ public void testWarehouseNotSpecified() throws IOException { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + testEnv.baseUri() + "/api/catalog", OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + fixture.snowmanCredentials.clientId() + + ":" + + fixture.snowmanCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", emptyEnvironmentVariable, "header." + REALM_PROPERTY_KEY, - realm))) + fixture.realm))) .isInstanceOf(BadRequestException.class) .hasMessage("Malformed request: Please specify a warehouse"); } @@ -677,34 +645,35 @@ public void testWarehouseNotSpecified() throws IOException { @Test public void testRequestHeaderTooLarge() { - Invocation.Builder request = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json"); - - // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely - // exceeds the limit - for (int i = 0; i < 1500; i++) { - request = request.header("header" + i, "" + i); - } + try (Client client = ClientBuilder.newClient()) { + Invocation.Builder request = + client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) + .request("application/json"); + + // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely + // exceeds the limit + for (int i = 0; i < 1500; i++) { + request = request.header("header" + i, "" + i); + } - try { - try (Response response = - request - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(new PrincipalRole("r")))) { - assertThat(response) - .returns( - Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), - Response::getStatus); + try { + try (Response response = + request + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .post(Entity.json(new PrincipalRole("r")))) { + assertThat(response) + .returns( + Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), + Response::getStatus); + } + } catch (ProcessingException e) { + // In some runtime environments the request above will return a 431 but in others it'll + // result in a ProcessingException from the socket being closed. The test asserts that one + // of those things happens. + assertThat(e).hasMessageContaining("Connection was closed"); } - } catch (ProcessingException e) { - // In some runtime environments the request above will return a 431 but in others it'll result - // in a ProcessingException from the socket being closed. The test asserts that one of those - // things happens. } } @@ -716,39 +685,38 @@ public void testRequestBodyTooLarge() { // JSON overhead. Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(largeRequest)) { - // Note we only validate the status code here because per RFC 9110, the server MAY not provide - // a response body. The HTTP status line is still expected to be provided. - assertThat(response.getStatus()) - .isEqualTo(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode()); - } catch (ProcessingException e) { - // Per RFC 9110 servers MAY close the connection in case of 413 responses, which - // might cause the client to fail to read the status code (cf. RFC 9112, section 9.6). - // TODO: servers are expected to close connections gracefully. It might be worth investigating - // whether "connection closed" exceptions are a client-side bug. - assertThat(e).hasMessageContaining("Connection was closed"); + try (Client client = ClientBuilder.newClient()) { + try (Response response = + client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) + .request("application/json") + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .post(largeRequest)) { + // Note we only validate the status code here because per RFC 9110, the server MAY not + // provide a response body. The HTTP status line is still expected to be provided. + assertThat(response.getStatus()) + .isEqualTo(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode()); + } catch (ProcessingException e) { + // Per RFC 9110 servers MAY close the connection in case of 413 responses, which + // might cause the client to fail to read the status code (cf. RFC 9112, section 9.6). + // TODO: servers are expected to close connections gracefully. It might be worth + // investigating whether "connection closed" exceptions are a client-side bug. + assertThat(e).hasMessageContaining("Connection was closed"); + } } } @Test public void testRefreshToken() throws IOException { - String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", EXT.getLocalPort()); + String path = String.format("%s/api/catalog/v1/oauth/tokens", testEnv.baseUri()); try (RESTClient client = - HTTPClient.builder(ImmutableMap.of()) - .withHeader(REALM_PROPERTY_KEY, realm) + HTTPClient.builder(Map.of()) + .withHeader(REALM_PROPERTY_KEY, fixture.realm) .uri(path) .build()) { String credentialString = - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(); + fixture.snowmanCredentials.clientId() + ":" + fixture.snowmanCredentials.clientSecret(); String expiredToken = JWT.create().withExpiresAt(Instant.EPOCH).sign(Algorithm.HMAC256("irrelevant-secret")); var authConfig = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TestServices.java similarity index 87% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TestServices.java index c9dacaf45..a31d4bb47 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TestServices.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.quarkus; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; @@ -37,7 +37,6 @@ import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; import org.apache.polaris.service.catalog.IcebergCatalogAdapter; @@ -48,8 +47,8 @@ import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TaskExecutor; import org.mockito.Mockito; @@ -59,6 +58,7 @@ public record TestServices( PolarisCatalogsApi catalogsApi, RealmContext realmContext, SecurityContext securityContext) { + private static final RealmContext testRealm = () -> "test-realm"; public static TestServices inMemory(Map config) { @@ -71,17 +71,27 @@ public static TestServices inMemory(FileIOFactory ioFactory) { public static TestServices inMemory(FileIOFactory ioFactory, Map config) { InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - metaStoreManagerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); + new InMemoryPolarisMetaStoreManagerFactory( + new PolarisStorageIntegrationProviderImpl( + Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(testRealm); - EntityCache cache = new EntityCache(metaStoreManager); + PolarisMetaStoreSession session = + metaStoreManagerFactory.getOrCreateSessionSupplier(testRealm).get(); + + PolarisCallContext context = + new PolarisCallContext( + session, + Mockito.mock(PolarisDiagnostics.class), + new DefaultConfigurationStore(config), + Clock.systemDefaultZone()); + + CallContext callContext = CallContext.of(testRealm, context); + RealmEntityManagerFactory realmEntityManagerFactory = - new RealmEntityManagerFactory(metaStoreManagerFactory, () -> cache) {}; + new RealmEntityManagerFactory(metaStoreManagerFactory) {}; CallContextCatalogFactory callContextFactory = new PolarisCallContextCatalogFactory( realmEntityManagerFactory, @@ -91,17 +101,13 @@ public static TestServices inMemory(FileIOFactory ioFactory, Map PolarisAuthorizer authorizer = Mockito.mock(PolarisAuthorizer.class); IcebergRestCatalogApiService service = new IcebergCatalogAdapter( - callContextFactory, realmEntityManagerFactory, metaStoreManagerFactory, authorizer); + callContext, + callContextFactory, + realmEntityManagerFactory, + metaStoreManagerFactory, + authorizer); IcebergRestCatalogApi restApi = new IcebergRestCatalogApi(service); - PolarisMetaStoreSession session = - metaStoreManagerFactory.getOrCreateSessionSupplier(testRealm).get(); - PolarisCallContext context = - new PolarisCallContext( - session, - Mockito.mock(PolarisDiagnostics.class), - new DefaultConfigurationStore(config), - Clock.systemDefaultZone()); PolarisMetaStoreManager.CreatePrincipalResult createdPrincipal = metaStoreManager.createPrincipal( context, @@ -140,7 +146,8 @@ public String getAuthenticationScheme() { PolarisCatalogsApi catalogsApi = new PolarisCatalogsApi( - new PolarisServiceImpl(realmEntityManagerFactory, metaStoreManagerFactory, authorizer)); + new PolarisServiceImpl( + realmEntityManagerFactory, metaStoreManagerFactory, authorizer, callContext)); CallContext.setCurrentContext(CallContext.of(testRealm, context)); return new TestServices(restApi, catalogsApi, testRealm, securityContext); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TimedApplicationEventListenerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TimedApplicationEventListenerTest.java new file mode 100644 index 000000000..fe5e574bc --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TimedApplicationEventListenerTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus; + +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response; +import java.util.Map; +import org.apache.polaris.service.quarkus.TimedApplicationEventListenerTest.Profile; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.test.TestMetricsUtil; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.Summary; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; + +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) +@TestProfile(Profile.class) +public class TimedApplicationEventListenerTest { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of("polaris.metrics.tags.environment", "prod"); + } + } + + private static final int ERROR_CODE = Response.Status.NOT_FOUND.getStatusCode(); + private static final String ENDPOINT = "api/management/v1/principals"; + private static final String METRIC_NAME = "polaris_principals_getPrincipal_seconds"; + + @Inject PolarisIntegrationTestHelper helper; + @Inject MeterRegistry registry; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; + + @BeforeAll + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @BeforeEach + public void clearMetrics() { + registry.clear(); + } + + @Test + public void testMetricsEmittedOnSuccessfulRequest() { + sendSuccessfulRequest(); + Map allMetrics = + TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri()); + assertThat(allMetrics).containsKey(METRIC_NAME); + assertThat(allMetrics.get(METRIC_NAME).getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry( + "class", "org.apache.polaris.service.admin.api.PolarisPrincipalsApi"), + Map.entry("exception", "none"), + Map.entry("method", "getPrincipal")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(1L); + }); + } + + @Test + public void testMetricsEmittedOnFailedRequest() { + sendFailingRequest(); + Map allMetrics = + TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri()); + assertThat(allMetrics).containsKey(METRIC_NAME); + assertThat(allMetrics.get(METRIC_NAME).getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry( + "class", "org.apache.polaris.service.admin.api.PolarisPrincipalsApi"), + Map.entry("exception", "NotFoundException"), + Map.entry("method", "getPrincipal")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(1L); + }); + } + + private int sendRequest(String principalName) { + try (Response response = + fixture + .client + .target(String.format("%s/%s/%s", testEnv.baseUri(), ENDPOINT, principalName)) + .request("application/json") + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .get()) { + return response.getStatus(); + } + } + + private void sendSuccessfulRequest() { + Assertions.assertEquals( + Response.Status.OK.getStatusCode(), + sendRequest(fixture.snowmanCredentials.identifier().principalName())); + } + + private void sendFailingRequest() { + Assertions.assertEquals(ERROR_CODE, sendRequest("notarealprincipal")); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/ManagementServiceTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java similarity index 96% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/ManagementServiceTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java index a9e27c1c6..a3f9e7c9f 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/ManagementServiceTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.quarkus.admin; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -32,8 +32,8 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; -import org.apache.polaris.service.dropwizard.TestServices; -import org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory; +import org.apache.polaris.service.quarkus.TestServices; +import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory; import org.junit.jupiter.api.Test; public class ManagementServiceTest { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java similarity index 99% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java index 292b9bc5a..8075d6264 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java @@ -16,8 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.quarkus.admin; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; import java.util.List; import java.util.Map; import java.util.Set; @@ -36,6 +38,8 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +@QuarkusTest +@TestProfile(PolarisAuthzTestBase.Profile.class) public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase { private PolarisAdminService newTestAdminService() { return newTestAdminService(Set.of()); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java similarity index 88% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 58a2e6ee1..b2b5ece86 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -16,15 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.quarkus.admin; import static org.apache.iceberg.types.Types.NestedField.required; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTestProfile; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import jakarta.inject.Inject; import java.io.IOException; import java.time.Clock; import java.util.Date; @@ -42,7 +47,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; @@ -62,27 +66,39 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; +import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.RealmEntityManagerFactory; +import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.dropwizard.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.apache.polaris.service.task.TaskExecutor; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; /** Base class for shared test setup logic used by various Polaris authz-related tests. */ public abstract class PolarisAuthzTestBase { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Set> getEnabledAlternatives() { + return Set.of(TestPolarisCallContextCatalogFactory.class); + } + } + protected static final String CATALOG_NAME = "polaris-catalog"; protected static final String PRINCIPAL_NAME = "snowman"; @@ -139,6 +155,12 @@ public abstract class PolarisAuthzTestBase { PolarisConfiguration.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING.key, true))); + @Inject protected MetaStoreManagerFactory managerFactory; + @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; + @Inject protected CallContextCatalogFactory callContextCatalogFactory; + @Inject protected PolarisDiagnostics diagServices; + @Inject protected Clock clock; + protected BasePolarisCatalog baseCatalog; protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; @@ -147,38 +169,39 @@ public abstract class PolarisAuthzTestBase { protected PrincipalEntity principalEntity; protected CallContext callContext; protected AuthenticatedPolarisPrincipal authenticatedRoot; - protected InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - metaStoreManagerFactory.setStorageIntegrationProvider( + private PolarisCallContext polarisContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); - RealmContext realmContext = () -> "realm"; - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date()))); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + + @BeforeEach + public void before(TestInfo testInfo) { + RealmContext realmContext = testInfo::getDisplayName; + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); Map configMap = Map.of( "ALLOW_SPECIFYING_FILE_IO_IMPL", true, "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true); - PolarisCallContext polarisContext = + polarisContext = new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), + managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, new PolarisConfigurationStore() { @Override public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); + @SuppressWarnings("unchecked") + var r = (T) configMap.get(configName); + return r; } }, - Clock.systemDefaultZone()); - this.entityManager = - new PolarisEntityManager( - metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); - this.metaStoreManager = metaStoreManager; + clock); + this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); @@ -295,13 +318,17 @@ public void before() { @AfterEach public void after() { - if (this.baseCatalog != null) { - try { - this.baseCatalog.close(); - this.baseCatalog = null; - } catch (IOException e) { - throw new RuntimeException(e); + try { + if (this.baseCatalog != null) { + try { + this.baseCatalog.close(); + this.baseCatalog = null; + } catch (IOException e) { + throw new RuntimeException(e); + } } + } finally { + metaStoreManager.purge(polarisContext); } } @@ -368,18 +395,22 @@ private void initBaseCatalog() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } - public class TestPolarisCallContextCatalogFactory extends PolarisCallContextCatalogFactory { + @Alternative + @ApplicationScoped + public static class TestPolarisCallContextCatalogFactory + extends PolarisCallContextCatalogFactory { + public TestPolarisCallContextCatalogFactory() { - super( - new RealmEntityManagerFactory() { - @Override - public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { - return entityManager; - } - }, - metaStoreManagerFactory, - Mockito.mock(), - new DefaultFileIOFactory()); + super(null, null, null, null); + } + + @Inject + public TestPolarisCallContextCatalogFactory( + RealmEntityManagerFactory entityManagerFactory, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskExecutor taskExecutor, + FileIOFactory fileIOFactory) { + super(entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); } @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisOverlappingCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingCatalogTest.java similarity index 97% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisOverlappingCatalogTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingCatalogTest.java index a37929a2b..505bc57d9 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisOverlappingCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingCatalogTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.quarkus.admin; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -33,8 +33,8 @@ import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.TestServices; -import org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory; +import org.apache.polaris.service.quarkus.TestServices; +import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisOverlappingTableTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java similarity index 97% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisOverlappingTableTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java index 876f9b3cd..9abbed940 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisOverlappingTableTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.quarkus.admin; import static org.apache.polaris.core.PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP; import static org.apache.polaris.core.PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION; -import static org.apache.polaris.service.dropwizard.admin.PolarisAuthzTestBase.SCHEMA; +import static org.apache.polaris.service.quarkus.admin.PolarisAuthzTestBase.SCHEMA; import static org.assertj.core.api.Assertions.assertThat; import jakarta.ws.rs.core.Response; @@ -36,7 +36,7 @@ import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.TestServices; +import org.apache.polaris.service.quarkus.TestServices; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisServiceImplIntegrationTest.java similarity index 81% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisServiceImplIntegrationTest.java index 2aa5bd6c4..87ae8ad33 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisServiceImplIntegrationTest.java @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.quarkus.admin; -import static io.dropwizard.jackson.Jackson.newObjectMapper; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; @@ -29,10 +28,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; @@ -84,28 +83,40 @@ import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.auth.TokenUtils; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestProfile(PolarisServiceImplIntegrationTest.Profile.class) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisServiceImplIntegrationTest { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + // disallow FILE urls for the sake of tests below + return Map.of( + "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"S3\",\"GCS\",\"AZURE\"]"); + } + } + private static final int MAX_IDENTIFIER_LENGTH = 256; private static final String ISSUER_KEY = "polaris"; private static final String CLAIM_KEY_ACTIVE = "active"; @@ -116,36 +127,27 @@ public class PolarisServiceImplIntegrationTest { // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh // slate on every test case; otherwise, leftover state from one test from failures will interfere // with other test cases. - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("gcp_credentials.access_token", "abc"), - ConfigOverride.config("gcp_credentials.expires_in", "12345")); - private static String userToken; - private static String realm; - private static String clientId; + + @Inject ObjectMapper mapper; + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, - PolarisPrincipalSecrets adminSecrets, - @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - clientId = adminSecrets.getPrincipalClientId(); - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @AfterAll + public void destroyFixture() { + fixture.destroy(); } @AfterEach - public void tearDown() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + public void after() { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { response .readEntity(Catalogs.class) .getCatalogs() @@ -155,11 +157,7 @@ public void tearDown() { // delete all the namespaces try (Response res = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces") - .get()) { + newRequest("%s/api/catalog/v1/" + catalog.getName() + "/namespaces").get()) { if (res.getStatus() != Response.Status.OK.getStatusCode()) { LoggerFactory.getLogger(getClass()) .warn( @@ -172,7 +170,7 @@ public void tearDown() { .forEach( namespace -> { newRequest( - "http://localhost:%d/api/catalog/v1/" + "%s/api/catalog/v1/" + catalog.getName() + "/namespaces/" + RESTUtil.encodeNamespace(namespace)) @@ -185,9 +183,7 @@ public void tearDown() { // delete all the catalog roles except catalog_admin try (Response res = newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles") + "%s/api/management/v1/catalogs/" + catalog.getName() + "/catalog-roles") .get()) { if (res.getStatus() != Response.Status.OK.getStatusCode()) { LoggerFactory.getLogger(getClass()) @@ -202,7 +198,7 @@ public void tearDown() { .forEach( cr -> newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + "%s/api/management/v1/catalogs/" + catalog.getName() + "/catalog-roles/" + cr.getName()) @@ -211,9 +207,7 @@ public void tearDown() { } Response deleteResponse = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalog.getName()) - .delete(); + newRequest("%s/api/management/v1/catalogs/" + catalog.getName()).delete(); if (deleteResponse.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { LoggerFactory.getLogger(getClass()) .warn( @@ -224,21 +218,19 @@ public void tearDown() { deleteResponse.close(); }); } - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = newRequest("%s/api/management/v1/principals").get()) { response.readEntity(Principals.class).getPrincipals().stream() .filter( principal -> !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) .forEach( principal -> { - newRequest( - "http://localhost:%d/api/management/v1/principals/" + principal.getName()) + newRequest("%s/api/management/v1/principals/" + principal.getName()) .delete() .close(); }); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles").get()) { response.readEntity(PrincipalRoles.class).getRoles().stream() .filter( principalRole -> @@ -247,9 +239,7 @@ public void tearDown() { .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) .forEach( principalRole -> { - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRole.getName()) + newRequest("%s/api/management/v1/principal-roles/" + principalRole.getName()) .delete() .close(); }); @@ -284,7 +274,7 @@ public void testCatalogSerializing() throws IOException { @Test public void testListCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -302,20 +292,18 @@ public void testListCatalogsUnauthorized() { Principal principal = new Principal("a_new_user"); String newToken = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { + newRequest("%s/api/management/v1/principals").post(Entity.json(principal))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); newToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), creds.getCredentials().getClientId(), creds.getCredentials().getClientSecret(), - realm); + fixture.realm); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs", newToken).get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -323,7 +311,7 @@ public void testListCatalogsUnauthorized() { @Test public void testCreateCatalog() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post( Entity.json( "{\"catalog\":{\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"},\"storageConfigInfo\":{\"storageType\":\"S3\",\"roleArn\":\"arn:aws:iam::123456789012:role/my-role\",\"externalId\":\"externalId\",\"userArn\":\"userArn\",\"allowedLocations\":[\"s3://my-old-bucket/path/to/data\"]}}}"))) { @@ -331,8 +319,7 @@ public void testCreateCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/my-catalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -350,8 +337,6 @@ public void testCreateCatalogWithInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - ObjectMapper mapper = newObjectMapper(); - Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -360,7 +345,7 @@ public void testCreateCatalogWithInvalidName() { .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(mapper.writeValueAsString(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } catch (JsonProcessingException e) { @@ -387,7 +372,7 @@ public void testCreateCatalogWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(mapper.writeValueAsString(catalog)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -419,12 +404,11 @@ public void testCreateCatalogWithAzureStorageConfig() { .setStorageConfigInfo(azureConfigInfo) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -452,12 +436,11 @@ public void testCreateCatalogWithGcpStorageConfig() { .setStorageConfigInfo(gcpConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -487,8 +470,7 @@ public void testCreateCatalogWithNullBaseLocation() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + newRequest("%s/api/management/v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -514,7 +496,7 @@ public void testCreateCatalogWithoutProperties() { requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -532,7 +514,7 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti String catalogString = "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -549,7 +531,7 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { String catalogString = "{\"catalog\": {{\"bad data}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -557,8 +539,7 @@ public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException assertThat(error) .isNotNull() .extracting(ErrorResponse::message) - .asString() - .startsWith("Invalid JSON: Unexpected character"); + .isEqualTo("HTTP 400 Bad Request"); } } @@ -581,7 +562,7 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -589,8 +570,7 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro // 200 successful GET after creation Catalog fetchedCatalog = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { + newRequest("%s/api/management/v1/catalogs/" + catalogName, fixture.adminToken).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -609,7 +589,7 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro // Successfully update Catalog updatedCatalog = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) + newRequest("%s/api/management/v1/catalogs/" + catalogName, fixture.adminToken) .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); updatedCatalog = response.readEntity(Catalog.class); @@ -644,8 +624,7 @@ public void testCreateExternalCatalog() { .build(); createCatalog(catalog); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog fetchedCatalog = response.readEntity(Catalog.class); assertThat(fetchedCatalog) @@ -661,8 +640,7 @@ public void testCreateExternalCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/" + catalogName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -690,8 +668,7 @@ public void testCreateCatalogWithoutDefaultLocation() { requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + newRequest("%s/api/management/v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -723,8 +700,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful GET after creation Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/myazurecatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -743,7 +719,7 @@ public void testCreateAndUpdateAzureCatalog() { Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), modifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") + newRequest("%s/api/management/v1/catalogs/myazurecatalog") .put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -763,7 +739,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") + newRequest("%s/api/management/v1/catalogs/myazurecatalog") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -774,8 +750,7 @@ public void testCreateAndUpdateAzureCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/myazurecatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -797,15 +772,14 @@ public void testCreateListUpdateAndDeleteCatalog() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -816,7 +790,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // Should list the catalog. - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -836,8 +810,7 @@ public void testCreateListUpdateAndDeleteCatalog() { Map.of("default-base-location", "s3://newbucket/"), invalidModifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(badUpdateRequest))) { + newRequest("%s/api/management/v1/catalogs/mycatalog").put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -862,8 +835,7 @@ public void testCreateListUpdateAndDeleteCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(updateRequest))) { + newRequest("%s/api/management/v1/catalogs/mycatalog").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -875,8 +847,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -885,19 +856,17 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -910,23 +879,23 @@ public void testCreateListUpdateAndDeleteCatalog() { } } - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) + private Invocation.Builder newRequest(String url, String token) { + return fixture + .client + .target(String.format(url, testEnv.baseUri())) .request("application/json") .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); + .header(REALM_PROPERTY_KEY, fixture.realm); } - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); + private Invocation.Builder newRequest(String url) { + return newRequest(url, fixture.adminToken); } @Test public void testGetCatalogNotFound() { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } } @@ -945,8 +914,7 @@ public void testGetCatalogInvalidName() { for (String invalidCatalogName : invalidCatalogNames) { // there's no catalog yet. Expect 404 try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + invalidCatalogName) - .get()) { + newRequest("%s/api/management/v1/catalogs/" + invalidCatalogName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -981,7 +949,7 @@ public void testCatalogRoleInvalidName() { for (String invalidCatalogRoleName : invalidCatalogRoleNames) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/" + "%s/api/management/v1/catalogs/mycatalog1/catalog-roles/" + invalidCatalogRoleName) .get()) { @@ -999,20 +967,18 @@ public void testListPrincipalsUnauthorized() { Principal principal = new Principal("new_admin"); String newToken = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { + newRequest("%s/api/management/v1/principals").post(Entity.json(principal))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); newToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), creds.getCredentials().getClientId(), creds.getCredentials().getClientSecret(), - realm); + fixture.realm); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -1028,7 +994,7 @@ public void testCreatePrincipalAndRotateCredentials() { PrincipalWithCredentialsCredentials creds = null; Principal returnedPrincipal = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, true)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); @@ -1044,21 +1010,18 @@ public void testCreatePrincipalAndRotateCredentials() { // newly created principal's credentials, we should fail; rotateCredentials is only // a "self" privilege that even admins can't inherit. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal/rotate") - .post(Entity.json(""))) { + newRequest("%s/api/management/v1/principals/myprincipal/rotate").post(Entity.json(""))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } // Get a fresh token associate with the principal itself. String newPrincipalToken = TokenUtils.getTokenFromSecrets( - EXT.client(), EXT.getLocalPort(), oldClientId, oldSecret, realm); + fixture.client, testEnv.baseUri(), oldClientId, oldSecret, fixture.realm); // Any call should initially fail with error indicating that rotation is needed. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal", newPrincipalToken) - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal", newPrincipalToken).get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); assertThat(error) @@ -1070,9 +1033,7 @@ public void testCreatePrincipalAndRotateCredentials() { // Now try to rotate using the principal's token. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal/rotate", - newPrincipalToken) + newRequest("%s/api/management/v1/principals/myprincipal/rotate", newPrincipalToken) .post(Entity.json(""))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); @@ -1097,22 +1058,21 @@ public void testCreateListUpdateAndDeletePrincipal() { .setProperties(Map.of("custom-tag", "foo")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation Principal fetchedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1122,7 +1082,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // Should list the principal. - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = newRequest("%s/api/management/v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1137,8 +1097,7 @@ public void testCreateListUpdateAndDeletePrincipal() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal") - .put(Entity.json(updateRequest))) { + newRequest("%s/api/management/v1/principals/myprincipal").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1146,8 +1105,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1155,19 +1113,17 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").delete()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = newRequest("%s/api/management/v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1186,7 +1142,7 @@ public void testCreatePrincipalWithInvalidName() { .setProperties(Map.of("custom-tag", "good_principal")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -1209,7 +1165,7 @@ public void testCreatePrincipalWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1233,8 +1189,7 @@ public void testGetPrincipalWithInvalidName() { for (String invalidPrincipalName : invalidPrincipalNames) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/" + invalidPrincipalName) - .get()) { + newRequest("%s/api/management/v1/principals/" + invalidPrincipalName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1252,7 +1207,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + newRequest("%s/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); @@ -1261,7 +1216,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful GET after creation PrincipalRole fetchedPrincipalRole = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1272,8 +1227,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // Should list the principalRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1289,7 +1243,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") + newRequest("%s/api/management/v1/principal-roles/myprincipalrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1299,7 +1253,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 GET after update should show new properties try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1308,22 +1262,20 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 204 Successful delete try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1357,7 +1309,7 @@ public void testCreatePrincipalRoleInvalidName() { invalidPrincipalRoleName, Map.of("custom-tag", "bad_principal_role"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + newRequest("%s/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1381,10 +1333,7 @@ public void testGetPrincipalRoleInvalidName() { for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + invalidPrincipalRoleName) - .get()) { + newRequest("%s/api/management/v1/principal-roles/" + invalidPrincipalRoleName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1421,7 +1370,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { CatalogRole catalogRole = new CatalogRole("mycatalogrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1429,7 +1378,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); @@ -1438,9 +1387,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful GET after creation CatalogRole fetchedCatalogRole = null; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1452,8 +1399,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Should list the catalogRole. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1465,8 +1411,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Empty list if listing in catalog2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2/catalog-roles") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog2/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1487,8 +1432,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful update try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1498,9 +1442,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 GET after update should show new properties try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1509,8 +1451,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 204 Successful delete try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1518,17 +1459,14 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // NOT_FOUND after deletion try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1539,15 +1477,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1558,7 +1494,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Create two Principals Principal principal1 = new Principal("myprincipal1"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal1, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1566,7 +1502,7 @@ public void testAssignListAndRevokePrincipalRoles() { Principal principal2 = new Principal("myprincipal2"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal2, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1578,7 +1514,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Assign the role to myprincipal1 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles") .put(Entity.json(principalRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1586,8 +1522,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1601,9 +1536,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipal1 if listing assignees of myprincipalrole try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1616,8 +1549,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list if listing in principal2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2/principal-roles") - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal2/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1627,8 +1559,7 @@ public void testAssignListAndRevokePrincipalRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1636,8 +1567,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1645,9 +1575,7 @@ public void testAssignListAndRevokePrincipalRoles() { .returns(List.of(), PrincipalRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1656,23 +1584,20 @@ public void testAssignListAndRevokePrincipalRoles() { } // 204 Successful delete myprincipal1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1").delete()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipal2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2").delete()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1701,7 +1626,7 @@ public void testAssignListAndRevokeCatalogRoles() { CatalogRole catalogRole = new CatalogRole("mycr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1721,7 +1646,7 @@ public void testAssignListAndRevokeCatalogRoles() { CatalogRole otherCatalogRole = new CatalogRole("myothercr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles") + newRequest("%s/api/management/v1/catalogs/othercatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1729,15 +1654,13 @@ public void testAssignListAndRevokeCatalogRoles() { // Assign both the roles to mypr1 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") .put(Entity.json(new GrantCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1745,9 +1668,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list only mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1760,8 +1681,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list mypr1 if listing assignees of mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") .get()) { assertThat(response) @@ -1775,9 +1695,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list if listing in principalRole2 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog") - .get()) { + newRequest("%s/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1787,8 +1705,7 @@ public void testAssignListAndRevokeCatalogRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1796,9 +1713,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1806,8 +1721,7 @@ public void testAssignListAndRevokeCatalogRoles() { .returns(List.of(), CatalogRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") .get()) { assertThat(response) @@ -1817,46 +1731,39 @@ public void testAssignListAndRevokeCatalogRoles() { } // 204 Successful delete mypr1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr1").delete()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles/mypr1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mypr2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr2").delete()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles/mypr2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycr try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr") - .delete()) { + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles/mycr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myothercr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr") - .delete()) { + newRequest("%s/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete othercatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/othercatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1883,7 +1790,8 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { createCatalog(catalog); CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, fixture.adminToken); PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -1891,11 +1799,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // Create a second principal role. Use the catalog admin principal to list principal roles and // grant a catalog role to the new principal role @@ -1923,7 +1831,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName @@ -1938,13 +1846,13 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE privilege try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" + catalogRoleName, - userToken) + fixture.adminToken) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1971,7 +1879,8 @@ public void testServiceAdminCanTransferCatalogAdmin() { createCatalog(catalog); CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, fixture.adminToken); PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -1979,20 +1888,20 @@ public void testServiceAdminCanTransferCatalogAdmin() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // service_admin revokes the catalog_admin privilege from its principal role try { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" + "%s/api/management/v1/principal-roles/service_admin/catalog-roles/" + catalogName + "/catalog_admin", - userToken) + fixture.adminToken) .delete()) { assertThat(response) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -2001,7 +1910,7 @@ public void testServiceAdminCanTransferCatalogAdmin() { // the service_admin can not revoke the catalog_admin privilege from the new principal role try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName @@ -2054,11 +1963,12 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // create a catalog role *in the second catalog* and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, userToken); + createCatalogRole(catalogName2, catalogRoleName, fixture.adminToken); // Get the catalog admin role from the *first* catalog and grant that role to the principal role CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, fixture.adminToken); // Create a principal and grant the principal role to it PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -2066,11 +1976,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // Create a second principal role. String principalRoleName2 = "mypr2"; @@ -2081,7 +1991,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // catalog role is in the wrong catalog try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName2, @@ -2112,7 +2022,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { createCatalog(catalog); // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", userToken); + createCatalogRole(catalogName, "target_catalog_role", fixture.adminToken); // create a second catalog String catalogName2 = "anothertablemanagecatalog"; @@ -2128,7 +2038,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { createCatalog(catalog2); // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", userToken); + createCatalogRole(catalogName2, "invalid_target_catalog_role", fixture.adminToken); // create the namespace "c" in *both* namespaces String namespaceName = "c"; @@ -2139,7 +2049,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // namespace level // grant that role to the PrincipalRole String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, userToken); + createCatalogRole(catalogName, catalogRoleName, fixture.adminToken); grantPrivilegeToCatalogRole( catalogName, catalogRoleName, @@ -2147,11 +2057,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { List.of(namespaceName), NamespacePrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.NAMESPACE), - userToken, + fixture.adminToken, Response.Status.CREATED); grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), userToken); + principalRoleName, catalogName, new CatalogRole(catalogRoleName), fixture.adminToken); // Create a principal and grant the principal role to it PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); @@ -2159,11 +2069,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { String manageAccessUserToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target @@ -2186,7 +2096,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // as a securable try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName, @@ -2240,7 +2150,7 @@ public void testTokenExpiry() { .untilAsserted( () -> { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2252,8 +2162,7 @@ public void testTokenInactive() { // InvalidClaimException - if a claim contained a different value than the expected one. String newToken = defaultJwt().withClaim(CLAIM_KEY_ACTIVE, false).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2263,8 +2172,7 @@ public void testTokenInactive() { public void testTokenInvalidSignature() { // SignatureVerificationException - if the signature is invalid. String newToken = defaultJwt().sign(Algorithm.HMAC256("invalid_secret")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2274,8 +2182,21 @@ public void testTokenInvalidSignature() { public void testTokenInvalidPrincipalId() { String newToken = defaultJwt().withClaim(CLAIM_KEY_PRINCIPAL_ID, 0).sign(Algorithm.HMAC256("polaris")); + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { + assertThat(response) + .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); + } + } + + @Test + public void testNoAuth() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + fixture + .client + .target(String.format("%s/api/management/v1/principals", testEnv.baseUri())) + .request("application/json") + .header(REALM_PROPERTY_KEY, fixture.realm) + .get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2303,11 +2224,8 @@ public void testNamespaceExistsStatus() { // check if a namespace existed try (Response response = newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) + "%s/api/catalog/v1/" + catalogName + "/namespaces/" + namespaceName, + fixture.adminToken) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -2335,17 +2253,14 @@ public void testDropNamespaceStatus() { // drop a namespace try (Response response = newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) + "%s/api/catalog/v1/" + catalogName + "/namespaces/" + namespaceName, + fixture.adminToken) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } - public static JWTCreator.Builder defaultJwt() { + public JWTCreator.Builder defaultJwt() { Instant now = Instant.now(); return JWT.create() .withIssuer(ISSUER_KEY) @@ -2354,14 +2269,14 @@ public static JWTCreator.Builder defaultJwt() { .withExpiresAt(now.plus(10, ChronoUnit.SECONDS)) .withJWTId(UUID.randomUUID().toString()) .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) + .withClaim(CLAIM_KEY_CLIENT_ID, fixture.adminSecrets.getPrincipalClientId()) .withClaim(CLAIM_KEY_PRINCIPAL_ID, 1) .withClaim(CLAIM_KEY_SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL); } - private static void createNamespace(String catalogName, String namespaceName) { + private void createNamespace(String catalogName, String namespaceName) { try (Response response = - newRequest("http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", userToken) + newRequest("%s/api/catalog/v1/" + catalogName + "/namespaces", fixture.adminToken) .post( Entity.json( CreateNamespaceRequest.builder() @@ -2371,16 +2286,16 @@ private static void createNamespace(String catalogName, String namespaceName) { } } - private static void createCatalog(Catalog catalog) { + private void createCatalog(Catalog catalog) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } - private static void grantPrivilegeToCatalogRole( + private void grantPrivilegeToCatalogRole( String catalogName, String catalogRoleName, GrantResource grant, @@ -2388,7 +2303,7 @@ private static void grantPrivilegeToCatalogRole( Response.Status expectedStatus) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + "%s/api/management/v1/catalogs/" + catalogName + "/catalog-roles/" + catalogRoleName @@ -2399,33 +2314,29 @@ private static void grantPrivilegeToCatalogRole( } } - private static void createCatalogRole( + private void createCatalogRole( String catalogName, String catalogRoleName, String catalogAdminToken) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", + "%s/api/management/v1/catalogs/" + catalogName + "/catalog-roles", catalogAdminToken) .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } - private static void grantPrincipalRoleToPrincipal( - String principalName, PrincipalRole principalRole) { + private void grantPrincipalRoleToPrincipal(String principalName, PrincipalRole principalRole) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/" - + principalName - + "/principal-roles") + newRequest("%s/api/management/v1/principals/" + principalName + "/principal-roles") .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } - private static PrincipalWithCredentials createPrincipal(String principalName) { + private PrincipalWithCredentials createPrincipal(String principalName) { PrincipalWithCredentials catalogAdminPrincipal; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(new Principal(principalName), false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); catalogAdminPrincipal = response.readEntity(PrincipalWithCredentials.class); @@ -2433,11 +2344,11 @@ private static PrincipalWithCredentials createPrincipal(String principalName) { return catalogAdminPrincipal; } - private static void grantCatalogRoleToPrincipalRole( + private void grantCatalogRoleToPrincipalRole( String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName, @@ -2447,13 +2358,9 @@ private static void grantCatalogRoleToPrincipalRole( } } - private static CatalogRole readCatalogRole(String catalogName, String roleName) { + private CatalogRole readCatalogRole(String catalogName, String roleName) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + roleName) + newRequest("%s/api/management/v1/catalogs/" + catalogName + "/catalog-roles/" + roleName) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); @@ -2461,9 +2368,9 @@ private static CatalogRole readCatalogRole(String catalogName, String roleName) } } - private static void createPrincipalRole(PrincipalRole principalRole1) { + private void createPrincipalRole(PrincipalRole principalRole1) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + newRequest("%s/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java similarity index 59% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java index a5519e288..ba624e621 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java @@ -16,29 +16,21 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.auth; +package org.apache.polaris.service.quarkus.auth; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import com.auth0.jwt.JWT; import com.auth0.jwt.JWTVerifier; import com.auth0.jwt.algorithms.Algorithm; import com.auth0.jwt.interfaces.DecodedJWT; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.security.KeyPair; -import java.security.KeyPairGenerator; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; -import java.util.Base64; import java.util.HashMap; -import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -46,6 +38,7 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.auth.JWTRSAKeyPair; import org.apache.polaris.service.auth.LocalRSAKeyProvider; +import org.apache.polaris.service.auth.PemUtils; import org.apache.polaris.service.auth.TokenBroker; import org.apache.polaris.service.auth.TokenRequestValidator; import org.apache.polaris.service.auth.TokenResponse; @@ -55,70 +48,17 @@ public class JWTRSAKeyPairTest { - private void writePemToTmpFile(String privateFileLocation, String publicFileLocation) - throws Exception { - new File(privateFileLocation).delete(); - new File(publicFileLocation).delete(); - KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); - kpg.initialize(2048); - KeyPair kp = kpg.generateKeyPair(); - try (BufferedWriter writer = - new BufferedWriter(new FileWriter(privateFileLocation, UTF_8, true))) { - writer.write("-----BEGIN PRIVATE KEY-----"); // pragma: allowlist secret - writer.newLine(); - writer.write(Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded())); - writer.newLine(); - writer.write("-----END PRIVATE KEY-----"); - writer.newLine(); - } - try (BufferedWriter writer = - new BufferedWriter(new FileWriter(publicFileLocation, UTF_8, true))) { - writer.write("-----BEGIN PUBLIC KEY-----"); - writer.newLine(); - writer.write(Base64.getMimeEncoder().encodeToString(kp.getPublic().getEncoded())); - writer.newLine(); - writer.write("-----END PUBLIC KEY-----"); - writer.newLine(); - } - } - - public CallContext getTestCallContext(PolarisCallContext polarisCallContext) { - return CallContext.setCurrentContext( - new CallContext() { - @Override - public RealmContext getRealmContext() { - return () -> "realm"; - } - - @Override - public PolarisCallContext getPolarisCallContext() { - return polarisCallContext; - } - - @Override - public Map contextVariables() { - return Map.of("token", "me"); - } - }); - } - @Test public void testSuccessfulTokenGeneration() throws Exception { - String privateFileLocation = "/tmp/test-private.pem"; - String publicFileLocation = "/tmp/test-public.pem"; - writePemToTmpFile(privateFileLocation, publicFileLocation); + Path privateFileLocation = Files.createTempFile("test-private", ".pem"); + Path publicFileLocation = Files.createTempFile("test-public", ".pem"); + PemUtils.generateKeyPair(privateFileLocation, publicFileLocation); final String clientId = "test-client-id"; final String scope = "PRINCIPAL_ROLE:TEST"; - Map config = new HashMap<>(); - - config.put("LOCAL_PRIVATE_KEY_LOCATION_KEY", privateFileLocation); - config.put("LOCAL_PUBLIC_LOCATION_KEY", publicFileLocation); - - DefaultConfigurationStore store = new DefaultConfigurationStore(config); + DefaultConfigurationStore store = new DefaultConfigurationStore(new HashMap<>()); PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, store, null); - CallContext.setCurrentContext(getTestCallContext(polarisCallContext)); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "client-secret"; PolarisPrincipalSecrets principalSecrets = @@ -135,14 +75,19 @@ public void testSuccessfulTokenGeneration() throws Exception { "principal"); Mockito.when(metastoreManager.loadEntity(polarisCallContext, 0L, 1L)) .thenReturn(new PolarisMetaStoreManager.EntityResult(principal)); - TokenBroker tokenBroker = new JWTRSAKeyPair(metastoreManager, 420); + TokenBroker tokenBroker = + new JWTRSAKeyPair(metastoreManager, 420, publicFileLocation, privateFileLocation); TokenResponse token = tokenBroker.generateFromClientSecrets( - clientId, mainSecret, TokenRequestValidator.CLIENT_CREDENTIALS, scope); + clientId, + mainSecret, + TokenRequestValidator.CLIENT_CREDENTIALS, + scope, + polarisCallContext); assertThat(token).isNotNull(); assertThat(token.getExpiresIn()).isEqualTo(420); - LocalRSAKeyProvider provider = new LocalRSAKeyProvider(); + LocalRSAKeyProvider provider = new LocalRSAKeyProvider(publicFileLocation, privateFileLocation); assertThat(provider.getPrivateKey()).isNotNull(); assertThat(provider.getPublicKey()).isNotNull(); JWTVerifier verifier = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java similarity index 92% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java index 3384a89e7..aaa62643d 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.auth; +package org.apache.polaris.service.quarkus.auth; import static org.assertj.core.api.Assertions.assertThat; @@ -26,7 +26,6 @@ import com.auth0.jwt.interfaces.DecodedJWT; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -70,7 +69,7 @@ public Map contextVariables() { PolarisPrincipalSecrets principalSecrets = new PolarisPrincipalSecrets(1L, clientId, mainSecret, "otherSecret"); Mockito.when(metastoreManager.loadPrincipalSecrets(polarisCallContext, clientId)) - .thenReturn(new PrincipalSecretsResult(principalSecrets)); + .thenReturn(new PolarisMetaStoreManager.PrincipalSecretsResult(principalSecrets)); PolarisBaseEntity principal = new PolarisBaseEntity( 0L, @@ -84,7 +83,11 @@ public Map contextVariables() { TokenBroker generator = new JWTSymmetricKeyBroker(metastoreManager, 666, () -> "polaris"); TokenResponse token = generator.generateFromClientSecrets( - clientId, mainSecret, TokenRequestValidator.CLIENT_CREDENTIALS, "PRINCIPAL_ROLE:TEST"); + clientId, + mainSecret, + TokenRequestValidator.CLIENT_CREDENTIALS, + "PRINCIPAL_ROLE:TEST", + polarisCallContext); assertThat(token).isNotNull(); JWTVerifier verifier = JWT.require(Algorithm.HMAC256("polaris")).withIssuer("polaris").build(); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenRequestValidatorTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/TokenRequestValidatorTest.java similarity index 98% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenRequestValidatorTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/TokenRequestValidatorTest.java index 4fa519eee..54e07f486 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenRequestValidatorTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/TokenRequestValidatorTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.auth; +package org.apache.polaris.service.quarkus.auth; import org.apache.polaris.service.auth.OAuthTokenErrorResponse; import org.apache.polaris.service.auth.TokenRequestValidator; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/TokenUtils.java similarity index 83% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/TokenUtils.java index db3043b25..606d87a14 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/TokenUtils.java @@ -16,8 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.auth; +package org.apache.polaris.service.quarkus.auth; +import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; @@ -26,6 +27,7 @@ import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.MultivaluedHashMap; import jakarta.ws.rs.core.Response; +import java.net.URI; import java.util.Map; import org.apache.iceberg.rest.responses.OAuthTokenResponse; @@ -33,13 +35,7 @@ public class TokenUtils { /** Get token against specified realm */ public static String getTokenFromSecrets( - Client client, int port, String clientId, String clientSecret, String realm) { - return getTokenFromSecrets( - client, String.format("http://localhost:%d", port), clientId, clientSecret, realm); - } - - public static String getTokenFromSecrets( - Client client, String baseUrl, String clientId, String clientSecret, String realm) { + Client client, URI baseUrl, String clientId, String clientSecret, String realm) { String token; Invocation.Builder builder = @@ -58,7 +54,7 @@ public static String getTokenFromSecrets( "grant_type", "client_credentials", "scope", - "PRINCIPAL_ROLE:ALL", + PRINCIPAL_ROLE_ALL, "client_id", clientId, "client_secret", diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/AccessDelegationModeTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AccessDelegationModeTest.java similarity index 97% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/AccessDelegationModeTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AccessDelegationModeTest.java index 7e85a9a81..1a24fe3ae 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/AccessDelegationModeTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/AccessDelegationModeTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import static org.apache.polaris.service.catalog.AccessDelegationMode.*; import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java similarity index 97% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 1b7c59e22..b0b5d6fe9 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.iceberg.types.Types.NestedField.required; @@ -25,12 +25,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; -import jakarta.annotation.Nullable; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; +import java.lang.reflect.Method; import java.time.Clock; import java.util.Arrays; import java.util.EnumMap; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +62,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -77,6 +78,7 @@ import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; @@ -91,27 +93,29 @@ import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory; import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory; +import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.task.TaskFileIOSupplier; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; -import org.assertj.core.api.AbstractBooleanAssert; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; +@QuarkusTest public class BasePolarisCatalogTest extends CatalogTests { protected static final Namespace NS = Namespace.of("newdb"); protected static final TableIdentifier TABLE = TableIdentifier.of(NS, "table"); @@ -124,9 +128,15 @@ public class BasePolarisCatalogTest extends CatalogTests { public static final String SECRET_ACCESS_KEY = "secret_access_key"; public static final String SESSION_TOKEN = "session_token"; + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject PolarisDiagnostics diagServices; + private BasePolarisCatalog catalog; private AwsStorageConfigInfo storageConfigModel; private StsClient stsClient; + private String realmName; private PolarisMetaStoreManager metaStoreManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; @@ -134,29 +144,27 @@ public class BasePolarisCatalogTest extends CatalogTests { private AuthenticatedPolarisPrincipal authenticatedRoot; private PolarisEntity catalogEntity; + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + @BeforeEach @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - RealmContext realmContext = () -> "realm"; - PolarisStorageIntegrationProvider storageIntegrationProvider = Mockito.mock(); - InMemoryPolarisMetaStoreManagerFactory managerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - managerFactory.setStorageIntegrationProvider(storageIntegrationProvider); + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); - Map configMap = new HashMap<>(); - configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true); - configMap.put("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST", true); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, + configurationStore, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( @@ -246,6 +254,7 @@ public void before() { @AfterEach public void after() throws IOException { catalog().close(); + metaStoreManager.purge(polarisContext); } @Override @@ -291,7 +300,13 @@ public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext rea } @Override - public Map bootstrapRealms(List realms) { + public EntityCache getOrCreateEntityCache(RealmContext realmContext) { + return new EntityCache(metaStoreManager); + } + + @Override + public Map bootstrapRealms( + List realms, PolarisCredentialsBootstrap credentialsBootstrap) { throw new NotImplementedException("Bootstrapping realms is not supported"); } @@ -1330,8 +1345,8 @@ public void testDropTableWithPurge() { TableMetadata tableMetadata = ((BaseTable) table).operations().current(); boolean dropped = catalog.dropTable(TABLE, true); - ((AbstractBooleanAssert) - Assertions.assertThat(dropped).as("Should drop a table that does exist", new Object[0])) + Assertions.assertThat(dropped) + .as("Should drop a table that does exist", new Object[0]) .isTrue(); Assertions.assertThatPredicate(catalog::tableExists) .as("Table should not exist after drop") diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java similarity index 73% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java index 17f1cf019..01fb70497 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java @@ -16,17 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableMap; -import jakarta.annotation.Nullable; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Path; import java.time.Clock; -import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; @@ -34,7 +35,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -47,6 +47,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.cache.EntityCache; @@ -54,47 +55,64 @@ import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; +@QuarkusTest public class BasePolarisCatalogViewTest extends ViewCatalogTests { public static final String CATALOG_NAME = "polaris-catalog"; + + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisDiagnostics diagServices; + private BasePolarisCatalog catalog; + private String realmName; + private PolarisMetaStoreManager metaStoreManager; + private PolarisCallContext polarisContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + @BeforeEach - @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - RealmContext realmContext = () -> "realm"; - InMemoryPolarisMetaStoreManagerFactory managerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - managerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); - PolarisMetaStoreManager metaStoreManager = - managerFactory.getOrCreateMetaStoreManager(realmContext); - Map configMap = new HashMap<>(); - configMap.put("ALLOW_WILDCARD_LOCATION", true); - configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true); - PolarisCallContext polarisContext = + public void setUpTempDir(@TempDir Path tempDir) throws Exception { + // see https://github.com/quarkusio/quarkus/issues/13261 + Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); + field.setAccessible(true); + field.set(this, tempDir); + } + + @BeforeEach + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; + + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, + configurationStore, Clock.systemDefaultZone()); PolarisEntityManager entityManager = new PolarisEntityManager( metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); - CallContext callContext = CallContext.of(null, polarisContext); + CallContext callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); PrincipalEntity rootEntity = @@ -149,6 +167,12 @@ public void before() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } + @AfterEach + public void after() throws IOException { + catalog().close(); + metaStoreManager.purge(polarisContext); + } + @Override protected BasePolarisCatalog catalog() { return catalog; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java similarity index 98% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java index db80ac1f6..fa3c214b7 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; import java.time.Instant; import java.util.List; import java.util.Map; @@ -63,8 +65,9 @@ import org.apache.polaris.service.catalog.PolarisCatalogHandlerWrapper; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; +import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.dropwizard.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.quarkus.admin.PolarisAuthzTestBase; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; @@ -72,20 +75,28 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +@QuarkusTest +@TestProfile(PolarisCatalogHandlerWrapperAuthzTest.Profile.class) public class PolarisCatalogHandlerWrapperAuthzTest extends PolarisAuthzTestBase { + + public static class Profile extends PolarisAuthzTestBase.Profile { + + @Override + public Map getConfigOverrides() { + return Map.of("polaris.features.defaults.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", "true"); + } + } + private PolarisCatalogHandlerWrapper newWrapper() { return newWrapper(Set.of()); } private PolarisCatalogHandlerWrapper newWrapper(Set activatedPrincipalRoles) { - return newWrapper( - activatedPrincipalRoles, CATALOG_NAME, new TestPolarisCallContextCatalogFactory()); + return newWrapper(activatedPrincipalRoles, CATALOG_NAME, callContextCatalogFactory); } private PolarisCatalogHandlerWrapper newWrapper( - Set activatedPrincipalRoles, - String catalogName, - PolarisCallContextCatalogFactory factory) { + Set activatedPrincipalRoles, String catalogName, CallContextCatalogFactory factory) { final AuthenticatedPolarisPrincipal authenticatedPrincipal = new AuthenticatedPolarisPrincipal(principalEntity, activatedPrincipalRoles); return new PolarisCatalogHandlerWrapper( @@ -230,7 +241,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, authenticatedPrincipal, - new TestPolarisCallContextCatalogFactory(), + callContextCatalogFactory, CATALOG_NAME, polarisAuthorizer); @@ -261,7 +272,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, authenticatedPrincipal1, - new TestPolarisCallContextCatalogFactory(), + callContextCatalogFactory, CATALOG_NAME, polarisAuthorizer); @@ -1676,13 +1687,13 @@ public void testSendNotificationSufficientPrivileges() { PolarisCallContextCatalogFactory factory = new PolarisCallContextCatalogFactory( - new RealmEntityManagerFactory() { + new RealmEntityManagerFactory(null) { @Override public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { return entityManager; } }, - metaStoreManagerFactory, + managerFactory, Mockito.mock(), new DefaultFileIOFactory()) { @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisPassthroughResolutionView.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisPassthroughResolutionView.java similarity index 99% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisPassthroughResolutionView.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisPassthroughResolutionView.java index 549184c59..dc9dec0d1 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisPassthroughResolutionView.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisPassthroughResolutionView.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import java.util.Arrays; import org.apache.iceberg.catalog.Namespace; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogIntegrationTest.java similarity index 82% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogIntegrationTest.java index 948f6d581..e6afe06f3 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogIntegrationTest.java @@ -16,22 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,36 +77,32 @@ import org.apache.polaris.core.admin.model.ViewPrivilege; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisRestCatalogIntegrationTest extends CatalogTests { private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) @@ -116,22 +110,16 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String S3_BUCKET_BASE = Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) .orElse("file:///tmp/buckets/my-bucket"); - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; private RESTCatalog restCatalog; private String currentCatalogName; - private String userToken; - private String realm; + private Path tempDir; private final String catalogBaseLocation = S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; @@ -157,25 +145,18 @@ String[] properties() default { } @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @BeforeEach + public void destroyFixture(@TempDir Path tempDir) { + this.tempDir = tempDir; } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - this.realm = realm; - userToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - snowmanCredentials.clientId(), - snowmanCredentials.clientSecret(), - realm); + void before(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( @@ -237,15 +218,15 @@ public void before( }); restCatalog = TestUtil.createSnowmanManagedCatalog( - EXT, - adminToken, - snowmanCredentials, - realm, - catalog, - extraPropertiesBuilder.build()); + testEnv, fixture, catalog, extraPropertiesBuilder.build()); }); } + @AfterAll + public void tearDown() { + fixture.destroy(); + } + @Override protected RESTCatalog catalog() { return restCatalog; @@ -274,14 +255,15 @@ protected boolean overridesRequestedLocation() { private void createCatalogRole(String catalogRoleName) { CatalogRole catalogRole = new CatalogRole(catalogRoleName); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s/catalog-roles", + testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(catalogRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -289,14 +271,15 @@ private void createCatalogRole(String catalogRoleName) { private void addGrant(String catalogRoleName, GrantResource grant) { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, catalogRoleName)) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, catalogRoleName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put(Entity.json(grant))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -424,14 +407,15 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { // List grants for catalogrole1 try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, "catalogrole1")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -443,14 +427,15 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { // List grants for catalogrole2 try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole2")) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, "catalogrole2")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -494,14 +479,15 @@ public void testListGrantsAfterRename() { GrantResource.TypeEnum.TABLE); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, "catalogrole1")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -513,16 +499,16 @@ public void testListGrantsAfterRename() { } @Test - public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocation() { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -530,14 +516,14 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -569,17 +555,16 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { } @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( - PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -587,14 +572,14 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -635,17 +620,16 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( } @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( - PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -653,14 +637,14 @@ public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -699,16 +683,17 @@ public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisabled() { Namespace ns1 = Namespace.of("ns1"); restCatalog.createNamespace(ns1); + String baseLocation = "file://" + tempDir.toString() + "/ns1/my_table"; TableMetadata tableMetadata = TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + baseLocation, Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = baseLocation + "/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -734,16 +719,17 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisabled() { Namespace ns1 = Namespace.of("ns1"); restCatalog.createNamespace(ns1); + String baseLocation = "file://" + tempDir.toString() + "/ns1/my_table"; TableMetadata tableMetadata = TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + baseLocation, Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = baseLocation + "/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -768,16 +754,17 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabledForCatalog() { Namespace ns1 = Namespace.of("ns1"); restCatalog.createNamespace(ns1); + String baseLocation = "file://" + tempDir.toString() + "/ns1/my_table"; TableMetadata tableMetadata = TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + baseLocation, Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = baseLocation + "/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -802,14 +789,15 @@ public void testSendNotificationInternalCatalog() { restCatalog.createNamespace(Namespace.of("ns1")); String notificationUrl = String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - EXT.getLocalPort(), currentCatalogName); + "%s/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", + testEnv.baseUri(), currentCatalogName); try (Response response = - EXT.client() + fixture + .client .target(notificationUrl) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(notification))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) @@ -820,11 +808,12 @@ public void testSendNotificationInternalCatalog() { // NotificationType.VALIDATE should also surface the same error. notification.setNotificationType(NotificationType.VALIDATE); try (Response response = - EXT.client() + fixture + .client .target(notificationUrl) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(notification))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) @@ -1033,14 +1022,15 @@ public void testTableExistsStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) + "%s/api/catalog/v1/%s/namespaces/%s/tables/%s", + testEnv.baseUri(), currentCatalogName, namespace.toString(), tableName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1059,14 +1049,15 @@ public void testDropTableStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) + "%s/api/catalog/v1/%s/namespaces/%s/tables/%s", + testEnv.baseUri(), currentCatalogName, namespace.toString(), tableName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1093,14 +1084,15 @@ public void testViewExistsStatus() { .create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, viewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1127,14 +1119,15 @@ public void testDropViewStatus() { .create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, viewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1168,42 +1161,44 @@ public void testRenameViewStatus() { // Perform view rename try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/views/rename", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/catalog/v1/%s/views/rename", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(payload))) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Original view should no longer exists try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, viewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // New view should exists try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, newViewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, newViewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewAwsIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewAwsIntegrationTest.java index b7fdbee57..60c8af18c 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewAwsIntegrationTest.java @@ -16,8 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.Optional; import java.util.stream.Stream; @@ -26,6 +27,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on AWS. */ +@QuarkusTest public class PolarisRestCatalogViewAwsIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String ROLE_ARN = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewAzureIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewAzureIntegrationTest.java index be755526f..fddf9d8f1 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewAzureIntegrationTest.java @@ -16,8 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.stream.Stream; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; @@ -25,6 +26,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on Azure. */ +@QuarkusTest public class PolarisRestCatalogViewAzureIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID"); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewFileIntegrationTest.java similarity index 93% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewFileIntegrationTest.java index c7853df97..8b847d860 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewFileIntegrationTest.java @@ -16,13 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; /** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */ +@QuarkusTest public class PolarisRestCatalogViewFileIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket"; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewGcpIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewGcpIntegrationTest.java index b74677275..06266e100 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewGcpIntegrationTest.java @@ -16,8 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.stream.Stream; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; @@ -25,6 +26,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on GCP. */ +@QuarkusTest public class PolarisRestCatalogViewGcpIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String SERVICE_ACCOUNT = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewIntegrationTest.java similarity index 64% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewIntegrationTest.java index 215288dbc..4bbb721d2 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisRestCatalogViewIntegrationTest.java @@ -16,16 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; -import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Path; import java.util.Map; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.view.ViewCatalogTests; @@ -34,75 +32,64 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironment; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; private RESTCatalog restCatalog; @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + Assumptions.assumeFalse(shouldSkip()); + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm, - TestEnvironment testEnv) { - - Assumptions.assumeFalse(shouldSkip()); + public void setUpTempDir(@TempDir Path tempDir) throws Exception { + // see https://github.com/quarkusio/quarkus/issues/13261 + Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); + field.setAccessible(true); + field.set(this, tempDir); + } - String userToken = adminToken.token(); + @BeforeEach + void before(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( method -> { - String catalogName = method.getName() + testEnv.testId(); + String catalogName = method.getName(); try (Response response = - testEnv - .apiClient() + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { if (response.getStatus() == Response.Status.OK.getStatusCode()) { // Already exists! Must be in a parameterized test. @@ -139,17 +126,17 @@ public void before( .setStorageConfigInfo(storageConfig) .build(); restCatalog = - TestUtil.createSnowmanManagedCatalog( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminToken, - snowmanCredentials, - realm, - catalog, - Map.of()); + TestUtil.createSnowmanManagedCatalog(testEnv, fixture, catalog, Map.of()); }); } + @AfterAll + public void destroyFixture() { + if (fixture != null) { + fixture.destroy(); + } + } + /** * @return The catalog's storage config. */ diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisSparkIntegrationTest.java similarity index 77% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisSparkIntegrationTest.java index 173e2bda6..87ecdd91b 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisSparkIntegrationTest.java @@ -16,20 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.time.Instant; import java.util.List; import java.util.Map; @@ -41,11 +38,10 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; @@ -58,52 +54,43 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisSparkIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism public static final String CATALOG_NAME = "mycatalog"; public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; - private static final S3MockContainer s3Container = + + private final S3MockContainer s3Container = new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - private static PolarisConnectionExtension.PolarisToken polarisToken; - private static SparkSession spark; - private String realm; + + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; + private SparkSession spark; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken polarisToken, @PolarisRealm String realm) - throws IOException { + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; s3Container.start(); - PolarisSparkIntegrationTest.polarisToken = polarisToken; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + fixture = helper.createFixture(testEnv, testInfo); } @AfterAll - public static void cleanup() { + public void destroyFixture() { + fixture.destroy(); s3Container.stop(); } @BeforeEach - public void before(@PolarisRealm String realm) { - this.realm = realm; + public void before() { AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -140,12 +127,12 @@ public void before(@PolarisRealm String realm) { .build(); try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -178,12 +165,12 @@ public void before(@PolarisRealm String realm) { .setRemoteUrl("http://dummy_url") .build(); try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(externalCatalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -215,11 +202,11 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") .config( String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + EXT.getLocalPort() + "/api/catalog") + testEnv.baseUri() + "/api/catalog") .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), polarisToken.token()) + .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), fixture.realm) + .config(String.format("spark.sql.catalog.%s.token", catalogName), fixture.adminToken) .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") .config( String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") @@ -254,14 +241,13 @@ private void cleanupCatalog(String catalogName) { onSpark("DROP NAMESPACE " + namespace.getString(0)); } try (Response response = - EXT.client() + fixture + .client .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - EXT.getLocalPort())) + String.format("%s/api/management/v1/catalogs/" + catalogName, testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -303,16 +289,17 @@ public void testCreateAndUpdateExternalTable() { LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); try (Response registerResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/" + "%s/api/catalog/v1/" + EXTERNAL_CATALOG_NAME + "/namespaces/externalns1/register", - EXT.getLocalPort())) + testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post( Entity.json( ImmutableRegisterTableRequest.builder() @@ -344,14 +331,15 @@ public void testCreateAndUpdateExternalTable() { notificationRequest.setPayload(updateNotification); notificationRequest.setNotificationType(NotificationType.UPDATE); try (Response notifyResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - EXT.getLocalPort(), EXTERNAL_CATALOG_NAME)) + "%s/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", + testEnv.baseUri(), EXTERNAL_CATALOG_NAME)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(notificationRequest))) { assertThat(notifyResponse) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -378,21 +366,22 @@ public void testCreateView() { private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), catalog, namespace, table)) + "%s/api/catalog/v1/%s/namespaces/%s/tables/%s", + testEnv.baseUri(), catalog, namespace, table)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); return response.readEntity(LoadTableResponse.class); } } - private static Dataset onSpark(@Language("SQL") String sql) { + private Dataset onSpark(@Language("SQL") String sql) { return spark.sql(sql); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/TestUtil.java similarity index 75% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/TestUtil.java index c2342d8b8..de9c0aab9 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/TestUtil.java @@ -16,16 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.quarkus.catalog; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; +import java.net.URI; import java.util.Map; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.SessionCatalog; @@ -38,50 +37,45 @@ import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.TestEnvironment; /** Test utilities for catalog tests */ public class TestUtil { - /** Performs createSnowmanManagedCatalog() on a Dropwizard instance of Polaris */ + + /** + * Creates a catalog and grants the snowman principal permission to manage it. + * + * @return A client to interact with the catalog. + */ public static RESTCatalog createSnowmanManagedCatalog( - DropwizardAppExtension EXT, - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - String realm, + TestEnvironment testEnv, + PolarisIntegrationTestFixture fixture, Catalog catalog, Map extraProperties) { return createSnowmanManagedCatalog( - EXT.client(), - String.format("http://localhost:%d", EXT.getLocalPort()), - adminToken, - snowmanCredentials, - realm, - catalog, - extraProperties); + testEnv, fixture, testEnv.baseUri(), fixture.realm, catalog, extraProperties); } /** - * Creates a catalog and grants the snowman principal from SnowmanCredentialsExtension permission - * to manage it. + * Creates a catalog and grants the snowman principal permission to manage it. * * @return A client to interact with the catalog. */ public static RESTCatalog createSnowmanManagedCatalog( - Client client, - String baseUrl, - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, + TestEnvironment testEnv, + PolarisIntegrationTestFixture fixture, + URI baseUri, String realm, Catalog catalog, Map extraProperties) { String currentCatalogName = catalog.getName(); try (Response response = - client - .target(String.format("%s/api/management/v1/catalogs", baseUrl)) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", baseUri)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .post(Entity.json(catalog))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -90,12 +84,13 @@ public static RESTCatalog createSnowmanManagedCatalog( // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS CatalogRole newRole = new CatalogRole("custom-admin"); try (Response response = - client + fixture + .client .target( String.format( - "%s/api/management/v1/catalogs/%s/catalog-roles", baseUrl, currentCatalogName)) + "%s/api/management/v1/catalogs/%s/catalog-roles", baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .post(Entity.json(newRole))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -103,13 +98,14 @@ public static RESTCatalog createSnowmanManagedCatalog( CatalogGrant grantResource = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); try (Response response = - client + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", - baseUrl, currentCatalogName)) + baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(grantResource))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -117,13 +113,14 @@ public static RESTCatalog createSnowmanManagedCatalog( CatalogGrant grantAccessResource = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); try (Response response = - client + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", - baseUrl, currentCatalogName)) + baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(grantAccessResource))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -131,28 +128,30 @@ public static RESTCatalog createSnowmanManagedCatalog( // Assign this new CatalogRole to the service_admin PrincipalRole try (Response response = - client + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin", - baseUrl, currentCatalogName)) + baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .get()) { assertStatusCode(response, Response.Status.OK.getStatusCode()); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - client + fixture + .client .target( String.format( "%s/api/management/v1/principal-roles/%s/catalog-roles/%s", - baseUrl, - snowmanCredentials.identifier().principalRoleName(), + baseUri, + fixture.snowmanCredentials.identifier().principalRoleName(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(catalogRole))) { assertStatusCode(assignResponse, Response.Status.CREATED.getStatusCode()); @@ -167,10 +166,12 @@ public static RESTCatalog createSnowmanManagedCatalog( ImmutableMap.Builder propertiesBuilder = ImmutableMap.builder() - .put(CatalogProperties.URI, baseUrl + "/api/catalog") + .put(CatalogProperties.URI, baseUri + "/api/catalog") .put( OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret()) + fixture.snowmanCredentials.clientId() + + ":" + + fixture.snowmanCredentials.clientSecret()) .put(OAuth2Properties.SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL) .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") .put("warehouse", currentCatalogName) diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/FileIOExceptionsTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java similarity index 97% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/FileIOExceptionsTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java index 59918500f..bc1f0f9e6 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/FileIOExceptionsTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog.io; +package org.apache.polaris.service.quarkus.catalog.io; import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; @@ -38,7 +38,7 @@ import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.TestServices; +import org.apache.polaris.service.quarkus.TestServices; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIO.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestFileIO.java similarity index 98% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIO.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestFileIO.java index a5d60da93..8acf2efd8 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIO.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestFileIO.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog.io; +package org.apache.polaris.service.quarkus.catalog.io; import java.util.Map; import java.util.Optional; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestFileIOFactory.java similarity index 91% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestFileIOFactory.java index fa59f5678..750475599 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestFileIOFactory.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog.io; +package org.apache.polaris.service.quarkus.catalog.io; -import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.inject.Vetoed; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -27,14 +27,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; -import org.apache.polaris.service.catalog.io.FileIOFactory; +import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; /** * A FileIOFactory that measures the number of bytes read, files written, and files deleted. It can * inject exceptions at various parts of the IO construction. */ -@Identifier("test") -public class TestFileIOFactory implements FileIOFactory { +@Vetoed +public class TestFileIOFactory extends DefaultFileIOFactory { private final List ios = new ArrayList<>(); // When present, the following will be used to throw exceptions at various parts of the IO diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestInputFile.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestInputFile.java similarity index 96% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestInputFile.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestInputFile.java index ebeb7145d..da050c119 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestInputFile.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/TestInputFile.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog.io; +package org.apache.polaris.service.quarkus.catalog.io; import java.util.Optional; import java.util.function.Supplier; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/config/DefaultConfigurationStoreTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java similarity index 98% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/config/DefaultConfigurationStoreTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java index 29fc0da60..ac7c0116a 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/config/DefaultConfigurationStoreTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.config; +package org.apache.polaris.service.quarkus.config; import static org.assertj.core.api.Assertions.assertThat; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/entity/CatalogEntityTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java similarity index 99% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/entity/CatalogEntityTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java index d7dcffd66..2d4667d10 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/entity/CatalogEntityTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.entity; +package org.apache.polaris.service.quarkus.entity; import java.util.List; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/MockTokenBucketFactory.java similarity index 71% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/MockTokenBucketFactory.java index 6516e7c79..bff955c9c 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/MockTokenBucketFactory.java @@ -16,25 +16,29 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.ratelimiter; +package org.apache.polaris.service.quarkus.ratelimiter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import jakarta.inject.Inject; import java.time.Instant; import java.time.ZoneOffset; import org.apache.polaris.service.ratelimiter.DefaultTokenBucketFactory; +import org.apache.polaris.service.ratelimiter.RateLimiterConfiguration; import org.threeten.extra.MutableClock; /** TokenBucketFactory with a mock clock */ -@Identifier("mock") +@Alternative +@ApplicationScoped public class MockTokenBucketFactory extends DefaultTokenBucketFactory { public static MutableClock CLOCK = MutableClock.of(Instant.now(), ZoneOffset.UTC); - @JsonCreator - public MockTokenBucketFactory( - @JsonProperty("requestsPerSecond") long requestsPerSecond, - @JsonProperty("windowSeconds") long windowSeconds) { - super(requestsPerSecond, windowSeconds, CLOCK); + public MockTokenBucketFactory() { + super(0, null, CLOCK); + } + + @Inject + public MockTokenBucketFactory(RateLimiterConfiguration configuration) { + super(configuration, CLOCK); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/RateLimiterFilterTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/RateLimiterFilterTest.java new file mode 100644 index 000000000..6e39a5d09 --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/RateLimiterFilterTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.ratelimiter; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response.Status; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.polaris.service.quarkus.ratelimiter.RateLimiterFilterTest.Profile; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.quarkus.test.TestEnvironment; +import org.apache.polaris.service.quarkus.test.TestEnvironmentExtension; +import org.apache.polaris.service.quarkus.test.TestMetricsUtil; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.Summary; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.extension.ExtendWith; + +/** Main integration tests for rate limiting */ +@QuarkusTest +@TestInstance(Lifecycle.PER_CLASS) +@TestProfile(Profile.class) +@ExtendWith(TestEnvironmentExtension.class) +public class RateLimiterFilterTest { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Set> getEnabledAlternatives() { + return Set.of(MockTokenBucketFactory.class); + } + + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.rate-limiter.type", + "default", + "polaris.rate-limiter.token-bucket.type", + "default", + "polaris.rate-limiter.token-bucket.requests-per-second", + String.valueOf(REQUESTS_PER_SECOND), + "polaris.rate-limiter.token-bucket.window", + WINDOW.toString(), + "polaris.metrics.tags.environment", + "prod"); + } + } + + private static final long REQUESTS_PER_SECOND = 5; + private static final Duration WINDOW = Duration.ofSeconds(10); + + @Inject PolarisIntegrationTestHelper helper; + @Inject MeterRegistry meterRegistry; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; + + @BeforeAll + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @AfterAll + public void destroyFixture() { + if (fixture != null) { + fixture.destroy(); + } + } + + @BeforeEach + @AfterEach + public void resetRateLimiter() { + MockTokenBucketFactory.CLOCK.add( + WINDOW.multipliedBy(2)); // Clear any counters from before/after this test + } + + @BeforeEach + public void resetMeterRegistry() { + meterRegistry.clear(); + } + + @Test + public void testRateLimiter() { + Consumer requestAsserter = + TestUtil.constructRequestAsserter(testEnv, fixture, fixture.realm); + + for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.toSeconds(); i++) { + requestAsserter.accept(Status.OK); + } + requestAsserter.accept(Status.TOO_MANY_REQUESTS); + + // Ensure that a different realm identifier gets a separate limit + Consumer requestAsserter2 = + TestUtil.constructRequestAsserter(testEnv, fixture, fixture.realm + "2"); + requestAsserter2.accept(Status.OK); + } + + @Test + public void testMetricsAreEmittedWhenRateLimiting() { + Consumer requestAsserter = + TestUtil.constructRequestAsserter(testEnv, fixture, fixture.realm); + + for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.toSeconds(); i++) { + requestAsserter.accept(Status.OK); + } + requestAsserter.accept(Status.TOO_MANY_REQUESTS); + + // Examples of expected metrics: + // http_server_requests_seconds_count{application="Polaris",environment="prod",method="GET",outcome="CLIENT_ERROR",realm_id="org_apache_polaris_service_ratelimiter_RateLimiterFilterTest",status="429",uri="/api/management/v1/principal-roles"} 1.0 + // polaris_principal_roles_listPrincipalRoles_seconds_count{application="Polaris",class="org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi",environment="prod",exception="none",method="listPrincipalRoles"} 50.0 + + Map metrics = + TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri()); + + assertThat(metrics) + .isNotEmpty() + .containsKey("http_server_requests_seconds") + .containsKey("polaris_principal_roles_listPrincipalRoles_seconds"); + + assertThat(metrics.get("http_server_requests_seconds").getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry("method", "GET"), + Map.entry("outcome", "CLIENT_ERROR"), + Map.entry("status", String.valueOf(Status.TOO_MANY_REQUESTS.getStatusCode())), + Map.entry("uri", "/api/management/v1/principal-roles")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(1L); + }); + + assertThat(metrics.get("polaris_principal_roles_listPrincipalRoles_seconds").getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry( + "class", "org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi"), + Map.entry("exception", "none"), + Map.entry("method", "listPrincipalRoles")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(REQUESTS_PER_SECOND * WINDOW.toSeconds()); + }); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/TestUtil.java similarity index 70% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/TestUtil.java index 1cd5789df..a27f1c311 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/TestUtil.java @@ -16,15 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.ratelimiter; +package org.apache.polaris.service.quarkus.ratelimiter; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; -import io.dropwizard.testing.junit5.DropwizardAppExtension; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Response.Status; import java.util.function.Consumer; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.quarkus.test.TestEnvironment; /** Common test utils for testing rate limiting */ public class TestUtil { @@ -33,20 +34,15 @@ public class TestUtil { * of the response. This is a relatively simple type of request that can be used for validating * whether the rate limiter intervenes. */ - public static Consumer constructRequestAsserter( - DropwizardAppExtension dropwizardAppExtension, - String userToken, - String realm) { + public static Consumer constructRequestAsserter( + TestEnvironment testEnv, PolarisIntegrationTestFixture fixture, String realm) { return (Response.Status status) -> { try (Response response = - dropwizardAppExtension - .client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", - dropwizardAppExtension.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) .request("application/json") - .header("Authorization", "Bearer " + userToken) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .get()) { assertThat(response).returns(status.getStatusCode(), Response::getStatus); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketRateLimiterTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/TokenBucketTest.java similarity index 95% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketRateLimiterTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/TokenBucketTest.java index 472b79aca..3364b4cde 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketRateLimiterTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/ratelimiter/TokenBucketTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.ratelimiter; +package org.apache.polaris.service.quarkus.ratelimiter; import java.time.Clock; import java.time.Duration; @@ -31,8 +31,8 @@ import org.junit.jupiter.api.Test; import org.threeten.extra.MutableClock; -/** Main unit test class for TokenBucketRateLimiter */ -public class TokenBucketRateLimiterTest { +/** Main unit test class for TokenBucket */ +public class TokenBucketTest { @Test void testBasic() { MutableClock clock = MutableClock.of(Instant.now(), ZoneOffset.UTC); @@ -70,7 +70,6 @@ void testConcurrent() throws InterruptedException { try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { for (int i = 0; i < numTasks; i++) { - int i_ = i; executor.submit( () -> { try { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java similarity index 97% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java index b594b8e01..18196d476 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java @@ -16,12 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.task; +package org.apache.polaris.service.quarkus.task; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatPredicate; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -50,21 +52,16 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; import org.apache.polaris.service.task.TaskUtils; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +@QuarkusTest class ManifestFileCleanupTaskHandlerTest { - private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - private RealmContext realmContext; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - void setUp() { - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - realmContext = () -> "realmName"; - } + private final RealmContext realmContext = () -> "realmName"; @Test public void testCleanupFileNotExists() throws IOException { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java similarity index 98% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index 1fc788c34..42efb95d2 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -16,10 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.task; +package org.apache.polaris.service.quarkus.task; import static org.assertj.core.api.Assertions.assertThat; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; import java.util.List; import java.util.UUID; @@ -42,25 +44,20 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskUtils; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.LoggerFactory; +@QuarkusTest class TableCleanupTaskHandlerTest { - private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - private RealmContext realmContext; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - void setUp() { - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - realmContext = () -> "realmName"; - } + private final RealmContext realmContext = () -> "realmName"; @Test public void testTableCleanup() throws IOException { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TaskTestUtils.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java similarity index 99% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TaskTestUtils.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java index c1f6021ed..9cd0ddd9e 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TaskTestUtils.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.task; +package org.apache.polaris.service.quarkus.task; import jakarta.annotation.Nonnull; import java.io.IOException; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TestSnapshot.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TestSnapshot.java similarity index 98% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TestSnapshot.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TestSnapshot.java index f10db52d6..1a9064e96 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TestSnapshot.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TestSnapshot.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.task; +package org.apache.polaris.service.quarkus.task; import com.google.common.collect.Lists; import java.io.IOException; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/DefaultTestEnvironmentResolver.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/DefaultTestEnvironmentResolver.java new file mode 100644 index 000000000..4f793f63d --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/DefaultTestEnvironmentResolver.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.test; + +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DefaultTestEnvironmentResolver implements TestEnvironmentResolver { + + private final int localPort = Integer.getInteger("quarkus.http.port"); + private final int localManagementPort = Integer.getInteger("quarkus.management.port"); + + /** Resolves the TestEnvironment to point to the local Quarkus Application instance. */ + @Override + public TestEnvironment resolveTestEnvironment(ExtensionContext extensionContext) { + return new TestEnvironment( + String.format("http://localhost:%d", localPort), + String.format("http://localhost:%d", localManagementPort)); + } +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java new file mode 100644 index 000000000..74f1cc4cf --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.test; + +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.auth.TokenUtils; +import org.junit.jupiter.api.TestInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PolarisIntegrationTestFixture { + + public record SnowmanIdentifier(String principalName, String principalRoleName) {} + + public record SnowmanCredentials( + String clientId, String clientSecret, SnowmanIdentifier identifier) {} + + private static final Logger LOGGER = LoggerFactory.getLogger(PolarisIntegrationTestFixture.class); + + private final PolarisIntegrationTestHelper helper; + + public final String realm; + public final PolarisPrincipalSecrets adminSecrets; + public final SnowmanCredentials snowmanCredentials; + public final String adminToken; + public final String userToken; + public final Client client; + + private final URI baseUri; + + public PolarisIntegrationTestFixture( + PolarisIntegrationTestHelper helper, TestEnvironment testEnv, TestInfo testInfo) { + this.helper = helper; + this.client = ClientBuilder.newClient(); + this.baseUri = testEnv.baseUri(); + // Generate unique realm using test name for each test since the tests can run in parallel + realm = testInfo.getTestClass().orElseThrow().getName().replace('.', '_'); + adminSecrets = fetchAdminSecrets(); + adminToken = + TokenUtils.getTokenFromSecrets( + client, + baseUri, + adminSecrets.getPrincipalClientId(), + adminSecrets.getMainSecret(), + realm); + snowmanCredentials = createSnowmanCredentials(testEnv); + userToken = + TokenUtils.getTokenFromSecrets( + client, + baseUri, + snowmanCredentials.clientId(), + snowmanCredentials.clientSecret(), + realm); + } + + private PolarisPrincipalSecrets fetchAdminSecrets() { + if (!(helper.metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { + helper.metaStoreManagerFactory.bootstrapRealms( + List.of(realm), PolarisCredentialsBootstrap.fromEnvironment()); + } + + RealmContext realmContext = + helper.realmContextResolver.resolveRealmContext( + baseUri.toString(), "GET", "/", Map.of(REALM_PROPERTY_KEY, realm)); + + PolarisMetaStoreSession metaStoreSession = + helper.metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); + PolarisCallContext polarisContext = + new PolarisCallContext( + metaStoreSession, helper.diagServices, helper.configurationStore, helper.clock); + try (CallContext ctx = CallContext.of(realmContext, polarisContext)) { + CallContext.setCurrentContext(ctx); + PolarisMetaStoreManager metaStoreManager = + helper.metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); + PolarisMetaStoreManager.EntityResult principal = + metaStoreManager.readEntityByName( + ctx.getPolarisCallContext(), + null, + PolarisEntityType.PRINCIPAL, + PolarisEntitySubType.NULL_SUBTYPE, + PolarisEntityConstants.getRootPrincipalName()); + + Map propertiesMap = readInternalProperties(principal); + return metaStoreManager + .loadPrincipalSecrets(ctx.getPolarisCallContext(), propertiesMap.get("client_id")) + .getPrincipalSecrets(); + } finally { + CallContext.unsetCurrentContext(); + } + } + + private SnowmanCredentials createSnowmanCredentials(TestEnvironment testEnv) { + + SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); + PrincipalRole principalRole = new PrincipalRole(snowmanIdentifier.principalRoleName()); + + try (Response createPrResponse = + client + .target(String.format("%s/api/management/v1/principal-roles", baseUri)) + .request("application/json") + .header("Authorization", "Bearer " + adminToken) + .header(REALM_PROPERTY_KEY, realm) + .post(Entity.json(principalRole))) { + assertThat(createPrResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + Principal principal = new Principal(snowmanIdentifier.principalName()); + SnowmanCredentials snowmanCredentials; + + try (Response createPResponse = + client + .target(String.format("%s/api/management/v1/principals", baseUri)) + .request("application/json") + .header("Authorization", "Bearer " + adminToken) // how is token getting used? + .header(REALM_PROPERTY_KEY, realm) + .post(Entity.json(principal))) { + assertThat(createPResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + + PrincipalWithCredentials snowmanWithCredentials = + createPResponse.readEntity(PrincipalWithCredentials.class); + try (Response rotateResp = + client + .target( + String.format( + "%s/api/management/v1/principals/%s/rotate", baseUri, principal.getName())) + .request(MediaType.APPLICATION_JSON) + .header( + "Authorization", + "Bearer " + + TokenUtils.getTokenFromSecrets( + client, + baseUri, + snowmanWithCredentials.getCredentials().getClientId(), + snowmanWithCredentials.getCredentials().getClientSecret(), + realm)) + .header(REALM_PROPERTY_KEY, realm) + .post(Entity.json(snowmanWithCredentials))) { + + assertThat(rotateResp).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + + // Use the rotated credentials. + snowmanWithCredentials = rotateResp.readEntity(PrincipalWithCredentials.class); + } + snowmanCredentials = + new SnowmanCredentials( + snowmanWithCredentials.getCredentials().getClientId(), + snowmanWithCredentials.getCredentials().getClientSecret(), + snowmanIdentifier); + } + try (Response assignPrResponse = + client + .target( + String.format( + "%s/api/management/v1/principals/%s/principal-roles", + baseUri, principal.getName())) + .request("application/json") + .header("Authorization", "Bearer " + adminToken) // how is token getting used? + .header(REALM_PROPERTY_KEY, realm) + .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { + assertThat(assignPrResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + return snowmanCredentials; + } + + public void destroy() { + try { + if (realm != null) { + helper.metaStoreManagerFactory.purgeRealms(List.of(realm)); + } + } catch (Exception e) { + LOGGER.error("Failed to purge realm", e); + } finally { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + LOGGER.error("Failed to close client", e); + } + } + } + } + + private Map readInternalProperties( + PolarisMetaStoreManager.EntityResult principal) { + try { + return helper.objectMapper.readValue( + principal.getEntity().getInternalProperties(), new TypeReference<>() {}); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private static SnowmanIdentifier getSnowmanIdentifier(TestEnvironment testEnv) { + return new SnowmanIdentifier("snowman" + testEnv.testId(), "catalog-admin" + testEnv.testId()); + } +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java new file mode 100644 index 000000000..f7f5ed719 --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.time.Clock; +import org.apache.polaris.core.PolarisConfigurationStore; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.context.RealmContextResolver; +import org.junit.jupiter.api.TestInfo; + +@Singleton +public class PolarisIntegrationTestHelper { + + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject RealmContextResolver realmContextResolver; + @Inject ObjectMapper objectMapper; + @Inject PolarisDiagnostics diagServices; + @Inject PolarisConfigurationStore configurationStore; + @Inject Clock clock; + + public PolarisIntegrationTestFixture createFixture(TestEnvironment testEnv, TestInfo testInfo) { + return new PolarisIntegrationTestFixture(this, testEnv, testInfo); + } +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironment.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironment.java new file mode 100644 index 000000000..72808de41 --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironment.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.test; + +import java.net.URI; +import java.util.UUID; + +/** Defines the test environment that a test should run in. */ +public class TestEnvironment { + + private final URI baseUri; + private final URI baseManagementUri; + private final String testId; + + public TestEnvironment(String baseUri, String baseManagementUri) { + this( + URI.create(baseUri), + URI.create(baseManagementUri), + UUID.randomUUID().toString().replace("-", "")); + } + + public TestEnvironment(URI baseUri, URI baseManagementUri, String testId) { + this.baseUri = baseUri; + this.baseManagementUri = baseManagementUri; + this.testId = testId; + } + + /** The base URL that requests should target, for example {@code http://localhost:8181} */ + public URI baseUri() { + return baseUri; + } + + /** + * The base URL that requests should target for the management interface, for example {@code + * http://localhost:8182} + */ + public URI baseManagementUri() { + return baseManagementUri; + } + + /** + * An ID unique to this test. This can be used to prefix resource names, such as catalog names, to + * prevent collision. + */ + public String testId() { + return testId; + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironmentExtension.java similarity index 92% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironmentExtension.java index 0131fbf3a..729db42df 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironmentExtension.java @@ -16,11 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.test; +package org.apache.polaris.service.quarkus.test; import java.lang.reflect.InvocationTargetException; import java.util.Optional; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; import org.junit.jupiter.api.extension.ParameterResolver; @@ -43,7 +44,7 @@ public static TestEnvironment getEnv(ExtensionContext extensionContext) throws IllegalAccessException { // This must be cached because the TestEnvironment has a randomly generated ID return extensionContext - .getStore(ExtensionContext.Namespace.create(extensionContext.getRequiredTestClass())) + .getStore(Namespace.create(extensionContext.getRequiredTestClass())) .getOrComputeIfAbsent( ENV_PROPERTY_KEY, (k) -> getTestEnvironmentResolver().resolveTestEnvironment(extensionContext), @@ -71,7 +72,7 @@ public Object resolveParameter( private static TestEnvironmentResolver getTestEnvironmentResolver() { String impl = Optional.ofNullable(System.getenv(ENV_TEST_ENVIRONMENT_RESOLVER_IMPL)) - .orElse(DropwizardTestEnvironmentResolver.class.getName()); + .orElse(DefaultTestEnvironmentResolver.class.getName()); try { return (TestEnvironmentResolver) (Class.forName(impl).getDeclaredConstructor().newInstance()); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentResolver.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironmentResolver.java similarity index 95% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentResolver.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironmentResolver.java index 3c4a44e90..617862912 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentResolver.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestEnvironmentResolver.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.test; +package org.apache.polaris.service.quarkus.test; import org.junit.jupiter.api.extension.ExtensionContext; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestMetricsUtil.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestMetricsUtil.java new file mode 100644 index 000000000..be61bb79f --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/TestMetricsUtil.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.core.Response; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.stream.Collectors; +import org.hawkular.agent.prometheus.text.TextPrometheusMetricsProcessor; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.walkers.CollectorPrometheusMetricsWalker; + +/** Utils for working with metrics in tests */ +public class TestMetricsUtil { + + public static Map fetchMetrics(Client client, URI baseManagementUri) { + Response response = + client.target(String.format("%s/q/metrics", baseManagementUri)).request().get(); + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + String body = response.readEntity(String.class); + InputStream inputStream = new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8)); + CollectorPrometheusMetricsWalker walker = new CollectorPrometheusMetricsWalker(); + new TextPrometheusMetricsProcessor(inputStream, walker).walk(); + return walker.getAllMetricFamilies().stream() + .collect( + Collectors.toMap( + MetricFamily::getName, + metricFamily -> metricFamily, + (mf1, mf2) -> { + throw new IllegalStateException("Duplicate metric family: " + mf1.getName()); + })); + } +} diff --git a/server-templates/api.mustache b/server-templates/api.mustache index 5979bf8b0..b817bc495 100644 --- a/server-templates/api.mustache +++ b/server-templates/api.mustache @@ -23,6 +23,7 @@ import {{import}}; {{/imports}} import io.micrometer.core.annotation.Timed; +import io.micrometer.core.aop.MeterTag; import java.util.Map; import java.util.List; @@ -107,7 +108,7 @@ public class {{classname}} { @Produces({ {{#produces}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/produces}} }){{/hasProduces}}{{#hasAuthMethods}} {{#authMethods}}{{#isOAuth}}@RolesAllowed({ {{#scopes}}"{{scope}}"{{^-last}}, {{/-last}}{{/scopes}} }){{/isOAuth}}{{/authMethods}}{{/hasAuthMethods}} @Timed("{{metricsPrefix}}.{{baseName}}.{{nickname}}") - public Response {{nickname}}({{#isMultipart}}MultipartFormDataInput input,{{/isMultipart}}{{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{^isMultipart}}{{>formParams}},{{/isMultipart}}{{#isMultipart}}{{^isFormParam}},{{/isFormParam}}{{/isMultipart}}{{/allParams}}@Context RealmContext realmContext,@Context SecurityContext securityContext) { + public Response {{nickname}}({{#isMultipart}}MultipartFormDataInput input,{{/isMultipart}}{{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{^isMultipart}}{{>formParams}},{{/isMultipart}}{{#isMultipart}}{{^isFormParam}},{{/isFormParam}}{{/isMultipart}}{{/allParams}}@Context @MeterTag(key="realm_id",expression="realmIdentifier") RealmContext realmContext,@Context SecurityContext securityContext) { {{! Don't log form or header params in case there are secrets, e.g., OAuth tokens }} LOGGER.atDebug().setMessage("Invoking {{baseName}} with params") .addKeyValue("operation", "{{nickname}}"){{#allParams}}{{^isHeaderParam}}{{^isFormParam}} diff --git a/service/common/build.gradle.kts b/service/common/build.gradle.kts index e027641a8..d9aff4b19 100644 --- a/service/common/build.gradle.kts +++ b/service/common/build.gradle.kts @@ -17,7 +17,10 @@ * under the License. */ -plugins { id("polaris-server") } +plugins { + id("polaris-server") + alias(libs.plugins.jandex) +} dependencies { implementation(project(":polaris-core")) @@ -41,19 +44,21 @@ dependencies { exclude("com.sun.jersey", "jersey-core") exclude("com.sun.jersey", "jersey-server") exclude("com.sun.jersey", "jersey-servlet") + exclude("com.sun.jersey", "jersey-servlet") + exclude("io.dropwizard.metrics", "metrics-core") } implementation(libs.hadoop.hdfs.client) - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.jakarta.servlet.api) - compileOnly(libs.jakarta.validation.api) - compileOnly(libs.jakarta.ws.rs.api) + implementation(libs.jakarta.annotation.api) + implementation(libs.jakarta.enterprise.cdi.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.servlet.api) + implementation(libs.jakarta.validation.api) + implementation(libs.jakarta.ws.rs.api) - compileOnly(libs.smallrye.common.annotation) + implementation(libs.smallrye.common.annotation) implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml") implementation("com.fasterxml.jackson.core:jackson-annotations") implementation(libs.caffeine) @@ -82,3 +87,5 @@ dependencies { implementation(platform(libs.azuresdk.bom)) implementation("com.azure:azure-core") } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 49b7a3bba..d20c53cee 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.admin; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -75,6 +76,7 @@ import org.slf4j.LoggerFactory; /** Concrete implementation of the Polaris API services */ +@RequestScoped public class PolarisServiceImpl implements PolarisCatalogsApiService, PolarisPrincipalsApiService, @@ -83,15 +85,20 @@ public class PolarisServiceImpl private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final CallContext callContext; @Inject public PolarisServiceImpl( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, - PolarisAuthorizer polarisAuthorizer) { + PolarisAuthorizer polarisAuthorizer, + CallContext callContext) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.polarisAuthorizer = polarisAuthorizer; + this.callContext = callContext; + // FIXME: This is a hack to set the current context for downstream calls. + CallContext.setCurrentContext(callContext); } private PolarisAdminService newAdminService( @@ -107,12 +114,7 @@ private PolarisAdminService newAdminService( PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); return new PolarisAdminService( - // FIXME remove call to CallContext.getCurrentContext() - CallContext.getCurrentContext(), - entityManager, - metaStoreManager, - authenticatedPrincipal, - polarisAuthorizer); + callContext, entityManager, metaStoreManager, authenticatedPrincipal, polarisAuthorizer); } /** From PolarisCatalogsApiService */ @@ -130,7 +132,7 @@ public Response createCatalog( } private void validateStorageConfig(StorageConfigInfo storageConfigInfo) { - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); + PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); List allowedStorageTypes = polarisCallContext .getConfigurationStore() diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/AuthenticationConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/auth/AuthenticationConfiguration.java new file mode 100644 index 000000000..21fd912a7 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/auth/AuthenticationConfiguration.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.auth; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.Optional; + +public interface AuthenticationConfiguration { + + /** The configuration for the authenticator. */ + AuthenticatorConfiguration authenticator(); + + interface AuthenticatorConfiguration {} + + /** The configuration for the OAuth2 service that delivers OAuth2 tokens. */ + TokenServiceConfiguration tokenService(); + + interface TokenServiceConfiguration {} + + /** + * The configuration for the token broker factory. Token brokers are used by both the + * authenticator and the token service. + */ + TokenBrokerConfiguration tokenBroker(); + + interface TokenBrokerConfiguration { + + /** The maximum token duration. */ + Duration maxTokenGeneration(); + + /** Configuration for the rsa-key-pair token broker factory. */ + Optional rsaKeyPair(); + + /** Configuration for the symmetric-key token broker factory. */ + Optional symmetricKey(); + + interface RSAKeyPairConfiguration { + + /** The path to the public key file. */ + Path publicKeyFile(); + + /** The path to the private key file. */ + Path privateKeyFile(); + } + + interface SymmetricKeyConfiguration { + + /** + * The secret to use for both signing and verifying signatures. Either this option of {@link + * #file()} must be provided. + */ + Optional secret(); + + /** + * The file to read the secret from. Either this option of {@link #secret()} must be provided. + */ + Optional file(); + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java b/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java index 28dd9e0ef..f7845d464 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java @@ -18,14 +18,12 @@ */ package org.apache.polaris.service.auth; -import jakarta.inject.Inject; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -51,27 +49,29 @@ public abstract class BasePolarisAuthenticator public static final String PRINCIPAL_ROLE_PREFIX = "PRINCIPAL_ROLE:"; private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisAuthenticator.class); - @Inject protected MetaStoreManagerFactory metaStoreManagerFactory; + protected final MetaStoreManagerFactory metaStoreManagerFactory; + protected final CallContext callContext; - public PolarisCallContext getCurrentPolarisContext() { - return CallContext.getCurrentContext().getPolarisCallContext(); + protected BasePolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, CallContext callContext) { + this.metaStoreManagerFactory = metaStoreManagerFactory; + this.callContext = callContext; } protected Optional getPrincipal(DecodedToken tokenInfo) { LOGGER.debug("Resolving principal for tokenInfo client_id={}", tokenInfo.getClientId()); - RealmContext realmContext = CallContext.getCurrentContext().getRealmContext(); PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()); PolarisEntity principal; try { principal = tokenInfo.getPrincipalId() > 0 ? PolarisEntity.of( metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0L, tokenInfo.getPrincipalId())) + callContext.getPolarisCallContext(), 0L, tokenInfo.getPrincipalId())) : PolarisEntity.of( metaStoreManager.readEntityByName( - getCurrentPolarisContext(), + callContext.getPolarisCallContext(), null, PolarisEntityType.PRINCIPAL, PolarisEntitySubType.NULL_SUBTYPE, @@ -108,9 +108,7 @@ protected Optional getPrincipal(DecodedToken toke AuthenticatedPolarisPrincipal authenticatedPrincipal = new AuthenticatedPolarisPrincipal(new PrincipalEntity(principal), activatedPrincipalRoles); LOGGER.debug("Populating authenticatedPrincipal into CallContext: {}", authenticatedPrincipal); - CallContext.getCurrentContext() - .contextVariables() - .put(CallContext.AUTHENTICATED_PRINCIPAL, authenticatedPrincipal); + callContext.contextVariables().put(CallContext.AUTHENTICATED_PRINCIPAL, authenticatedPrincipal); return Optional.of(authenticatedPrincipal); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java index 5dd3b8b7c..da9a5c1c7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java @@ -21,11 +21,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import org.apache.commons.codec.binary.Base64; import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; import org.apache.polaris.service.types.TokenType; @@ -36,6 +38,7 @@ * Default implementation of the {@link IcebergRestOAuth2ApiService} that generates a JWT token for * the client if the client secret matches. */ +@RequestScoped @Identifier("default") public class DefaultOAuth2ApiService implements IcebergRestOAuth2ApiService { @@ -44,9 +47,14 @@ public class DefaultOAuth2ApiService implements IcebergRestOAuth2ApiService { private static final String CLIENT_CREDENTIALS = "client_credentials"; private static final String BEARER = "bearer"; - @Inject private TokenBrokerFactory tokenBrokerFactory; + private final TokenBrokerFactory tokenBrokerFactory; + private final CallContext callContext; - public DefaultOAuth2ApiService() {} + @Inject + public DefaultOAuth2ApiService(TokenBrokerFactory tokenBrokerFactory, CallContext callContext) { + this.tokenBrokerFactory = tokenBrokerFactory; + this.callContext = callContext; + } @Override public Response getToken( @@ -99,13 +107,18 @@ public Response getToken( // secret and treat it as a new token request if (clientId != null && clientSecret != null) { yield tokenBroker.generateFromClientSecrets( - clientId, clientSecret, CLIENT_CREDENTIALS, scope); + clientId, + clientSecret, + CLIENT_CREDENTIALS, + scope, + callContext.getPolarisCallContext()); } else { yield tokenBroker.generateFromToken(subjectTokenType, subjectToken, grantType, scope); } } case null -> - tokenBroker.generateFromClientSecrets(clientId, clientSecret, grantType, scope); + tokenBroker.generateFromClientSecrets( + clientId, clientSecret, grantType, scope, callContext.getPolarisCallContext()); }; if (tokenResponse == null) { return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.unsupported_grant_type); diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java index eff07c2a5..900a4ddd3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java @@ -19,19 +19,35 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import java.util.Optional; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +@RequestScoped @Identifier("default") public class DefaultPolarisAuthenticator extends BasePolarisAuthenticator { - @Inject private TokenBrokerFactory tokenBrokerFactory; + + private final TokenBrokerFactory tokenBrokerFactory; + + public DefaultPolarisAuthenticator() { + this(null, null, null); + } + + @Inject + public DefaultPolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, + TokenBrokerFactory tokenBrokerFactory, + CallContext callContext) { + super(metaStoreManagerFactory, callContext); + this.tokenBrokerFactory = tokenBrokerFactory; + } @Override public Optional authenticate(String credentials) { - TokenBroker handler = - tokenBrokerFactory.apply(CallContext.getCurrentContext().getRealmContext()); + TokenBroker handler = tokenBrokerFactory.apply(callContext.getRealmContext()); DecodedToken decodedToken = handler.verify(credentials); return getPrincipal(decodedToken); } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java index 74f6491ca..f1945e3c4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java @@ -29,6 +29,7 @@ import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; @@ -119,7 +120,11 @@ public TokenResponse generateFromToken( @Override public TokenResponse generateFromClientSecrets( - String clientId, String clientSecret, String grantType, String scope) { + String clientId, + String clientSecret, + String grantType, + String scope, + PolarisCallContext polarisCallContext) { // Initial sanity checks TokenRequestValidator validator = new TokenRequestValidator(); Optional initialValidationResponse = @@ -129,7 +134,8 @@ public TokenResponse generateFromClientSecrets( } Optional principal = - TokenBroker.findPrincipalEntity(metaStoreManager, clientId, clientSecret); + TokenBroker.findPrincipalEntity( + metaStoreManager, clientId, clientSecret, polarisCallContext); if (principal.isEmpty()) { return new TokenResponse(OAuthTokenErrorResponse.Error.unauthorized_client); } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java index 497877445..18f270238 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.auth; import com.auth0.jwt.algorithms.Algorithm; +import java.nio.file.Path; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -26,17 +27,19 @@ /** Generates a JWT using a Public/Private RSA Key */ public class JWTRSAKeyPair extends JWTBroker { - public JWTRSAKeyPair(PolarisMetaStoreManager metaStoreManager, int maxTokenGenerationInSeconds) { - super(metaStoreManager, maxTokenGenerationInSeconds); - } + private final KeyProvider keyProvider; - KeyProvider getKeyProvider() { - return new LocalRSAKeyProvider(); + public JWTRSAKeyPair( + PolarisMetaStoreManager metaStoreManager, + int maxTokenGenerationInSeconds, + Path publicKeyFile, + Path privateKeyFile) { + super(metaStoreManager, maxTokenGenerationInSeconds); + keyProvider = new LocalRSAKeyProvider(publicKeyFile, privateKeyFile); } @Override public Algorithm getAlgorithm() { - KeyProvider keyProvider = getKeyProvider(); return Algorithm.RSA256( (RSAPublicKey) keyProvider.getPublicKey(), (RSAPrivateKey) keyProvider.getPrivateKey()); } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java index cb49ea2b6..d9537b95d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java @@ -19,27 +19,62 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.auth.AuthenticationConfiguration.TokenBrokerConfiguration; +import org.apache.polaris.service.auth.AuthenticationConfiguration.TokenBrokerConfiguration.RSAKeyPairConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +@ApplicationScoped @Identifier("rsa-key-pair") public class JWTRSAKeyPairFactory implements TokenBrokerFactory { - private final TokenBrokerFactoryConfig config; + private static final Logger LOGGER = LoggerFactory.getLogger(JWTRSAKeyPairFactory.class); + private final MetaStoreManagerFactory metaStoreManagerFactory; + private final TokenBrokerConfiguration tokenBrokerConfiguration; + private final RSAKeyPairConfiguration keyPairConfiguration; @Inject public JWTRSAKeyPairFactory( - TokenBrokerFactoryConfig config, MetaStoreManagerFactory metaStoreManagerFactory) { - this.config = config; + MetaStoreManagerFactory metaStoreManagerFactory, + AuthenticationConfiguration authenticationConfiguration) { this.metaStoreManagerFactory = metaStoreManagerFactory; + this.tokenBrokerConfiguration = authenticationConfiguration.tokenBroker(); + this.keyPairConfiguration = + tokenBrokerConfiguration.rsaKeyPair().orElseGet(this::generateKeyPair); } @Override public TokenBroker apply(RealmContext realmContext) { return new JWTRSAKeyPair( metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - config.maxTokenGenerationInSeconds()); + (int) tokenBrokerConfiguration.maxTokenGeneration().toSeconds(), + keyPairConfiguration.publicKeyFile(), + keyPairConfiguration.privateKeyFile()); + } + + private RSAKeyPairConfiguration generateKeyPair() { + LOGGER.warn( + "No public and private key files were provided; these will be generated. " + + "This should not be done in production!"); + try { + Path privateFileLocation = Files.createTempFile("polaris-private", ".pem"); + Path publicFileLocation = Files.createTempFile("polaris-public", ".pem"); + PemUtils.generateKeyPair(privateFileLocation, publicFileLocation); + return new GeneratedKeyPair(privateFileLocation, publicFileLocation); + } catch (IOException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } } + + private record GeneratedKeyPair(Path privateKeyFile, Path publicKeyFile) + implements RSAKeyPairConfiguration {} } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java index 061a8a180..d08754ef0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java @@ -21,48 +21,56 @@ import static com.google.common.base.Preconditions.checkState; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.time.Duration; import java.util.function.Supplier; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.auth.AuthenticationConfiguration.TokenBrokerConfiguration.SymmetricKeyConfiguration; +@ApplicationScoped @Identifier("symmetric-key") public class JWTSymmetricKeyFactory implements TokenBrokerFactory { private final MetaStoreManagerFactory metaStoreManagerFactory; - private final TokenBrokerFactoryConfig config; + private final Duration maxTokenGeneration; private final Supplier secretSupplier; @Inject public JWTSymmetricKeyFactory( - MetaStoreManagerFactory metaStoreManagerFactory, TokenBrokerFactoryConfig config) { + MetaStoreManagerFactory metaStoreManagerFactory, + AuthenticationConfiguration authenticationConfiguration) { this.metaStoreManagerFactory = metaStoreManagerFactory; - this.config = config; - - String secret = config.secret(); - String file = config.file(); + this.maxTokenGeneration = authenticationConfiguration.tokenBroker().maxTokenGeneration(); + SymmetricKeyConfiguration symmetricKeyConfiguration = + authenticationConfiguration + .tokenBroker() + .symmetricKey() + .orElseThrow(() -> new IllegalStateException("Symmetric key configuration is missing")); + String secret = symmetricKeyConfiguration.secret().orElse(null); + Path file = symmetricKeyConfiguration.file().orElse(null); checkState(secret != null || file != null, "Either file or secret must be set"); - this.secretSupplier = secret != null ? () -> secret : readSecretFromDisk(Paths.get(file)); + this.secretSupplier = secret != null ? () -> secret : readSecretFromDisk(file); } @Override public TokenBroker apply(RealmContext realmContext) { return new JWTSymmetricKeyBroker( metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - config.maxTokenGenerationInSeconds(), + (int) maxTokenGeneration.toSeconds(), secretSupplier); } - private Supplier readSecretFromDisk(Path path) { + private static Supplier readSecretFromDisk(Path file) { return () -> { try { - return Files.readString(path); + return Files.readString(file); } catch (IOException e) { - throw new RuntimeException("Failed to read secret from file: " + config.file(), e); + throw new RuntimeException("Failed to read secret from file: " + file, e); } }; } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java b/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java index 666af1f2a..a42736844 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java @@ -19,10 +19,9 @@ package org.apache.polaris.service.auth; import java.io.IOException; +import java.nio.file.Path; import java.security.PrivateKey; import java.security.PublicKey; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,19 +31,14 @@ */ public class LocalRSAKeyProvider implements KeyProvider { - private static final String LOCAL_PRIVATE_KEY_LOCATION_KEY = "LOCAL_PRIVATE_KEY_LOCATION_KEY"; - private static final String LOCAL_PUBLIC_KEY_LOCATION_KEY = "LOCAL_PUBLIC_LOCATION_KEY"; - private static final Logger LOGGER = LoggerFactory.getLogger(LocalRSAKeyProvider.class); - private String getLocation(String configKey) { - CallContext callContext = CallContext.getCurrentContext(); - PolarisCallContext pCtx = callContext.getPolarisCallContext(); - String fileLocation = pCtx.getConfigurationStore().getConfiguration(pCtx, configKey); - if (fileLocation == null) { - throw new RuntimeException("Cannot find location for key " + configKey); - } - return fileLocation; + private final Path publicKeyFileLocation; + private final Path privateKeyFileLocation; + + public LocalRSAKeyProvider(Path publicKeyFileLocation, Path privateKeyFileLocation) { + this.publicKeyFileLocation = publicKeyFileLocation; + this.privateKeyFileLocation = privateKeyFileLocation; } /** @@ -54,7 +48,6 @@ private String getLocation(String configKey) { */ @Override public PublicKey getPublicKey() { - final String publicKeyFileLocation = getLocation(LOCAL_PUBLIC_KEY_LOCATION_KEY); try { return PemUtils.readPublicKeyFromFile(publicKeyFileLocation, "RSA"); } catch (IOException e) { @@ -70,7 +63,6 @@ public PublicKey getPublicKey() { */ @Override public PrivateKey getPrivateKey() { - final String privateKeyFileLocation = getLocation(LOCAL_PRIVATE_KEY_LOCATION_KEY); try { return PemUtils.readPrivateKeyFromFile(privateKeyFileLocation, "RSA"); } catch (IOException e) { diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java b/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java index c39124feb..311175176 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java @@ -19,41 +19,52 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.service.types.TokenType; /** Default {@link TokenBrokerFactory} that produces token brokers that do not do anything. */ +@ApplicationScoped @Identifier("none") public class NoneTokenBrokerFactory implements TokenBrokerFactory { + + public static final TokenBroker NONE_TOKEN_BROKER = + new TokenBroker() { + @Override + public boolean supportsGrantType(String grantType) { + return false; + } + + @Override + public boolean supportsRequestedTokenType(TokenType tokenType) { + return false; + } + + @Override + public TokenResponse generateFromClientSecrets( + String clientId, + String clientSecret, + String grantType, + String scope, + PolarisCallContext polarisCallContext) { + return null; + } + + @Override + public TokenResponse generateFromToken( + TokenType tokenType, String subjectToken, String grantType, String scope) { + return null; + } + + @Override + public DecodedToken verify(String token) { + return null; + } + }; + @Override public TokenBroker apply(RealmContext realmContext) { - return new TokenBroker() { - @Override - public boolean supportsGrantType(String grantType) { - return false; - } - - @Override - public boolean supportsRequestedTokenType(TokenType tokenType) { - return false; - } - - @Override - public TokenResponse generateFromClientSecrets( - String clientId, String clientSecret, String grantType, String scope) { - return null; - } - - @Override - public TokenResponse generateFromToken( - TokenType tokenType, String subjectToken, String grantType, String scope) { - return null; - } - - @Override - public DecodedToken verify(String token) { - return null; - } - }; + return NONE_TOKEN_BROKER; } } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java b/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java index 24c43a7f0..fd4a0be62 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java @@ -20,11 +20,14 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import java.io.File; +import java.io.BufferedWriter; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.security.PublicKey; @@ -32,64 +35,81 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; import org.bouncycastle.util.io.pem.PemObject; import org.bouncycastle.util.io.pem.PemReader; public class PemUtils { - private static byte[] parsePEMFile(File pemFile) throws IOException { - if (!pemFile.isFile() || !pemFile.exists()) { + private static byte[] parsePEMFile(Path pemPath) throws IOException { + if (!Files.isRegularFile(pemPath) || !Files.exists(pemPath)) { throw new FileNotFoundException( - String.format("The file '%s' doesn't exist.", pemFile.getAbsolutePath())); + String.format("The file '%s' doesn't exist.", pemPath.toAbsolutePath())); + } + try (PemReader reader = new PemReader(Files.newBufferedReader(pemPath, UTF_8))) { + PemObject pemObject = reader.readPemObject(); + return pemObject.getContent(); } - PemReader reader = new PemReader(new FileReader(pemFile, UTF_8)); - PemObject pemObject = reader.readPemObject(); - byte[] content = pemObject.getContent(); - reader.close(); - return content; } private static PublicKey getPublicKey(byte[] keyBytes, String algorithm) { - PublicKey publicKey = null; try { KeyFactory kf = KeyFactory.getInstance(algorithm); EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); - publicKey = kf.generatePublic(keySpec); + return kf.generatePublic(keySpec); } catch (NoSuchAlgorithmException e) { - System.out.println( - "Could not reconstruct the public key, the given algorithm could not be found."); + throw new RuntimeException( + "Could not reconstruct the public key, the given algorithm could not be found", e); } catch (InvalidKeySpecException e) { - System.out.println("Could not reconstruct the public key"); + throw new RuntimeException("Could not reconstruct the public key", e); } - - return publicKey; } private static PrivateKey getPrivateKey(byte[] keyBytes, String algorithm) { - PrivateKey privateKey = null; try { KeyFactory kf = KeyFactory.getInstance(algorithm); EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); - privateKey = kf.generatePrivate(keySpec); + return kf.generatePrivate(keySpec); } catch (NoSuchAlgorithmException e) { - System.out.println( - "Could not reconstruct the private key, the given algorithm could not be found."); + throw new RuntimeException( + "Could not reconstruct the private key, the given algorithm could not be found", e); } catch (InvalidKeySpecException e) { - System.out.println("Could not reconstruct the private key"); + throw new RuntimeException("Could not reconstruct the private key", e); } - - return privateKey; } - public static PublicKey readPublicKeyFromFile(String filepath, String algorithm) + public static PublicKey readPublicKeyFromFile(Path filepath, String algorithm) throws IOException { - byte[] bytes = PemUtils.parsePEMFile(new File(filepath)); + byte[] bytes = PemUtils.parsePEMFile(filepath); return PemUtils.getPublicKey(bytes, algorithm); } - public static PrivateKey readPrivateKeyFromFile(String filepath, String algorithm) + public static PrivateKey readPrivateKeyFromFile(Path filepath, String algorithm) throws IOException { - byte[] bytes = PemUtils.parsePEMFile(new File(filepath)); + byte[] bytes = PemUtils.parsePEMFile(filepath); return PemUtils.getPrivateKey(bytes, algorithm); } + + public static void generateKeyPair(Path privateFileLocation, Path publicFileLocation) + throws NoSuchAlgorithmException, IOException { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + try (BufferedWriter writer = Files.newBufferedWriter(privateFileLocation, UTF_8)) { + writer.write("-----BEGIN PRIVATE KEY-----"); + writer.newLine(); + writer.write(Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded())); + writer.newLine(); + writer.write("-----END PRIVATE KEY-----"); + writer.newLine(); + } + try (BufferedWriter writer = Files.newBufferedWriter(publicFileLocation, UTF_8)) { + writer.write("-----BEGIN PUBLIC KEY-----"); + writer.newLine(); + writer.write(Base64.getMimeEncoder().encodeToString(kp.getPublic().getEncoded())); + writer.newLine(); + writer.write("-----END PUBLIC KEY-----"); + writer.newLine(); + } + } } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java b/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java index 26e35f344..cd4b914aa 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java @@ -20,15 +20,17 @@ import com.google.common.base.Splitter; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,18 +47,27 @@ * This class does not expect a client to be either present or correct. Lookup is delegated to the * {@link PolarisMetaStoreManager} for the current realm. */ +@RequestScoped @Identifier("test") public class TestInlineBearerTokenPolarisAuthenticator extends BasePolarisAuthenticator { private static final Logger LOGGER = LoggerFactory.getLogger(TestInlineBearerTokenPolarisAuthenticator.class); + public TestInlineBearerTokenPolarisAuthenticator() { + this(null, null); + } + + @Inject + public TestInlineBearerTokenPolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, CallContext callContext) { + super(metaStoreManagerFactory, callContext); + } + @Override public Optional authenticate(String credentials) { Map properties = extractPrincipal(credentials); PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager( - CallContext.getCurrentContext().getRealmContext()); - PolarisCallContext callContext = CallContext.getCurrentContext().getPolarisCallContext(); + metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()); String principal = properties.get("principal"); LOGGER.info("Checking for existence of principal {} in map {}", principal, properties); @@ -71,7 +82,9 @@ public Optional authenticate(String credentials) } PolarisPrincipalSecrets secrets = - metaStoreManager.loadPrincipalSecrets(callContext, principal).getPrincipalSecrets(); + metaStoreManager + .loadPrincipalSecrets(callContext.getPolarisCallContext(), principal) + .getPrincipalSecrets(); if (secrets == null) { // For test scenarios, if we're allowing short-circuiting into the bearer flow, there may // not be a clientId/clientSecret, and instead we'll let the BasePolarisAuthenticator diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java b/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java index b23530bc2..910defbfa 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -39,11 +40,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@RequestScoped @Identifier("test") public class TestOAuth2ApiService implements IcebergRestOAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(TestOAuth2ApiService.class); - @Inject private MetaStoreManagerFactory metaStoreManagerFactory; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject CallContext callContext; @Override public Response getToken( @@ -80,8 +83,7 @@ public Response getToken( private String getPrincipalName(String clientId, RealmContext realmContext) { PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - // FIXME remove call to CallContext.getCurrentContext() - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); + PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); PrincipalSecretsResult secretsResult = metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); if (secretsResult.isSuccess()) { diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java b/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java index 190a21e54..2ace9bb92 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java @@ -22,7 +22,6 @@ import java.util.Optional; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -36,7 +35,11 @@ public interface TokenBroker { boolean supportsRequestedTokenType(TokenType tokenType); TokenResponse generateFromClientSecrets( - final String clientId, final String clientSecret, final String grantType, final String scope); + final String clientId, + final String clientSecret, + final String grantType, + final String scope, + PolarisCallContext polarisCallContext); TokenResponse generateFromToken( TokenType tokenType, String subjectToken, final String grantType, final String scope); @@ -44,9 +47,11 @@ TokenResponse generateFromToken( DecodedToken verify(String token); static @Nonnull Optional findPrincipalEntity( - PolarisMetaStoreManager metaStoreManager, String clientId, String clientSecret) { + PolarisMetaStoreManager metaStoreManager, + String clientId, + String clientSecret, + PolarisCallContext polarisCallContext) { // Validate the principal is present and secrets match - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); PrincipalSecretsResult principalSecrets = metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); if (!principalSecrets.isSuccess()) { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 483778ccd..176e78e3a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -74,6 +75,7 @@ * org.apache.iceberg.rest.CatalogHandlers} after finding the appropriate {@link Catalog} for the * current {@link RealmContext}. */ +@RequestScoped public class IcebergCatalogAdapter implements IcebergRestCatalogApiService, IcebergRestConfigurationApiService { @@ -109,6 +111,7 @@ public class IcebergCatalogAdapter .add(Endpoint.create("POST", ResourcePaths.V1_TRANSACTIONS_COMMIT)) .build(); + private final CallContext callContext; private final CallContextCatalogFactory catalogFactory; private final MetaStoreManagerFactory metaStoreManagerFactory; private final RealmEntityManagerFactory entityManagerFactory; @@ -116,14 +119,18 @@ public class IcebergCatalogAdapter @Inject public IcebergCatalogAdapter( + CallContext callContext, CallContextCatalogFactory catalogFactory, RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, PolarisAuthorizer polarisAuthorizer) { + this.callContext = callContext; this.catalogFactory = catalogFactory; this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.polarisAuthorizer = polarisAuthorizer; + // FIXME: This is a hack to set the current context for downstream calls. + CallContext.setCurrentContext(callContext); } private PolarisCatalogHandlerWrapper newHandlerWrapper( @@ -138,8 +145,7 @@ private PolarisCatalogHandlerWrapper newHandlerWrapper( entityManagerFactory.getOrCreateEntityManager(realmContext); return new PolarisCatalogHandlerWrapper( - // FIXME remove call to CallContext.getCurrentContext() - CallContext.getCurrentContext(), + callContext, entityManager, metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), authenticatedPrincipal, @@ -540,8 +546,6 @@ public Response getConfig( if (warehouse == null) { throw new BadRequestException("Please specify a warehouse"); } - // FIXME remove call to CallContext.getCurrentContext() - CallContext callContext = CallContext.getCurrentContext(); Resolver resolver = entityManager.prepareResolver(callContext, authenticatedPrincipal, warehouse); ResolverStatus resolverStatus = resolver.resolveAll(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index b930fb35a..b962ec79d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -19,12 +19,14 @@ package org.apache.polaris.service.catalog.io; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; /** A simple FileIOFactory implementation that defers all the work to the Iceberg SDK */ +@ApplicationScoped @Identifier("default") public class DefaultFileIOFactory implements FileIOFactory { @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java index 64999bbf5..ca3c08511 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java @@ -18,15 +18,10 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.util.Map; import org.apache.iceberg.io.FileIO; /** Interface for providing a way to construct FileIO objects, such as for reading/writing S3. */ -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "factoryType") public interface FileIOFactory { FileIO loadFileIO(String impl, Map properties); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java index 016afa54c..469587a39 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java @@ -19,18 +19,19 @@ package org.apache.polaris.service.catalog.io; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; /** A {@link FileIOFactory} that translates WASB paths to ABFS ones */ +@ApplicationScoped @Identifier("wasb") public class WasbTranslatingFileIOFactory implements FileIOFactory { @Override public FileIO loadFileIO(String ioImpl, Map properties) { - WasbTranslatingFileIO wrapped = - new WasbTranslatingFileIO(CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); - return wrapped; + return new WasbTranslatingFileIO( + CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java b/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java index 35a54a5f0..3db2fd998 100644 --- a/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java +++ b/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java @@ -18,33 +18,51 @@ */ package org.apache.polaris.service.config; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; +@ApplicationScoped public class DefaultConfigurationStore implements PolarisConfigurationStore { - private final Map config; - private final Map> realmConfig; - public DefaultConfigurationStore(Map config) { - this.config = config; - this.realmConfig = Map.of(); + private final Map defaults; + private final Map> realmOverrides; + + // FIXME the whole PolarisConfigurationStore + PolarisConfiguration needs to be refactored + // to become a proper Quarkus configuration object + @Inject + public DefaultConfigurationStore( + ObjectMapper objectMapper, FeaturesConfiguration configurations) { + this( + configurations.parseDefaults(objectMapper), + configurations.parseRealmOverrides(objectMapper)); + } + + public DefaultConfigurationStore(Map defaults) { + this(defaults, Map.of()); } public DefaultConfigurationStore( - Map config, Map> realmConfig) { - this.config = config; - this.realmConfig = realmConfig; + Map defaults, Map> realmOverrides) { + this.defaults = Map.copyOf(defaults); + this.realmOverrides = Map.copyOf(realmOverrides); } - @SuppressWarnings("unchecked") @Override public @Nullable T getConfiguration(@Nonnull PolarisCallContext ctx, String configName) { String realm = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); - return (T) - realmConfig.getOrDefault(realm, Map.of()).getOrDefault(configName, config.get(configName)); + @SuppressWarnings("unchecked") + T confgValue = + (T) + realmOverrides + .getOrDefault(realm, Map.of()) + .getOrDefault(configName, defaults.get(configName)); + return confgValue; } } diff --git a/service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java new file mode 100644 index 000000000..c17159888 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.config; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// FIXME replace with dynamic sources and/or feature flags +public interface FeaturesConfiguration { + + interface RealmOverrides { + Map overrides(); + } + + Map defaults(); + + Map realmOverrides(); + + default Map parseDefaults(ObjectMapper objectMapper) { + return convertMap(objectMapper, defaults()); + } + + default Map> parseRealmOverrides(ObjectMapper objectMapper) { + Map> m = new HashMap<>(); + for (String realm : realmOverrides().keySet()) { + m.put(realm, convertMap(objectMapper, realmOverrides().get(realm).overrides())); + } + return m; + } + + private static Map convertMap( + ObjectMapper objectMapper, Map properties) { + Map m = new HashMap<>(); + for (String configName : properties.keySet()) { + String json = properties.get(configName); + try { + JsonNode node = objectMapper.readTree(json); + m.put(configName, configValue(node)); + } catch (JsonProcessingException e) { + throw new RuntimeException( + "Invalid JSON value for feature configuration: " + configName, e); + } + } + return m; + } + + private static Object configValue(JsonNode node) { + return switch (node.getNodeType()) { + case BOOLEAN -> node.asBoolean(); + case STRING -> node.asText(); + case NUMBER -> + switch (node.numberType()) { + case INT, LONG -> node.asLong(); + case FLOAT, DOUBLE -> node.asDouble(); + default -> + throw new IllegalArgumentException("Unsupported number type: " + node.numberType()); + }; + case ARRAY -> { + List list = new ArrayList<>(); + node.elements().forEachRemaining(n -> list.add(configValue(n))); + yield List.copyOf(list); + } + default -> + throw new IllegalArgumentException( + "Unsupported feature configuration JSON type: " + node.getNodeType()); + }; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java index 620e67900..03799bd40 100644 --- a/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java @@ -18,37 +18,30 @@ */ package org.apache.polaris.service.config; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import jakarta.inject.Provider; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.cache.EntityCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Gets or creates PolarisEntityManager instances based on config values and RealmContext. */ +@ApplicationScoped public class RealmEntityManagerFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(RealmEntityManagerFactory.class); + private final MetaStoreManagerFactory metaStoreManagerFactory; // Key: realmIdentifier private final Map cachedEntityManagers = new ConcurrentHashMap<>(); - private final Provider entityCache; - - // Subclasses for test injection. - protected RealmEntityManagerFactory() { - this.metaStoreManagerFactory = null; - this.entityCache = null; - } @Inject - public RealmEntityManagerFactory( - MetaStoreManagerFactory metaStoreManagerFactory, Provider entityCache) { + public RealmEntityManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; - this.entityCache = entityCache; } public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { @@ -63,7 +56,7 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { return new PolarisEntityManager( metaStoreManagerFactory.getOrCreateMetaStoreManager(context), metaStoreManagerFactory.getOrCreateStorageCredentialCache(context), - entityCache.get()); + metaStoreManagerFactory.getOrCreateEntityCache(context)); }); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java deleted file mode 100644 index fee41e71b..000000000 --- a/service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; - -public class TaskHandlerConfiguration { - private int poolSize = 10; - private boolean fixedSize = true; - private String threadNamePattern = "taskHandler-%d"; - - public void setPoolSize(int poolSize) { - this.poolSize = poolSize; - } - - public void setFixedSize(boolean fixedSize) { - this.fixedSize = fixedSize; - } - - public void setThreadNamePattern(String threadNamePattern) { - this.threadNamePattern = threadNamePattern; - } - - public ExecutorService executorService() { - return fixedSize - ? Executors.newFixedThreadPool(poolSize, threadFactory()) - : Executors.newCachedThreadPool(threadFactory()); - } - - private ThreadFactory threadFactory() { - return new ThreadFactoryBuilder().setNameFormat(threadNamePattern).setDaemon(true).build(); - } -} diff --git a/dropwizard/service/src/test/resources/META-INF/hk2-locator/default b/service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java similarity index 66% rename from dropwizard/service/src/test/resources/META-INF/hk2-locator/default rename to service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java index 92b32e5af..cd72cf2e8 100644 --- a/dropwizard/service/src/test/resources/META-INF/hk2-locator/default +++ b/service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java @@ -16,12 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -[org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory]S -contract={org.apache.polaris.service.catalog.io.FileIOFactory} -name=test -qualifier={io.smallrye.common.annotation.Identifier} +package org.apache.polaris.service.context; -[org.apache.polaris.service.dropwizard.ratelimiter.MockTokenBucketFactory]S -contract={org.apache.polaris.service.ratelimiter.TokenBucketFactory} -name=mock -qualifier={io.smallrye.common.annotation.Identifier} +public interface ContextConfiguration { + + /** The configuration for the realm context resolver. */ + RealmContextResolverConfiguration realmContextResolver(); + + interface RealmContextResolverConfiguration { + + /** The default realm to use when no realm is specified. */ + String defaultRealm(); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index cbaaabf3f..ff3949724 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -19,13 +19,12 @@ package org.apache.polaris.service.context; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.time.Clock; -import java.time.ZoneId; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -40,12 +39,15 @@ * *

Example: principal:data-engineer;password:test;realm:acct123 */ +@ApplicationScoped @Identifier("default") public class DefaultCallContextResolver implements CallContextResolver { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultCallContextResolver.class); - @Inject private MetaStoreManagerFactory metaStoreManagerFactory; - @Inject private PolarisConfigurationStore configurationStore; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisDiagnostics diagnostics; + @Inject Clock clock; @Override public CallContext resolveCallContext( @@ -58,15 +60,10 @@ public CallContext resolveCallContext( .addKeyValue("headers", headers) .log("Resolving CallContext"); - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); PolarisMetaStoreSession metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = - new PolarisCallContext( - metaStoreSession, - diagServices, - configurationStore, - Clock.system(ZoneId.systemDefault())); + new PolarisCallContext(metaStoreSession, diagnostics, configurationStore, clock); return CallContext.of(realmContext, polarisContext); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java index 9b4f40eb8..8cdf2e9f2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java @@ -20,6 +20,8 @@ import com.google.common.base.Splitter; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; import org.apache.polaris.core.context.RealmContext; @@ -32,13 +34,19 @@ * *

Example: principal:data-engineer;password:test;realm:acct123 */ +@ApplicationScoped @Identifier("default") public class DefaultRealmContextResolver implements RealmContextResolver { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultRealmContextResolver.class); public static final String REALM_PROPERTY_KEY = "realm"; - private String defaultRealm = "default-realm"; + private final ContextConfiguration configuration; + + @Inject + public DefaultRealmContextResolver(ContextConfiguration configuration) { + this.configuration = configuration; + } @Override public RealmContext resolveRealmContext( @@ -48,7 +56,7 @@ public RealmContext resolveRealmContext( // sure to only log non-sensitive contents. LOGGER.debug( "Resolving RealmContext for method: {}, path: {}, headers: {}", method, path, headers); - final Map parsedProperties = parseBearerTokenAsKvPairs(headers); + Map parsedProperties = parseBearerTokenAsKvPairs(headers); if (!parsedProperties.containsKey(REALM_PROPERTY_KEY) && headers.containsKey(REALM_PROPERTY_KEY)) { @@ -57,20 +65,13 @@ public RealmContext resolveRealmContext( if (!parsedProperties.containsKey(REALM_PROPERTY_KEY)) { LOGGER.warn( - "Failed to parse {} from headers; using {}", REALM_PROPERTY_KEY, getDefaultRealm()); - parsedProperties.put(REALM_PROPERTY_KEY, getDefaultRealm()); + "Failed to parse {} from headers; using {}", + REALM_PROPERTY_KEY, + configuration.realmContextResolver().defaultRealm()); + parsedProperties.put(REALM_PROPERTY_KEY, configuration.realmContextResolver().defaultRealm()); } - return () -> parsedProperties.get(REALM_PROPERTY_KEY); - } - - @Override - public void setDefaultRealm(String defaultRealm) { - this.defaultRealm = defaultRealm; - } - - @Override - public String getDefaultRealm() { - return this.defaultRealm; + String realmId = parsedProperties.get(REALM_PROPERTY_KEY); + return () -> realmId; } /** diff --git a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index a72f71431..a23802431 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.context; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.nio.file.Paths; import java.util.HashMap; @@ -39,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@ApplicationScoped public class PolarisCallContextCatalogFactory implements CallContextCatalogFactory { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisCallContextCatalogFactory.class); diff --git a/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java index 23a8b64ea..dae00f377 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java @@ -25,8 +25,4 @@ public interface RealmContextResolver { RealmContext resolveRealmContext( String requestURL, String method, String path, Map headers); - - void setDefaultRealm(String defaultRealm); - - String getDefaultRealm(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 7714fc3df..b5808b92d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -18,9 +18,10 @@ */ package org.apache.polaris.service.persistence; -import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.util.Collections; import java.util.HashSet; @@ -31,18 +32,31 @@ import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.PolarisTreeMapStore; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +@ApplicationScoped @Identifier("in-memory") public class InMemoryPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - @Inject protected PolarisStorageIntegrationProvider storageIntegration; - final Set bootstrappedRealms = new HashSet<>(); + private final PolarisStorageIntegrationProvider storageIntegration; + + public InMemoryPolarisMetaStoreManagerFactory() { + this(null); + } + + @Inject + public InMemoryPolarisMetaStoreManagerFactory( + PolarisStorageIntegrationProvider storageIntegration) { + this.storageIntegration = storageIntegration; + } + + private final Set bootstrappedRealms = new HashSet<>(); @Override protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { @@ -51,9 +65,11 @@ protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics dia @Override protected PolarisMetaStoreSession createMetaStoreSession( - @Nonnull PolarisTreeMapStore store, @Nonnull RealmContext realmContext) { + @Nonnull PolarisTreeMapStore store, + @Nonnull RealmContext realmContext, + @Nullable PolarisCredentialsBootstrap credentialsBootstrap) { return new PolarisTreeMapMetaStoreSessionImpl( - store, storageIntegration, secretsGenerator(realmContext)); + store, storageIntegration, secretsGenerator(realmContext, credentialsBootstrap)); } @Override @@ -78,7 +94,8 @@ public synchronized Supplier getOrCreateSessionSupplier private void bootstrapRealmAndPrintCredentials(String realmId) { Map results = - this.bootstrapRealms(Collections.singletonList(realmId)); + this.bootstrapRealms( + Collections.singletonList(realmId), PolarisCredentialsBootstrap.fromEnvironment()); bootstrappedRealms.add(realmId); PrincipalSecretsResult principalSecrets = results.get(realmId); @@ -91,10 +108,4 @@ private void bootstrapRealmAndPrintCredentials(String realmId) { principalSecrets.getPrincipalSecrets().getMainSecret()); System.out.println(msg); } - - @VisibleForTesting - public void setStorageIntegrationProvider( - PolarisStorageIntegrationProvider storageIntegrationProvider) { - this.storageIntegration = storageIntegrationProvider; - } } diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java index 8393e8f00..a1a9683e8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java @@ -18,32 +18,35 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.time.Clock; +import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.polaris.core.context.RealmContext; +@ApplicationScoped @Identifier("default") public class DefaultTokenBucketFactory implements TokenBucketFactory { private final long requestsPerSecond; - private final long windowSeconds; + private final Duration window; private final Clock clock; private final Map perRealmBuckets = new ConcurrentHashMap<>(); - @JsonCreator - public DefaultTokenBucketFactory( - @JsonProperty("requestsPerSecond") long requestsPerSecond, - @JsonProperty("windowSeconds") long windowSeconds) { - this(requestsPerSecond, windowSeconds, Clock.systemUTC()); + @Inject + public DefaultTokenBucketFactory(RateLimiterConfiguration configuration, Clock clock) { + this( + configuration.tokenBucket().requestsPerSecond(), + configuration.tokenBucket().window(), + clock); } - public DefaultTokenBucketFactory(long requestsPerSecond, long windowSeconds, Clock clock) { + public DefaultTokenBucketFactory(long requestsPerSecond, Duration window, Clock clock) { this.requestsPerSecond = requestsPerSecond; - this.windowSeconds = windowSeconds; + this.window = window; this.clock = clock; } @@ -54,6 +57,8 @@ public TokenBucket getOrCreateTokenBucket(RealmContext realmContext) { realmId, k -> new TokenBucket( - requestsPerSecond, Math.multiplyExact(requestsPerSecond, windowSeconds), clock)); + requestsPerSecond, + Math.multiplyExact(requestsPerSecond, window.toSeconds()), + clock)); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java index b20c01693..4d0ff7893 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java @@ -19,9 +19,11 @@ package org.apache.polaris.service.ratelimiter; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; /** Rate limiter that always allows the request */ @Identifier("no-op") +@ApplicationScoped public class NoOpRateLimiter implements RateLimiter { @Override public boolean canProceed() { diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java similarity index 73% rename from dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java rename to service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java index 32174bc87..521795ef0 100644 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java @@ -16,14 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.ratelimiter; -import com.codahale.metrics.health.HealthCheck; +import java.time.Duration; -/** Default {@link HealthCheck} implementation. */ -public class PolarisHealthCheck extends HealthCheck { - @Override - protected Result check() throws Exception { - return Result.healthy(); +public interface RateLimiterConfiguration { + + TokenBucketConfiguration tokenBucket(); + + interface TokenBucketConfiguration { + + long requestsPerSecond(); + + Duration window(); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java index d72def5fa..99a7d1ade 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java @@ -22,8 +22,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.ext.Provider; import java.io.IOException; -import javax.ws.rs.ext.Provider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java index 0d1deb88f..ee451f8cd 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java @@ -18,19 +18,28 @@ */ package org.apache.polaris.service.ratelimiter; -import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; -import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; /** * Rate limiter that maps the request's realm identifier to its own TokenBucket, with its own * capacity. */ -@Identifier("realm-token-bucket") +@Identifier("default") +@RequestScoped public class RealmTokenBucketRateLimiter implements RateLimiter { - @Inject protected TokenBucketFactory tokenBucketFactory; + private final TokenBucketFactory tokenBucketFactory; + private final RealmContext realmContext; + + @Inject + public RealmTokenBucketRateLimiter( + TokenBucketFactory tokenBucketFactory, RealmContext realmContext) { + this.tokenBucketFactory = tokenBucketFactory; + this.realmContext = realmContext; + } /** * This signifies that a request is being made. That is, the rate limiter should count the request @@ -40,13 +49,6 @@ public class RealmTokenBucketRateLimiter implements RateLimiter { */ @Override public boolean canProceed() { - return tokenBucketFactory - .getOrCreateTokenBucket(CallContext.getCurrentContext().getRealmContext()) - .tryAcquire(); - } - - @VisibleForTesting - public void setTokenBucketFactory(TokenBucketFactory tokenBucketFactory) { - this.tokenBucketFactory = tokenBucketFactory; + return tokenBucketFactory.getOrCreateTokenBucket(realmContext).tryAcquire(); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 98195c0e5..cde57535b 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -22,9 +22,10 @@ import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.ServiceOptions; -import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.EnumMap; import java.util.Map; import java.util.Set; @@ -40,12 +41,17 @@ import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import software.amazon.awssdk.services.sts.StsClient; -@Identifier("default") +@ApplicationScoped public class PolarisStorageIntegrationProviderImpl implements PolarisStorageIntegrationProvider { private final Supplier stsClientSupplier; private final Supplier gcpCredsProvider; + @Inject + public PolarisStorageIntegrationProviderImpl(StorageConfiguration storageConfiguration) { + this(storageConfiguration.stsClientSupplier(), storageConfiguration.gcpCredentialsSupplier()); + } + public PolarisStorageIntegrationProviderImpl( Supplier stsClientSupplier, Supplier gcpCredsProvider) { this.stsClientSupplier = stsClientSupplier; diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java new file mode 100644 index 000000000..711cf032e --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage; + +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.GoogleCredentials; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.function.Supplier; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; + +public interface StorageConfiguration { + + Duration DEFAULT_TOKEN_LIFESPAN = Duration.ofHours(1); + + /** + * The AWS access key to use for authentication. If not present, the default credentials provider + * chain will be used. + */ + Optional awsAccessKey(); + + /** + * The AWS secret key to use for authentication. If not present, the default credentials provider + * chain will be used. + */ + Optional awsSecretKey(); + + /** + * The GCP access token to use for authentication. If not present, the default credentials + * provider chain will be used. + */ + Optional gcpAccessToken(); + + /** + * The lifespan of the GCP access token. If not present, the {@linkplain #DEFAULT_TOKEN_LIFESPAN + * default token lifespan} will be used. + */ + Optional gcpAccessTokenLifespan(); + + default Supplier stsClientSupplier() { + return () -> { + StsClientBuilder stsClientBuilder = StsClient.builder(); + if (awsAccessKey().isPresent() && awsSecretKey().isPresent()) { + LoggerFactory.getLogger(StorageConfiguration.class) + .warn("Using hard-coded AWS credentials - this is not recommended for production"); + StaticCredentialsProvider awsCredentialsProvider = + StaticCredentialsProvider.create( + AwsBasicCredentials.create(awsAccessKey().get(), awsSecretKey().get())); + stsClientBuilder.credentialsProvider(awsCredentialsProvider); + } + return stsClientBuilder.build(); + }; + } + + default Supplier gcpCredentialsSupplier() { + return () -> { + if (gcpAccessToken().isEmpty()) { + try { + return GoogleCredentials.getApplicationDefault(); + } catch (IOException e) { + throw new RuntimeException("Failed to get GCP credentials", e); + } + } else { + AccessToken accessToken = + new AccessToken( + gcpAccessToken().get(), + new Date( + Instant.now() + .plus(gcpAccessTokenLifespan().orElse(DEFAULT_TOKEN_LIFESPAN)) + .toEpochMilli())); + return GoogleCredentials.create(accessToken); + } + }; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java b/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java index 920e5e21d..cc084a4d2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java @@ -19,12 +19,13 @@ package org.apache.polaris.service.task; import jakarta.annotation.Nonnull; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -42,15 +43,27 @@ */ public class TaskExecutorImpl implements TaskExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(TaskExecutorImpl.class); - public static final long TASK_RETRY_DELAY = 1000; - private final ExecutorService executorService; + private static final long TASK_RETRY_DELAY = 1000; + + private final Executor executor; private final MetaStoreManagerFactory metaStoreManagerFactory; - private final List taskHandlers = new ArrayList<>(); + private final TaskFileIOSupplier fileIOSupplier; + private final List taskHandlers = new CopyOnWriteArrayList<>(); public TaskExecutorImpl( - ExecutorService executorService, MetaStoreManagerFactory metaStoreManagerFactory) { - this.executorService = executorService; + Executor executor, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskFileIOSupplier fileIOSupplier) { + this.executor = executor; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.fileIOSupplier = fileIOSupplier; + } + + public void init() { + addTaskHandler(new TableCleanupTaskHandler(this, metaStoreManagerFactory, fileIOSupplier)); + addTaskHandler( + new ManifestFileCleanupTaskHandler( + fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); } /** @@ -68,69 +81,70 @@ public void addTaskHandler(TaskHandler taskHandler) { */ @Override public void addTaskHandlerContext(long taskEntityId, CallContext callContext) { + // Unfortunately CallContext is a request-scoped bean and must be cloned now, + // because its usage inside the TaskExecutor thread pool will outlive its + // lifespan, so the original CallContext will eventually be closed while + // the task is still running. + // Note: PolarisCallContext has request-scoped beans as well, and must be cloned. + // FIXME replace with context propagation? CallContext clone = CallContext.copyOf(callContext); tryHandleTask(taskEntityId, clone, null, 1); } private @Nonnull CompletableFuture tryHandleTask( - long taskEntityId, CallContext clone, Throwable e, int attempt) { + long taskEntityId, CallContext callContext, Throwable e, int attempt) { if (attempt > 3) { return CompletableFuture.failedFuture(e); } return CompletableFuture.runAsync( - () -> { - // set the call context INSIDE the async task - try (CallContext ctx = CallContext.setCurrentContext(CallContext.copyOf(clone))) { - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); - PolarisBaseEntity taskEntity = - metaStoreManager - .loadEntity(ctx.getPolarisCallContext(), 0L, taskEntityId) - .getEntity(); - if (!PolarisEntityType.TASK.equals(taskEntity.getType())) { - throw new IllegalArgumentException("Provided taskId must be a task entity type"); - } - TaskEntity task = TaskEntity.of(taskEntity); - Optional handlerOpt = - taskHandlers.stream().filter(th -> th.canHandleTask(task)).findFirst(); - if (handlerOpt.isEmpty()) { - LOGGER - .atWarn() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("taskType", task.getTaskType()) - .log("Unable to find handler for task type"); - return; - } - TaskHandler handler = handlerOpt.get(); - boolean success = handler.handleTask(task); - if (success) { - LOGGER - .atInfo() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("handlerClass", handler.getClass()) - .log("Task successfully handled"); - metaStoreManager.dropEntityIfExists( - ctx.getPolarisCallContext(), - null, - PolarisEntity.toCore(taskEntity), - Map.of(), - false); - } else { - LOGGER - .atWarn() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("taskEntityName", taskEntity.getName()) - .log("Unable to execute async task"); - } - } - }, - executorService) + () -> handleTask(taskEntityId, callContext, attempt), executor) .exceptionallyComposeAsync( (t) -> { LOGGER.warn("Failed to handle task entity id {}", taskEntityId, t); - return tryHandleTask(taskEntityId, clone, t, attempt + 1); + return tryHandleTask(taskEntityId, callContext, t, attempt + 1); }, CompletableFuture.delayedExecutor( - TASK_RETRY_DELAY * (long) attempt, TimeUnit.MILLISECONDS, executorService)); + TASK_RETRY_DELAY * (long) attempt, TimeUnit.MILLISECONDS, executor)); + } + + protected void handleTask(long taskEntityId, CallContext ctx, int attempt) { + // set the call context INSIDE the async task + CallContext.setCurrentContext(ctx); + LOGGER.info("Handling task entity id {}", taskEntityId); + PolarisMetaStoreManager metaStoreManager = + metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); + PolarisBaseEntity taskEntity = + metaStoreManager.loadEntity(ctx.getPolarisCallContext(), 0L, taskEntityId).getEntity(); + if (!PolarisEntityType.TASK.equals(taskEntity.getType())) { + throw new IllegalArgumentException("Provided taskId must be a task entity type"); + } + TaskEntity task = TaskEntity.of(taskEntity); + Optional handlerOpt = + taskHandlers.stream().filter(th -> th.canHandleTask(task)).findFirst(); + if (handlerOpt.isEmpty()) { + LOGGER + .atWarn() + .addKeyValue("taskEntityId", taskEntityId) + .addKeyValue("taskType", task.getTaskType()) + .log("Unable to find handler for task type"); + return; + } + TaskHandler handler = handlerOpt.get(); + boolean success = handler.handleTask(task); + if (success) { + LOGGER + .atInfo() + .addKeyValue("taskEntityId", taskEntityId) + .addKeyValue("handlerClass", handler.getClass()) + .log("Task successfully handled"); + metaStoreManager.dropEntityIfExists( + ctx.getPolarisCallContext(), null, PolarisEntity.toCore(taskEntity), Map.of(), false); + } else { + LOGGER + .atWarn() + .addKeyValue("taskEntityId", taskEntityId) + .addKeyValue("taskEntityName", taskEntity.getName()) + .log("Unable to execute async task"); + } } } diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index 75ceff8ec..e926e2d60 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.task; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -33,11 +35,13 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.catalog.io.FileIOFactory; +@ApplicationScoped public class TaskFileIOSupplier implements Function { private final MetaStoreManagerFactory metaStoreManagerFactory; private final FileIOFactory fileIOFactory; private final PolarisConfigurationStore configurationStore; + @Inject public TaskFileIOSupplier( MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory, diff --git a/extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default b/service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java similarity index 76% rename from extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default rename to service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java index ccebc9f69..7b278c1dc 100644 --- a/extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java @@ -16,8 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -[org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory]S -contract={org.apache.polaris.core.persistence.MetaStoreManagerFactory} -name=eclipse-link -qualifier={io.smallrye.common.annotation.Identifier} +package org.apache.polaris.service.task; +public interface TaskHandlerConfiguration { + + int maxConcurrentTasks(); + + int maxQueuedTasks(); +} diff --git a/site/content/in-dev/unreleased/metastores.md b/site/content/in-dev/unreleased/metastores.md index 74766c9c8..8b287ffb2 100644 --- a/site/content/in-dev/unreleased/metastores.md +++ b/site/content/in-dev/unreleased/metastores.md @@ -76,7 +76,7 @@ A single `persistence.xml` can describe multiple [persistence units](https://ecl To build Polaris with the necessary H2 dependency and start the Polaris service, run the following: ```bash polaris> ./gradlew --no-daemon --info -PeclipseLink=true -PeclipseLinkDeps=com.h2database:h2:2.3.232 clean shadowJar -polaris> java -jar dropwizard/service/build/libs/polaris-dropwizard-service-*.jar server ./polaris-server.yml +polaris> java -jar quarkus/service/build/quarkus-app/quarkus-run.jar ``` ### Postgres @@ -108,5 +108,5 @@ The following shows a sample configuration for integrating Polaris with Postgres To build Polaris with the necessary Postgres dependency and start the Polaris service, run the following: ```bash polaris> ./gradlew --no-daemon --info -PeclipseLink=true -PeclipseLinkDeps=org.postgresql:postgresql:42.7.4 clean shadowJar -polaris> java -jar dropwizard/service/build/libs/polaris-dropwizard-service-*.jar server ./polaris-server.yml +polaris> java -jar quarkus/service/build/quarkus-app/quarkus-run.jar ``` \ No newline at end of file