Skip to content

Commit

Permalink
transition away from StorageAdapter (#16985)
Browse files Browse the repository at this point in the history
* transition away from StorageAdapter
changes:
* CursorHolderFactory has been renamed to CursorFactory and moved off of StorageAdapter, instead fetched directly from the segment via 'asCursorFactory'. The previous deprecated CursorFactory interface has been merged into StorageAdapter
* StorageAdapter is no longer used by any engines or tests and has been marked as deprecated with default implementations of all methods that throw exceptions indicating the new methods to call instead
* StorageAdapter methods not covered by CursorFactory (CursorHolderFactory prior to this change) have been moved into interfaces which are retrieved by Segment.as, the primary classes are the previously existing Metadata, as well as new interfaces PhysicalSegmentInspector and TopNOptimizationInspector
* added UnnestSegment and FilteredSegment that extend WrappedSegmentReference since their StorageAdapter implementations were previously provided by WrappedSegmentReference
* added PhysicalSegmentInspector which covers some of the previous StorageAdapter functionality which was primarily used for segment metadata queries and other metadata uses, and is implemented for QueryableIndexSegment and IncrementalIndexSegment
* added TopNOptimizationInspector to cover the oddly specific StorageAdapter.hasBuiltInFilters implementation, which is implemented for HashJoinSegment, UnnestSegment, and FilteredSegment
* Updated all engines and tests to no longer use StorageAdapter
clintropolis authored Sep 9, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent f4261c0 commit f57cd6f
Showing 242 changed files with 3,293 additions and 3,722 deletions.
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.generator.GeneratorColumnSchema;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -161,9 +161,8 @@ public void queryUsingNative(Blackhole blackhole)

private double compute(final Function<ColumnSelectorFactory, BufferAggregator> aggregatorFactory)
{
final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);

try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();

final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory());
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.generator.GeneratorColumnSchema;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -148,7 +148,9 @@ public void expressionFilter(Blackhole blackhole)
final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
.setFilter(expressionFilter.toFilter())
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);

try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();


@@ -166,7 +168,9 @@ public void nativeFilter(Blackhole blackhole)
final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
.setFilter(nativeFilter.toFilter())
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);

try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
while (!cursor.isDone()) {
Original file line number Diff line number Diff line change
@@ -42,9 +42,8 @@
import org.apache.druid.segment.Cursors;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
@@ -160,7 +159,8 @@ public void timeFloorUsingExpression(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();

final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
@@ -171,7 +171,8 @@ public void timeFloorUsingExpression(Blackhole blackhole)
@Benchmark
public void timeFloorUsingExtractionFn(Blackhole blackhole)
{
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();

final DimensionSelector selector = cursor
@@ -190,15 +191,15 @@ public void timeFloorUsingExtractionFn(Blackhole blackhole)
@Benchmark
public void timeFloorUsingCursor(Blackhole blackhole)
{
final StorageAdapter adapter = new QueryableIndexStorageAdapter(index);
try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();
final CursorGranularizer granularizer = CursorGranularizer.create(
cursor,
QueryableIndexTimeBoundaryInspector.create(index),
Cursors.getTimeOrdering(index.getOrdering()),
Granularities.HOUR,
adapter.getInterval()
index.getDataInterval()
);
final Sequence<Long> results =
Sequences.simple(granularizer.getBucketIterable())
@@ -241,7 +242,8 @@ public void timeFormatUsingExpression(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(
DefaultDimensionSpec.of("v")
@@ -253,7 +255,8 @@ public void timeFormatUsingExpression(Blackhole blackhole)
@Benchmark
public void timeFormatUsingExtractionFn(Blackhole blackhole)
{
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor
.getColumnSelectorFactory()
@@ -284,7 +287,8 @@ public void strlenUsingExpressionAsLong(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -307,7 +311,8 @@ public void strlenUsingExpressionAsString(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor
.getColumnSelectorFactory()
@@ -320,7 +325,8 @@ public void strlenUsingExpressionAsString(Blackhole blackhole)
@Benchmark
public void strlenUsingExtractionFn(Blackhole blackhole)
{
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor
.getColumnSelectorFactory()
@@ -346,7 +352,8 @@ public void arithmeticOnLong(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -368,7 +375,8 @@ public void stringConcatAndCompareOnLong(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -390,7 +398,8 @@ public void caseSearched1(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -412,7 +421,8 @@ public void caseSearched2(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -447,7 +457,8 @@ public void caseSearched100(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -476,7 +487,8 @@ public void caseSearchedWithLookup(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
@@ -513,7 +525,8 @@ public void caseSearchedWithLookup2(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -155,7 +155,7 @@ public void scan(Blackhole blackhole)
.setVirtualColumns(virtualColumns)
.build();
final CursorHolder cursorHolder = closer.register(
new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)
new QueryableIndexCursorFactory(index).makeCursorHolder(buildSpec)
);
if (vectorize) {
VectorCursor cursor = cursorHolder.asVectorCursor();
Loading

0 comments on commit f57cd6f

Please sign in to comment.