diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java index 36c51c09d53..749ba813e20 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.metadata.schema.SchemaProvider; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.PlanCacheTable; import org.apache.drill.exec.store.PluginRulesProviderImpl; import org.apache.drill.exec.store.StoragePluginRulesSupplier; import org.apache.drill.exec.store.dfs.FileSelection; @@ -41,6 +42,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.hadoop.HadoopTables; import java.io.IOException; import java.util.Collections; @@ -104,6 +109,27 @@ public boolean supportsRead() { return true; } + @Override + public boolean supportPlanCache(FileSelection selection) { + return selection != null && config.getSnapshot() == null + && !(selection instanceof IcebergMetadataFileSelection) && selection.getSelectionRoot() != null; + } + + @Override + public PlanCacheTable planCacheTable(FileSelection selection) throws IOException { + if (!supportPlanCache(selection)) { + return null; + } + String identifier = selection.getSelectionRoot().toString(); + Table table = new HadoopTables(fsConf).load(identifier); + if (!(table instanceof HasTableOperations)) { + return null; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + return metadata == null || metadata.uuid() == null ? null + : new PlanCacheTable(identifier, metadata.uuid() + ":" + metadata.currentSchemaId()); + } + @Override public boolean supportsWrite() { return false; diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java index 3449935e4a1..4eeb8a9502d 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.store.iceberg.plan; import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; @@ -32,6 +34,46 @@ public class DrillExprToIcebergTranslator extends AbstractExprVisitor INSTANCE = new DrillExprToIcebergTranslator(); + @Override + public Expression visitBooleanOperator(BooleanOperator op, Void value) { + Expression result = null; + for (LogicalExpression argument : op.args()) { + Expression next = argument.accept(this, value); + if (next == null) { + return null; + } + result = result == null ? next : FunctionNames.AND.equals(op.getName()) + ? Expressions.and(result, next) : Expressions.or(result, next); + } + return result; + } + + @Override + public Expression visitCastExpression(CastExpression cast, Void value) { + Expression input = cast.getInput().accept(this, value); + if (!(input instanceof ConstantExpression)) { + return null; + } + Object literal = ((ConstantExpression) input).getValue(); + if (!(literal instanceof Integer) && !(literal instanceof Long)) { + return null; + } + long millis = ((Number) literal).longValue(); + // Drill serializes temporal constants as casts of epoch milliseconds. + // Iceberg's DATE uses days; TIME and TIMESTAMP use microseconds. + switch (cast.getMajorType().getMinorType()) { + case BIGINT: + return new ConstantExpression<>(millis); + case DATE: + return new ConstantExpression<>(Math.toIntExact(Math.floorDiv(millis, 86_400_000L))); + case TIME: + case TIMESTAMP: + return new ConstantExpression<>(Math.multiplyExact(millis, 1_000L)); + default: + return null; + } + } + @Override public Expression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { switch (call.getName()) { @@ -103,7 +145,7 @@ public Expression visitFunctionCall(FunctionCall call, Void value) throws Runtim } case FunctionNames.GE: { LogicalExpression nameRef = call.args().get(0); - Expression expression = call.args().get(0).accept(this, null); + Expression expression = call.args().get(1).accept(this, null); if (nameRef instanceof SchemaPath && expression instanceof ConstantExpression) { String name = IcebergGroupScan.getPath((SchemaPath) nameRef); return Expressions.greaterThanOrEqual(name, ((ConstantExpression) expression).getValue()); @@ -174,17 +216,17 @@ public Expression visitVarDecimalConstant(ValueExpressions.VarDecimalExpression @Override public Expression visitDateConstant(ValueExpressions.DateExpression dateExpr, Void value) throws RuntimeException { - return new ConstantExpression<>(dateExpr.getDate()); + return new ConstantExpression<>(Math.toIntExact(Math.floorDiv(dateExpr.getDate(), 86_400_000L))); } @Override public Expression visitTimeConstant(ValueExpressions.TimeExpression timeExpr, Void value) throws RuntimeException { - return new ConstantExpression<>(timeExpr.getTime()); + return new ConstantExpression<>((long) timeExpr.getTime() * 1_000L); } @Override public Expression visitTimeStampConstant(ValueExpressions.TimeStampExpression timestampExpr, Void value) throws RuntimeException { - return new ConstantExpression<>(timestampExpr.getTimeStamp()); + return new ConstantExpression<>(Math.multiplyExact(timestampExpr.getTimeStamp(), 1_000L)); } @Override diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java index a4863273d89..2606d44b42f 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java @@ -25,6 +25,8 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; +import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.FunctionCallFactory; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.base.GroupScan; @@ -46,6 +48,7 @@ import java.io.IOException; import java.math.BigDecimal; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -67,6 +70,10 @@ public void implement(PluginFilterRel filter) throws IOException { new DrillParseContext(PrelUtil.getPlannerSettings(filter.getCluster().getPlanner())), filter.getInput(), condition); + if (groupScan.getCondition() != null) { + expression = FunctionCallFactory.createBooleanOperator(FunctionNames.AND, + Arrays.asList(groupScan.getCondition(), expression)); + } groupScan = groupScan.toBuilder().condition(expression).build(); } diff --git a/contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java b/contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java new file mode 100644 index 00000000000..eb658650dc4 --- /dev/null +++ b/contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.iceberg; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.apache.drill.common.config.DrillProperties; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.sql.PlanCache; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.iceberg.format.IcebergFormatPluginConfig; +import org.apache.drill.test.ClientFixture; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.TestBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.GenericAppenderFactory; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class IcebergPlanCacheTest extends ClusterTest { + private static FileSystemConfig config; + private final HadoopTables tables = new HadoopTables(new Configuration()); + private Table table; + private String name; + + @BeforeClass + public static void setupCluster() throws Exception { + startCluster(ClusterFixture.builder(dirTestWatcher)); + FileSystemConfig original = (FileSystemConfig) cluster.drillbit().getContext() + .getStorage().getPlugin("dfs").getConfig(); + Map formats = new HashMap<>(original.getFormats()); + formats.put("iceberg", IcebergFormatPluginConfig.builder().build()); + config = original.copyWithFormats(formats); + } + + @Before + public void setupTable() throws Exception { + cluster.drillbit().getContext().getStorage().put("dfs", config); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + cache().awaitWrites(); + cache().clear(); + name = "cache_" + UUID.randomUUID().toString().replace("-", ""); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "label", Types.StringType.get())); + table = tables.create(schema, dirTestWatcher.getDfsTestTmpDir().toPath().resolve(name).toString()); + append(1, "one", 2, "two", 3, "three"); + } + + @Test + public void testDifferentValuesAcrossConnectionsAndEmptyHit() throws Exception { + try (ClientFixture first = cluster.clientBuilder().property(DrillProperties.USER, "iceberg-cache").build(); + ClientFixture second = cluster.clientBuilder().property(DrillProperties.USER, "iceberg-cache").build()) { + String sql = pointSql(1); + first.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + rows(first, sql, 1L, "one"); + first.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + second.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + rows(first, sql, 1L, "one"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(second, pointSql(2), 2L, "two"); + assertEquals(hits + 1, cache().getHitCount()); + // Keep the literal precision unchanged so this tests an empty cache hit. + rows(second, pointSql(9)); + assertEquals(hits + 2, cache().getHitCount()); + } + } + + @Test + public void testRangePruningRebuiltForNewValues() throws Exception { + rows(client, rangeSql(1, 2), 1L, "one", 2L, "two"); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + rows(client, rangeSql(1, 2), 1L, "one", 2L, "two"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(client, rangeSql(2, 3), 2L, "two", 3L, "three"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testCompatibleAppendVisibleOnHit() throws Exception { + warmPoint(); + append(4, "four"); + long hits = cache().getHitCount(); + rows(client, pointSql(4), 4L, "four"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testSchemaChangeInvalidatesAndReplans() throws Exception { + warmPoint(); + table.updateSchema().addColumn("extra", Types.StringType.get()).commit(); + append(4, "four"); + long hits = cache().getHitCount(); + long invalidations = invalidations(); + rows(client, pointSql(4), 4L, "four"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(invalidations + 1, invalidations()); + rows(client, pointSql(2), 2L, "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testReplacementAtSamePathInvalidatesAndReadsNewTable() throws Exception { + warmPoint(); + Schema schema = table.schema(); + String location = table.location(); + tables.dropTable(location, true); + table = tables.create(schema, location); + append(1, "replacement"); + long hits = cache().getHitCount(); + long invalidations = invalidations(); + rows(client, pointSql(1), 1L, "replacement"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(invalidations + 1, invalidations()); + rows(client, pointSql(1), 1L, "replacement"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testPluginConfigurationsKeepSeparateEntries() throws Exception { + warmPoint(); + Map formats = new HashMap<>(config.getFormats()); + formats.put("iceberg", IcebergFormatPluginConfig.builder().includeColumnStats(true).build()); + cluster.drillbit().getContext().getStorage().put("dfs", config.copyWithFormats(formats)); + long hits = cache().getHitCount(); + rows(client, pointSql(2), 2L, "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(client, pointSql(3), 3L, "three"); + assertEquals(hits + 1, cache().getHitCount()); + cluster.drillbit().getContext().getStorage().put("dfs", config); + rows(client, pointSql(1), 1L, "one"); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testJoinWithUnsupportedJsonBypassesCache() throws Exception { + String json = name + ".json"; + Files.write(dirTestWatcher.getDfsTestTmpDir().toPath().resolve(json), + Arrays.asList("{\"id\":1}", "{\"id\":2}"), StandardCharsets.UTF_8); + String sql = "SELECT t.id, t.label FROM dfs.tmp.`" + name + "` t JOIN dfs.tmp.`" + json + + "` j ON t.id = j.id WHERE t.id = %d"; + rows(client, String.format(sql, 1), 1L, "one"); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + rows(client, String.format(sql, 1), 1L, "one"); + rows(client, String.format(sql, 2), 2L, "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(0L, (long) DrillMetrics.getRegistry().getGauges() + .get("drill.plan_cache.entries").getValue()); + } + + private void warmPoint() throws Exception { + rows(client, pointSql(1), 1L, "one"); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + rows(client, pointSql(1), 1L, "one"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(client, pointSql(2), 2L, "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + private String pointSql(long id) { + return "SELECT id, label FROM dfs.tmp.`" + name + "` WHERE id = " + id; + } + + private String rangeSql(long start, long end) { + return "SELECT id, label FROM dfs.tmp.`" + name + "` WHERE id >= " + start + " AND id <= " + end; + } + + private void rows(ClientFixture target, String sql, Object... values) throws Exception { + TestBuilder builder = target.testBuilder().sqlQuery(sql).unOrdered(); + if (values.length == 0) { + builder.expectsEmptyResultSet(); + } else { + builder.baselineColumns("id", "label"); + for (int i = 0; i < values.length; i += 2) { + builder.baselineValues(values[i], values[i + 1]); + } + } + builder.go(); + } + + private void append(Object... values) throws IOException { + OutputFile output = table.io().newOutputFile(table.location() + "/" + + FileFormat.PARQUET.addExtension(UUID.randomUUID().toString())); + FileAppender appender = new GenericAppenderFactory(table.schema()).newAppender(output, FileFormat.PARQUET); + try (FileAppender closeable = appender) { + for (int i = 0; i < values.length; i += 2) { + Record record = GenericRecord.create(table.schema()); + record.setField("id", ((Number) values[i]).longValue()); + record.setField("label", values[i + 1]); + closeable.add(record); + } + } + table.newAppend().appendFile(DataFiles.builder(table.spec()).withInputFile(output.toInputFile()) + .withMetrics(appender.metrics()).build()).commit(); + } + + private PlanCache cache() { + return cluster.drillbit().getContext().getPlanCache(); + } + + private long invalidations() { + return (Long) DrillMetrics.getRegistry().getGauges().get("drill.plan_cache.invalidations").getValue(); + } +} diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc index 273b3fc328b..1c147c07dc4 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc @@ -243,6 +243,7 @@ constexpr QueryProfile::QueryProfile( , planend_(PROTOBUF_LONGLONG(0)) , queuewaitend_(PROTOBUF_LONGLONG(0)) , total_cost_(0) + , plan_cache_hit_(false) , type_(1) {} struct QueryProfileDefaultTypeInternal { @@ -583,8 +584,9 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_UserBitShared_2eproto::offsets PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, queryid_), PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, autolimit_), PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, scanned_plugins_), + PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, plan_cache_hit_), 10, - 21, + 22, 12, 13, 0, @@ -607,6 +609,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_UserBitShared_2eproto::offsets 9, 17, ~0u, + 21, PROTOBUF_FIELD_OFFSET(::exec::shared::MajorFragmentProfile, _has_bits_), PROTOBUF_FIELD_OFFSET(::exec::shared::MajorFragmentProfile, _internal_metadata_), ~0u, // no _extensions_ @@ -729,15 +732,15 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 124, 132, sizeof(::exec::shared::QueryResult)}, { 135, 144, sizeof(::exec::shared::QueryData)}, { 148, 161, sizeof(::exec::shared::QueryInfo)}, - { 169, 198, sizeof(::exec::shared::QueryProfile)}, - { 222, 229, sizeof(::exec::shared::MajorFragmentProfile)}, - { 231, 247, sizeof(::exec::shared::MinorFragmentProfile)}, - { 258, 272, sizeof(::exec::shared::OperatorProfile)}, - { 281, 289, sizeof(::exec::shared::StreamProfile)}, - { 292, 300, sizeof(::exec::shared::MetricValue)}, - { 303, -1, sizeof(::exec::shared::Registry)}, - { 309, 316, sizeof(::exec::shared::Jar)}, - { 318, 326, sizeof(::exec::shared::SaslMessage)}, + { 169, 199, sizeof(::exec::shared::QueryProfile)}, + { 224, 231, sizeof(::exec::shared::MajorFragmentProfile)}, + { 233, 249, sizeof(::exec::shared::MinorFragmentProfile)}, + { 260, 274, sizeof(::exec::shared::OperatorProfile)}, + { 283, 291, sizeof(::exec::shared::StreamProfile)}, + { 294, 302, sizeof(::exec::shared::MetricValue)}, + { 305, -1, sizeof(::exec::shared::Registry)}, + { 311, 318, sizeof(::exec::shared::Jar)}, + { 320, 328, sizeof(::exec::shared::SaslMessage)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -823,7 +826,7 @@ const char descriptor_table_protodef_UserBitShared_2eproto[] PROTOBUF_SECTION_VA "red.QueryResult.QueryState\022\017\n\004user\030\004 \001(\t" ":\001-\022\'\n\007foreman\030\005 \001(\0132\026.exec.DrillbitEndp" "oint\022\024\n\014options_json\030\006 \001(\t\022\022\n\ntotal_cost" - "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\337\004\n\014QueryP" + "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\367\004\n\014QueryP" "rofile\022 \n\002id\030\001 \001(\0132\024.exec.shared.QueryId" "\022$\n\004type\030\002 \001(\0162\026.exec.shared.QueryType\022\r" "\n\005start\030\003 \001(\003\022\013\n\003end\030\004 \001(\003\022\r\n\005query\030\005 \001(" @@ -838,46 +841,46 @@ const char descriptor_table_protodef_UserBitShared_2eproto[] PROTOBUF_SECTION_VA "tions_json\030\021 \001(\t\022\017\n\007planEnd\030\022 \001(\003\022\024\n\014que" "ueWaitEnd\030\023 \001(\003\022\022\n\ntotal_cost\030\024 \001(\001\022\025\n\nq" "ueue_name\030\025 \001(\t:\001-\022\017\n\007queryId\030\026 \001(\t\022\021\n\ta" - "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\"" - "t\n\024MajorFragmentProfile\022\031\n\021major_fragmen" - "t_id\030\001 \001(\005\022A\n\026minor_fragment_profile\030\002 \003" - "(\0132!.exec.shared.MinorFragmentProfile\"\350\002" - "\n\024MinorFragmentProfile\022)\n\005state\030\001 \001(\0162\032." - "exec.shared.FragmentState\022(\n\005error\030\002 \001(\013" - "2\031.exec.shared.DrillPBError\022\031\n\021minor_fra" - "gment_id\030\003 \001(\005\0226\n\020operator_profile\030\004 \003(\013" - "2\034.exec.shared.OperatorProfile\022\022\n\nstart_" - "time\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_u" - "sed\030\007 \001(\003\022\027\n\017max_memory_used\030\010 \001(\003\022(\n\010en" - "dpoint\030\t \001(\0132\026.exec.DrillbitEndpoint\022\023\n\013" - "last_update\030\n \001(\003\022\025\n\rlast_progress\030\013 \001(\003" - "\"\237\002\n\017OperatorProfile\0221\n\rinput_profile\030\001 " - "\003(\0132\032.exec.shared.StreamProfile\022\023\n\013opera" - "tor_id\030\003 \001(\005\022\031\n\roperator_type\030\004 \001(\005B\002\030\001\022" - "\023\n\013setup_nanos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 " - "\001(\003\022#\n\033peak_local_memory_allocated\030\007 \001(\003" - "\022(\n\006metric\030\010 \003(\0132\030.exec.shared.MetricVal" - "ue\022\022\n\nwait_nanos\030\t \001(\003\022\032\n\022operator_type_" - "name\030\n \001(\t\"B\n\rStreamProfile\022\017\n\007records\030\001" - " \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J" - "\n\013MetricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong" - "_value\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001\")\n\010Re" - "gistry\022\035\n\003jar\030\001 \003(\0132\020.exec.shared.Jar\"/\n" - "\003Jar\022\014\n\004name\030\001 \001(\t\022\032\n\022function_signature" - "\030\002 \003(\t\"W\n\013SaslMessage\022\021\n\tmechanism\030\001 \001(\t" - "\022\014\n\004data\030\002 \001(\014\022\'\n\006status\030\003 \001(\0162\027.exec.sh" - "ared.SaslStatus*5\n\nRpcChannel\022\017\n\013BIT_CON" - "TROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004USER\020\002*V\n\tQueryT" - "ype\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022" - "\r\n\tEXECUTION\020\004\022\026\n\022PREPARED_STATEMENT\020\005*\207" - "\001\n\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAITIN" - "G_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020" - "\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLA" - "TION_REQUESTED\020\006*g\n\nSaslStatus\022\020\n\014SASL_U" - "NKNOWN\020\000\022\016\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROG" - "RESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020" - "\004B.\n\033org.apache.drill.exec.protoB\rUserBi" - "tSharedH\001" + "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\022" + "\026\n\016plan_cache_hit\030\031 \001(\010\"t\n\024MajorFragment" + "Profile\022\031\n\021major_fragment_id\030\001 \001(\005\022A\n\026mi" + "nor_fragment_profile\030\002 \003(\0132!.exec.shared" + ".MinorFragmentProfile\"\350\002\n\024MinorFragmentP" + "rofile\022)\n\005state\030\001 \001(\0162\032.exec.shared.Frag" + "mentState\022(\n\005error\030\002 \001(\0132\031.exec.shared.D" + "rillPBError\022\031\n\021minor_fragment_id\030\003 \001(\005\0226" + "\n\020operator_profile\030\004 \003(\0132\034.exec.shared.O" + "peratorProfile\022\022\n\nstart_time\030\005 \001(\003\022\020\n\010en" + "d_time\030\006 \001(\003\022\023\n\013memory_used\030\007 \001(\003\022\027\n\017max" + "_memory_used\030\010 \001(\003\022(\n\010endpoint\030\t \001(\0132\026.e" + "xec.DrillbitEndpoint\022\023\n\013last_update\030\n \001(" + "\003\022\025\n\rlast_progress\030\013 \001(\003\"\237\002\n\017OperatorPro" + "file\0221\n\rinput_profile\030\001 \003(\0132\032.exec.share" + "d.StreamProfile\022\023\n\013operator_id\030\003 \001(\005\022\031\n\r" + "operator_type\030\004 \001(\005B\002\030\001\022\023\n\013setup_nanos\030\005" + " \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033peak_loca" + "l_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(\013" + "2\030.exec.shared.MetricValue\022\022\n\nwait_nanos" + "\030\t \001(\003\022\032\n\022operator_type_name\030\n \001(\t\"B\n\rSt" + "reamProfile\022\017\n\007records\030\001 \001(\003\022\017\n\007batches\030" + "\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013MetricValue\022\021\n" + "\tmetric_id\030\001 \001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014" + "double_value\030\003 \001(\001\")\n\010Registry\022\035\n\003jar\030\001 " + "\003(\0132\020.exec.shared.Jar\"/\n\003Jar\022\014\n\004name\030\001 \001" + "(\t\022\032\n\022function_signature\030\002 \003(\t\"W\n\013SaslMe" + "ssage\022\021\n\tmechanism\030\001 \001(\t\022\014\n\004data\030\002 \001(\014\022\'" + "\n\006status\030\003 \001(\0162\027.exec.shared.SaslStatus*" + "5\n\nRpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DA" + "TA\020\001\022\010\n\004USER\020\002*V\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007" + "LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022\r\n\tEXECUTION\020\004\022\026" + "\n\022PREPARED_STATEMENT\020\005*\207\001\n\rFragmentState" + "\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013" + "\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004" + "\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATION_REQUESTED\020\006" + "*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL" + "_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_S" + "UCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.apache." + "drill.exec.protoB\rUserBitSharedH\001" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_UserBitShared_2eproto_deps[3] = { &::descriptor_table_Coordination_2eproto, @@ -886,7 +889,7 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor }; static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_UserBitShared_2eproto_once; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_UserBitShared_2eproto = { - false, false, 4449, descriptor_table_protodef_UserBitShared_2eproto, "UserBitShared.proto", + false, false, 4473, descriptor_table_protodef_UserBitShared_2eproto, "UserBitShared.proto", &descriptor_table_UserBitShared_2eproto_once, descriptor_table_UserBitShared_2eproto_deps, 3, 22, schemas, file_default_instances, TableStruct_UserBitShared_2eproto::offsets, file_level_metadata_UserBitShared_2eproto, file_level_enum_descriptors_UserBitShared_2eproto, file_level_service_descriptors_UserBitShared_2eproto, @@ -5427,7 +5430,7 @@ class QueryProfile::_Internal { (*has_bits)[0] |= 1024u; } static void set_has_type(HasBits* has_bits) { - (*has_bits)[0] |= 2097152u; + (*has_bits)[0] |= 4194304u; } static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 4096u; @@ -5490,6 +5493,9 @@ class QueryProfile::_Internal { static void set_has_autolimit(HasBits* has_bits) { (*has_bits)[0] |= 131072u; } + static void set_has_plan_cache_hit(HasBits* has_bits) { + (*has_bits)[0] |= 2097152u; + } }; const ::exec::shared::QueryId& @@ -5599,8 +5605,8 @@ queue_name_.UnsafeSetDefault(nullptr); queryid_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(reinterpret_cast(this) + static_cast( reinterpret_cast(&id_) - reinterpret_cast(this)), - 0, static_cast(reinterpret_cast(&total_cost_) - - reinterpret_cast(&id_)) + sizeof(total_cost_)); + 0, static_cast(reinterpret_cast(&plan_cache_hit_) - + reinterpret_cast(&id_)) + sizeof(plan_cache_hit_)); type_ = 1; } @@ -5692,10 +5698,10 @@ void QueryProfile::Clear() { reinterpret_cast(&total_fragments_) - reinterpret_cast(&start_)) + sizeof(total_fragments_)); } - if (cached_has_bits & 0x003f0000u) { + if (cached_has_bits & 0x007f0000u) { ::memset(&finished_fragments_, 0, static_cast( - reinterpret_cast(&total_cost_) - - reinterpret_cast(&finished_fragments_)) + sizeof(total_cost_)); + reinterpret_cast(&plan_cache_hit_) - + reinterpret_cast(&finished_fragments_)) + sizeof(plan_cache_hit_)); type_ = 1; } _has_bits_.Clear(); @@ -5949,6 +5955,14 @@ const char* QueryProfile::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_I } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<194>(ptr)); } else goto handle_unusual; continue; + // optional bool plan_cache_hit = 25; + case 25: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 200)) { + _Internal::set_has_plan_cache_hit(&has_bits); + plan_cache_hit_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { @@ -5989,7 +6003,7 @@ ::PROTOBUF_NAMESPACE_ID::uint8* QueryProfile::_InternalSerialize( } // optional .exec.shared.QueryType type = 2; - if (cached_has_bits & 0x00200000u) { + if (cached_has_bits & 0x00400000u) { target = stream->EnsureSpace(target); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray( 2, this->_internal_type(), target); @@ -6176,6 +6190,12 @@ ::PROTOBUF_NAMESPACE_ID::uint8* QueryProfile::_InternalSerialize( target = stream->WriteString(24, s, target); } + // optional bool plan_cache_hit = 25; + if (cached_has_bits & 0x00200000u) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(25, this->_internal_plan_cache_hit(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); @@ -6323,7 +6343,7 @@ size_t QueryProfile::ByteSizeLong() const { } } - if (cached_has_bits & 0x003f0000u) { + if (cached_has_bits & 0x007f0000u) { // optional int32 finished_fragments = 10; if (cached_has_bits & 0x00010000u) { total_size += 1 + @@ -6357,8 +6377,13 @@ size_t QueryProfile::ByteSizeLong() const { total_size += 2 + 8; } - // optional .exec.shared.QueryType type = 2; + // optional bool plan_cache_hit = 25; if (cached_has_bits & 0x00200000u) { + total_size += 2 + 1; + } + + // optional .exec.shared.QueryType type = 2; + if (cached_has_bits & 0x00400000u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->_internal_type()); } @@ -6451,7 +6476,7 @@ void QueryProfile::MergeFrom(const QueryProfile& from) { } _has_bits_[0] |= cached_has_bits; } - if (cached_has_bits & 0x003f0000u) { + if (cached_has_bits & 0x007f0000u) { if (cached_has_bits & 0x00010000u) { finished_fragments_ = from.finished_fragments_; } @@ -6468,6 +6493,9 @@ void QueryProfile::MergeFrom(const QueryProfile& from) { total_cost_ = from.total_cost_; } if (cached_has_bits & 0x00200000u) { + plan_cache_hit_ = from.plan_cache_hit_; + } + if (cached_has_bits & 0x00400000u) { type_ = from.type_; } _has_bits_[0] |= cached_has_bits; @@ -6509,8 +6537,8 @@ void QueryProfile::InternalSwap(QueryProfile* other) { queue_name_.Swap(&other->queue_name_, nullptr, GetArena()); queryid_.Swap(&other->queryid_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::memswap< - PROTOBUF_FIELD_OFFSET(QueryProfile, total_cost_) - + sizeof(QueryProfile::total_cost_) + PROTOBUF_FIELD_OFFSET(QueryProfile, plan_cache_hit_) + + sizeof(QueryProfile::plan_cache_hit_) - PROTOBUF_FIELD_OFFSET(QueryProfile, id_)>( reinterpret_cast(&id_), reinterpret_cast(&other->id_)); diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h index ffbc7cffa84..b963dcd0ab5 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.h +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h @@ -3202,6 +3202,7 @@ class QueryProfile PROTOBUF_FINAL : kPlanEndFieldNumber = 18, kQueueWaitEndFieldNumber = 19, kTotalCostFieldNumber = 20, + kPlanCacheHitFieldNumber = 25, kTypeFieldNumber = 2, }; // repeated .exec.shared.MajorFragmentProfile fragment_profile = 11; @@ -3579,6 +3580,19 @@ class QueryProfile PROTOBUF_FINAL : void _internal_set_total_cost(double value); public: + // optional bool plan_cache_hit = 25; + bool has_plan_cache_hit() const; + private: + bool _internal_has_plan_cache_hit() const; + public: + void clear_plan_cache_hit(); + bool plan_cache_hit() const; + void set_plan_cache_hit(bool value); + private: + bool _internal_plan_cache_hit() const; + void _internal_set_plan_cache_hit(bool value); + public: + // optional .exec.shared.QueryType type = 2; bool has_type() const; private: @@ -3626,6 +3640,7 @@ class QueryProfile PROTOBUF_FINAL : ::PROTOBUF_NAMESPACE_ID::int64 planend_; ::PROTOBUF_NAMESPACE_ID::int64 queuewaitend_; double total_cost_; + bool plan_cache_hit_; int type_; friend struct ::TableStruct_UserBitShared_2eproto; }; @@ -7808,7 +7823,7 @@ inline void QueryProfile::set_allocated_id(::exec::shared::QueryId* id) { // optional .exec.shared.QueryType type = 2; inline bool QueryProfile::_internal_has_type() const { - bool value = (_has_bits_[0] & 0x00200000u) != 0; + bool value = (_has_bits_[0] & 0x00400000u) != 0; return value; } inline bool QueryProfile::has_type() const { @@ -7816,7 +7831,7 @@ inline bool QueryProfile::has_type() const { } inline void QueryProfile::clear_type() { type_ = 1; - _has_bits_[0] &= ~0x00200000u; + _has_bits_[0] &= ~0x00400000u; } inline ::exec::shared::QueryType QueryProfile::_internal_type() const { return static_cast< ::exec::shared::QueryType >(type_); @@ -7827,7 +7842,7 @@ inline ::exec::shared::QueryType QueryProfile::type() const { } inline void QueryProfile::_internal_set_type(::exec::shared::QueryType value) { assert(::exec::shared::QueryType_IsValid(value)); - _has_bits_[0] |= 0x00200000u; + _has_bits_[0] |= 0x00400000u; type_ = value; } inline void QueryProfile::set_type(::exec::shared::QueryType value) { @@ -8852,6 +8867,34 @@ QueryProfile::mutable_scanned_plugins() { return &scanned_plugins_; } +// optional bool plan_cache_hit = 25; +inline bool QueryProfile::_internal_has_plan_cache_hit() const { + bool value = (_has_bits_[0] & 0x00200000u) != 0; + return value; +} +inline bool QueryProfile::has_plan_cache_hit() const { + return _internal_has_plan_cache_hit(); +} +inline void QueryProfile::clear_plan_cache_hit() { + plan_cache_hit_ = false; + _has_bits_[0] &= ~0x00200000u; +} +inline bool QueryProfile::_internal_plan_cache_hit() const { + return plan_cache_hit_; +} +inline bool QueryProfile::plan_cache_hit() const { + // @@protoc_insertion_point(field_get:exec.shared.QueryProfile.plan_cache_hit) + return _internal_plan_cache_hit(); +} +inline void QueryProfile::_internal_set_plan_cache_hit(bool value) { + _has_bits_[0] |= 0x00200000u; + plan_cache_hit_ = value; +} +inline void QueryProfile::set_plan_cache_hit(bool value) { + _internal_set_plan_cache_hit(value); + // @@protoc_insertion_point(field_set:exec.shared.QueryProfile.plan_cache_hit) +} + // ------------------------------------------------------------------- // MajorFragmentProfile diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java index d4867da24ca..f9bebaae3f3 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java @@ -39,6 +39,9 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.parser.LogicalExpressionParser; +import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.GroupScan; @@ -84,6 +87,11 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst private HBaseScanSpec hbaseScanSpec; + /** Selection before predicate pushdown, needed to rebuild value-dependent HBase filters. */ + private HBaseScanSpec baseScanSpec; + private LogicalExpression pushedFilter; + private boolean pushedFilterFullyConverted; + private HBaseStoragePlugin storagePlugin; private final Stopwatch watch = Stopwatch.createUnstarted(); @@ -108,18 +116,44 @@ public HBaseGroupScan(@JsonProperty("userName") String userName, @JsonProperty("storage") HBaseStoragePluginConfig storagePluginConfig, @JsonProperty("columns") List columns, @JsonProperty("maxRecords") int maxRecords, + @JsonProperty("baseScanSpec") HBaseScanSpec baseScanSpec, + @JsonProperty("pushedFilter") String pushedFilter, + @JsonProperty("pushedFilterFullyConverted") boolean pushedFilterFullyConverted, @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { - this (userName, pluginRegistry.resolve(storagePluginConfig, HBaseStoragePlugin.class), hbaseScanSpec, columns, maxRecords); + this(userName, pluginRegistry.resolve(storagePluginConfig, HBaseStoragePlugin.class), + hbaseScanSpec, columns, maxRecords, baseScanSpec, + pushedFilter == null ? null : LogicalExpressionParser.parse(pushedFilter), + pushedFilterFullyConverted, true); } public HBaseGroupScan(String userName, HBaseStoragePlugin storagePlugin, HBaseScanSpec scanSpec, List columns, int maxRecords) { + this(userName, storagePlugin, scanSpec, columns, maxRecords, null, null, false, false); + } + + HBaseGroupScan(String userName, HBaseStoragePlugin storagePlugin, HBaseScanSpec scanSpec, + List columns, int maxRecords, HBaseScanSpec baseScanSpec, + LogicalExpression pushedFilter, boolean pushedFilterFullyConverted, boolean rebuild) { super(userName); this.storagePlugin = storagePlugin; this.storagePluginConfig = storagePlugin.getConfig(); - this.hbaseScanSpec = scanSpec; + this.baseScanSpec = baseScanSpec; + this.pushedFilter = pushedFilter; + this.pushedFilterFullyConverted = pushedFilterFullyConverted; + this.hbaseScanSpec = rebuild && pushedFilter != null ? baseScanSpec : scanSpec; this.columns = columns == null ? ALL_COLUMNS : columns; this.maxRecords = maxRecords; + if (rebuild && pushedFilter != null) { + if (baseScanSpec == null) { + throw new IllegalArgumentException("Cached HBase filter has no base scan specification"); + } + HBaseFilterBuilder builder = new HBaseFilterBuilder(this, pushedFilter); + HBaseScanSpec rebuilt = builder.parseTree(); + if (rebuilt == null || (pushedFilterFullyConverted && !builder.isAllExpressionsConverted())) { + throw new IllegalArgumentException("Cached HBase filter cannot be rebuilt safely"); + } + this.hbaseScanSpec = rebuilt; + } init(); } @@ -131,6 +165,9 @@ private HBaseGroupScan(HBaseGroupScan that) { super(that); this.columns = that.columns == null ? ALL_COLUMNS : that.columns; this.hbaseScanSpec = that.hbaseScanSpec; + this.baseScanSpec = that.baseScanSpec; + this.pushedFilter = that.pushedFilter; + this.pushedFilterFullyConverted = that.pushedFilterFullyConverted; this.endpointFragmentMapping = that.endpointFragmentMapping; this.regionsToScan = that.regionsToScan; this.storagePlugin = that.storagePlugin; @@ -146,6 +183,9 @@ private HBaseGroupScan(HBaseGroupScan that, int maxRecords) { super(that); this.columns = that.columns == null ? ALL_COLUMNS : that.columns; this.hbaseScanSpec = that.hbaseScanSpec; + this.baseScanSpec = that.baseScanSpec; + this.pushedFilter = that.pushedFilter; + this.pushedFilterFullyConverted = that.pushedFilterFullyConverted; this.endpointFragmentMapping = that.endpointFragmentMapping; this.regionsToScan = that.regionsToScan; this.storagePlugin = that.storagePlugin; @@ -449,6 +489,21 @@ public HBaseScanSpec getHBaseScanSpec() { return hbaseScanSpec; } + @JsonProperty("baseScanSpec") + public HBaseScanSpec getBaseScanSpec() { + return baseScanSpec; + } + + @JsonProperty("pushedFilter") + public String getPushedFilter() { + return pushedFilter == null ? null : ExpressionStringBuilder.toString(pushedFilter); + } + + @JsonProperty("pushedFilterFullyConverted") + public boolean isPushedFilterFullyConverted() { + return pushedFilterFullyConverted; + } + @Override @JsonIgnore public boolean canPushdownProjects(List columns) { diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java index 835d36a8a11..a07126b87e8 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java @@ -122,7 +122,8 @@ protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel fi } final HBaseGroupScan newGroupsScan = new HBaseGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), - newScanSpec, groupScan.getColumns(), groupScan.getMaxRecords()); + newScanSpec, groupScan.getColumns(), groupScan.getMaxRecords(), groupScan.getHBaseScanSpec(), + conditionExp, hbaseFilterBuilder.isAllExpressionsConverted(), false); newGroupsScan.setFilterPushedDown(true); final ScanPrel newScanPrel = new ScanPrel(scan.getCluster(), filter.getTraitSet(), newGroupsScan, scan.getRowType(), scan.getTable()); diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java index 48c4fe4d6d5..53280a545ed 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java @@ -25,13 +25,20 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.planner.PlannerPhase; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.PlanCacheTable; import org.apache.drill.exec.store.SchemaConfig; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hashing; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import com.fasterxml.jackson.core.type.TypeReference; @@ -58,6 +65,26 @@ public boolean supportsRead() { return true; } + @Override + public boolean supportPlanCache(DrillTableSelection selection) { + return selection instanceof HBaseScanSpec; + } + + @Override + public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { + if (!supportPlanCache(selection)) { + return null; + } + String identifier = ((HBaseScanSpec) selection).getTableName(); + try (Table table = getConnection().getTable(TableName.valueOf(identifier))) { + String version = Hashing.sha256().hashBytes( + TableDescriptorBuilder.toByteArray(table.getDescriptor())).toString(); + return new PlanCacheTable(identifier, version); + } catch (TableNotFoundException e) { + return null; + } + } + @Override public HBaseGroupScan getPhysicalScan(String userName, JSONOptions selection) throws IOException { HBaseScanSpec scanSpec = selection.getListWith(new TypeReference() {}); diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index 39c1e3b1f94..60657781887 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -45,6 +45,8 @@ TestHBaseCFAsJSONString.class, TestHBaseConnectionManager.class, TestHBaseFilterPushDown.class, + TestHBasePlanCache.class, + TestHBasePlanCacheMetadata.class, TestHBaseProjectPushDown.class, TestHBaseQueries.class, TestHBaseRegexParser.class, diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java new file mode 100644 index 00000000000..ed2aef3c7d4 --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.apache.drill.categories.HbaseStorageTest; +import org.apache.drill.categories.SlowTest; +import org.apache.drill.exec.planner.sql.PlanCache; +import org.apache.drill.exec.store.hbase.HBaseStoragePluginConfig; +import org.apache.drill.test.TestBuilder; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category({SlowTest.class, HbaseStorageTest.class}) +public class TestHBasePlanCache extends BaseHBaseTest { + private TableName tableName; + + @Before + public void createTable() throws Exception { + test("ALTER SESSION SET `planner.enable_plan_cache` = false"); + cache().awaitWrites(); + cache().clear(); + tableName = TableName.valueOf("plan_cache_" + UUID.randomUUID().toString().replace("-", "")); + HBaseTestsSuite.getAdmin().createTable(TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build()); + put("a", "one"); + put("b", "two"); + put("c", "three"); + } + + @After + public void dropTable() throws Exception { + cache().awaitWrites(); + getDrillbitContext().getStorage().put(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig); + test("ALTER SESSION SET `planner.enable_plan_cache` = false"); + if (HBaseTestsSuite.getAdmin().tableExists(tableName)) { + HBaseTestsSuite.getAdmin().disableTable(tableName); + HBaseTestsSuite.getAdmin().deleteTable(tableName); + } + } + + @Test + public void testPointBoundsReboundAndEmptyHit() throws Exception { + rows(pointSql("a"), "a", "one"); + enableCache(); + long hits = cache().getHitCount(); + rows(pointSql("a"), "a", "one"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(pointSql("b"), "b", "two"); + rows(pointSql("z")); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testRangeBoundsRebuiltForNewValues() throws Exception { + rows(rangeSql("a", "b"), "a", "one", "b", "two"); + enableCache(); + rows(rangeSql("a", "b"), "a", "one", "b", "two"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(rangeSql("b", "c"), "b", "two", "c", "three"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testDataUpdatesVisibleOnHit() throws Exception { + warmPoint(); + put("a", "updated"); + put("d", "four"); + long hits = cache().getHitCount(); + rows(pointSql("a"), "a", "updated"); + rows(pointSql("d"), "d", "four"); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testSchemaChangeInvalidatesAndReplans() throws Exception { + warmPoint(); + HBaseTestsSuite.getAdmin().addColumnFamily(tableName, ColumnFamilyDescriptorBuilder.of("extra")); + long hits = cache().getHitCount(); + rows(pointSql("a"), "a", "one"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(pointSql("b"), "b", "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testPluginConfigurationsKeepSeparateEntries() throws Exception { + warmPoint(); + Map properties = new HashMap<>(storagePluginConfig.getConfig()); + properties.put("hbase.client.operation.timeout", "90000"); + HBaseStoragePluginConfig changed = new HBaseStoragePluginConfig(properties, false); + changed.setEnabled(true); + getDrillbitContext().getStorage().put(HBASE_STORAGE_PLUGIN_NAME, changed); + long hits = cache().getHitCount(); + rows(pointSql("b"), "b", "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(pointSql("c"), "c", "three"); + assertEquals(hits + 1, cache().getHitCount()); + getDrillbitContext().getStorage().put(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig); + rows(pointSql("a"), "a", "one"); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testJoinWithUnsupportedJsonBypassesCache() throws Exception { + String json = tableName.getNameAsString() + ".json"; + Files.write(dirTestWatcher.getDfsTestTmpDir().toPath().resolve(json), + Arrays.asList("{\"k\":\"a\"}", "{\"k\":\"b\"}"), StandardCharsets.UTF_8); + String sql = projection() + " JOIN dfs.tmp.`" + json + + "` j ON CONVERT_FROM(t.row_key, 'UTF8') = j.k WHERE t.row_key = '%s'"; + rows(String.format(sql, "a"), "a", "one"); + enableCache(); + long hits = cache().getHitCount(); + rows(String.format(sql, "a"), "a", "one"); + cache().awaitWrites(); + rows(String.format(sql, "a"), "a", "one"); + rows(String.format(sql, "b"), "b", "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + } + + private void warmPoint() throws Exception { + rows(pointSql("a"), "a", "one"); + enableCache(); + rows(pointSql("a"), "a", "one"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(pointSql("b"), "b", "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + private void enableCache() throws Exception { + test("ALTER SESSION SET `planner.enable_plan_cache` = true"); + } + + private String projection() { + return "SELECT CONVERT_FROM(t.row_key, 'UTF8') AS k, CONVERT_FROM(t.f.v, 'UTF8') AS val FROM hbase.`" + + tableName.getNameAsString() + "` t"; + } + + private String pointSql(String key) { + return projection() + " WHERE t.row_key = '" + key + "'"; + } + + private String rangeSql(String start, String end) { + return projection() + " WHERE t.row_key >= '" + start + "' AND t.row_key <= '" + end + "'"; + } + + private void rows(String sql, Object... values) throws Exception { + TestBuilder builder = testBuilder().sqlQuery(sql).unOrdered(); + if (values.length == 0) { + builder.expectsEmptyResultSet(); + } else { + builder.baselineColumns("k", "val"); + for (int i = 0; i < values.length; i += 2) { + builder.baselineValues(values[i], values[i + 1]); + } + } + builder.go(); + } + + private void put(String key, String value) throws Exception { + try (Table table = HBaseTestsSuite.getConnection().getTable(tableName)) { + table.put(new Put(Bytes.toBytes(key)).addColumn(Bytes.toBytes("f"), Bytes.toBytes("v"), Bytes.toBytes(value))); + } + } + + private PlanCache cache() { + return getDrillbitContext().getPlanCache(); + } +} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java new file mode 100644 index 00000000000..97f645dcffe --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import java.io.IOException; + +import com.google.common.hash.Hashing; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.PlanCacheTable; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.hbase.HBaseStoragePlugin; +import org.apache.drill.exec.store.hbase.HBaseStoragePluginConfig; +import org.apache.drill.test.BaseTest; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class TestHBasePlanCacheMetadata extends BaseTest { + private static final TableName NAME = TableName.valueOf("plan_cache_metadata"); + private final HBaseScanSpec selection = new HBaseScanSpec(NAME.getNameAsString()); + private Connection connection; + private Table table; + private HBaseStoragePlugin plugin; + + @Before + public void setup() throws Exception { + connection = mock(Connection.class); + table = mock(Table.class); + when(connection.getTable(NAME)).thenReturn(table); + plugin = spy(new HBaseStoragePlugin(new HBaseStoragePluginConfig(null, false), + mock(DrillbitContext.class), "hbase")); + doReturn(connection).when(plugin).getConnection(); + } + + @Test + public void testDescriptorFingerprintPreservesExistingCompatibilityVersion() throws Exception { + TableDescriptor descriptor = descriptor("cf1"); + when(table.getDescriptor()).thenReturn(descriptor); + PlanCacheTable metadata = plugin.planCacheTable(selection); + assertEquals(NAME.getNameAsString(), metadata.getIdentifier()); + assertEquals(Hashing.sha256().hashBytes(new HTableDescriptor(descriptor).toByteArray()).toString(), + metadata.getVersion()); + verify(table).getDescriptor(); + verify(table).close(); + verifyNoMoreInteractions(table); + verify(connection).getTable(NAME); + // No Admin handle, separate existence check or connection close is needed. + verifyNoMoreInteractions(connection); + } + + @Test + public void testSchemaChangeChangesCompatibilityVersion() throws Exception { + when(table.getDescriptor()).thenReturn(descriptor("cf1"), descriptor("cf2")); + assertNotEquals(plugin.planCacheTable(selection).getVersion(), + plugin.planCacheTable(selection).getVersion()); + } + + @Test + public void testMissingDescriptorReturnsNoMetadataAndClosesTable() throws Exception { + when(table.getDescriptor()).thenThrow(new TableNotFoundException(NAME)); + assertNull(plugin.planCacheTable(selection)); + verify(table).close(); + } + + @Test + public void testMissingTableHandleReturnsNoMetadata() throws Exception { + when(connection.getTable(NAME)).thenThrow(new TableNotFoundException(NAME)); + assertNull(plugin.planCacheTable(selection)); + verifyNoInteractions(table); + } + + @Test + public void testOtherIoFailuresPropagateAndCloseTable() throws Exception { + IOException failure = new IOException("descriptor read failed"); + when(table.getDescriptor()).thenThrow(failure); + assertSame(failure, assertThrows(IOException.class, () -> plugin.planCacheTable(selection))); + verify(table).close(); + } + + private TableDescriptor descriptor(String family) { + return TableDescriptorBuilder.newBuilder(NAME) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes(family))).build(); + } +} diff --git a/docs/dev/DevDocs.md b/docs/dev/DevDocs.md index eef8105b4d6..03951d2a505 100644 --- a/docs/dev/DevDocs.md +++ b/docs/dev/DevDocs.md @@ -27,3 +27,8 @@ For information about the Jetty 12 upgrade, known limitations, and developer gui ## Materialized Views For information about materialized view support, including SQL syntax, query rewriting, and metastore integration, see [MaterializedViews.md](MaterializedViews.md) + +## Physical Plan Cache + +For cache design, configuration and trade-offs, see [PLAN_CACHE_DESIGN.md](PLAN_CACHE_DESIGN.md). +For storage-plugin integration and scan reconstruction requirements, see [PLAN_CACHE_PLUGIN_GUIDE.md](PLAN_CACHE_PLUGIN_GUIDE.md). diff --git a/docs/dev/PLAN_CACHE_DESIGN.md b/docs/dev/PLAN_CACHE_DESIGN.md new file mode 100644 index 00000000000..14927f345e6 --- /dev/null +++ b/docs/dev/PLAN_CACHE_DESIGN.md @@ -0,0 +1,225 @@ + + +# Physical plan cache + +Drill normally validates and optimizes every SQL query, even when the same query +shape has just been executed. The plan cache reuses that work for repeated reads. +It caches a physical plan, not query results: every execution reads the current +source data. + +The feature is opt-in and disabled by default: + +```sql +ALTER SESSION SET planner.enable_plan_cache = true; +``` + +Use `ALTER SYSTEM` instead of `ALTER SESSION` to set the default for new queries. + +## The basic idea + +These queries can share a plan when the scan plugin supports it: + +```sql +SELECT row_key FROM hbase.usertable WHERE row_key = 'user00000101'; +SELECT row_key FROM hbase.usertable WHERE row_key = 'user00000250'; +``` + +Drill replaces eligible data literals with typed slots when building the cache +key. Each slot carries its current value during the first planning pass. The +resulting physical expressions retain the slot number, so a later query can +replace the value without running the optimizer again. Literal type, numeric +precision and scale, and string length are part of the template. + +```mermaid +flowchart LR + SQL[Parse SQL and build template] --> Lookup{Compatible cached plan?} + Lookup -->|Yes| Bind[Bind current values and rebuild scans] + Lookup -->|No| Plan[Validate and optimize normally] + Bind --> Execute[Execute against current data] + Plan --> Execute + Execute -->|Successful first execution| Save[Publish immutable plan snapshot] +``` + +The cache belongs to one Drillbit and can be shared across connections. Its key +includes the query user, default schema and SQL template. Each entry also records +effective planner options, plugin configurations and the compatibility version +of every referenced table, including tables inside joins, CTEs and subqueries. +Effective-option and plugin-configuration fingerprints are also part of the key, +so sessions with different settings retain separate entries. Table versions stay +in the entry's context snapshot and trigger invalidation when compatibility changes. +A mismatch or failure falls back to ordinary planning. + +The cached representation is immutable physical-plan JSON. A hit creates a new +plan object graph and binds the new values throughout its expressions. Each +participating scan must also rebuild any state derived from those values. +HBase rebuilds row-key bounds and remote filters, then discovers current regions. +Iceberg reloads table metadata and plans tasks for the current snapshot and +predicate. Reusing the first execution's filters or file list would be incorrect. + +## What can be reused + +Read queries can include joins, aggregates, subqueries, CTEs, ordering and limits. +Every source plugin must explicitly support caching and guarantee safe reconstruction of its scans. This change opts in HBase and Iceberg; other plugins retain the default of no support. + +Constants that determine SQL structure stay in the key: group/order expressions, +window definitions, limit/offset, field selectors, and function configuration +arguments such as `DATE_PART`'s unit or `CONVERT_FROM`'s encoding. DATE, TIME, +TIMESTAMP and INTERVAL literals also stay in the key because their parameter +binding is not supported. Other eligible literals in the same query can still +be rebound. Queries with no slots can reuse their complete template. + +When SELECT expressions are preserved for grouping, ordering or window definitions, +HAVING and QUALIFY expressions are also preserved to keep matching expressions +consistent during validation and rewriting. Their literals remain part of the key; +eligible WHERE literals can still be rebound. + +Writes, DDL, existing dynamic parameters, volatile/query-context functions and +sessions containing temporary tables or mutable aliases bypass the cache. +Unsupported scans, views without a compatible table identity, Iceberg metadata tables and explicit +snapshot selections use normal planning. + +## Storage plugin roadmap + +HBase was chosen first because short queries under high concurrency are expected +to benefit most from plan caching: validation and optimization can account for a +substantial share of their latency. Since Drill is also an analytical engine, +Iceberg was included to evaluate the benefit for analytical workloads over a +data lake format. These initial integrations cover both point/range reads and +analytical scans. + +Support for other commonly used plugins is planned in follow-up changes, in the +following order: + +1. **DFS formats without filter pushdown**, such as CSV/TSV, JSON and Avro. + Filtering remains in Drill's Filter operator. On bind, reconstruct the current + file selection and redo directory partition pruning for the new values, with + a table compatibility version that detects schema changes. +2. **Parquet.** Rebuild partition, file and row-group pruning for the current + values and data files. Reuse the existing Parquet metadata cache where possible + to reduce the cost of reconstruction. +3. **JDBC.** Rebuild pushed-down SQL for the current parameter values rather than + reuse SQL containing the first execution's literals. Evaluate shared plugin + hooks for rebuilding pushdown on bind, so JDBC and other plugins that construct + native queries can avoid duplicating the integration logic. + +Each integration must retain the inputs needed to reconstruct value-dependent +scan state and provide reliable compatibility checks, following the +[plugin developer guide](PLAN_CACHE_PLUGIN_GUIDE.md). Benchmark cache-hit planning +and end-to-end latency against ordinary planning, including metadata reads and +bind-time reconstruction. For DFS and Parquet in particular, measure file listing +and pruning costs to establish how much planning work a hit actually saves. + +## Lifetime and observation + +An immutable physical-plan JSON snapshot is serialized immediately after planning, +before parallelization or execution can mutate the operators. Publication happens +only after the first query succeeds. A bounded background queue holds JSON strings +and performs read-back validation; a full queue simply skips publication. + +This is intentionally a general plan for the parameterized template. Constant +folding that depends on bindable parameter values cannot be applied without +specializing the plan to one execution and losing the slots needed by later queries. +Structural constants can still be optimized. Value-dependent selectivity estimates, +join ordering and distribution choices may therefore be less effective than when +planning literal SQL, including on the first cache miss. This is an expected trade-off +for avoiding repeated optimization. Cached row-count estimates come from the first optimization; +new values and statistics changes alone do not trigger a new cost-based plan choice. + +The following boot options can be overridden in `drill-override.conf`. They are +read once per Drillbit and changing them requires a restart. The effective settings +are logged at startup. `planner.enable_plan_cache` remains the runtime on/off switch. + +```hocon +drill.exec.plan_cache: { + max_size_bytes: 33554432, + expire_after_write: 0, + expire_after_access: 10m +} +``` + +- `max_size_bytes` bounds the total UTF-8 byte size of cache keys, physical-plan JSON, + and optional explain text per Drillbit. The default is 32 MiB; entry metadata and + Java object overhead are not included in this limit. A value of `0` disables cache storage. +- `expire_after_write` sets a fixed lifetime since creation or replacement. + The default of `0` disables this policy; reads do not extend a configured lifetime. +- `expire_after_access` sets an idle lifetime since the last read or write. + The default of `10m` removes unused entries while allowing active plans to remain + cached, subject to capacity eviction and compatibility checks. + +Durations accept HOCON units such as `30s`, `10m`, or `1h`. A duration of `0` +disables only that expiration policy. When both policies are enabled, an entry +expires as soon as either deadline is reached. When both are disabled, capacity +eviction and explicit invalidation still apply. All three settings must be +non-negative; invalid settings fail Drillbit startup. + +Capacity eviction favors recently accessed entries using Guava's segmented LRU +policy, so entries can be evicted before their expiration deadlines or before the +global size limit is reached. Expired entries are no longer returned by lookups; +physical removal occurs during cache maintenance rather than on a background timer. +Without write-based expiration, hot plans do not periodically re-optimize. Operators +who need that behavior can configure a nonzero `expire_after_write` lifetime. + +The query profile's **Plan Cache Hit** field, or JSON `planCacheHit`, identifies a +successful hit. Explain shows slot names and this execution's parameter values; +those alone do not prove a hit. + +DEBUG logging from `DrillSqlWorker` reports cache hits, misses and invalidation due +to context changes or binding failures. `PlanCache` logs publication and skipped +publication caused by serialization errors or a busy writer. Foreman thread names +include the query ID, allowing these messages to be correlated with query profiles. + +The following per-cache gauges are registered in `DrillMetrics` under +`drill.plan_cache.`: + +| Gauge | Meaning | +| --- | --- | +| `hits` | Plans successfully rebound and returned by eligible lookups. | +| `misses` | Eligible lookups without a usable entry, including stale entries and binding failures. Queries that bypass caching are excluded. | +| `invalidations` | Explicit invalidation requests, including whole-cache clear commands. | +| `evictions` | Guava automatic removals due to capacity or expiration. | +| `entries` | Current entry count reported by Guava; expired entries may remain counted until maintenance. | + +JMX reporting is enabled by default and can be controlled with the JVM property +`drill.metrics.jmx.enabled`. Periodic metric logging can be enabled with +`-Ddrill.metrics.log.enabled=true`; `drill.metrics.log.interval` defaults to 60 seconds. +Closing the cache unregisters its gauges. + +Administrators can clear the receiving Drillbit's cache with: + +```sql +ALTER SYSTEM CLEAR PLAN CACHE; +``` + +The command follows the SYSTEM option administrator policy: when user +authentication is enabled, only configured administrator users or members of +administrator groups can run it. With authentication disabled, the command is +available to all users. It returns `ok` and a `summary` identifying the Drillbit +whose cache was cleared. It does not broadcast to other Drillbits; to clear a +cluster, connect to each Drillbit and issue the command there. + +Clearing also advances a cache generation. Plans prepared before the clear, +including queued writes and publications pending query completion, cannot +repopulate the cache. Already-bound queries continue normally. Queries planned +after the clear can populate it again. Repeated clears, including while plan +caching is disabled for the session, succeed. Hit/miss counters remain cumulative; +each clear counts as one invalidation request. + +See the [plugin developer guide](PLAN_CACHE_PLUGIN_GUIDE.md) for the integration +contract. The accompanying pull request description contains validation results +and benchmark methodology. diff --git a/docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md b/docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md new file mode 100644 index 00000000000..61d67ad603f --- /dev/null +++ b/docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md @@ -0,0 +1,109 @@ + + +# Adding plan-cache support to a plugin + +Plan caching is disabled by default. A plugin opts in through its metadata interface and guarantees that every scan of an eligible selection can be safely rebuilt. The engine binds SQL literals in serialized Drill expressions; the plugin remains responsible for rebuilding scan state derived from those expressions. + +## Storage-plugin API + +Implement these methods in [StoragePlugin](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java): + +```java +boolean supportPlanCache(DrillTableSelection selection); +PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException; +``` + +`planCacheTable` is called before each lookup to capture the current query's table dependencies. Return a stable, opaque physical table identifier and a compatibility version. Return `null` for missing tables or unsupported selections. Failed metadata reads also cause a fallback to normal planning. The engine compares this complete context snapshot, including effective options and storage-plugin configurations, with the cached entry's snapshot. A miss retains the same snapshot for publication after successful execution. + +The engine first calls `supportPlanCache(selection)` for every participating plugin and returns immediately if any selection is unsupported. Its default returns `false`. A plugin can cast `DrillTableSelection` to its own selection type to decide eligibility. Keep this check free of table-version reads. Only after all checks pass does the engine call `planCacheTable` and compute configuration and option fingerprints. + +The version must change whenever the cached plan's table definition becomes +incompatible. Do not use a data snapshot number if compatible data updates should +keep hitting. A path alone is insufficient when replacement can change the +schema or table identity. + +DFS formats use the same two-method contract in [FormatPlugin](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java), with `FileSelection` as the input: + +```java +boolean supportPlanCache(FileSelection selection); +PlanCacheTable planCacheTable(FileSelection selection) throws IOException; +``` + +[FileSystemPlugin](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) delegates eligibility and metadata reads to the selected format plugin. It prefixes the returned table identifier with the format name to distinguish formats within the storage plugin. + +## Scan API and JSON reconstruction + +Declaring plugin support covers every `GroupScan` produced for an eligible selection. Ensure that all planning, pushdown and cloning paths preserve the inputs needed for reconstruction. Unsupported selections must be rejected during snapshot construction. + +Keep the bound `LogicalExpression` in the scan's JSON. Converting a predicate to +a native filter or file list and dropping the source expression loses the +information needed to bind the next query. On deserialization: + +1. Resolve the plugin and reload current table metadata. +2. Translate the newly bound expression into the plugin's native predicate. +3. Recompute value-dependent row bounds, filters, files/splits, region locations + and assignments. Preserve residual filtering for unsupported predicates. +4. Reject reconstruction if the original pushdown guarantees cannot be met. + +Serialize stable inputs rather than live connections, execution resources or old +endpoint assignments. Validate the round trip with Drill's `PhysicalPlanReader`. +If a parameter changes projection, schema, limit or another derived field that +cannot be rebuilt, keep that argument structural or leave the selection ineligible. + +Keep the regular expression serialization for JSON so parameter slot numbers survive the round trip. + +## Reference implementations + +| Plugin | Compatibility version | Reconstruction | +| --- | --- | --- | +| [HBaseStoragePlugin](../../contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java) / [HBaseGroupScan](../../contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java) | Table descriptor fingerprint | Retains the selection before pushdown and the original predicate; rebuilds native row-key bounds/filters and discovers current regions. | +| [IcebergFormatPlugin](../../contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java) / [IcebergGroupScan](../../contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java) | Table UUID and schema ID | Reloads the table and runs `planTasks()` for the current predicate and snapshot. Metadata tables and explicit snapshot selection return no support. | + +HBase descriptor equality means definition compatibility; a replacement with an +identical descriptor can still be read safely because regions and filters are +rebuilt. Iceberg uses a UUID to distinguish tables recreated at the same path. + +## Required correctness checks + +Compare complete results with caching disabled, after the first planning pass, +and after a hit with different literals. Exercise cross-connection reuse, +pushdown and residual filtering, and an empty scan after rebinding. Verify that +compatible data updates are visible on hits, while incompatible schema changes, +deletions, plugin configuration changes and failed metadata reads reject the old +entry. Include joins or subqueries so every table dependency is checked, and +confirm that queries involving an unsupported plugin or selection bypass caching. + +Check `planCacheHit` in the query profile. In isolated tests, the Drillbit's hit +counter can also establish reuse: + +```java +PlanCache cache = cluster.drillbit().getContext().getPlanCache(); +first.queryBuilder().sql(firstSql).run(); +cache.awaitWrites(); // Test synchronization for asynchronous publication. +long before = cache.getHitCount(); +second.queryBuilder().sql(sameTemplateWithNewValues).run(); +assertEquals(before + 1, cache.getHitCount()); +``` + +The counter is shared across the Drillbit; use per-query profiles for concurrent +workloads. `awaitWrites()` is a test helper, not a SQL operation. + +## Scope of the contract + +The cache compares the current query's physical identifiers and versions with those captured for the cached plan. Sessions with temporary tables or mutable user/public aliases bypass caching. Statistics changes alone do not trigger re-optimization. diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd index ede92c22aca..b0eecadea78 100644 --- a/exec/java-exec/src/main/codegen/data/Parser.tdd +++ b/exec/java-exec/src/main/codegen/data/Parser.tdd @@ -30,6 +30,8 @@ # List of keywords. keywords: [ + "CLEAR", + "CACHE", "DAFFODIL", "DATABASES", "SCHEMAS", @@ -111,6 +113,7 @@ # List of methods for parsing extensions to "ALTER " calls. # Each must accept arguments "(SqlParserPos pos, String scope)". alterStatementParserMethods: [ + "SqlClearPlanCache", "DrillSqlSetOption", "DrillSqlResetOption" ] @@ -124,6 +127,8 @@ # Copied from calcite-core config.fmpp # For details please see comment under CALCITE-2405. nonReservedKeywords: [ + "CLEAR" + "CACHE" "A" "ABSENT" "ABSOLUTE" diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl index fe1d3159c31..0ad4bbcad27 100644 --- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl +++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl @@ -976,12 +976,26 @@ DrillSqlSetOption DrillSqlSetOption(Span s, String scope) : } } +/** + * Parses CLEAR PLAN CACHE after ALTER SYSTEM. + */ +SqlClearPlanCache SqlClearPlanCache(Span s, String scope) : +{ +} +{ + + { + if (!"SYSTEM".equalsIgnoreCase(scope)) { + throw new ParseException("CLEAR PLAN CACHE requires ALTER SYSTEM."); + } + return new SqlClearPlanCache(s.end(this)); + } +} + /** * Parses a RESET statement without a leading "ALTER ": * * RESET { | ALL } - *

- * Statement handles in: {@link SetAndResetOptionHandler} */ DrillSqlResetOption DrillSqlResetOption(Span s, String scope) : { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index b511daa9c2f..535981b5df0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -48,6 +48,10 @@ private ExecConstants() { // Don't allow instantiation } + public static final String PLAN_CACHE_MAX_SIZE_BYTES = "drill.exec.plan_cache.max_size_bytes"; + public static final String PLAN_CACHE_EXPIRE_AFTER_WRITE = "drill.exec.plan_cache.expire_after_write"; + public static final String PLAN_CACHE_EXPIRE_AFTER_ACCESS = "drill.exec.plan_cache.expire_after_access"; + public static final String ZK_RETRY_TIMES = "drill.exec.zk.retry.count"; public static final String ZK_RETRY_DELAY = "drill.exec.zk.retry.delay"; public static final String ZK_CONNECTION = "drill.exec.zk.connect"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index 90cb45e4138..d40abd0eaba 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -86,6 +86,8 @@ public enum SqlStatementType {OTHER, ANALYZE, CTAS, EXPLAIN, DESCRIBE_TABLE, DES /** Stores constants and their holders by type */ private final Map> constantValueHolderCache; private SqlStatementType stmtType; + private Runnable pendingPlanCacheInsert; + private boolean planCacheHit; /* * Flag to indicate if close has been called, after calling close the first @@ -381,6 +383,24 @@ public void close() throws Exception { } } + public void setPendingPlanCacheInsert(Runnable insert) { + pendingPlanCacheInsert = insert; + } + + public void setPlanCacheHit() { + planCacheHit = true; + } + + public boolean isPlanCacheHit() { + return planCacheHit; + } + + public Runnable takePendingPlanCacheInsert() { + Runnable insert = pendingPlanCacheInsert; + pendingPlanCacheInsert = null; + return insert; + } + /** * @param stmtType : Sets the type {@link SqlStatementType} of the statement e.g. CTAS, ANALYZE */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java index 0828bea676c..6097d22c4aa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.base.Sender; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; @@ -40,7 +41,7 @@ public class HashToMergeExchange extends AbstractExchange { @JsonCreator public HashToMergeExchange(@JsonProperty("child") PhysicalOperator child, @JsonProperty("expr") LogicalExpression expr, - @JsonProperty("orderings") List orderExprs) { + @JsonProperty("orderings") @JsonAlias("orderExpr") List orderExprs) { super(child); this.distExpr = expr; this.orderExprs = orderExprs; @@ -62,7 +63,12 @@ protected PhysicalOperator getNewWithChild(PhysicalOperator child) { return new HashToMergeExchange(child, distExpr, orderExprs); } - @JsonProperty("orderExpr") + @JsonProperty("expr") + public LogicalExpression getExpression() { + return distExpr; + } + + @JsonProperty("orderings") public List getOrderExpressions(){ return orderExprs; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java index 7808c67e42b..9ef3e303a12 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.physical.base.Receiver; import org.apache.drill.common.logical.data.Order.Ordering; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonTypeName; /** @@ -34,6 +35,7 @@ public class OrderedMuxExchange extends AbstractMuxExchange { private final List orderings; + @JsonCreator public OrderedMuxExchange(@JsonProperty("child") PhysicalOperator child, @JsonProperty("orderings")List orderings) { super(child); this.orderings = orderings; @@ -48,4 +50,9 @@ public Receiver getReceiver(int minorFragmentId) { protected PhysicalOperator getNewWithChild(PhysicalOperator child) { return new OrderedMuxExchange(child, orderings); } + + @JsonProperty("orderings") + public List getOrderings() { + return orderings; + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java index 96c29162fd8..f527e651715 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java @@ -93,6 +93,10 @@ public String writeJson(PhysicalOperator op) throws JsonProcessingException{ return mapper.writeValueAsString(op); } + public String writeJson(PhysicalPlan plan) throws JsonProcessingException { + return mapper.writeValueAsString(plan); + } + public PhysicalPlan readPhysicalPlan(String json) throws IOException { logger.debug("Reading physical plan {}", json); return physicalPlanReader.readValue(json); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java index bed9a2716fe..cd41c307abf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java @@ -51,8 +51,17 @@ public ExprToRex(RelNode inputRel, RelDataType newRowType, RexBuilder builder) { } public static RelDataTypeField findField(String fieldName, RelDataType rowType) { + // A projected wildcard can be named "*" in a plugin row type. It is a + // literal field name here, not an expression to parse. + for (RelDataTypeField f : rowType.getFieldList()) { + if (fieldName.equalsIgnoreCase(f.getName())) { + return f; + } + } + if ("*".equals(fieldName)) { + return null; + } final String rootPart = SchemaPath.parseFromString(fieldName).getRootSegmentPath(); - for (RelDataTypeField f : rowType.getFieldList()) { if (rootPart.equalsIgnoreCase(f.getName())) { return f; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java index 349ba2a02f9..5e0eea1f21f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java @@ -40,6 +40,7 @@ import org.apache.drill.common.expression.FunctionCallFactory; import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.IfExpression.IfCondition; +import org.apache.drill.common.expression.LiteralExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; @@ -76,6 +77,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam; import org.apache.drill.exec.work.ExecErrorConstants; import static org.apache.drill.exec.planner.physical.PlannerSettings.ENABLE_DECIMAL_DATA_TYPE; @@ -469,6 +471,15 @@ public LogicalExpression visitCorrelVariable(RexCorrelVariable correlVariable) { @Override public LogicalExpression visitDynamicParam(RexDynamicParam dynamicParam) { + if (dynamicParam instanceof RexBoundDynamicParam) { + RexBoundDynamicParam bound = (RexBoundDynamicParam) dynamicParam; + LogicalExpression expression = bound.getLiteral().accept(this); + if (!(expression instanceof LiteralExpression)) { + return doUnknown(dynamicParam); + } + ((LiteralExpression) expression).setDynamicParamIndex(bound.getIndex()); + return expression; + } return doUnknown(dynamicParam); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 4b5202d8366..deb8cdd1924 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -58,6 +58,9 @@ public class PlannerSettings implements Context{ public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", new OptionDescription("If one side of a filter condition is a constant expression, constant folding evaluates the expression in the planning phase and replaces the expression with the constant value. For example, Drill can rewrite WHERE age + 5 < 42 as WHERE age < 37.")); + public static final String ENABLE_PLAN_CACHE_OPTION = "planner.enable_plan_cache"; + public static final BooleanValidator PLAN_CACHE = new BooleanValidator(ENABLE_PLAN_CACHE_OPTION, + new OptionDescription("Enables reuse of compatible physical plans for parameterized SQL queries.")); public static final String DISABLE_EXCHANGE_OPTION = "planner.disable_exchanges"; public static final OptionValidator EXCHANGE = new BooleanValidator(DISABLE_EXCHANGE_OPTION, new OptionDescription("Toggles the state of hashing to a random exchange.")); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index c706f8f3733..d0d3e238721 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.planner.sql; import java.io.IOException; +import java.util.function.Consumer; import org.apache.calcite.sql.SqlDescribeSchema; import org.apache.calcite.sql.SqlKind; @@ -31,11 +32,15 @@ import org.apache.calcite.tools.ValidationException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.alias.AliasRegistry; import org.apache.drill.exec.exception.MetadataException; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.ops.QueryContext.SqlStatementType; import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler; +import org.apache.drill.exec.planner.sql.handlers.ClearPlanCacheHandler; import org.apache.drill.exec.planner.sql.handlers.AnalyzeTableHandler; import org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler; import org.apache.drill.exec.planner.sql.handlers.DescribeSchemaHandler; @@ -51,6 +56,7 @@ import org.apache.drill.exec.planner.sql.parser.DrillSqlCall; import org.apache.drill.exec.planner.sql.parser.DrillSqlDescribeTable; import org.apache.drill.exec.planner.sql.parser.DrillSqlResetOption; +import org.apache.drill.exec.planner.sql.parser.SqlClearPlanCache; import org.apache.drill.exec.planner.sql.parser.SqlSchema; import org.apache.drill.exec.planner.sql.conversion.SqlConverter; import org.apache.drill.exec.proto.UserBitShared.DrillPBError; @@ -218,7 +224,6 @@ private static PhysicalPlan getPhysicalPlan(QueryContext context, String sql, Po */ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Pointer textPlan) throws ForemanSetupException, RelConversionException, IOException, ValidationException { - final SqlConverter parser = new SqlConverter(context); injector.injectChecked(context.getExecutionControls(), "sql-parsing", ForemanSetupException.class); final SqlNode sqlNode = checkAndApplyAutoLimit(parser, context, sql); @@ -270,6 +275,11 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point case DROP_VIEW: case OTHER_DDL: case OTHER: + if (sqlNode instanceof SqlClearPlanCache) { + handler = new ClearPlanCacheHandler(context); + context.setSQLStatementType(SqlStatementType.OTHER); + break; + } if (sqlNode instanceof DrillSqlCall) { handler = ((DrillSqlCall) sqlNode).getSqlHandler(config); if (handler instanceof AnalyzeTableHandler || handler instanceof MetastoreAnalyzeTableHandler) { @@ -295,7 +305,95 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point context.getOptions().setLocalOption(ExecConstants.RETURN_RESULT_SET_FOR_DDL, true); } - return handler.getPlan(sqlNode); + // Keep the original SQL if no safe snapshot is available or cache preparation fails. + SqlNode planningSql = sqlNode; + Consumer prepareCacheInsert = null; + if (context.getOptions().getOption(PlannerSettings.PLAN_CACHE) + && !context.getSession().hasTemporaryTables() + && !hasMutableAliases(context) + && PlanCacheEligibility.isSafeToCache(sqlNode, context.getFunctionRegistry())) { + try { + PlanCache cache = context.getDrillbitContext().getPlanCache(); + long cacheGeneration = cache.getGeneration(); + // Capture the context once, before lookup and planning. Changes during + // planning invalidate this entry on its first hit. + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( + parser.getDefaultSchema(), sqlNode, context); + if (snapshot != null) { + PlanCacheParameterizer.Candidate candidate = + PlanCacheParameterizer.parameterize(sqlNode, parser.getTypeFactory()); + String key = context.getQueryUserName() + '\n' + + context.getSession().getDefaultSchemaPath() + '\n' + + snapshot.keyFingerprint() + '\n' + + candidate.template; + PlanCache.Entry entry = cache.get(key); + if (entry != null) { + try { + if (entry.matchesContext(snapshot)) { + PhysicalPlan bound = entry.bind(candidate.literals, + context.getDrillbitContext().getPlanReader()); + PlanCache.bindTextPlan(textPlan, entry.getTextPlan(), candidate.literals); + cache.recordBind(); + context.setPlanCacheHit(); + logger.debug("Plan cache hit"); + return bound; + } + logger.debug("Plan cache entry invalidated because its context changed; replanning"); + } catch (RuntimeException | IOException e) { + logger.debug("Plan cache entry invalidated because rebinding failed; replanning", e); + } + cache.invalidate(key); + } else { + logger.debug("Plan cache miss"); + } + cache.recordMiss(); + final PhysicalPlanReader cacheReader = + context.getDrillbitContext().getPlanReader(); + // Prepare insertion after planning; Foreman publishes it only after the query succeeds. + prepareCacheInsert = planned -> { + String templateTextPlan = PlanCache.bindTextPlan(textPlan, candidate.literals); + final String json; + try { + // Capture before parallelization/execution mutates the operators. + // Pending publication and the writer queue retain only immutable JSON. + json = cacheReader.writeJson(planned); + } catch (IOException | RuntimeException e) { + logger.debug("Plan could not be serialized for the plan cache; skipping publication", e); + return; + } + context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, json, + templateTextPlan, cacheReader, snapshot, cacheGeneration)); + }; + // On a cache miss, plan the parameterized SQL to produce a reusable template. + planningSql = candidate.sql; + } + } catch (RuntimeException e) { + logger.debug("Plan cache attempt fell back to ordinary planning", e); + } + } + + // Plan exactly once, outside cache fallback. Validation can mutate the SQL tree, + // so retrying a failed planning attempt could reuse partially rewritten nodes. + PhysicalPlan planned = handler.getPlan(planningSql); + if (prepareCacheInsert != null) { + prepareCacheInsert.accept(planned); + } + return planned; + } + + private static boolean hasMutableAliases(QueryContext context) { + // Table-version checks cannot detect an alias redirected to another table. + AliasRegistry[] registries = { + context.getAliasRegistryProvider().getStorageAliasesRegistry(), + context.getAliasRegistryProvider().getTableAliasesRegistry() + }; + for (AliasRegistry registry : registries) { + if (registry.getPublicAliases().getAllAliases().hasNext() + || registry.getUserAliases(context.getQueryUserName()).getAllAliases().hasNext()) { + return true; + } + } + return false; } private static boolean isAutoLimitShouldBeApplied(SqlNode sqlNode, int queryMaxRows) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java new file mode 100644 index 00000000000..e45ad0d82d0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -0,0 +1,744 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.calcite.util.NlsString; +import org.apache.drill.common.expression.AnyValueExpression; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.CastExpression; +import org.apache.drill.common.expression.ConvertExpression; +import org.apache.drill.common.expression.ExpressionStringBuilder; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.IfExpression; +import org.apache.drill.common.expression.LiteralExpression; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.NullExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedNullConstant; +import org.apache.drill.common.expression.ValueExpressions; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.parser.LogicalExpressionParser; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; +import org.apache.drill.exec.rpc.NamedThreadFactory; +import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.PlanCacheTable; +import org.apache.drill.exec.util.Pointer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ticker; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.hash.Hashing; +import com.google.common.primitives.Ints; +import com.codahale.metrics.Gauge; + +/** Drillbit-scoped, immutable JSON snapshots of query plans. */ +public final class PlanCache implements AutoCloseable { + private static final Logger logger = LoggerFactory.getLogger(PlanCache.class); + private static final ObjectMapper JSON = new ObjectMapper(); + private static final String MARKER = ExpressionStringBuilder.BOUND_DYNAMIC_PARAM + "("; + private final Cache entries; + private final AtomicLong successfulBinds = new AtomicLong(); + private final AtomicLong misses = new AtomicLong(); + private final AtomicLong invalidations = new AtomicLong(); + private final Map> metrics = new LinkedHashMap<>(); + private volatile long generation; + private final ThreadPoolExecutor writer = new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(64), + new NamedThreadFactory("plan-cache-writer-")); + + /** A zero expiration duration disables that policy; a zero capacity disables storage. */ + public PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAfterAccess) { + this(maxSizeBytes, expireAfterWrite, expireAfterAccess, Ticker.systemTicker()); + } + + @VisibleForTesting + PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAfterAccess, Ticker ticker) { + checkArgument(maxSizeBytes >= 0, "%s must be non-negative", ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES); + checkArgument(!expireAfterWrite.isNegative(), "%s must be non-negative", + ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE); + checkArgument(!expireAfterAccess.isNegative(), "%s must be non-negative", + ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS); + CacheBuilder builder = CacheBuilder.newBuilder() + .maximumWeight(maxSizeBytes) + .weigher((String key, Entry value) -> Ints.saturatedCast( + key.getBytes(StandardCharsets.UTF_8).length + value.contentSizeBytes)) + .ticker(ticker) + .recordStats(); + // Passing zero to Guava would expire entries immediately, rather than disable the policy. + if (!expireAfterWrite.isZero()) { + builder.expireAfterWrite(expireAfterWrite); + } + if (!expireAfterAccess.isZero()) { + builder.expireAfterAccess(expireAfterAccess); + } + entries = builder.build(); + logger.info("Plan cache settings: max_size_bytes={}, expire_after_write={}, expire_after_access={}" + + " (zero expiration disables that policy)", maxSizeBytes, expireAfterWrite, expireAfterAccess); + registerMetric("hits", successfulBinds::get); + registerMetric("misses", misses::get); + registerMetric("invalidations", invalidations::get); + registerMetric("evictions", () -> entries.stats().evictionCount()); + registerMetric("entries", entries::size); + } + + private void registerMetric(String suffix, Gauge gauge) { + String name = "drill.plan_cache." + suffix; + metrics.put(name, gauge); + DrillMetrics.register(name, gauge); + } + + public long getHitCount() { + return successfulBinds.get(); + } + + public void recordBind() { + successfulBinds.incrementAndGet(); + } + + /** Counts an eligible lookup that could not produce a compatible, bound plan. */ + public void recordMiss() { + misses.incrementAndGet(); + } + + /** Updates optional explain output using the cached template. */ + static void bindTextPlan(Pointer textPlan, String template, List literals) { + if (textPlan != null) { + textPlan.value = textWithBindings(template, literals); + } + } + + /** Updates optional explain output and returns its original template for cache publication. */ + static String bindTextPlan(Pointer textPlan, List literals) { + String template = textPlan == null ? null : textPlan.value; + bindTextPlan(textPlan, template, literals); + return template; + } + + /** Appends values from this query without changing the cached explain template. */ + private static String textWithBindings(String template, List literals) { + if (template == null || literals.isEmpty()) { + return template; + } + StringBuilder text = new StringBuilder(template); + if (text.length() > 0 && text.charAt(text.length() - 1) != '\n') { + text.append('\n'); + } + text.append("Parameters: "); + for (int i = 0; i < literals.size(); i++) { + if (i > 0) { + text.append(", "); + } + text.append('?').append(i).append(" = ") + .append(literals.get(i).toString().replace("\r", "\\r").replace("\n", "\\n")); + } + return text.toString(); + } + + public Entry get(String key) { + return entries.getIfPresent(key); + } + + /** Waits for writes already submitted by completed queries; intended for tests. */ + public void awaitWrites() throws InterruptedException, ExecutionException, TimeoutException { + writer.submit(() -> { }).get(10, TimeUnit.SECONDS); + } + + public void writeAfterSuccess(String key, String json, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context, long expectedGeneration) { + try { + writer.execute(() -> { + try { + put(key, json, textPlan, reader, context, expectedGeneration); + } catch (RuntimeException | IOException e) { + logger.debug("Plan cache entry could not be published", e); + } + }); + } catch (RejectedExecutionException e) { + logger.debug("Plan cache writer is busy; skipping cache entry", e); + } + } + + public void invalidate(String key) { + invalidations.incrementAndGet(); + entries.invalidate(key); + } + + long getGeneration() { + return generation; + } + + /** Clears local entries and prevents pre-clear plans from being published later. */ + public synchronized void clear() { + generation++; + invalidations.incrementAndGet(); + entries.invalidateAll(); + entries.cleanUp(); + logger.info("Local plan cache cleared; generation={}", generation); + } + + public boolean put(String key, String json, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context) throws IOException { + return put(key, json, textPlan, reader, context, generation); + } + + private boolean put(String key, String json, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context, long expectedGeneration) throws IOException { + // The engine checks generic context and expression compatibility. Each + // opted-in plugin is responsible for rebuilding its scans' value-dependent state. + if (context == null || expectedGeneration != generation) { + return false; + } + // Confirm that standard PhysicalPlan JSON can be read back before publishing. + reader.readPhysicalPlan(json); + // Readback may perform plugin I/O. Keep it outside the clear/publication lock, + // then check again so a concurrent clear cannot be undone by that readback. + synchronized (this) { + if (expectedGeneration != generation) { + logger.debug("Skipping plan cache publication after a cache clear"); + return false; + } + entries.put(key, new Entry(json, textPlan, context)); + } + logger.debug("Plan cache entry published; entries={}", entries.size()); + return true; + } + + @Override + public void close() { + writer.shutdownNow(); + // Test JVMs may contain several Drillbits. A closing cache must not remove + // gauges registered by a newer cache under the same names. + synchronized (DrillMetrics.class) { + metrics.forEach((name, gauge) -> { + if (DrillMetrics.getRegistry().getMetrics().get(name) == gauge) { + DrillMetrics.getRegistry().remove(name); + } + }); + } + } + + public static final class Entry { + private final String json; + private final long contentSizeBytes; + private final String textPlan; + private final ContextSnapshot context; + + private Entry(String json, String textPlan, ContextSnapshot context) { + this.json = json; + this.contentSizeBytes = (long) json.getBytes(StandardCharsets.UTF_8).length + + (textPlan == null ? 0 : textPlan.getBytes(StandardCharsets.UTF_8).length); + this.textPlan = textPlan; + this.context = context; + } + + public boolean matchesContext(ContextSnapshot current) { + return context.matches(current); + } + + public String getTextPlan() { + return textPlan; + } + + public PhysicalPlan bind(List literals, PhysicalPlanReader reader) + throws IOException { + JsonNode tree = JSON.readTree(json); + walk(tree, literals); + return reader.readPhysicalPlan(JSON.writeValueAsString(tree)); + } + } + + /** Options, plugin configurations, and table versions captured before lookup and planning. */ + public static final class ContextSnapshot { + private final String optionsFingerprint; + private final Map tableVersions; + private final Map pluginConfigs; + + private ContextSnapshot(String optionsFingerprint, Map tableVersions, + Map pluginConfigs) { + this.optionsFingerprint = Objects.requireNonNull(optionsFingerprint, "optionsFingerprint"); + this.tableVersions = Collections.unmodifiableMap(tableVersions); + this.pluginConfigs = Collections.unmodifiableMap(pluginConfigs); + } + + private static final class TableIdentifier { + private final String storageName; + private final String tableId; + + private TableIdentifier(String storageName, String tableId) { + this.storageName = storageName; + this.tableId = tableId; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof TableIdentifier)) { + return false; + } + TableIdentifier that = (TableIdentifier) other; + return Objects.equals(storageName, that.storageName) + && Objects.equals(tableId, that.tableId); + } + + @Override + public int hashCode() { + return Objects.hash(storageName, tableId); + } + } + + private static final class ResolvedTable { + private final List tableNames; + private final String storageName; + private final StoragePlugin plugin; + private final DrillTableSelection selection; + + private ResolvedTable(List tableNames, String storageName, + StoragePlugin plugin, DrillTableSelection selection) { + this.tableNames = tableNames; + this.storageName = storageName; + this.plugin = plugin; + this.selection = selection; + } + } + + /** Resolves every physical table in a query, including joins and subqueries. */ + public static ContextSnapshot resolve(SchemaPlus defaultSchema, SqlNode query, + QueryContext context) { + StoragePluginRegistry plugins = context.getStorage(); + List dependencies = new ArrayList<>(); + try { + // Check every participating plugin before reading any table version. + if (!collectQuery(defaultSchema, query, new HashSet<>(), dependencies)) { + return null; + } + } catch (RuntimeException e) { + logger.debug("Query sources could not be validated for the plan cache", e); + return null; + } + Map versions = new LinkedHashMap<>(); + Map configs = new LinkedHashMap<>(); + for (ResolvedTable dependency : dependencies) { + try { + PlanCacheTable source = dependency.plugin.planCacheTable(dependency.selection); + if (source == null) { + return null; + } + TableIdentifier identifier = new TableIdentifier(dependency.storageName, + source.getIdentifier()); + String previous = versions.putIfAbsent(identifier, source.getVersion()); + if (previous != null && !previous.equals(source.getVersion())) { + return null; + } + String fingerprint = configFingerprint(plugins, dependency.plugin.getConfig()); + String previousConfig = configs.putIfAbsent(dependency.storageName, fingerprint); + if (previousConfig != null && !previousConfig.equals(fingerprint)) { + return null; + } + } catch (IOException | RuntimeException e) { + logger.debug("Table context could not be captured for the plan cache", e); + return null; + } + } + return new ContextSnapshot(optionFingerprint(context), versions, configs); + } + + private static boolean collectQuery(SchemaPlus schema, SqlNode node, + Set ctes, List dependencies) { + if (node == null) { + return true; + } + if (node instanceof SqlNodeList) { + for (SqlNode child : (SqlNodeList) node) { + if (!collectQuery(schema, child, ctes, dependencies)) { + return false; + } + } + return true; + } + if (node instanceof SqlWith) { + SqlWith with = (SqlWith) node; + Set local = new HashSet<>(ctes); + for (SqlNode itemNode : with.withList) { + SqlWithItem item = (SqlWithItem) itemNode; + if (!collectQuery(schema, item.query, local, dependencies)) { + return false; + } + local.add(item.name.getSimple()); + } + return collectQuery(schema, with.body, local, dependencies); + } + if (node instanceof SqlSelect) { + SqlSelect select = (SqlSelect) node; + if (!collectFrom(schema, select.getFrom(), ctes, dependencies)) { + return false; + } + for (SqlNode operand : select.getOperandList()) { + if (operand != select.getFrom() + && !collectQuery(schema, operand, ctes, dependencies)) { + return false; + } + } + return true; + } + if (node instanceof SqlCall) { + for (SqlNode operand : ((SqlCall) node).getOperandList()) { + if (!collectQuery(schema, operand, ctes, dependencies)) { + return false; + } + } + } + return true; + } + + private static boolean collectFrom(SchemaPlus schema, SqlNode from, + Set ctes, List dependencies) { + if (from == null || from.getKind() == SqlKind.VALUES) { + return true; + } + if (from instanceof SqlIdentifier) { + List names = ((SqlIdentifier) from).names; + if (names.size() == 1 && ctes.contains(names.get(0))) { + return true; + } + for (ResolvedTable dependency : dependencies) { + if (dependency.tableNames.equals(names)) { + return true; + } + } + ResolvedTable source = resolveTable(schema, names); + if (source == null) { + return false; + } + dependencies.add(source); + return true; + } + if (from instanceof SqlCall) { + SqlCall call = (SqlCall) from; + if (call.getKind() == SqlKind.AS || call.getKind() == SqlKind.LATERAL) { + return collectFrom(schema, call.operand(0), ctes, dependencies); + } + if (call.getKind() == SqlKind.JOIN) { + return collectFrom(schema, call.operand(0), ctes, dependencies) + && collectFrom(schema, call.operand(3), ctes, dependencies) + && collectQuery(schema, call.operand(5), ctes, dependencies); + } + if (call.getKind().belongsTo(SqlKind.QUERY)) { + return collectQuery(schema, call, ctes, dependencies); + } + } + return false; + } + + private static ResolvedTable resolveTable(SchemaPlus defaultSchema, List names) { + if (names.isEmpty() || names.get(names.size() - 1).contains("*") + || names.get(names.size() - 1).contains("?")) { + return null; + } + SchemaPlus schema = SchemaUtilities.findSchema(defaultSchema, + names.subList(0, names.size() - 1)); + if (schema == null) { + return null; + } + Table table = schema.getTable(names.get(names.size() - 1)); + if (!(table instanceof DrillTable)) { + return null; + } + DrillTable drillTable = (DrillTable) table; + Object selection = drillTable.getSelection(); + if (!(selection instanceof DrillTableSelection)) { + return null; + } + StoragePlugin storagePlugin = drillTable.getPlugin(); + if (!storagePlugin.supportPlanCache((DrillTableSelection) selection)) { + return null; + } + return new ResolvedTable( + Collections.unmodifiableList(new ArrayList<>(names)), + drillTable.getStorageEngineName(), storagePlugin, (DrillTableSelection) selection); + } + + private boolean matches(ContextSnapshot current) { + return current != null + && optionsFingerprint.equals(current.optionsFingerprint) + && tableVersions.equals(current.tableVersions) + && pluginConfigs.equals(current.pluginConfigs); + } + + /** Options and plugin configurations identify separate reusable templates. */ + String keyFingerprint() { + StringBuilder configs = new StringBuilder(); + new TreeMap<>(pluginConfigs).forEach((name, fingerprint) -> configs + .append(name.length()).append(':').append(name).append('=').append(fingerprint).append('\n')); + // Keep table versions outside the key so incompatible entries can be invalidated. + return optionsFingerprint + '\n' + + Hashing.sha256().hashString(configs, StandardCharsets.UTF_8).toString(); + } + + private static String optionFingerprint(QueryContext context) { + // OptionManager iteration walks system defaults, system overrides, session + // overrides, and query overrides in precedence order. getOptionList() only + // exposes local session/query values and misses system-level changes. + Map options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (OptionValue option : context.getOptions()) { + options.put(option.name, option); + } + StringBuilder fingerprint = new StringBuilder(); + for (OptionValue option : options.values()) { + fingerprint.append(option.name).append(':').append(option.kind).append('='); + switch (option.kind) { + case BOOLEAN: + fingerprint.append(option.bool_val); + break; + case LONG: + fingerprint.append(option.num_val); + break; + case DOUBLE: + fingerprint.append(option.float_val); + break; + case STRING: + fingerprint.append(option.string_val.length()).append(':').append(option.string_val); + break; + default: + throw new IllegalArgumentException("Unsupported option kind"); + } + fingerprint.append('\n'); + } + // Keep the cache key bounded even when the system has many registered options. + return Hashing.sha256().hashString(fingerprint, StandardCharsets.UTF_8).toString(); + } + + private static String configFingerprint(StoragePluginRegistry plugins, + StoragePluginConfig config) { + if (config == null) { + throw new IllegalArgumentException("Missing plugin configuration"); + } + String encoded = plugins.encode(config); + if (encoded == null) { + throw new IllegalArgumentException("Plugin configuration could not be encoded"); + } + return Hashing.sha256().hashString(encoded, StandardCharsets.UTF_8).toString(); + } + } + + private static LiteralExpression bindLiteral(SqlLiteral literal, MajorType type, int index) { + Object value = literal.getValue(); + LogicalExpression expression; + if (value instanceof BigDecimal) { + BigDecimal number = (BigDecimal) value; + if (type.getMinorType() == MinorType.VARDECIMAL) { + BigDecimal decimal = number.setScale(type.getScale(), RoundingMode.UNNECESSARY); + if (decimal.precision() > type.getPrecision()) { + throw new IllegalArgumentException("Decimal parameter exceeds cached precision"); + } + expression = ValueExpressions.getVarDecimal(decimal, + type.getPrecision(), type.getScale()); + } else if (type.getMinorType() == MinorType.FLOAT8) { + expression = ValueExpressions.getFloat8(number.doubleValue()); + } else if (type.getMinorType() == MinorType.FLOAT4) { + expression = ValueExpressions.getFloat4(number.floatValue()); + } else if (type.getMinorType() == MinorType.BIGINT) { + expression = ValueExpressions.getBigInt(number.longValueExact()); + } else if (type.getMinorType() == MinorType.INT) { + expression = ValueExpressions.getInt(number.intValueExact()); + } else { + throw new IllegalArgumentException("Unsupported numeric parameter type"); + } + } else if (value instanceof Boolean) { + expression = ValueExpressions.getBit((Boolean) value); + } else if (value instanceof NlsString) { + String string = ((NlsString) value).getValue(); + expression = ValueExpressions.getChar(string, type.getPrecision()); + } else { + throw new IllegalArgumentException("Unsupported parameter type"); + } + LiteralExpression bound = (LiteralExpression) expression; + bound.setDynamicParamIndex(index); + return bound; + } + + /** Visits every serialized field, including nested plugin-specific fields. */ + private static void walk(JsonNode node, List replacements) { + if (node.isObject()) { + ObjectNode object = (ObjectNode) node; + object.fields().forEachRemaining(field -> { + JsonNode value = field.getValue(); + if (value.isTextual() && value.asText().contains(MARKER)) { + object.put(field.getKey(), rewrite(value.asText(), replacements)); + } else { + walk(value, replacements); + } + }); + } else if (node.isArray()) { + ArrayNode array = (ArrayNode) node; + for (int i = 0; i < array.size(); i++) { + JsonNode value = array.get(i); + if (value.isTextual() && value.asText().contains(MARKER)) { + array.set(i, JSON.getNodeFactory().textNode(rewrite(value.asText(), replacements))); + } else { + walk(value, replacements); + } + } + } + } + + private static String rewrite(String input, List replacements) { + // Bind expression nodes rather than text: a quoted field name or string + // containing the marker must never be interpreted as a parameter slot. + LogicalExpression original = LogicalExpressionParser.parse(input); + Map originalTypes = new HashMap<>(); + collectTypes(original, originalTypes); + if (originalTypes.isEmpty()) { + return input; + } + String result = ExpressionStringBuilder.toString(bindExpression(original, replacements)); + LogicalExpression parsed = LogicalExpressionParser.parse(result); + Map newTypes = new HashMap<>(); + collectTypes(parsed, newTypes); + if (!originalTypes.equals(newTypes)) { + throw new IllegalArgumentException("Cached parameter type changed"); + } + return result; + } + + private static LogicalExpression bindExpression(LogicalExpression expression, + List replacements) { + if (expression instanceof LiteralExpression) { + LiteralExpression literal = (LiteralExpression) expression; + if (!literal.isDynamicParam()) { + return literal; + } + int index = literal.getDynamicParamIndex(); + if (index >= replacements.size()) { + throw new IllegalArgumentException("Unexpected cached parameter index"); + } + return bindLiteral(replacements.get(index), literal.getMajorType(), index); + } + if (expression instanceof FunctionCall) { + FunctionCall call = (FunctionCall) expression; + List args = new ArrayList<>(call.args().size()); + for (LogicalExpression arg : call.args()) { + args.add(bindExpression(arg, replacements)); + } + return call instanceof BooleanOperator + ? new BooleanOperator(call.getName(), args, call.getPosition()) + : new FunctionCall(call.getName(), args, call.getPosition()); + } + if (expression instanceof CastExpression) { + CastExpression cast = (CastExpression) expression; + return new CastExpression(bindExpression(cast.getInput(), replacements), + cast.getMajorType(), cast.getPosition()); + } + if (expression instanceof ConvertExpression) { + ConvertExpression convert = (ConvertExpression) expression; + return new ConvertExpression(convert.getConvertFunction(), convert.getEncodingType(), + bindExpression(convert.getInput(), replacements), convert.getPosition()); + } + if (expression instanceof IfExpression) { + IfExpression conditional = (IfExpression) expression; + return IfExpression.newBuilder() + .setPosition(conditional.getPosition()) + .setIfCondition(new IfExpression.IfCondition( + bindExpression(conditional.ifCondition.condition, replacements), + bindExpression(conditional.ifCondition.expression, replacements))) + .setElse(bindExpression(conditional.elseExpression, replacements)) + .build(); + } + if (expression instanceof AnyValueExpression) { + AnyValueExpression any = (AnyValueExpression) expression; + return new AnyValueExpression(bindExpression(any.getInput(), replacements), any.getPosition()); + } + if (expression instanceof SchemaPath || expression instanceof NullExpression + || expression instanceof TypedNullConstant) { + return expression; + } + throw new IllegalArgumentException("Unsupported cached expression: " + expression.getClass().getName()); + } + + private static void collectTypes(LogicalExpression expression, + Map types) { + if (expression instanceof LiteralExpression) { + LiteralExpression literal = (LiteralExpression) expression; + if (literal.isDynamicParam()) { + int index = literal.getDynamicParamIndex(); + MajorType type = expression.getMajorType(); + MajorType previous = types.putIfAbsent(index, type); + if (previous != null && !previous.equals(type)) { + throw new IllegalArgumentException("Conflicting cached parameter types"); + } + } + } + for (LogicalExpression child : expression) { + collectTypes(child, types); + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java new file mode 100644 index 00000000000..d9b70dc60bb --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.util.Locale; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.drill.exec.expr.fn.DrillFuncHolder; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; + +/** Checks whether a parsed query is safe to reuse without SQL validation. */ +final class PlanCacheEligibility { + private PlanCacheEligibility() { } + + static boolean isSafeToCache(SqlNode parsed, FunctionImplementationRegistry functions) { + // SqlKind.OTHER_FUNCTION is a classification, not evidence of volatility. + if (!parsed.getKind().belongsTo(SqlKind.QUERY)) { + return false; + } + try { + parsed.accept(new SqlShuttle() { + @Override + public SqlNode visit(SqlCall call) { + if (!call.getOperator().isDeterministic() + || call.getOperator().isDynamicFunction() + || isQueryContextFunction(call) + || (call.getKind() == SqlKind.OTHER_FUNCTION + && hasVolatileOverload(call, functions))) { + throw UnsafeSql.INSTANCE; + } + return super.visit(call); + } + + @Override + public SqlNode visit(SqlDynamicParam param) { + throw UnsafeSql.INSTANCE; + } + + @Override + public SqlNode visit(SqlIdentifier id) { + // Niladic functions can be parsed as identifiers rather than calls. + if (id.isSimple() && isNiladicQueryContextName(id.getSimple())) { + throw UnsafeSql.INSTANCE; + } + return super.visit(id); + } + }); + return true; + } catch (UnsafeSql ignored) { + return false; + } + } + + private static boolean isQueryContextFunction(SqlCall call) { + String name = call.getOperator().getName().toUpperCase(Locale.ROOT); + switch (name) { + case "NOW": + case "CURRENT_TIMESTAMP": + case "LOCALTIMESTAMP": + case "STATEMENT_TIMESTAMP": + case "TRANSACTION_TIMESTAMP": + case "CURRENT_DATE": + case "CURRENT_TIME": + case "LOCALTIME": + case "SESSION_ID": + case "CURRENT_SCHEMA": + case "USER": + case "SESSION_USER": + case "SYSTEM_USER": + return true; + case "UNIX_TIMESTAMP": + return call.getOperandList().isEmpty(); + default: + return false; + } + } + + private static boolean isNiladicQueryContextName(String name) { + switch (name.toUpperCase(Locale.ROOT)) { + case "CURRENT_TIMESTAMP": + case "LOCALTIMESTAMP": + case "CURRENT_DATE": + case "CURRENT_TIME": + case "LOCALTIME": + case "SESSION_ID": + case "CURRENT_SCHEMA": + case "USER": + case "SESSION_USER": + case "SYSTEM_USER": + return true; + default: + return false; + } + } + + private static boolean hasVolatileOverload(SqlCall call, + FunctionImplementationRegistry functions) { + // Before validation, Drill functions may still be unresolved Calcite + // functions whose determinism flag is always true. + for (DrillFuncHolder holder : functions.getLocalFunctionRegistry() + .getMethods(call.getOperator().getName())) { + if (!holder.isDeterministic()) { + return true; + } + } + return false; + } + + private static final class UnsafeSql extends RuntimeException { + private static final UnsafeSql INSTANCE = new UnsafeSql(); + + private UnsafeSql() { + super(null, null, false, false); + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java new file mode 100644 index 00000000000..815b50a572d --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.calcite.util.NlsString; +import org.apache.drill.exec.planner.sql.conversion.SqlBoundDynamicParam; + +/** Extracts SQL literals into typed slots independently of cache eligibility. */ +final class PlanCacheParameterizer extends SqlShuttle { + private final List literals = new ArrayList<>(); + private final List types = new ArrayList<>(); + private final RelDataTypeFactory typeFactory; + private boolean preserveProjection; + + private PlanCacheParameterizer(RelDataTypeFactory typeFactory) { + this.typeFactory = Objects.requireNonNull(typeFactory, "typeFactory"); + } + + static Candidate parameterize(SqlNode parsed, RelDataTypeFactory typeFactory) { + Objects.requireNonNull(parsed, "parsed"); + PlanCacheParameterizer visitor = new PlanCacheParameterizer(typeFactory); + SqlNode parameterized = parsed.accept(visitor); + if (visitor.literals.isEmpty()) { + parameterized = parsed; + } + return new Candidate(parameterized, + Collections.unmodifiableList(visitor.literals), + parameterized.toString() + visitor.types); + } + + @Override + public SqlNode visit(SqlCall call) { + int configurationOperand = configurationOperand(call); + if (configurationOperand >= 0 && configurationOperand < call.operandCount()) { + // These operands become function names, encodings or resolved plugin + // references during conversion. They are not bindable expressions in + // the physical plan, so keep their entire subtree in the template key. + SqlCall parameterized = (SqlCall) call.clone(call.getParserPosition()); + for (int i = 0; i < call.operandCount(); i++) { + if (i != configurationOperand) { + parameterized.setOperand(i, visitNullable(call.operand(i))); + } + } + return parameterized; + } + if ("COALESCE".equalsIgnoreCase(call.getOperator().getName()) + && call.getOperandList().stream() + .anyMatch(operand -> operand instanceof SqlLiteral + && ((SqlLiteral) operand).getTypeName() == SqlTypeName.NULL)) { + // Calcite cannot infer a dynamic parameter's type alongside an untyped + // NULL. Keep this call in the key; explicit casts still allow slots. + return call; + } + if (call instanceof SqlJoin) { + SqlJoin join = (SqlJoin) call; + return new SqlJoin(join.getParserPosition(), visitNullable(join.getLeft()), + join.isNaturalNode(), join.getJoinTypeNode(), visitNullable(join.getRight()), + join.getConditionTypeNode(), visitNullable(join.getCondition())); + } + if (call.getKind() == SqlKind.ITEM && call.operand(1) instanceof SqlLiteral) { + // A map/array key identifies a field in the input schema. Changing it can + // change the scan projection and the HBase column used by pushdown. + return call.getOperator().createCall(call.getParserPosition(), + visitNullable(call.operand(0)), call.operand(1)); + } + // These literals control rows, ordinals, window frames or operator + // configuration. Keep them verbatim in the template key. + if (call.getKind() == SqlKind.VALUES || call.getKind() == SqlKind.ORDER_BY + || call.getKind() == SqlKind.OVER) { + if (call instanceof SqlOrderBy) { + SqlOrderBy orderBy = (SqlOrderBy) call; + boolean previous = preserveProjection; + preserveProjection = true; + SqlNode query; + try { + query = orderBy.query.accept(this); + } finally { + preserveProjection = previous; + } + return new SqlOrderBy(call.getParserPosition(), query, orderBy.orderList, + orderBy.offset, orderBy.fetch); + } + return call; + } + if (call instanceof SqlSelect) { + SqlSelect select = (SqlSelect) call; + // Preserve matching expressions across SELECT, GROUP BY, HAVING and + // QUALIFY. Parameterizing only some occurrences breaks validation or rewrites. + boolean structuralProjection = preserveProjection + || (select.getGroup() != null && !select.getGroup().isEmpty()) + || (select.getOrderList() != null && !select.getOrderList().isEmpty()) + || (select.getWindowList() != null && !select.getWindowList().isEmpty()); + return new SqlSelect(select.getParserPosition(), + (SqlNodeList) select.getOperandList().get(0), + structuralProjection ? select.getSelectList() + : (SqlNodeList) visitNullable(select.getSelectList()), + visitNullable(select.getFrom()), visitNullable(select.getWhere()), + select.getGroup(), + structuralProjection ? select.getHaving() : visitNullable(select.getHaving()), + select.getWindowList(), + structuralProjection ? select.getQualify() : visitNullable(select.getQualify()), + select.getOrderList(), select.getOffset(), select.getFetch(), + select.getHints()); + } + return super.visit(call); + } + + private static int configurationOperand(SqlCall call) { + // Keep this list aligned with the value-dependent rewrites in DrillOptiq + // and PreProcessLogicalRel; ordinary data operands still get slots. + switch (call.getOperator().getName().toUpperCase(Locale.ROOT)) { + case "DATE_PART": + case "DATE_TRUNC": + case "EXTRACT": + case "TIMESTAMPDIFF": + case "TRIM": + case "HTTPREQUEST": + case "HTTP_REQUEST": + return 0; + case "CONVERT_FROM": + case "CONVERT_TO": + return 1; + case "LENGTH": + return call.operandCount() == 2 ? 1 : -1; + default: + return -1; + } + } + + private SqlNode visitNullable(SqlNode node) { + return node == null ? null : node.accept(this); + } + + @Override + public SqlNode visit(SqlLiteral literal) { + if (literal.getValue() == null) { + // A NULL without an inferred type is part of the template, not a slot. + return literal; + } + Object value = literal.getValue(); + if (!(value instanceof BigDecimal) && !(value instanceof Boolean) + && !(value instanceof NlsString)) { + // Temporal literals, including unresolved DATE literals, and structural + // literals such as EXTRACT's time unit stay in the template key. + return literal; + } + int index = literals.size(); + literals.add(literal); + // SqlNumericLiteral reports DECIMAL even for INT and BIGINT values. Resolve + // its actual type so equal precision and scale cannot hide a width change. + String shape = literal.createSqlType(typeFactory).getSqlTypeName().name(); + if (value instanceof BigDecimal) { + BigDecimal number = (BigDecimal) value; + shape += ":" + number.precision() + ":" + number.scale(); + } else if (value instanceof NlsString) { + NlsString string = (NlsString) value; + shape += ":" + string.getValue().length() + ":" + string.getCharsetName() + + ":" + string.getCollation(); + } + types.add(shape); + return new SqlBoundDynamicParam(index, literal.getParserPosition(), literal); + } + + static final class Candidate { + final SqlNode sql; + final List literals; + final String template; + + Candidate(SqlNode sql, List literals, String template) { + this.sql = sql; + this.literals = literals; + this.template = template; + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java new file mode 100644 index 00000000000..9545b21a783 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.SqlRexConvertletTable; +import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.util.NlsString; + +/** Converts Drill SQL to relational expressions, preserving bound cache parameters. */ +public final class DrillSqlToRelConverter extends SqlToRelConverter { + public DrillSqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, + DrillCalciteCatalogReader catalog, RelOptCluster cluster, + SqlRexConvertletTable convertletTable, Config config) { + super(viewExpander, validator, catalog, cluster, convertletTable, config); + } + + @Override + public RexDynamicParam convertDynamicParam(SqlDynamicParam param) { + if (!(param instanceof SqlBoundDynamicParam)) { + return super.convertDynamicParam(param); + } + // Register the parameter with Calcite before reading its inferred type. + RexDynamicParam converted = super.convertDynamicParam(param); + SqlLiteral source = ((SqlBoundDynamicParam) param).getLiteral(); + Object literalValue = source.getValue(); + if (literalValue instanceof NlsString) { + literalValue = ((NlsString) literalValue).getValue(); + } + // Calcite may leave a dynamic parameter as ANY. RexBuilder then guesses a + // type from the Java value and can turn an exact decimal into BIGINT. + // Preserve the SQL literal's own precision and scale in that case. + RelDataType valueType = converted.getType().getSqlTypeName() == SqlTypeName.ANY + ? source.createSqlType(rexBuilder.getTypeFactory()) : converted.getType(); + RexLiteral value = rexBuilder.makeLiteral(literalValue, valueType); + return new RexBoundDynamicParam(converted.getType(), param.getIndex(), value); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java new file mode 100644 index 00000000000..b8f6566f910 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; +import org.apache.calcite.sql.validate.SqlValidatorImpl; +import org.apache.calcite.sql.validate.SqlValidatorScope; + +/** Gives bound plan-cache parameters a type during SQL validation. */ +final class DrillSqlValidator extends SqlValidatorImpl { + DrillSqlValidator(SqlOperatorTable operators, SqlValidatorCatalogReader catalog, + RelDataTypeFactory types, Config config) { + super(operators, catalog, types, config); + } + + private RelDataType boundType(SqlNode node) { + SqlBoundDynamicParam param = (SqlBoundDynamicParam) node; + return param.getLiteral().createSqlType(typeFactory); + } + + @Override + public RelDataType getValidatedNodeTypeIfKnown(SqlNode node) { + RelDataType type = super.getValidatedNodeTypeIfKnown(node); + if (type == null && node instanceof SqlBoundDynamicParam) { + type = boundType(node); + setValidatedNodeType(node, type); + } + return type; + } + + @Override + public RelDataType deriveType(SqlValidatorScope scope, SqlNode node) { + if (node instanceof SqlBoundDynamicParam) { + return getValidatedNodeType(node); + } + return super.deriveType(scope, node); + } + + @Override + protected void inferUnknownTypes(RelDataType inferredType, SqlValidatorScope scope, SqlNode node) { + if (node instanceof SqlBoundDynamicParam) { + // Unlike an unbound JDBC parameter, this slot already has a concrete + // non-null SQL literal. Do not reject it when the parent has no type. + getValidatedNodeType(node); + return; + } + super.inferUnknownTypes(inferredType, scope, node); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java new file mode 100644 index 00000000000..e19d2411235 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexLiteral; + +/** A SQL parameter whose value is available during the first plan build. */ +public final class RexBoundDynamicParam extends RexDynamicParam { + private final RexLiteral literal; + + public RexBoundDynamicParam(RelDataType type, int index, RexLiteral literal) { + super(type, index); + this.literal = literal; + } + + public RexLiteral getLiteral() { + return literal; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java new file mode 100644 index 00000000000..b1de3cbae37 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.parser.SqlParserPos; + +/** SQL parameter that retains its value for the first physical plan build. */ +public final class SqlBoundDynamicParam extends SqlDynamicParam { + private final SqlLiteral literal; + + public SqlBoundDynamicParam(int index, SqlParserPos pos, SqlLiteral literal) { + super(index, pos); + this.literal = literal; + } + + public SqlLiteral getLiteral() { + return literal; + } + + @Override + public SqlBoundDynamicParam clone(SqlParserPos pos) { + return new SqlBoundDynamicParam(getIndex(), pos, literal); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java index 25ed545c687..93ee87271e0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java @@ -42,7 +42,6 @@ import org.apache.calcite.sql.util.ChainedSqlOperatorTable; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlValidator; -import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; @@ -152,7 +151,7 @@ public SqlConverter(QueryContext context) { ); this.opTab = new ChainedSqlOperatorTable(Arrays.asList(context.getDrillOperatorTable(), catalog)); this.costFactory = (settings.useDefaultCosting()) ? null : new DrillCostBase.DrillCostFactory(); - this.validator = SqlValidatorUtil.newValidator(opTab, catalog, typeFactory, + this.validator = new DrillSqlValidator(opTab, catalog, typeFactory, SqlValidator.Config.DEFAULT.withConformance(parserConfig.conformance()) .withTypeCoercionEnabled(true) .withIdentifierExpansion(true)); @@ -176,7 +175,7 @@ public SqlConverter(QueryContext context) { this.catalog = catalog; this.opTab = parent.opTab; this.planner = parent.planner; - this.validator = SqlValidatorUtil.newValidator(opTab, catalog, typeFactory, + this.validator = new DrillSqlValidator(opTab, catalog, typeFactory, SqlValidator.Config.DEFAULT.withConformance(parserConfig.conformance()) .withTypeCoercionEnabled(true) .withIdentifierExpansion(true)); @@ -225,7 +224,7 @@ public RelRoot toRel(final SqlNode validatedNode) { initCluster(initPlanner()); DrillViewExpander viewExpander = new DrillViewExpander(this); util.getViewExpansionContext().setViewExpander(viewExpander); - final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter( + final SqlToRelConverter sqlToRelConverter = new DrillSqlToRelConverter( viewExpander, validator, catalog, cluster, DrillConvertletTable.INSTANCE, sqlToRelConverterConfig); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java index 4169a58f561..fd049442cd0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java @@ -27,12 +27,28 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.calcite.sql.SqlNode; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.server.options.QueryOptionManager; +import org.apache.drill.exec.util.ImpersonationUtil; public abstract class AbstractSqlHandler { static final Logger logger = LoggerFactory.getLogger(AbstractSqlHandler.class); public abstract PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException; + /** Applies the same administrator policy to SYSTEM options and cache management. */ + protected static void checkAdminPrivileges(QueryContext context, QueryOptionManager options, String message) { + if (context.isUserAuthenticationEnabled() + && !ImpersonationUtil.hasAdminPrivileges( + context.getQueryUserName(), + ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options), + ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) { + throw UserException.permissionError().message(message).build(logger); + } + } + @SuppressWarnings("unchecked") public static T unwrap(Object o, Class clazz) throws ForemanSetupException { if (clazz.isAssignableFrom(o.getClass())) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java index e6bf3700e97..315e60bab93 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java @@ -23,7 +23,6 @@ import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.QueryOptionManager; -import org.apache.drill.exec.util.ImpersonationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,16 +73,6 @@ OptionValue.OptionScope getScope(SqlSetOption statement, QueryOptionManager opti * @param options Options object */ void checkAdminPrivileges(QueryOptionManager options) { - if (context.isUserAuthenticationEnabled() - && !ImpersonationUtil.hasAdminPrivileges( - context.getQueryUserName(), - ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options), - ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) { - - throw UserException - .permissionError() - .message("Not authorized to change SYSTEM options.") - .build(logger); - } + checkAdminPrivileges(context, options, "Not authorized to change SYSTEM options."); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java new file mode 100644 index 00000000000..aba3535651f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.handlers; + +import org.apache.calcite.sql.SqlNode; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.sql.DirectPlan; +import org.apache.drill.exec.planner.sql.parser.SqlClearPlanCache; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.work.foreman.ForemanSetupException; + +/** Clears the cache on the Drillbit that plans this command. */ +public final class ClearPlanCacheHandler extends AbstractSqlHandler { + private final QueryContext context; + + public ClearPlanCacheHandler(QueryContext context) { + this.context = context; + } + + @Override + public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException { + unwrap(sqlNode, SqlClearPlanCache.class); + checkAdminPrivileges(context, context.getOptions(), "Not authorized to clear the SYSTEM plan cache."); + context.getDrillbitContext().getPlanCache().clear(); + DrillbitEndpoint endpoint = context.getCurrentEndpoint(); + return DirectPlan.createDirectPlan(context, true, String.format( + "Plan cache cleared on Drillbit %s:%d.", endpoint.getAddress(), endpoint.getUserPort())); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java new file mode 100644 index 00000000000..e2a7ce20df5 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.parser; + +import java.util.Collections; +import java.util.List; + +import org.apache.calcite.sql.SqlAlter; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; + +/** Parse tree for clearing the receiving Drillbit's physical plan cache. */ +public final class SqlClearPlanCache extends SqlAlter { + public static final SqlSpecialOperator OPERATOR = + new SqlSpecialOperator("CLEAR_PLAN_CACHE", SqlKind.OTHER_DDL) { + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) { + return new SqlClearPlanCache(pos); + } + }; + + public SqlClearPlanCache(SqlParserPos pos) { + super(pos, "SYSTEM"); + } + + @Override + public SqlOperator getOperator() { + return OPERATOR; + } + + @Override + public List getOperandList() { + return Collections.emptyList(); + } + + @Override + protected void unparseAlterOperation(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword("CLEAR PLAN CACHE"); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java index fc632df06d9..b1a3fc52888 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java @@ -309,6 +309,11 @@ public String resolveTemporaryTableName(String tableName) { return temporaryTables.get(tableName.toLowerCase()); } + /** Temporary tables can shadow names in plans shared across sessions. */ + public boolean hasTemporaryTables() { + return !temporaryTables.isEmpty(); + } + public String getOriginalTableNameFromTemporaryTable(String tableName) { for (String originalTableName : temporaryTables.keySet()) { if (temporaryTables.get(originalTableName).equals(tableName)) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 3fa9bcede8c..44c98c30a83 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -33,6 +33,7 @@ import org.apache.drill.exec.oauth.OAuthTokenProvider; import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.sql.PlanCache; import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.rpc.control.Controller; @@ -63,6 +64,7 @@ public class DrillbitContext implements AutoCloseable { private final BootStrapContext context; private final PhysicalPlanReader reader; + private final PlanCache planCache; private final ClusterCoordinator coord; private final DataConnectionCreator connectionsPool; private final DrillbitEndpoint endpoint; @@ -122,6 +124,9 @@ public DrillbitContext( ExecConstants.STORAGE_PLUGIN_REGISTRY_IMPL, StoragePluginRegistry.class, this); reader = new PhysicalPlanReader(config, classpathScan, lpPersistence, endpoint, storagePlugins); + planCache = new PlanCache(config.getLong(ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES), + config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE), + config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS)); operatorCreatorRegistry = new OperatorCreatorRegistry(classpathScan); systemOptions = new SystemOptionManager(lpPersistence, provider, config, context.getDefinitions()); functionRegistry = new FunctionImplementationRegistry(config, classpathScan, systemOptions); @@ -252,6 +257,10 @@ public PhysicalPlanReader getPlanReader() { return reader; } + public PlanCache getPlanCache() { + return planCache; + } + public PersistentStoreProvider getStoreProvider() { return provider; } @@ -325,6 +334,7 @@ public Set> getUserConnect @Override public void close() throws Exception { + planCache.close(); getOptionManager().close(); getFunctionImplementationRegistry().close(); getRemoteFunctionRegistry().close(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 6cda5c872c0..d4502cee418 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -83,6 +83,7 @@ public static CaseInsensitiveMap createDefaultOptionDefinition @SuppressWarnings("deprecation") final OptionDefinition[] definitions = new OptionDefinition[]{ new OptionDefinition(PlannerSettings.CONSTANT_FOLDING), + new OptionDefinition(PlannerSettings.PLAN_CACHE), new OptionDefinition(PlannerSettings.EXCHANGE), new OptionDefinition(PlannerSettings.HASHAGG), new OptionDefinition(PlannerSettings.STREAMAGG), diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java index 02b0890cc99..472c6f94f77 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java @@ -179,6 +179,10 @@ public QueryProfile getProfile() { return profile; } + public String getPlanCacheStatus() { + return profile.hasPlanCacheHit() ? (profile.getPlanCacheHit() ? "Yes" : "No") : "Unknown"; + } + public String getProfileDuration() { return (new SimpleDurationFormat(profile.getStart(), profile.getEnd())).verbose(); } @@ -377,6 +381,9 @@ private void generateOpMap(String plan) { //[e.g ] operatorLine = "01-03 Flatten(flattenField=[$1]) : rowType = RecordType(ANY rfsSpecCode, ..." String[] operatorLine = plan.split("\\n"); for (String line : operatorLine) { + if (line.startsWith("Parameters: ")) { + continue; + } String[] lineToken = line.split("\\s+", 3); if (lineToken.length < 2) { continue; //Skip due to possible invalid entry diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java new file mode 100644 index 00000000000..89572f31c33 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import java.util.Objects; + +/** Immutable plan-cache identity and compatibility version of one resolved table. */ +public final class PlanCacheTable { + private final String identifier; + private final String version; + + public PlanCacheTable(String identifier, String version) { + this.identifier = Objects.requireNonNull(identifier, "identifier"); + this.version = Objects.requireNonNull(version, "version"); + } + + public String getIdentifier() { + return identifier; + } + + public String getVersion() { + return version; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index b3bf92813c9..b810510c264 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -28,6 +28,7 @@ import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.planner.PlannerPhase; import org.apache.drill.exec.metastore.MetadataProviderManager; import org.apache.drill.exec.server.options.SessionOptionManager; @@ -71,6 +72,27 @@ public interface StoragePlugin extends SchemaFactory, AutoCloseable { */ boolean supportsWrite(); + /** + * Opts this plugin into plan caching. The engine validates query context and + * rebinds expression parameters; the plugin must provide a plan-compatible + * table version and safely rebuild every scan for an eligible selection, + * including value-dependent state for each execution. Check selection + * eligibility without reading table versions or building cache metadata. + */ + default boolean supportPlanCache(DrillTableSelection selection) { + return false; + } + + /** + * Returns a stable table identifier and a version that changes whenever the + * cached operator structure is no longer compatible. Data-only changes may + * keep the version only if the scan reads current data on deserialization. + * Return null when that contract cannot be met. + */ + default PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { + return null; + } + /** * Indicates if Drill can insert to a table to this plugin. */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 5665ae72a4a..1791c27d554 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -51,11 +51,13 @@ import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.planner.PlannerPhase; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.SessionOptionManager; import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.ClassPathFileSystem; import org.apache.drill.exec.store.LocalSyncableFileSystem; +import org.apache.drill.exec.store.PlanCacheTable; import org.apache.drill.exec.store.SchemaConfig; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; @@ -330,6 +332,39 @@ public FormatPlugin getFormatPlugin(String name) { return formatCreator.getFormatPluginByName(name); } + @Override + public boolean supportPlanCache(DrillTableSelection selection) { + if (!(selection instanceof FormatSelection)) { + return false; + } + FormatSelection formatSelection = (FormatSelection) selection; + FormatPlugin format = getFormatPlugin(formatSelection.getFormat()); + return format != null && format.supportPlanCache(formatSelection.getSelection()); + } + + @Override + public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { + if (!(selection instanceof FormatSelection)) { + return null; + } + FormatSelection formatSelection = (FormatSelection) selection; + FileSelection files = formatSelection.getSelection(); + FormatPlugin format = getFormatPlugin(formatSelection.getFormat()); + if (format == null || !format.supportPlanCache(files)) { + return null; + } + PlanCacheTable table = format.planCacheTable(files); + if (table == null) { + return null; + } + String formatName = format.getName(); + if (formatName == null || formatName.isEmpty()) { + return null; + } + String identifier = formatName.length() + ":" + formatName + table.getIdentifier(); + return new PlanCacheTable(identifier, table.getVersion()); + } + /** * If format plugin configuration is for named format plugin, will return * format plugin from pre-loaded list by name. For other cases will try to diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java index 364af39bc6c..d72bbd5d0a0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -34,6 +34,7 @@ import org.apache.drill.exec.planner.common.DrillStatsTable.TableStatistics; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.store.PlanCacheTable; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -48,6 +49,22 @@ public interface FormatPlugin { boolean supportsWrite(); + /** + * Whether every scan of this selection can be safely rebuilt for a cached plan. + * Check eligibility without reading table versions or building cache metadata. + */ + default boolean supportPlanCache(FileSelection selection) { + return false; + } + + /** + * Returns a stable table identifier and a physical-plan compatibility version, + * or null when the selection cannot be cached. + */ + default PlanCacheTable planCacheTable(FileSelection selection) throws IOException { + return null; + } + /** * Indicates whether this FormatPlugin supports auto-partitioning for CTAS statements * @return true if auto-partitioning is supported diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java index 1d4fff8df14..7334ad6cd0d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java @@ -75,7 +75,7 @@ public boolean needsFinalColumnReordering() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupScan", groupScan); + return super.explainTerms(pw).item("groupScan", groupScan.getDigest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index a099b96b123..7fb7d764135 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -759,6 +759,7 @@ public void close() { drillbitContext.getWorkBus().removeFragmentStatusListener(queryId); drillbitContext.getClusterCoordinator().removeDrillbitStatusListener(queryManager.getDrillbitStatusListener()); + final Runnable pendingPlanCacheInsert = queryContext.takePendingPlanCacheInsert(); suppressingClose(queryContext); /* @@ -800,6 +801,10 @@ public void close() { queryManager.writeFinalProfile(uex); } + if (resultState == QueryState.COMPLETED && pendingPlanCacheInsert != null) { + pendingPlanCacheInsert.run(); + } + /* * If sending the result fails, we don't really have any way to modify the result we tried to send; * it is possible it got sent but the result came from a later part of the code path. It is also diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java index a919ed90059..10147955dd2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java @@ -372,6 +372,8 @@ private QueryProfile getQueryProfile(UserException ex) { .setQueueName(queueName == null ? "-" : queueName) .setOptionsJson(getQueryOptionsAsJson()); + profileBuilder.setPlanCacheHit(queryCtx.isPlanCacheHit()); + if (ex != null) { profileBuilder.setError(ex.getMessage(false)); profileBuilder.setVerboseError(ex.getVerboseMessage(false)); diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 887fa15bff4..7781aa486e1 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -281,6 +281,16 @@ drill.exec: { max_size: 500000000 } }, + plan_cache: { + # Maximum total UTF-8 byte size of cache keys, plan JSON, and explain text per Drillbit. + # 0 disables storage. Entry metadata and Java object overhead are not included. + max_size_bytes: 33554432, + # Fixed lifetime since creation or replacement; 0 disables this expiration policy. + expire_after_write: 0, + # Idle lifetime since the last read or write; 0 disables this expiration policy. + # When both policies are enabled, either deadline expires the entry. + expire_after_access: 10m + }, compile: { codegen.debug.topn: false, compiler: "DEFAULT", @@ -663,6 +673,7 @@ drill.exec.options: { planner.disable_exchanges: false, planner.enable_broadcast_join: true, planner.enable_constant_folding: true, + planner.enable_plan_cache: false, planner.enable_decimal_data_type: true, planner.enable_demux_exchange: false, planner.enable_file_listing_limit0_optimization: false, diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl index 513e22cf592..129d79f0b02 100644 --- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl +++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl @@ -313,6 +313,7 @@ State Foreman Total Fragments + Plan Cache Hit <#if queued> Total Cost Queue @@ -324,6 +325,7 @@ ${model.getQueryStateDisplayName()} ${model.getProfile().getForeman().getAddress()} ${model.getProfile().getTotalFragments()} + ${model.getPlanCacheStatus()} <#if queued> ${model.getProfile().getTotalCost()} ${queueName} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java new file mode 100644 index 00000000000..70ca96f6848 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Ticker; +import org.apache.drill.categories.PlannerTest; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.test.BaseTest; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category(PlannerTest.class) +public class TestPlanCacheConfig extends BaseTest { + private final ManualTicker ticker = new ManualTicker(); + private PhysicalPlan plan; + private PhysicalPlanReader reader; + private PlanCache.ContextSnapshot context; + + @Before + public void setup() throws Exception { + plan = mock(PhysicalPlan.class); + reader = mock(PhysicalPlanReader.class); + context = mock(PlanCache.ContextSnapshot.class); + when(reader.writeJson(plan)).thenReturn("{}"); + } + + @Test + public void testDefaultSettingsKeepHotPlansAndExpireIdlePlans() throws Exception { + DrillConfig config = DrillConfig.create(); + long maxSizeBytes = config.getLong(ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES); + Duration write = config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE); + Duration access = config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS); + assertEquals(32L * 1024 * 1024, maxSizeBytes); + assertEquals(Duration.ZERO, write); + assertEquals(Duration.ofMinutes(10), access); + try (PlanCache cache = new PlanCache(maxSizeBytes, write, access, ticker)) { + put(cache, "hot"); + put(cache, "idle"); + for (int i = 0; i < 3; i++) { + ticker.advance(Duration.ofMinutes(5)); + assertNotNull(cache.get("hot")); + } + assertNull(cache.get("idle")); + ticker.advance(Duration.ofMinutes(10)); + assertNull(cache.get("hot")); + } + } + + @Test + public void testWriteExpirationDoesNotExtendOnAccess() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ofMinutes(10), Duration.ZERO, ticker)) { + put(cache, "plan"); + ticker.advance(Duration.ofMinutes(9)); + assertNotNull(cache.get("plan")); + ticker.advance(Duration.ofMinutes(1)); + assertNull(cache.get("plan")); + } + } + + @Test + public void testReplacementRestartsWriteExpiration() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ofMinutes(10), Duration.ZERO, ticker)) { + put(cache, "plan"); + ticker.advance(Duration.ofMinutes(9)); + put(cache, "plan"); + ticker.advance(Duration.ofMinutes(9)); + assertNotNull(cache.get("plan")); + ticker.advance(Duration.ofMinutes(1)); + assertNull(cache.get("plan")); + } + } + + @Test + public void testCombinedPoliciesExpireAtEitherDeadline() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ofMinutes(10), Duration.ofMinutes(3), ticker)) { + put(cache, "idle"); + ticker.advance(Duration.ofMinutes(3)); + assertNull(cache.get("idle")); + + put(cache, "hot"); + for (int i = 0; i < 4; i++) { + ticker.advance(Duration.ofMinutes(2)); + assertNotNull(cache.get("hot")); + } + ticker.advance(Duration.ofMinutes(2)); + assertNull(cache.get("hot")); + } + } + + @Test + public void testBothExpirationPoliciesCanBeDisabled() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + ticker.advance(Duration.ofDays(365)); + assertNotNull(cache.get("plan")); + } + } + + @Test + public void testCapacityCountsUtf8Bytes() throws Exception { + // Eight JSON bytes plus four key bytes when encoded as UTF-8. + when(reader.writeJson(plan)).thenReturn("\"ééé\""); + try (PlanCache cache = new PlanCache(11, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + assertNull(cache.get("plan")); + } + try (PlanCache cache = new PlanCache(12, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + assertNotNull(cache.get("plan")); + } + } + + @Test + public void testCapacityCountsUtf8KeyBytes() throws Exception { + // Six key bytes plus two JSON bytes. + try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "ééé"); + assertNull(cache.get("ééé")); + } + try (PlanCache cache = new PlanCache(8, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "ééé"); + assertNotNull(cache.get("ééé")); + } + } + + @Test + public void testCapacityCountsUtf8ExplainBytes() throws Exception { + // Two key bytes, two JSON bytes, and three explain text bytes. + try (PlanCache cache = new PlanCache(6, Duration.ZERO, Duration.ZERO, ticker)) { + assertTrue(cache.put("é", reader.writeJson(plan), "计", reader, context)); + assertNull(cache.get("é")); + } + try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { + assertTrue(cache.put("é", reader.writeJson(plan), "计", reader, context)); + assertNotNull(cache.get("é")); + assertEquals("计", cache.get("é").getTextPlan()); + } + } + + @Test + public void testNullAndEmptyExplainTextHaveZeroWeight() throws Exception { + try (PlanCache cache = new PlanCache(4, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "é"); + assertNotNull(cache.get("é")); + assertNull(cache.get("é").getTextPlan()); + assertTrue(cache.put("é", reader.writeJson(plan), "", reader, context)); + assertNotNull(cache.get("é")); + assertEquals("", cache.get("é").getTextPlan()); + } + } + + @Test + public void testCapacityEvictsBeforeIdleExpiration() throws Exception { + // Small limits use a single Guava segment, making eviction order deterministic. + try (PlanCache cache = new PlanCache(11, Duration.ZERO, Duration.ofMinutes(10), ticker)) { + when(reader.writeJson(plan)).thenReturn("\"aa\""); + put(cache, "a"); + when(reader.writeJson(plan)).thenReturn("\"bbb\""); + put(cache, "b"); + assertNotNull(cache.get("a")); + when(reader.writeJson(plan)).thenReturn("\"cc\""); + put(cache, "c"); + assertNull(cache.get("b")); + assertNotNull(cache.get("a")); + assertNotNull(cache.get("c")); + assertEquals(1L, metric("evictions")); + assertEquals(2L, metric("entries")); + } + } + + @Test + public void testRegisteredInvalidationAndEntryMetrics() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + assertEquals(0L, metric("invalidations")); + assertEquals(0L, metric("entries")); + put(cache, "key"); + assertEquals(1L, metric("entries")); + cache.invalidate("key"); + assertEquals(1L, metric("invalidations")); + assertEquals(0L, metric("entries")); + } + assertNull(DrillMetrics.getRegistry().getGauges().get("drill.plan_cache.entries")); + } + + @Test + public void testClosingOlderCacheKeepsNewerCacheMetrics() { + try (PlanCache older = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker); + PlanCache newer = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + older.close(); + newer.recordBind(); + assertEquals(1L, metric("hits")); + } + assertNull(DrillMetrics.getRegistry().getGauges().get("drill.plan_cache.hits")); + } + + @Test + public void testZeroCapacityDisablesStorage() throws Exception { + try (PlanCache cache = new PlanCache(0, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + assertNull(cache.get("plan")); + } + } + + @Test + public void testNegativeSettingsAreRejected() { + IllegalArgumentException size = assertThrows(IllegalArgumentException.class, + () -> new PlanCache(-1, Duration.ZERO, Duration.ZERO, ticker)); + assertTrue(size.getMessage().contains(ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES)); + IllegalArgumentException write = assertThrows(IllegalArgumentException.class, + () -> new PlanCache(1024, Duration.ofSeconds(-1), Duration.ZERO, ticker)); + assertTrue(write.getMessage().contains(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE)); + IllegalArgumentException access = assertThrows(IllegalArgumentException.class, + () -> new PlanCache(1024, Duration.ZERO, Duration.ofSeconds(-1), ticker)); + assertTrue(access.getMessage().contains(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS)); + } + + private void put(PlanCache cache, String key) throws Exception { + assertTrue(cache.put(key, reader.writeJson(plan), null, reader, context)); + } + + @Test + public void testClearDiscardsActiveAndQueuedPublications() throws Exception { + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch resume = new CountDownLatch(1); + when(reader.readPhysicalPlan("{}")).thenAnswer(invocation -> { + reading.countDown(); + assertTrue(resume.await(10, TimeUnit.SECONDS)); + return plan; + }); + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + long generation = cache.getGeneration(); + try { + cache.writeAfterSuccess("active", "{}", null, reader, context, generation); + assertTrue(reading.await(10, TimeUnit.SECONDS)); + cache.writeAfterSuccess("queued", "{}", null, reader, context, generation); + cache.clear(); + } finally { + resume.countDown(); + } + cache.awaitWrites(); + assertNull(cache.get("active")); + assertNull(cache.get("queued")); + assertEquals(0L, metric("entries")); + assertEquals(1L, metric("invalidations")); + assertEquals(0L, metric("evictions")); + + // A newly planned query can publish with the new generation. + cache.writeAfterSuccess("new", "{}", null, reader, context, cache.getGeneration()); + cache.awaitWrites(); + assertNotNull(cache.get("new")); + } + } + + private long metric(String name) { + return (Long) DrillMetrics.getRegistry().getGauges().get("drill.plan_cache." + name).getValue(); + } + + private static final class ManualTicker extends Ticker { + private long nanos; + + @Override + public long read() { + return nanos; + } + + void advance(Duration duration) { + nanos += duration.toNanos(); + } + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java new file mode 100644 index 00000000000..43e63087c7d --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.time.Duration; +import java.util.Collections; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.drill.categories.PlannerTest; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; +import org.apache.drill.exec.server.options.QueryOptionManager; +import org.apache.drill.exec.store.PlanCacheTable; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.test.BaseTest; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category(PlannerTest.class) +public class TestPlanCacheContext extends BaseTest { + private QueryContext context; + private SchemaPlus schema; + private StoragePluginRegistry registry; + private StoragePlugin pluginA; + private StoragePluginConfig configA; + private DrillTableSelection selectionA; + + @Before + public void setup() throws Exception { + context = mock(QueryContext.class); + schema = mock(SchemaPlus.class); + registry = mock(StoragePluginRegistry.class); + QueryOptionManager options = mock(QueryOptionManager.class); + when(options.iterator()).thenAnswer(call -> Collections.emptyIterator()); + when(context.getOptions()).thenReturn(options); + when(context.getStorage()).thenReturn(registry); + configA = mock(StoragePluginConfig.class); + pluginA = mock(StoragePlugin.class); + selectionA = mock(DrillTableSelection.class); + addTable("a", pluginA, configA, selectionA); + addTable("b", mock(StoragePlugin.class), mock(StoragePluginConfig.class), mock(DrillTableSelection.class)); + } + + @Test + public void testPluginConfigChangesPartitionCacheKey() throws Exception { + PlanCache.ContextSnapshot original = snapshot("SELECT * FROM \"a\""); + when(registry.encode(configA)).thenReturn("changed plugin configuration"); + PlanCache.ContextSnapshot changed = snapshot("SELECT * FROM \"a\""); + assertNotEquals(original.keyFingerprint(), changed.keyFingerprint()); + } + + @Test + public void testPluginTraversalOrderDoesNotChangeFingerprint() throws Exception { + PlanCache.ContextSnapshot first = snapshot("SELECT * FROM \"a\" JOIN \"b\" ON TRUE"); + PlanCache.ContextSnapshot reversed = snapshot("SELECT * FROM \"b\" JOIN \"a\" ON TRUE"); + assertEquals(first.keyFingerprint(), reversed.keyFingerprint()); + } + + @Test + public void testTableVersionChangeInvalidatesEntryWithoutChangingKey() throws Exception { + PlanCache.ContextSnapshot original = snapshot("SELECT * FROM \"a\""); + when(pluginA.planCacheTable(selectionA)).thenReturn(new PlanCacheTable("a", "version-2")); + PlanCache.ContextSnapshot changed = snapshot("SELECT * FROM \"a\""); + assertEquals(original.keyFingerprint(), changed.keyFingerprint()); + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO)) { + assertTrue(cache.put("key", "{}", null, mock(PhysicalPlanReader.class), original)); + assertTrue(cache.get("key").matchesContext(original)); + assertFalse(cache.get("key").matchesContext(changed)); + } + } + + private void addTable(String name, StoragePlugin plugin, StoragePluginConfig config, + DrillTableSelection selection) throws Exception { + DrillTable table = mock(DrillTable.class); + when(table.getSelection()).thenReturn(selection); + when(table.getPlugin()).thenReturn(plugin); + when(table.getStorageEngineName()).thenReturn("storage-" + name); + when(schema.getTable(name)).thenReturn(table); + when(plugin.supportPlanCache(selection)).thenReturn(true); + when(plugin.planCacheTable(selection)).thenReturn(new PlanCacheTable(name, "version-1")); + when(plugin.getConfig()).thenReturn(config); + when(registry.encode(config)).thenReturn("configuration-" + name); + } + + private PlanCache.ContextSnapshot snapshot(String sql) throws Exception { + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( + schema, SqlParser.create(sql).parseQuery(), context); + assertNotNull(snapshot); + return snapshot; + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java new file mode 100644 index 00000000000..85261a4c68f --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.calcite.sql.SqlNode; +import org.apache.drill.categories.SqlTest; +import org.apache.drill.common.config.DrillProperties; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.rowSet.DirectRowSet; +import org.apache.drill.exec.physical.rowSet.RowSetReader; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.sql.conversion.SqlConverter; +import org.apache.drill.exec.proto.UserBitShared.QueryId; +import org.apache.drill.exec.proto.UserBitShared.UserCredentials; +import org.apache.drill.exec.rpc.user.UserSession; +import org.apache.drill.exec.util.Pointer; +import org.apache.drill.test.ClientFixture; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@Category(SqlTest.class) +public class TestPlanCachePlanning extends ClusterTest { + @BeforeClass + public static void setupCluster() throws Exception { + startCluster(ClusterFixture.builder(dirTestWatcher)); + } + + @Before + public void disableCacheForBaseline() { + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + } + + @Test + public void testHavingMatchesGroupedConcatenationOnMissAndHit() throws Exception { + String sql = "SELECT x || 'a' AS k, COUNT(*) AS n " + + "FROM (VALUES ('b', 1), ('b', 2), ('c', 3)) AS t(x, v) " + + "WHERE v > %d GROUP BY x || 'a' HAVING x || 'a' = 'ba'"; + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "n").baselineValues("ba", 2L).go(); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + long misses = metric("misses"); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "n").baselineValues("ba", 2L).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(misses + 1, metric("misses")); + + // WHERE values remain bindable even though the matching HAVING literals are preserved. + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("k", "n").baselineValues("ba", 1L).go(); + assertEquals(hits + 1, cache().getHitCount()); + assertEquals(hits + 1, metric("hits")); + assertEquals(misses + 1, metric("misses")); + } + + @Test + public void testHavingMatchesGroupedArithmeticAndKeepsLiteralsInKey() throws Exception { + String sql = "SELECT x + 1 AS k, COUNT(*) AS n " + + "FROM (VALUES (1), (1), (2)) AS t(x) GROUP BY x + 1 HAVING x + 1 = %d"; + client.testBuilder().sqlQuery(sql, 2).unOrdered() + .baselineColumns("k", "n").baselineValues(2, 2L).go(); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 2).unOrdered() + .baselineColumns("k", "n").baselineValues(2, 2L).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 2).unOrdered() + .baselineColumns("k", "n").baselineValues(2, 2L).go(); + assertEquals(hits + 1, cache().getHitCount()); + + // A different preserved HAVING literal must not reuse the first template. + client.testBuilder().sqlQuery(sql, 3).unOrdered() + .baselineColumns("k", "n").baselineValues(3, 1L).go(); + cache().awaitWrites(); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testQualifyMatchesOrderedProjectionOnMissAndHit() throws Exception { + String sql = "SELECT x + 1 AS k, ROW_NUMBER() OVER (ORDER BY x + 1) AS rn " + + "FROM (VALUES (1, 1), (1, 2), (2, 3)) AS t(x, v) " + + "WHERE v > %d QUALIFY x + 1 = 2 " + + "AND ROW_NUMBER() OVER (ORDER BY x + 1) = 1 ORDER BY x + 1"; + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "rn").baselineValues(2, 1L).go(); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "rn").baselineValues(2, 1L).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("k", "rn").baselineValues(2, 1L).go(); + assertEquals(hits + 1, cache().getHitCount()); + } + + private PlanCache cache() { + return cluster.drillbit().getContext().getPlanCache(); + } + + @Test + public void testClearPlanCacheForcesMissThenAllowsNewHits() throws Exception { + String sql = "SELECT x AS clear_value FROM (VALUES (1), (2)) AS t(x) WHERE x > %d"; + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("clear_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("clear_value").baselineValues(2).go(); + assertEquals(hits + 1, cache().getHitCount()); + + long invalidations = metric("invalidations"); + clearCacheWithSql(); + assertEquals(0L, metric("entries")); + assertEquals(invalidations + 1, metric("invalidations")); + assertEquals(hits + 1, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("clear_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + assertEquals(hits + 1, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("clear_value").baselineValues(2).go(); + assertEquals(hits + 2, cache().getHitCount()); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + clearCacheWithSql(); + clearCacheWithSql(); + assertEquals(0L, metric("entries")); + } + + private void clearCacheWithSql() throws Exception { + client.testBuilder().sqlQuery("ALTER SYSTEM CLEAR PLAN CACHE").unOrdered() + .baselineColumns("ok", "summary") + .baselineValues(true, String.format("Plan cache cleared on Drillbit %s:%d.", + cluster.drillbit().getContext().getEndpoint().getAddress(), + cluster.drillbit().getContext().getEndpoint().getUserPort())).go(); + } + + private long metric(String name) { + return (Long) DrillMetrics.getRegistry().getGauges().get("drill.plan_cache." + name).getValue(); + } + + @Test + public void testDifferentSessionOptionsKeepSeparateCachedPlans() throws Exception { + String sql = "SELECT x AS option_value FROM (VALUES (1), (2)) AS t(x) WHERE x > %d"; + try (ClientFixture first = cluster.clientBuilder().property(DrillProperties.USER, "options-test").build(); + ClientFixture second = cluster.clientBuilder().property(DrillProperties.USER, "options-test").build()) { + first.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + second.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + first.alterSession(ExecConstants.SLICE_TARGET, 10000); + second.alterSession(ExecConstants.SLICE_TARGET, 20000); + long hits = cache().getHitCount(); + first.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("option_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + second.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("option_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + + // Alternating clients must keep hitting their own entry rather than evicting each other. + first.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("option_value").baselineValues(2).go(); + second.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("option_value").baselineValues(2).go(); + assertEquals(hits + 2, cache().getHitCount()); + } + } + + @Test + public void testPublicationUsesSnapshotBeforeExecutionMutatesOperators() throws Exception { + UserSession session = UserSession.Builder.newBuilder() + .withCredentials(UserCredentials.newBuilder().setUserName("snapshot-test").build()) + .withOptionManager(cluster.drillbit().getContext().getOptionManager()) + .setSupportComplexTypes(true).build(); + session.getOptions().setLocalOption(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + String sql = "SELECT x AS snapshot_value FROM (VALUES (1), (2)) AS t(x) WHERE x > 0"; + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan planned = DrillSqlWorker.getPlan(context, sql, new Pointer<>()); + Runnable publish = context.takePendingPlanCacheInsert(); + assertNotNull(publish); + List originalIds = planned.getSortedOperators().stream() + .map(PhysicalOperator::getOperatorId).collect(Collectors.toList()); + planned.getSortedOperators().forEach(operator -> operator.setOperatorId(operator.getOperatorId() + 1000)); + + publish.run(); + cache().awaitWrites(); + SqlConverter converter = new SqlConverter(context); + SqlNode parsed = converter.parse(sql); + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve(converter.getDefaultSchema(), parsed, context); + assertNotNull(snapshot); + PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(parsed, converter.getTypeFactory()); + String key = context.getQueryUserName() + '\n' + session.getDefaultSchemaPath() + '\n' + + snapshot.keyFingerprint() + '\n' + candidate.template; + PlanCache.Entry entry = cache().get(key); + assertNotNull(entry); + PhysicalPlan cached = entry.bind(candidate.literals, cluster.drillbit().getContext().getPlanReader()); + List cachedIds = cached.getSortedOperators().stream() + .map(PhysicalOperator::getOperatorId).collect(Collectors.toList()); + assertEquals(originalIds, cachedIds); + assertNotEquals(planned.getSortedOperators().stream().map(PhysicalOperator::getOperatorId) + .collect(Collectors.toList()), cachedIds); + } + } + + @Test + public void testClearPreventsPublicationPendingQueryCompletion() throws Exception { + UserSession session = UserSession.Builder.newBuilder() + .withCredentials(UserCredentials.newBuilder().setUserName("clear-pending-test").build()) + .withOptionManager(cluster.drillbit().getContext().getOptionManager()) + .setSupportComplexTypes(true).build(); + session.getOptions().setLocalOption(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + DrillSqlWorker.getPlan(context, + "SELECT x AS pending_value FROM (VALUES (1), (2)) AS t(x) WHERE x > 0", new Pointer<>()); + Runnable publish = context.takePendingPlanCacheInsert(); + assertNotNull(publish); + clearCacheWithSql(); + publish.run(); + cache().awaitWrites(); + assertEquals(0L, metric("entries")); + } + } + + @Test + public void testBindingFailureInvalidatesAndReplansBeforeExecution() throws Exception { + UserSession session = UserSession.Builder.newBuilder() + .withCredentials(UserCredentials.newBuilder().setUserName("binding-failure-test").build()) + .withOptionManager(cluster.drillbit().getContext().getOptionManager()) + .setSupportComplexTypes(true).build(); + session.getOptions().setLocalOption(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + String sql = "SELECT x AS fallback_value FROM (VALUES (1), (2)) AS t(x) WHERE x > %d"; + PhysicalPlanReader reader = cluster.drillbit().getContext().getPlanReader(); + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan planned = DrillSqlWorker.getPlan(context, String.format(sql, 0), new Pointer<>()); + context.takePendingPlanCacheInsert(); + SqlConverter converter = new SqlConverter(context); + SqlNode parsed = converter.parse(String.format(sql, 0)); + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve(converter.getDefaultSchema(), parsed, context); + PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(parsed, converter.getTypeFactory()); + String key = context.getQueryUserName() + '\n' + session.getDefaultSchemaPath() + '\n' + + snapshot.keyFingerprint() + '\n' + candidate.template; + String json = reader.writeJson(planned); + assertTrue(json.contains("bound_dynamic_param(0,")); + // Readable plan JSON with a nonexistent slot must fail binding before execution. + assertTrue(cache().put(key, json.replace("bound_dynamic_param(0,", "bound_dynamic_param(999,"), + null, reader, snapshot)); + } + + long hits = cache().getHitCount(); + long invalidations = metric("invalidations"); + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan replanned = DrillSqlWorker.getPlan(context, String.format(sql, 0), new Pointer<>()); + assertFalse(context.isPlanCacheHit()); + assertEquals(invalidations + 1, metric("invalidations")); + assertEquals(Arrays.asList(1, 2), executeValues(replanned)); + Runnable publish = context.takePendingPlanCacheInsert(); + assertNotNull(publish); + publish.run(); + cache().awaitWrites(); + } + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan rebound = DrillSqlWorker.getPlan(context, String.format(sql, 1), new Pointer<>()); + assertTrue(context.isPlanCacheHit()); + assertEquals(Arrays.asList(2), executeValues(rebound)); + assertEquals(hits + 1, cache().getHitCount()); + } + } + + private List executeValues(PhysicalPlan plan) throws Exception { + DirectRowSet rows = client.queryBuilder().physical(cluster.drillbit().getContext().getPlanReader().writeJson(plan)).rowSet(); + try { + List values = new ArrayList<>(); + RowSetReader reader = rows.reader(); + while (reader.next()) { + values.add(reader.scalar(0).getInt()); + } + return values; + } finally { + rows.clear(); + } + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java new file mode 100644 index 00000000000..9ea918a54ac --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.parser; + +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.drill.categories.SqlTest; +import org.apache.drill.exec.planner.sql.parser.impl.DrillParserImpl; +import org.apache.drill.test.BaseTest; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +@Category(SqlTest.class) +public class TestClearPlanCacheSqlParser extends BaseTest { + private SqlNode parse(String sql) throws SqlParseException { + return SqlParser.create(sql, SqlParser.config().withParserFactory(DrillParserImpl.FACTORY)).parseStmt(); + } + + @Test + public void testClearPlanCacheRoundTrip() throws Exception { + SqlNode node = parse("alter system clear plan cache"); + assertTrue(node instanceof SqlClearPlanCache); + assertEquals(SqlKind.OTHER_DDL, node.getKind()); + assertEquals("SYSTEM", ((SqlClearPlanCache) node).getScope()); + assertTrue(parse(node.toString()) instanceof SqlClearPlanCache); + assertTrue(SqlNode.clone(node) instanceof SqlClearPlanCache); + } + + @Test + public void testOnlySystemScopeIsAccepted() { + assertThrows(SqlParseException.class, () -> parse("ALTER SESSION CLEAR PLAN CACHE")); + assertThrows(SqlParseException.class, () -> parse("CLEAR PLAN CACHE")); + assertThrows(SqlParseException.class, () -> parse("ALTER SYSTEM CLEAR PLAN")); + assertThrows(SqlParseException.class, () -> parse("ALTER SYSTEM CLEAR PLAN CACHE extra")); + } + + @Test + public void testExistingOptionSyntaxAndIdentifiersStillParse() throws Exception { + assertEquals(SqlKind.SET_OPTION, parse("ALTER SYSTEM SET foo = 1").getKind()); + assertTrue(parse("ALTER SYSTEM RESET ALL") instanceof DrillSqlResetOption); + assertEquals(SqlKind.SET_OPTION, parse("ALTER SESSION SET foo = 1").getKind()); + assertTrue(parse("ALTER SESSION RESET foo") instanceof DrillSqlResetOption); + assertEquals(SqlKind.SELECT, parse("SELECT clear, cache FROM t").getKind()); + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java index 3f7f97c2b50..acaafcd6a4a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java @@ -93,6 +93,24 @@ public void updateSysOptAsUserInAdminGroup() throws Exception { setOptHelper(); } + @Test + public void clearPlanCacheAsAdminUser() throws Exception { + updateClient(ADMIN_USER, ADMIN_USER_PASSWORD); + test("ALTER SYSTEM CLEAR PLAN CACHE"); + } + + @Test + public void clearPlanCacheAsUserInAdminGroup() throws Exception { + updateClient(TEST_USER_1, TEST_USER_1_PASSWORD); + test("ALTER SYSTEM CLEAR PLAN CACHE"); + } + + @Test + public void cannotClearPlanCacheAsNonAdminUser() throws Exception { + updateClient(TEST_USER_2, TEST_USER_2_PASSWORD); + errorMsgTestHelper("ALTER SYSTEM CLEAR PLAN CACHE", "Not authorized to clear the SYSTEM plan cache."); + } + @Test public void trySettingAdminOptsAtSessionScopeAsAdmin() throws Exception { updateClient(ADMIN_USER, ADMIN_USER_PASSWORD); diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 index 53b39bd98a7..5288057d62a 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 @@ -94,6 +94,8 @@ Bool | 'false' ; +BoundDynamicParam : 'bound_dynamic_param'; + Number : Int ('.' Digit*)? (('e' | 'E') ('+' | '-')? Digit*)? ; diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 index d868cff3e7c..49f41525bd8 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 @@ -47,6 +47,7 @@ import org.apache.drill.common.exceptions.ExpressionParsingException; public ExpressionPosition pos(Token token){ return new ExpressionPosition(fullExpression, token.getTokenIndex()); } + } parse returns [LogicalExpression e] @@ -324,7 +325,11 @@ arraySegment returns [PathSegment seg] ; lookup returns [LogicalExpression e] - : functionCall {$e = $functionCall.e ;} + : BoundDynamicParam OParen Number Comma dataType Comma boundLiteralValue CParen { + $e = ValueExpressions.getBoundDynamicParam(Integer.parseInt($Number.text), + $dataType.type, $boundLiteralValue.value); + } + | functionCall {$e = $functionCall.e ;} | convertCall {$e = $convertCall.e; } | anyValueCall {$e = $anyValueCall.e; } | castCall {$e = $castCall.e; } @@ -333,6 +338,14 @@ lookup returns [LogicalExpression e] | OParen expression CParen {$e = $expression.e; } | SingleQuote Identifier SingleQuote {$e = new SchemaPath($Identifier.text, pos($Identifier) ); } ; + +boundLiteralValue returns [String value] + : s=String { $value = $s.text; } + | b=Bool { $value = $b.text; } + | sign=(Plus|Minus)? n=Number { + $value = ($sign == null ? "" : $sign.text) + $n.text; + } + ; diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 4133f174a9d..4682dbf63b8 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -45,8 +45,46 @@ public class ExpressionStringBuilder extends AbstractExprVisitor{ + /** Parser syntax for a literal carrying a parameter slot; not an executable function. */ + public static final String BOUND_DYNAMIC_PARAM = "bound_dynamic_param"; + static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder(); + /** + * Starts bound_dynamic_param(index, type, value) for a parameterized literal. + * The visitor writes the value and endBoundLiteral closes the wrapper. + * This format preserves the slot, type and current value for plan rebinding + * and deserialization, and is also used when displaying the expression. + * Ordinary literals have no slot and retain their usual representation. + */ + private void startBoundLiteral(LiteralExpression expression, StringBuilder sb) { + if (expression.isDynamicParam()) { + int index = expression.getDynamicParamIndex(); + MajorType type = expression.getMajorType(); + sb.append(BOUND_DYNAMIC_PARAM).append('(').append(index).append(", ") + .append(type.getMinorType().name()); + // Preserve length, precision and scale so rebinding restores the same type. + switch (type.getMinorType()) { + case VARCHAR: + sb.append('(').append(type.getPrecision()).append(')'); + break; + case VARDECIMAL: + sb.append('(').append(type.getPrecision()).append(", ") + .append(type.getScale()).append(')'); + break; + default: + break; + } + sb.append(", "); + } + } + + private static void endBoundLiteral(LiteralExpression expression, StringBuilder sb) { + if (expression.isDynamicParam()) { + sb.append(')'); + } + } + public static String toString(LogicalExpression expr) { StringBuilder sb = new StringBuilder(); expr.accept(INSTANCE, sb); @@ -147,7 +185,11 @@ public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeExc @Override public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws RuntimeException { + startBoundLiteral(lExpr, sb); + // Bound parameters carry BIGINT in their wrapper; ordinary literals keep + // the existing representation used by explain output and scan digests. sb.append(lExpr.getLong()); + endBoundLiteral(lExpr, sb); return null; } @@ -219,27 +261,35 @@ public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb @Override public Void visitVarDecimalConstant(VarDecimalExpression decExpr, StringBuilder sb) throws RuntimeException { + startBoundLiteral(decExpr, sb); sb.append(decExpr.getBigDecimal().toString()); + endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDoubleConstant(DoubleExpression dExpr, StringBuilder sb) throws RuntimeException { + startBoundLiteral(dExpr, sb); sb.append(dExpr.getDouble()); + endBoundLiteral(dExpr, sb); return null; } @Override public Void visitBooleanConstant(BooleanExpression e, StringBuilder sb) throws RuntimeException { + startBoundLiteral(e, sb); sb.append(e.getBoolean()); + endBoundLiteral(e, sb); return null; } @Override public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException { + startBoundLiteral(e, sb); sb.append("'"); sb.append(escapeSingleQuote(e.value)); sb.append("'"); + endBoundLiteral(e, sb); return null; } @@ -324,13 +374,17 @@ public Void visitCastExpression(CastExpression e, StringBuilder sb) throws Runti @Override public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws RuntimeException { + startBoundLiteral(fExpr, sb); sb.append(fExpr.getFloat()); + endBoundLiteral(fExpr, sb); return null; } @Override public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws RuntimeException { + startBoundLiteral(intExpr, sb); sb.append(intExpr.getInt()); + endBoundLiteral(intExpr, sb); return null; } diff --git a/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java b/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java new file mode 100644 index 00000000000..e591f2867cf --- /dev/null +++ b/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.expression; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +/** A literal that may carry a bound SQL parameter slot in a cached plan. */ +public abstract class LiteralExpression extends LogicalExpressionBase { + private int dynamicParamIndex = -1; + + protected LiteralExpression(ExpressionPosition position) { + super(position); + } + + @JsonIgnore + public int getDynamicParamIndex() { + return dynamicParamIndex; + } + + /** Whether this literal is bound to a zero-based SQL parameter slot. */ + @JsonIgnore + public boolean isDynamicParam() { + return dynamicParamIndex >= 0; + } + + public void setDynamicParamIndex(int dynamicParamIndex) { + if (dynamicParamIndex < 0) { + throw new IllegalArgumentException("Dynamic parameter index must be non-negative"); + } + this.dynamicParamIndex = dynamicParamIndex; + } +} diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java index 2f04a588ccd..8c96e2092d3 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java @@ -95,6 +95,49 @@ public static LogicalExpression getVarDecimal(BigDecimal input, int precision, i return new VarDecimalExpression(input, precision, scale, ExpressionPosition.UNKNOWN); } + /** Restores the exact literal type carried by a serialized bound parameter. */ + public static LiteralExpression getBoundDynamicParam(int index, MajorType type, String value) { + LiteralExpression expression; + switch (type.getMinorType()) { + case INT: + expression = (LiteralExpression) getInt(Integer.parseInt(value)); + break; + case BIGINT: + expression = (LiteralExpression) getBigInt(Long.parseLong(value)); + break; + case FLOAT4: + expression = (LiteralExpression) getFloat4(Float.parseFloat(value)); + break; + case FLOAT8: + expression = (LiteralExpression) getFloat8(Double.parseDouble(value)); + break; + case BIT: + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + throw new IllegalArgumentException("Invalid boolean parameter value"); + } + expression = (LiteralExpression) getBit(Boolean.parseBoolean(value)); + break; + case VARCHAR: + expression = (LiteralExpression) getChar(value, type.getPrecision()); + break; + case VARDECIMAL: + BigDecimal decimal = new BigDecimal(value); + if (decimal.scale() != type.getScale() || decimal.precision() > type.getPrecision()) { + throw new IllegalArgumentException("Invalid decimal parameter value"); + } + expression = (LiteralExpression) getVarDecimal(decimal, + type.getPrecision(), type.getScale()); + break; + default: + throw new IllegalArgumentException("Unsupported bound parameter type: " + type.getMinorType()); + } + if (!expression.getMajorType().equals(type)) { + throw new IllegalArgumentException("Bound parameter type changed"); + } + expression.setDynamicParamIndex(index); + return expression; + } + public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) { String numStr = (sign == null) ? s : sign+s; try { @@ -120,7 +163,7 @@ public static LogicalExpression getParameterExpression(String name, MajorType ty return new ParameterExpression(name, type, ExpressionPosition.UNKNOWN); } - protected static abstract class ValueExpression extends LogicalExpressionBase { + protected static abstract class ValueExpression extends LiteralExpression { public final V value; protected ValueExpression(String value, ExpressionPosition pos) { @@ -165,7 +208,7 @@ public boolean getBoolean() { } } - public static class FloatExpression extends LogicalExpressionBase { + public static class FloatExpression extends LiteralExpression { private final float f; private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4); @@ -195,7 +238,7 @@ public Iterator iterator() { } } - public static class IntExpression extends LogicalExpressionBase { + public static class IntExpression extends LiteralExpression { private static final MajorType INT_CONSTANT = Types.required(MinorType.INT); @@ -226,7 +269,7 @@ public Iterator iterator() { } } - public static class Decimal9Expression extends LogicalExpressionBase { + public static class Decimal9Expression extends LiteralExpression { private final int decimal; private final int scale; @@ -267,7 +310,7 @@ public Iterator iterator() { } } - public static class Decimal18Expression extends LogicalExpressionBase { + public static class Decimal18Expression extends LiteralExpression { private final long decimal; private final int scale; @@ -308,7 +351,7 @@ public Iterator iterator() { } } - public static class Decimal28Expression extends LogicalExpressionBase { + public static class Decimal28Expression extends LiteralExpression { private final BigDecimal bigDecimal; @@ -337,7 +380,7 @@ public Iterator iterator() { } } - public static class Decimal38Expression extends LogicalExpressionBase { + public static class Decimal38Expression extends LiteralExpression { private final BigDecimal bigDecimal; @@ -366,7 +409,7 @@ public Iterator iterator() { } } - public static class VarDecimalExpression extends LogicalExpressionBase { + public static class VarDecimalExpression extends LiteralExpression { private final BigDecimal bigDecimal; private final int precision; @@ -405,7 +448,7 @@ public Iterator iterator() { } } - public static class DoubleExpression extends LogicalExpressionBase { + public static class DoubleExpression extends LiteralExpression { private final double d; private static final MajorType DOUBLE_CONSTANT = Types.required(MinorType.FLOAT8); @@ -435,7 +478,7 @@ public Iterator iterator() { } } - public static class LongExpression extends LogicalExpressionBase { + public static class LongExpression extends LiteralExpression { private static final MajorType LONG_CONSTANT = Types.required(MinorType.BIGINT); diff --git a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java index 5f5ee0f2f23..83c85b2e774 100644 --- a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java +++ b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java @@ -18,8 +18,10 @@ package org.apache.drill.common.expression.parser; import org.apache.drill.common.exceptions.ExpressionParsingException; +import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.parser.LogicalExpressionParser; import org.apache.drill.test.DrillTest; import org.junit.jupiter.api.Test; @@ -28,6 +30,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class TreeTest extends DrillTest { @@ -108,6 +111,30 @@ void testFunctionCallWithoutParams() { testExpressionParsing(expr, expr); } + @Test + void testOrdinaryBigintSerializationKeepsExistingRepresentation() { + for (long value : new long[] {0, -1, Integer.MAX_VALUE, (long) Integer.MAX_VALUE + 1, Long.MIN_VALUE}) { + assertEquals(Long.toString(value), serializeExpression(new LongExpression(value))); + } + assertTrue(LogicalExpressionParser.parse("cast(1 as BIGINT)") instanceof CastExpression); + } + + @Test + void testBoundBigintPreservesWidthAndSlotOnRoundTrip() { + for (long value : new long[] {0, -1, Integer.MIN_VALUE, Integer.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE}) { + LongExpression original = new LongExpression(value); + original.setDynamicParamIndex(3); + String serialized = serializeExpression(original); + assertEquals("bound_dynamic_param(3, BIGINT, " + value + ")", serialized); + LogicalExpression parsed = LogicalExpressionParser.parse(serialized); + assertTrue(parsed instanceof LongExpression); + LongExpression restored = (LongExpression) parsed; + assertEquals(value, restored.getLong()); + assertEquals(original.getMajorType(), restored.getMajorType()); + assertEquals(3, restored.getDynamicParamIndex()); + } + } + /** * Attempt to parse an expression. Once parsed, convert it to a string and then parse it again to make sure serialization works. */ diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java index 296fc710cd2..45d0ff063a0 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java @@ -1832,6 +1832,8 @@ public void writeTo(io.protostuff.Output output, org.apache.drill.exec.proto.Use output.writeInt32(23, message.getAutoLimit(), false); for(String scannedPlugins : message.getScannedPluginsList()) output.writeString(24, scannedPlugins, true); + if(message.hasPlanCacheHit()) + output.writeBool(25, message.getPlanCacheHit(), false); } public boolean isInitialized(org.apache.drill.exec.proto.UserBitShared.QueryProfile message) { @@ -1946,6 +1948,9 @@ public void mergeFrom(io.protostuff.Input input, org.apache.drill.exec.proto.Use case 24: builder.addScannedPlugins(input.readString()); break; + case 25: + builder.setPlanCacheHit(input.readBool()); + break; default: input.handleUnknownField(number, this); } @@ -2010,6 +2015,7 @@ public static java.lang.String getFieldName(int number) case 22: return "queryId"; case 23: return "autoLimit"; case 24: return "scannedPlugins"; + case 25: return "planCacheHit"; default: return null; } } @@ -2045,6 +2051,7 @@ public static int getFieldNumber(java.lang.String name) fieldMap.put("queryId", 22); fieldMap.put("autoLimit", 23); fieldMap.put("scannedPlugins", 24); + fieldMap.put("planCacheHit", 25); } } diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java index 622490b67f1..b4f677535af 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java @@ -15955,6 +15955,17 @@ org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfileOrBuilder getFragm */ com.google.protobuf.ByteString getScannedPluginsBytes(int index); + + /** + * optional bool plan_cache_hit = 25; + * @return Whether the planCacheHit field is set. + */ + boolean hasPlanCacheHit(); + /** + * optional bool plan_cache_hit = 25; + * @return The planCacheHit. + */ + boolean getPlanCacheHit(); } /** * Protobuf type {@code exec.shared.QueryProfile} @@ -16808,6 +16819,25 @@ public java.lang.String getScannedPlugins(int index) { return scannedPlugins_.getByteString(index); } + public static final int PLAN_CACHE_HIT_FIELD_NUMBER = 25; + private boolean planCacheHit_; + /** + * optional bool plan_cache_hit = 25; + * @return Whether the planCacheHit field is set. + */ + @java.lang.Override + public boolean hasPlanCacheHit() { + return ((bitField0_ & 0x00400000) != 0); + } + /** + * optional bool plan_cache_hit = 25; + * @return The planCacheHit. + */ + @java.lang.Override + public boolean getPlanCacheHit() { + return planCacheHit_; + } + private byte memoizedIsInitialized = -1; @java.lang.Override public final boolean isInitialized() { @@ -16894,6 +16924,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) for (int i = 0; i < scannedPlugins_.size(); i++) { com.google.protobuf.GeneratedMessageV3.writeString(output, 24, scannedPlugins_.getRaw(i)); } + if (((bitField0_ & 0x00400000) != 0)) { + output.writeBool(25, planCacheHit_); + } getUnknownFields().writeTo(output); } @@ -16993,6 +17026,10 @@ public int getSerializedSize() { size += dataSize; size += 2 * getScannedPluginsList().size(); } + if (((bitField0_ & 0x00400000) != 0)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(25, planCacheHit_); + } size += getUnknownFields().getSerializedSize(); memoizedSize = size; return size; @@ -17121,6 +17158,11 @@ public boolean equals(final java.lang.Object obj) { } if (!getScannedPluginsList() .equals(other.getScannedPluginsList())) return false; + if (hasPlanCacheHit() != other.hasPlanCacheHit()) return false; + if (hasPlanCacheHit()) { + if (getPlanCacheHit() + != other.getPlanCacheHit()) return false; + } if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -17233,6 +17275,11 @@ public int hashCode() { hash = (37 * hash) + SCANNED_PLUGINS_FIELD_NUMBER; hash = (53 * hash) + getScannedPluginsList().hashCode(); } + if (hasPlanCacheHit()) { + hash = (37 * hash) + PLAN_CACHE_HIT_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean( + getPlanCacheHit()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -17430,6 +17477,8 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00400000); scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; bitField0_ = (bitField0_ & ~0x00800000); + planCacheHit_ = false; + bitField0_ = (bitField0_ & ~0x01000000); return this; } @@ -17568,6 +17617,10 @@ public org.apache.drill.exec.proto.UserBitShared.QueryProfile buildPartial() { bitField0_ = (bitField0_ & ~0x00800000); } result.scannedPlugins_ = scannedPlugins_; + if (((from_bitField0_ & 0x01000000) != 0)) { + result.planCacheHit_ = planCacheHit_; + to_bitField0_ |= 0x00400000; + } result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -17739,6 +17792,9 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile } onChanged(); } + if (other.hasPlanCacheHit()) { + setPlanCacheHit(other.getPlanCacheHit()); + } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); return this; @@ -17912,6 +17968,11 @@ public Builder mergeFrom( scannedPlugins_.add(bs); break; } // case 194 + case 200: { + planCacheHit_ = input.readBool(); + bitField0_ |= 0x01000000; + break; + } // case 200 default: { if (!super.parseUnknownField(input, extensionRegistry, tag)) { done = true; // was an endgroup tag @@ -19755,6 +19816,45 @@ public Builder addScannedPluginsBytes( onChanged(); return this; } + + private boolean planCacheHit_ ; + /** + * optional bool plan_cache_hit = 25; + * @return Whether the planCacheHit field is set. + */ + @java.lang.Override + public boolean hasPlanCacheHit() { + return ((bitField0_ & 0x01000000) != 0); + } + /** + * optional bool plan_cache_hit = 25; + * @return The planCacheHit. + */ + @java.lang.Override + public boolean getPlanCacheHit() { + return planCacheHit_; + } + /** + * optional bool plan_cache_hit = 25; + * @param value The planCacheHit to set. + * @return This builder for chaining. + */ + public Builder setPlanCacheHit(boolean value) { + bitField0_ |= 0x01000000; + planCacheHit_ = value; + onChanged(); + return this; + } + /** + * optional bool plan_cache_hit = 25; + * @return This builder for chaining. + */ + public Builder clearPlanCacheHit() { + bitField0_ = (bitField0_ & ~0x01000000); + planCacheHit_ = false; + onChanged(); + return this; + } @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { @@ -28607,7 +28707,7 @@ public org.apache.drill.exec.proto.UserBitShared.SaslMessage getDefaultInstanceF "red.QueryResult.QueryState\022\017\n\004user\030\004 \001(\t" + ":\001-\022\'\n\007foreman\030\005 \001(\0132\026.exec.DrillbitEndp" + "oint\022\024\n\014options_json\030\006 \001(\t\022\022\n\ntotal_cost" + - "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\337\004\n\014QueryP" + + "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\367\004\n\014QueryP" + "rofile\022 \n\002id\030\001 \001(\0132\024.exec.shared.QueryId" + "\022$\n\004type\030\002 \001(\0162\026.exec.shared.QueryType\022\r" + "\n\005start\030\003 \001(\003\022\013\n\003end\030\004 \001(\003\022\r\n\005query\030\005 \001(" + @@ -28622,46 +28722,46 @@ public org.apache.drill.exec.proto.UserBitShared.SaslMessage getDefaultInstanceF "tions_json\030\021 \001(\t\022\017\n\007planEnd\030\022 \001(\003\022\024\n\014que" + "ueWaitEnd\030\023 \001(\003\022\022\n\ntotal_cost\030\024 \001(\001\022\025\n\nq" + "ueue_name\030\025 \001(\t:\001-\022\017\n\007queryId\030\026 \001(\t\022\021\n\ta" + - "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\"" + - "t\n\024MajorFragmentProfile\022\031\n\021major_fragmen" + - "t_id\030\001 \001(\005\022A\n\026minor_fragment_profile\030\002 \003" + - "(\0132!.exec.shared.MinorFragmentProfile\"\350\002" + - "\n\024MinorFragmentProfile\022)\n\005state\030\001 \001(\0162\032." + - "exec.shared.FragmentState\022(\n\005error\030\002 \001(\013" + - "2\031.exec.shared.DrillPBError\022\031\n\021minor_fra" + - "gment_id\030\003 \001(\005\0226\n\020operator_profile\030\004 \003(\013" + - "2\034.exec.shared.OperatorProfile\022\022\n\nstart_" + - "time\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_u" + - "sed\030\007 \001(\003\022\027\n\017max_memory_used\030\010 \001(\003\022(\n\010en" + - "dpoint\030\t \001(\0132\026.exec.DrillbitEndpoint\022\023\n\013" + - "last_update\030\n \001(\003\022\025\n\rlast_progress\030\013 \001(\003" + - "\"\237\002\n\017OperatorProfile\0221\n\rinput_profile\030\001 " + - "\003(\0132\032.exec.shared.StreamProfile\022\023\n\013opera" + - "tor_id\030\003 \001(\005\022\031\n\roperator_type\030\004 \001(\005B\002\030\001\022" + - "\023\n\013setup_nanos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 " + - "\001(\003\022#\n\033peak_local_memory_allocated\030\007 \001(\003" + - "\022(\n\006metric\030\010 \003(\0132\030.exec.shared.MetricVal" + - "ue\022\022\n\nwait_nanos\030\t \001(\003\022\032\n\022operator_type_" + - "name\030\n \001(\t\"B\n\rStreamProfile\022\017\n\007records\030\001" + - " \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J" + - "\n\013MetricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong" + - "_value\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001\")\n\010Re" + - "gistry\022\035\n\003jar\030\001 \003(\0132\020.exec.shared.Jar\"/\n" + - "\003Jar\022\014\n\004name\030\001 \001(\t\022\032\n\022function_signature" + - "\030\002 \003(\t\"W\n\013SaslMessage\022\021\n\tmechanism\030\001 \001(\t" + - "\022\014\n\004data\030\002 \001(\014\022\'\n\006status\030\003 \001(\0162\027.exec.sh" + - "ared.SaslStatus*5\n\nRpcChannel\022\017\n\013BIT_CON" + - "TROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004USER\020\002*V\n\tQueryT" + - "ype\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022" + - "\r\n\tEXECUTION\020\004\022\026\n\022PREPARED_STATEMENT\020\005*\207" + - "\001\n\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAITIN" + - "G_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020" + - "\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLA" + - "TION_REQUESTED\020\006*g\n\nSaslStatus\022\020\n\014SASL_U" + - "NKNOWN\020\000\022\016\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROG" + - "RESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020" + - "\004B.\n\033org.apache.drill.exec.protoB\rUserBi" + - "tSharedH\001" + "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\022" + + "\026\n\016plan_cache_hit\030\031 \001(\010\"t\n\024MajorFragment" + + "Profile\022\031\n\021major_fragment_id\030\001 \001(\005\022A\n\026mi" + + "nor_fragment_profile\030\002 \003(\0132!.exec.shared" + + ".MinorFragmentProfile\"\350\002\n\024MinorFragmentP" + + "rofile\022)\n\005state\030\001 \001(\0162\032.exec.shared.Frag" + + "mentState\022(\n\005error\030\002 \001(\0132\031.exec.shared.D" + + "rillPBError\022\031\n\021minor_fragment_id\030\003 \001(\005\0226" + + "\n\020operator_profile\030\004 \003(\0132\034.exec.shared.O" + + "peratorProfile\022\022\n\nstart_time\030\005 \001(\003\022\020\n\010en" + + "d_time\030\006 \001(\003\022\023\n\013memory_used\030\007 \001(\003\022\027\n\017max" + + "_memory_used\030\010 \001(\003\022(\n\010endpoint\030\t \001(\0132\026.e" + + "xec.DrillbitEndpoint\022\023\n\013last_update\030\n \001(" + + "\003\022\025\n\rlast_progress\030\013 \001(\003\"\237\002\n\017OperatorPro" + + "file\0221\n\rinput_profile\030\001 \003(\0132\032.exec.share" + + "d.StreamProfile\022\023\n\013operator_id\030\003 \001(\005\022\031\n\r" + + "operator_type\030\004 \001(\005B\002\030\001\022\023\n\013setup_nanos\030\005" + + " \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033peak_loca" + + "l_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(\013" + + "2\030.exec.shared.MetricValue\022\022\n\nwait_nanos" + + "\030\t \001(\003\022\032\n\022operator_type_name\030\n \001(\t\"B\n\rSt" + + "reamProfile\022\017\n\007records\030\001 \001(\003\022\017\n\007batches\030" + + "\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013MetricValue\022\021\n" + + "\tmetric_id\030\001 \001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014" + + "double_value\030\003 \001(\001\")\n\010Registry\022\035\n\003jar\030\001 " + + "\003(\0132\020.exec.shared.Jar\"/\n\003Jar\022\014\n\004name\030\001 \001" + + "(\t\022\032\n\022function_signature\030\002 \003(\t\"W\n\013SaslMe" + + "ssage\022\021\n\tmechanism\030\001 \001(\t\022\014\n\004data\030\002 \001(\014\022\'" + + "\n\006status\030\003 \001(\0162\027.exec.shared.SaslStatus*" + + "5\n\nRpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DA" + + "TA\020\001\022\010\n\004USER\020\002*V\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007" + + "LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022\r\n\tEXECUTION\020\004\022\026" + + "\n\022PREPARED_STATEMENT\020\005*\207\001\n\rFragmentState" + + "\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013" + + "\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004" + + "\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATION_REQUESTED\020\006" + + "*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL" + + "_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_S" + + "UCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.apache." + + "drill.exec.protoB\rUserBitSharedH\001" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor .internalBuildGeneratedFileFrom(descriptorData, @@ -28753,7 +28853,7 @@ public org.apache.drill.exec.proto.UserBitShared.SaslMessage getDefaultInstanceF internal_static_exec_shared_QueryProfile_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_exec_shared_QueryProfile_descriptor, - new java.lang.String[] { "Id", "Type", "Start", "End", "Query", "Plan", "Foreman", "State", "TotalFragments", "FinishedFragments", "FragmentProfile", "User", "Error", "VerboseError", "ErrorId", "ErrorNode", "OptionsJson", "PlanEnd", "QueueWaitEnd", "TotalCost", "QueueName", "QueryId", "AutoLimit", "ScannedPlugins", }); + new java.lang.String[] { "Id", "Type", "Start", "End", "Query", "Plan", "Foreman", "State", "TotalFragments", "FinishedFragments", "FragmentProfile", "User", "Error", "VerboseError", "ErrorId", "ErrorNode", "OptionsJson", "PlanEnd", "QueueWaitEnd", "TotalCost", "QueueName", "QueryId", "AutoLimit", "ScannedPlugins", "PlanCacheHit", }); internal_static_exec_shared_MajorFragmentProfile_descriptor = getDescriptor().getMessageTypes().get(14); internal_static_exec_shared_MajorFragmentProfile_fieldAccessorTable = new diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto index dafd533cddf..dfdf6d5340b 100644 --- a/protocol/src/main/protobuf/UserBitShared.proto +++ b/protocol/src/main/protobuf/UserBitShared.proto @@ -262,6 +262,7 @@ message QueryProfile { optional string queryId = 22; optional int32 autoLimit = 23; repeated string scanned_plugins = 24; + optional bool plan_cache_hit = 25; } message MajorFragmentProfile {