diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index 7e72e6bd1a777a..255813230aa266 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -53,8 +53,24 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_ERROR", TYPE_STRING, sizeof(StringRef), true}, + {"WEIGHT_BOUNDED", TYPE_BOOLEAN, sizeof(bool), true}, + {"MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"EVICTION_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"WEIGHT_REJECT_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"GLOBAL_MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"GLOBAL_ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; +// Columns that every FE knows. The weight statistics columns appended after LAST_ERROR are +// only served by FEs that carry the memory-governance change; during a rolling upgrade an older +// FE rejects a projection that names them, so the scanner falls back to this prefix and leaves +// the newer columns NULL. +static constexpr size_t kLegacyMetaCacheStatsColumnCount = 23; + SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} @@ -67,9 +83,10 @@ Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { +Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, + TFetchSchemaTableDataResult* result) { TSchemaTableRequestParams schema_table_request_params; - for (int i = 0; i < _s_tbls_columns.size(); i++) { + for (size_t i = 0; i < column_count; i++) { schema_table_request_params.__isset.columns_name = true; schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); } @@ -79,20 +96,28 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); request.__set_schema_table_params(schema_table_request_params); - TFetchSchemaTableDataResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( _fe_addr.hostname, _fe_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->fetchSchemaTableData(result, request); + [&request, result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(*result, request); }, _rpc_timeout)); + return Status::create(result->status); +} - Status status(Status::create(result.status)); +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TFetchSchemaTableDataResult result; + Status status = _fetch_from_fe(_s_tbls_columns.size(), &result); if (!status.ok()) { - LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname - << ") failed, errmsg=" << status; - return status; + LOG(INFO) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") with all columns failed, retrying with the legacy column set: " << status; + result = TFetchSchemaTableDataResult(); + status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } } std::vector result_data = result.data_batch; @@ -106,19 +131,29 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { _block->reserve(_block_rows_limit); + size_t col_size = _s_tbls_columns.size(); if (result_data.size() > 0) { - auto col_size = result_data[0].column_value.size(); - if (col_size != _s_tbls_columns.size()) { + col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size() && col_size != kLegacyMetaCacheStatsColumnCount) { return Status::InternalError( "catalog meta cache stats schema is not match for FE and BE"); } } + int available_columns = static_cast(col_size); + int total_columns = static_cast(_s_tbls_columns.size()); for (int i = 0; i < result_data.size(); i++) { TRow row = result_data[i]; - for (int j = 0; j < _s_tbls_columns.size(); j++) { - RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), - _s_tbls_columns[j].type)); + for (int j = 0; j < total_columns; j++) { + if (j < available_columns) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } else { + // Column unknown to the serving FE: NULL. + auto column_guard = _block->mutate_column_scoped(j); + column_guard.mutable_column()->insert_default(); + column_guard.restore(); + } } } return Status::OK(); diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h index 836500fd97de85..7a2339baf44335 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h @@ -25,6 +25,7 @@ namespace doris { class RuntimeState; class Block; +class TFetchSchemaTableDataResult; class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); @@ -40,6 +41,7 @@ class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { private: Status _get_meta_cache_from_fe(); + Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result); TNetworkAddress _fe_addr; diff --git a/fe/fe-benchmark/pom.xml b/fe/fe-benchmark/pom.xml new file mode 100644 index 00000000000000..2de619284a9f69 --- /dev/null +++ b/fe/fe-benchmark/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + + org.apache.doris + fe + ${revision} + ../pom.xml + + + fe-benchmark + Doris FE Benchmarks + + + + ${project.groupId} + fe-core + ${project.version} + + + diff --git a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh new file mode 100755 index 00000000000000..7ce2e0be8e3d4f --- /dev/null +++ b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd) +CLASSPATH_FILE=$(mktemp) +trap 'rm -f "${CLASSPATH_FILE}"' EXIT + +( + cd "${FE_DIR}" + mvn -Pbenchmark -pl fe-benchmark -am compile -DskipTests -Dskip.clean=true + mvn -Pbenchmark -pl fe-benchmark -am dependency:build-classpath \ + -Dskip.clean=true \ + -DincludeScope=test \ + -Dmdep.outputFile="${CLASSPATH_FILE}" +) + +REACTOR_CLASSES= +while IFS= read -r -d '' CLASSES_DIR; do + REACTOR_CLASSES+="${CLASSES_DIR}:" +done < <(find "${FE_DIR}" -type d -path '*/target/classes' -print0) +DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") +BENCHMARK_FILTER=${BENCHMARK_FILTER:-'HivePartitionValuesSizeBenchmark|IcebergCacheSizeBenchmark|PaimonCacheSizeBenchmark|MetaCacheSoftValueBenchmark'} + +BENCHMARK_CLASSES=( + org.apache.doris.datasource.hive.HivePartitionValuesSizeBenchmark + org.apache.doris.datasource.iceberg.IcebergCacheSizeBenchmark + org.apache.doris.datasource.paimon.PaimonCacheSizeBenchmark + org.apache.doris.datasource.metacache.MetaCacheSoftValueBenchmark +) + +for BENCHMARK_CLASS in "${BENCHMARK_CLASSES[@]}"; do + if [[ "${BENCHMARK_CLASS##*.}" =~ ${BENCHMARK_FILTER} ]]; then + java \ + -Xms1g \ + -Xmx4g \ + -classpath "${REACTOR_CLASSES}${DEPENDENCY_CLASSES}" \ + "${BENCHMARK_CLASS}" \ + "$@" + fi +done diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java new file mode 100644 index 00000000000000..c1e9234c70fa67 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.benchmark; + +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** Small dependency-free harness for opt-in FE microbenchmarks. */ +public final class BenchmarkHarness { + private static final long WARMUP_MILLIS = Long.getLong("benchmark.warmup.millis", 500L); + private static final long MEASUREMENT_MILLIS = Long.getLong("benchmark.measurement.millis", 500L); + private static final int MEASUREMENT_ITERATIONS = Integer.getInteger("benchmark.iterations", 3); + private static final boolean PRINT_RESULT = Boolean.getBoolean("benchmark.print.result"); + private static volatile Object sink; + + private BenchmarkHarness() { + } + + @FunctionalInterface + public interface Operation { + Object run() throws Exception; + } + + public static void measure(String name, TimeUnit outputUnit, Operation operation) throws Exception { + runWindow(operation, WARMUP_MILLIS); + double totalNanosPerOperation = 0.0D; + long totalOperations = 0L; + for (int iteration = 0; iteration < MEASUREMENT_ITERATIONS; iteration++) { + Window result = runWindow(operation, MEASUREMENT_MILLIS); + totalNanosPerOperation += result.nanosPerOperation; + totalOperations += result.operations; + } + double averageNanos = totalNanosPerOperation / MEASUREMENT_ITERATIONS; + String result = PRINT_RESULT ? ", result=" + sink : ""; + System.out.printf(Locale.ROOT, "%-72s %12.3f %s/op (%d ops%s)%n", + name, convertFromNanos(averageNanos, outputUnit), unitName(outputUnit), totalOperations, result); + } + + private static Window runWindow(Operation operation, long minimumMillis) throws Exception { + long start = System.nanoTime(); + long deadline = start + TimeUnit.MILLISECONDS.toNanos(minimumMillis); + long operations = 0L; + do { + sink = operation.run(); + operations++; + } while (System.nanoTime() < deadline); + long elapsed = System.nanoTime() - start; + return new Window(operations, (double) elapsed / operations); + } + + private static double convertFromNanos(double nanos, TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return nanos; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return nanos / 1_000.0D; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return nanos / 1_000_000.0D; + } else if (outputUnit == TimeUnit.SECONDS) { + return nanos / 1_000_000_000.0D; + } + throw new IllegalArgumentException("unsupported benchmark time unit: " + outputUnit); + } + + private static String unitName(TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return "ns"; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return "us"; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return "ms"; + } else if (outputUnit == TimeUnit.SECONDS) { + return "s"; + } + return outputUnit.name().toLowerCase(Locale.ROOT); + } + + private static final class Window { + private final long operations; + private final double nanosPerOperation; + + private Window(long operations, double nanosPerOperation) { + this.operations = operations; + this.nanosPerOperation = nanosPerOperation; + } + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java new file mode 100644 index 00000000000000..57a2ebb0e1bdb8 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; +import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.MoreExecutors; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** Measures count publication, weighted preparation, and prepared-value admission separately. */ +public class HivePartitionValuesSizeBenchmark { + private static final int TAIL_PAYLOAD_BYTES = 1024 * 1024; + private static final long MAX_WEIGHT_BYTES = 4L * 1024L * 1024L * 1024L; + + public int countPublicationBaseline(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + return state.partitionValues.getSortedPartitionRanges() + .map(ranges -> ranges.sortedPartitions.size() + ranges.defaultPartitions.size()) + .orElse(0); + } + + public int sealPublicationWithoutEstimate(UnsealedState state) { + state.partitionValues.sealForPublication(); + return state.partitionValues.getIdToPartitionItem().size(); + } + + public long weightedPublication(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + state.partitionValues.prepareForCachePublication(state.key); + return requireComplete(state.partitionValues); + } + + public long eventCopySealAndEstimate(PreparedState state) { + HivePartitionValues copy = state.partitionValues.mutableCopy(); + copy.rebuildSortedPartitionRangesForPublication(); + copy.prepareForCachePublication(state.key); + return requireComplete(copy); + } + + public long preparedSizeProvider(PreparedState state) { + return state.partitionValues.getSizeEstimate().getBytes(); + } + + public long estimateFormula(PreparedState state) { + state.partitionValues.prepareSizeEstimate(state.key); + return requireComplete(state.partitionValues); + } + + public void replacementAdmission(PreparedState state) { + MetaCacheEntry.ReplaceResult result = state.cacheEntry.tryReplace( + state.key, state.currentPartitionValues, state.nextPartitionValues); + if (result != MetaCacheEntry.ReplaceResult.REPLACED) { + throw new IllegalStateException("replacement failed: " + result); + } + state.currentPartitionValues = state.nextPartitionValues; + state.nextPartitionValues = state.nextPartitionValues == state.partitionValues + ? state.replacementPartitionValues : state.partitionValues; + } + + public HivePartitionValues countStrongCacheHit(PreparedState state) { + return state.countCacheEntry.getIfPresent(state.key); + } + + public HivePartitionValues weightedSoftCacheHit(PreparedState state) { + return state.cacheEntry.getIfPresent(state.key); + } + + public static void main(String[] args) throws Exception { + HivePartitionValuesSizeBenchmark benchmark = new HivePartitionValuesSizeBenchmark(); + for (int partitionCount : new int[] {1000, 10000, 100000}) { + for (String distribution : new String[] {"uniform", "tail_skew"}) { + String suffix = "[partitions=" + partitionCount + ",distribution=" + distribution + "]"; + BenchmarkHarness.measure("hive.countPublicationBaseline" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.countPublicationBaseline(state); + }); + BenchmarkHarness.measure("hive.sealPublicationWithoutEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.sealPublicationWithoutEstimate(state); + }); + BenchmarkHarness.measure("hive.weightedPublication" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.weightedPublication(state); + }); + + PreparedState prepared = new PreparedState(); + prepared.partitionCount = partitionCount; + prepared.distribution = distribution; + prepared.setup(); + try { + BenchmarkHarness.measure("hive.eventCopySealAndEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> benchmark.eventCopySealAndEstimate(prepared)); + BenchmarkHarness.measure("hive.preparedSizeProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedSizeProvider(prepared)); + BenchmarkHarness.measure("hive.estimateFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.estimateFormula(prepared)); + BenchmarkHarness.measure("hive.countStrongCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.countStrongCacheHit(prepared)); + BenchmarkHarness.measure("hive.weightedSoftCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.weightedSoftCacheHit(prepared)); + BenchmarkHarness.measure("hive.replacementAdmission" + suffix, + TimeUnit.NANOSECONDS, () -> { + benchmark.replacementAdmission(prepared); + return null; + }); + } finally { + prepared.tearDown(); + } + } + } + } + + private static UnsealedState unsealedState(int partitionCount, String distribution) throws Exception { + UnsealedState state = new UnsealedState(); + state.partitionCount = partitionCount; + state.distribution = distribution; + state.setupInvocation(); + return state; + } + + /** Fresh graph per invocation so all publication work stays inside the measured method. */ + public static class UnsealedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + + public void setupInvocation() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + } + } + + public static class PreparedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + private HivePartitionValues replacementPartitionValues; + private HivePartitionValues currentPartitionValues; + private HivePartitionValues nextPartitionValues; + private MetaCacheEntry cacheEntry; + private MetaCacheEntry countCacheEntry; + private ExecutorService cacheExecutor; + + public void setup() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + partitionValues.prepareForCachePublication(key); + requireComplete(partitionValues); + + // A distinct value root makes Caffeine perform a real replacement while sharing + // immutable payload objects to keep the fixture's resident heap bounded. + replacementPartitionValues = new HivePartitionValues( + partitionValues.getIdToPartitionItem(), + partitionValues.getPartitionNameToIdMap(), + partitionValues.getPartitionValuesMap()); + replacementPartitionValues.prepareForCachePublication(key); + requireComplete(replacementPartitionValues); + + cacheExecutor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "hive", "partition_values_benchmark", OptionalLong.empty(), OptionalLong.empty()); + cacheEntry = new MetaCacheEntry<>( + "partition_values_benchmark", + ignored -> partitionValues, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 1L, MAX_WEIGHT_BYTES), + cacheExecutor, + false, + false, + (entryKey, value) -> value.prepareForCachePublication(entryKey), + entryBudget); + cacheEntry.put(key, partitionValues); + countCacheEntry = new MetaCacheEntry<>( + "partition_values_count_benchmark", + ignored -> partitionValues, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L), + cacheExecutor, + false, + false); + countCacheEntry.put(key, partitionValues); + currentPartitionValues = partitionValues; + nextPartitionValues = replacementPartitionValues; + } + + public void tearDown() { + if (cacheEntry != null) { + cacheEntry.close(); + } + if (countCacheEntry != null) { + countCacheEntry.close(); + } + if (cacheExecutor != null) { + cacheExecutor.shutdownNow(); + } + } + } + + private static List benchmarkTypes() { + return Collections.singletonList(Type.STRING); + } + + private static PartitionValueCacheKey benchmarkKey(List types) { + return new PartitionValueCacheKey( + NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"), types); + } + + private static long requireComplete(HivePartitionValues value) { + MetaCacheSizeEstimate estimate = value.getSizeEstimate(); + if (!estimate.isComplete()) { + throw new IllegalStateException("benchmark graph is not fully measurable: " + + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private static HivePartitionValues createPartitionValues( + int count, String distribution, List types) throws Exception { + HashBiMap nameToId = HashBiMap.create(count); + Map idToItem = Maps.newHashMapWithExpectedSize(count); + Map> idToValues = Maps.newHashMapWithExpectedSize(count); + String tailPayload = "tail_skew".equals(distribution) ? repeat('x', TAIL_PAYLOAD_BYTES) : null; + long partitionNamePayloadBytes = 0L; + + for (int i = 0; i < count; i++) { + long id = i; + String value = i == count - 1 && tailPayload != null ? tailPayload : "value-" + i; + String name = "p=" + value; + partitionNamePayloadBytes += MetaCacheWeightUtils.estimatedStringPayloadBytes(name); + List rawValues = Collections.singletonList(new PartitionValue(value)); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(rawValues, types, true); + List keys = new ArrayList<>(1); + keys.add(partitionKey); + + nameToId.put(name, id); + idToItem.put(id, new ListPartitionItem(keys)); + idToValues.put(id, new ArrayList<>(Collections.singletonList(value))); + } + return new HivePartitionValues( + idToItem, nameToId, idToValues, partitionNamePayloadBytes, types.size()); + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, value); + return new String(chars); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java new file mode 100644 index 00000000000000..77235312739695 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** Measures Iceberg table, long-history table, snapshot and manifest publication. */ +public class IcebergCacheSizeBenchmark { + public int tableValueConstruction(TablePublicationState state) { + return new IcebergTableCacheValue(state.table).getIcebergTable().schema().schemaId(); + } + + public long tablePublication(TablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(TablePublicationState state) { + return IcebergCacheSizeEstimator.retainedTablePayloadBytes(state.table); + } + + public long longHistoryTablePublication(LongHistoryTablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long snapshotPublication(SnapshotPublicationState state) { + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table); + value.prepareForCachePublication(state.snapshotKey); + return requireComplete(value.getSizeEstimate()); + } + + public int snapshotValueConstruction(SnapshotPublicationState state) { + return new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table) + .getPartitionInfo().getNameToIcebergPartition().size(); + } + + public long preparedSnapshotCacheHit(SnapshotPublicationState state) { + return state.preparedSnapshotValue.getIcebergTable().get() + .currentSnapshot().snapshotId(); + } + + public long manifestPublication(ManifestState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateManifestEntry(state.key, state.value)); + } + + public int manifestValueConstruction(ManifestState state) { + return ManifestCacheValue.forDataFiles(state.files).getDataFiles().size(); + } + + public int denseManifestReaderBaseline(DenseManifestState state) { + List collected = new ArrayList<>(); + for (DataFile file : state.files) { + collected.add(file.copy()); + } + return ImmutableList.copyOf(collected).size(); + } + + public int denseManifestValueConstruction(DenseManifestState state) { + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); + for (DataFile file : state.files) { + builder.addDataFile(file.copy()); + } + ManifestCacheValue value = builder.build(); + if (value.isAccountingComplete() != state.expectedAccountingComplete()) { + throw new IllegalStateException("unexpected dense manifest accounting state"); + } + return value.getDataFiles().size(); + } + + public long preparedWeightProvider(PreparedState state) { + return state.preparedTableValue.getSizeEstimate().getBytes(); + } + + public long tableFormula(PreparedState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateTableEntry( + state.mapping, state.preparedTableValue)); + } + + public int preparedTableCacheHit(PreparedState state) { + return state.preparedTableValue.getIcebergTable().schema().schemaId(); + } + + public static void main(String[] args) throws Exception { + IcebergCacheSizeBenchmark benchmark = new IcebergCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState tableState = new TablePublicationState(); + tableState.fieldCount = fieldCount; + tableState.setup(); + BenchmarkHarness.measure("iceberg.tableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(tableState)); + BenchmarkHarness.measure("iceberg.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(tableState)); + BenchmarkHarness.measure("iceberg.tablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(tableState)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.setup(); + BenchmarkHarness.measure("iceberg.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("iceberg.tableFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableFormula(prepared)); + BenchmarkHarness.measure("iceberg.preparedTableCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedTableCacheHit(prepared)); + } + for (int fieldCount : new int[] {100, 1000}) { + // Wide identity-partitioned specs drive the O(fields) fieldsBySourceId reservation and + // the secondary partition Schema formula; nested schemas drive the per-field path + // and lower-case String terms. Both must stay far below the value construction cost + // of the same table. + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState partitioned = new TablePublicationState(); + partitioned.fieldCount = fieldCount; + partitioned.identityPartitioned = true; + partitioned.setup(); + BenchmarkHarness.measure("iceberg.partitionedTableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(partitioned)); + BenchmarkHarness.measure("iceberg.partitionedTablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(partitioned)); + BenchmarkHarness.measure("iceberg.partitionedTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(partitioned)); + + TablePublicationState nested = new TablePublicationState(); + nested.fieldCount = fieldCount; + nested.nestedSchema = true; + nested.setup(); + BenchmarkHarness.measure("iceberg.nestedTablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(nested)); + BenchmarkHarness.measure("iceberg.nestedTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(nested)); + } + for (int snapshotCount : new int[] {1000, 10000}) { + String suffix = "[snapshots=" + snapshotCount + "]"; + LongHistoryTablePublicationState state = new LongHistoryTablePublicationState(); + state.snapshotCount = snapshotCount; + state.setup(); + BenchmarkHarness.measure("iceberg.longHistoryTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.longHistoryTablePublication(state)); + } + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + SnapshotPublicationState state = new SnapshotPublicationState(); + state.fieldCount = fieldCount; + state.partitionCount = partitionCount; + state.setup(); + BenchmarkHarness.measure("iceberg.snapshotValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotValueConstruction(state)); + BenchmarkHarness.measure("iceberg.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(state)); + BenchmarkHarness.measure("iceberg.preparedSnapshotCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedSnapshotCacheHit(state)); + } + } + for (int fileCount : new int[] {100, 10000}) { + ManifestState state = new ManifestState(); + state.fileCount = fileCount; + state.setup(); + BenchmarkHarness.measure("iceberg.manifestPublication[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestPublication(state)); + BenchmarkHarness.measure("iceberg.manifestValueConstruction[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestValueConstruction(state)); + } + for (int metricColumns : new int[] {100, 1000}) { + for (int fileCount : new int[] {100, 10000}) { + DenseManifestState state = new DenseManifestState(); + state.metricColumns = metricColumns; + state.fileCount = fileCount; + state.setup(); + String accounting = state.expectedAccountingComplete() ? "complete" : "rejected"; + String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + + ",accounting=" + accounting + "]"; + BenchmarkHarness.measure("iceberg.denseManifestReaderBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestReaderBaseline(state)); + BenchmarkHarness.measure("iceberg.denseManifestValueConstruction" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestValueConstruction(state)); + } + } + } + + public static class TablePublicationState { + public int fieldCount; + public boolean identityPartitioned; + public boolean nestedSchema; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount, identityPartitioned, nestedSchema); + } + } + + public static class SnapshotPublicationState { + public int fieldCount; + + public int partitionCount; + + private Table table; + private IcebergSnapshotEntryKey snapshotKey; + private IcebergPartitionInfo partitionInfo; + private IcebergSnapshotCacheValue preparedSnapshotValue; + + public void setup() { + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + partitionInfo = newPartitionInfo(partitionCount); + snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table) + .orElseThrow(() -> new IllegalStateException("benchmark table has no generation key")); + preparedSnapshotValue = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), table); + preparedSnapshotValue.prepareForCachePublication(snapshotKey); + requireComplete(preparedSnapshotValue.getSizeEstimate()); + } + } + + public static class LongHistoryTablePublicationState { + public int snapshotCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newLongHistoryTable(snapshotCount); + } + } + + public static class PreparedState { + public int fieldCount; + + private IcebergTableCacheValue preparedTableValue; + private NameMapping mapping; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + Table table = newTable(fieldCount); + preparedTableValue = new IcebergTableCacheValue(table); + preparedTableValue.prepareForCachePublication(mapping); + requireComplete(preparedTableValue.getSizeEstimate()); + } + } + + public static class ManifestState { + public int fileCount; + + private IcebergManifestEntryKey key; + private ManifestCacheValue value; + private List files; + + public void setup() { + key = new IcebergManifestEntryKey("/benchmark/manifest.avro", ManifestContent.DATA); + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/file-" + index + ".parquet") + .withFileSizeInBytes(1024L + index) + .withRecordCount(10L + index) + .build()); + } + value = ManifestCacheValue.forDataFiles(files); + } + } + + public static class DenseManifestState { + public int fileCount; + + public int metricColumns; + + private List files; + private boolean accountingComplete; + + public void setup() { + int poolSize = Math.min(fileCount, 256); + List filePool = new ArrayList<>(poolSize); + for (int fileIndex = 0; fileIndex < poolSize; fileIndex++) { + HashMap lowerBounds = new HashMap<>(metricColumns); + HashMap upperBounds = new HashMap<>(metricColumns); + for (int columnIndex = 0; columnIndex < metricColumns; columnIndex++) { + lowerBounds.put(columnIndex, ByteBuffer.allocate(16)); + upperBounds.put(columnIndex, ByteBuffer.allocate(32)); + } + Metrics metrics = new Metrics( + 10L, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + lowerBounds, + upperBounds); + filePool.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/dense-" + fileIndex + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build()); + } + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(filePool.get(index % poolSize)); + } + accountingComplete = ManifestCacheValue.forDataFiles(files).isAccountingComplete(); + } + + private boolean expectedAccountingComplete() { + return accountingComplete; + } + } + + private static Table newTable(int fieldCount) { + return newTable(fieldCount, false, false); + } + + private static Table newTable(int fieldCount, boolean identityPartitioned, boolean nestedSchema) { + List fields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional(index + 1, "field_" + index, Types.StringType.get())); + } + Schema schema; + if (nestedSchema) { + List nestedFields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + nestedFields.add(Types.NestedField.optional( + 1000 + index, "Nested_" + index, Types.StringType.get())); + } + schema = new Schema( + Types.NestedField.optional(1, "payload", Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "list", Types.ListType.ofOptional(3, + Types.StructType.of(Types.NestedField.optional( + 4, "leaf", Types.StringType.get())))), + Types.NestedField.optional(5, "id", Types.LongType.get())); + } else { + schema = new Schema(fields); + } + PartitionSpec spec = PartitionSpec.unpartitioned(); + if (identityPartitioned) { + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + for (Types.NestedField field : schema.columns()) { + specBuilder.identity(field.name()); + } + spec = specBuilder.build(); + } + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, spec, "file:/benchmark/table", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/benchmark/manifest-list.avro\",\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges() + .withMetadataLocation("file:/benchmark/table/metadata/v1.json").build(); + return new BaseTable(new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); + } + + private static Table newLongHistoryTable(int snapshotCount) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"benchmark-table\",") + .append("\"location\":\"file:/benchmark/table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index) + .append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/benchmark/history/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "file:/benchmark/table/metadata/v1.json", json.toString()); + return new BaseTable( + new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); + } + + private static IcebergPartitionInfo newPartitionInfo(int partitionCount) { + HashMap partitions = new HashMap<>(partitionCount); + long retainedPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=value_" + index; + IcebergPartition partition = new IcebergPartition(name, 0, 10L + index, 1024L + index, + 1L, 1_700_000_000_000L + index, 7L, + Collections.singletonList("value_" + index), Collections.singletonList("identity")); + partitions.put(name, partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); + } + return new IcebergPartitionInfo( + Collections.emptyMap(), partitions, Collections.emptyMap(), retainedPayloadBytes); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java new file mode 100644 index 00000000000000..b488f428e8b9a9 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.util.concurrent.MoreExecutors; + +import java.lang.ref.Reference; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Measures reservation cleanup after Caffeine reports soft values as COLLECTED. */ +public final class MetaCacheSoftValueBenchmark { + private static final int SAMPLE_COUNT = 3; + private static final int VALUE_COUNT = 10_000; + private static final long MAX_WEIGHT_BYTES = 16L * 1024L * 1024L; + + private MetaCacheSoftValueBenchmark() { + } + + public static void main(String[] args) throws Exception { + long totalNanos = 0L; + for (int sample = 0; sample < SAMPLE_COUNT; sample++) { + CollectedState state = CollectedState.create(VALUE_COUNT); + try { + state.enqueueAll(); + long start = System.nanoTime(); + state.cleanUp(); + totalNanos += System.nanoTime() - start; + } finally { + state.close(); + } + } + double averageNanos = (double) totalNanos / SAMPLE_COUNT; + System.out.printf(Locale.ROOT, + "%-72s %12.3f us/batch (%.3f ns/value, %d samples)%n", + "metacache.collectedCleanup[values=" + VALUE_COUNT + "]", + averageNanos / TimeUnit.MICROSECONDS.toNanos(1L), + averageNanos / VALUE_COUNT, + SAMPLE_COUNT); + } + + private static final class CollectedState implements AutoCloseable { + private final ExecutorService executor; + private final ExternalMetaCacheBudgetManager budgetManager; + private final MetaCacheEntry entry; + private final LoadingCache loadingCache; + private final List> references; + + private CollectedState(ExecutorService executor, + ExternalMetaCacheBudgetManager budgetManager, + MetaCacheEntry entry, + LoadingCache loadingCache, + List> references) { + this.executor = executor; + this.budgetManager = budgetManager; + this.entry = entry; + this.loadingCache = loadingCache; + this.references = references; + } + + private static CollectedState create(int valueCount) throws Exception { + ExecutorService executor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget budget = budgetManager.createEntryBudget( + 1L, "benchmark", "soft_cleanup", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft_cleanup", + key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, valueCount, MAX_WEIGHT_BYTES), + executor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), + budget); + for (int index = 0; index < valueCount; index++) { + entry.put("key-" + index, new byte[1]); + } + + LoadingCache loadingCache = (LoadingCache) readField(entry, "loadingData"); + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + List> references = new ArrayList<>(nodes.size()); + for (Object node : nodes.values()) { + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + references.add((Reference) valueReferenceMethod.invoke(node)); + } + if (references.size() != valueCount) { + entry.close(); + executor.shutdownNow(); + throw new IllegalStateException( + "benchmark admission retained " + references.size() + " of " + valueCount + " values"); + } + return new CollectedState(executor, budgetManager, entry, loadingCache, references); + } + + private void enqueueAll() { + for (Reference reference : references) { + reference.clear(); + reference.enqueue(); + } + } + + private void cleanUp() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); + while (budgetManager.getGlobalUsedWeight() != 0L + && System.nanoTime() < deadline) { + loadingCache.cleanUp(); + LockSupport.parkNanos(TimeUnit.MICROSECONDS.toNanos(100L)); + } + if (budgetManager.getGlobalUsedWeight() != 0L) { + throw new IllegalStateException( + "COLLECTED cleanup retained " + budgetManager.getGlobalUsedWeight() + " bytes"); + } + } + + @Override + public void close() { + entry.close(); + executor.shutdownNow(); + } + } + + private static Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private static Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java new file mode 100644 index 00000000000000..f73eca412f3c44 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.AppendOnlyFileStoreTable; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Measures Paimon 1.4.2 nested-schema/non-empty snapshot publication and prepared weight lookup. */ +public class PaimonCacheSizeBenchmark { + public long snapshotPublication(PublicationState state) { + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue(state.partitionInfo, state.snapshot); + value.prepareForCachePublication(state.key); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(PublicationState state) { + return PaimonCacheSizeEstimator.retainedTablePayloadBytes(state.snapshot.getTable()); + } + + public long preparedWeightProvider(PreparedState state) { + return state.value.getSizeEstimate().getBytes(); + } + + public long snapshotFormula(PreparedState state) { + return requireComplete(PaimonCacheSizeEstimator.estimateSnapshotEntry(state.key, state.value)); + } + + public long partitionMapBaseline(PartitionPayloadState state) { + return buildPartitionInfo(state, false); + } + + public long partitionMapWithRetainedCounter(PartitionPayloadState state) { + return buildPartitionInfo(state, true); + } + + public static void main(String[] args) throws Exception { + PaimonCacheSizeBenchmark benchmark = new PaimonCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + PublicationState publication = new PublicationState(); + publication.fieldCount = fieldCount; + publication.partitionCount = partitionCount; + publication.setup(); + BenchmarkHarness.measure("paimon.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(publication)); + BenchmarkHarness.measure("paimon.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(publication)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.partitionCount = partitionCount; + prepared.setup(); + BenchmarkHarness.measure("paimon.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("paimon.snapshotFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotFormula(prepared)); + } + } + for (int partitionCount : new int[] {1000, 10000}) { + for (boolean tailSkew : new boolean[] {false, true}) { + PartitionPayloadState state = new PartitionPayloadState(); + state.partitionCount = partitionCount; + state.tailSkew = tailSkew; + state.setup(); + String suffix = "[partitions=" + partitionCount + + ",distribution=" + (tailSkew ? "tail-skew" : "uniform") + "]"; + BenchmarkHarness.measure("paimon.partitionMapBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapBaseline(state)); + BenchmarkHarness.measure("paimon.partitionMapWithRetainedCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapWithRetainedCounter(state)); + } + } + } + + public static class PublicationState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonPartitionInfo partitionInfo; + private PaimonSnapshot snapshot; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + partitionInfo = fixture.value.getPartitionInfo(); + snapshot = fixture.value.getSnapshot(); + } + } + + public static class PreparedState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonSnapshotCacheValue value; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + value = fixture.value; + value.prepareForCachePublication(fixture.key); + requireComplete(value.getSizeEstimate()); + } + } + + public static class PartitionPayloadState { + public int partitionCount; + + public boolean tailSkew; + + private List partitions; + + public void setup() { + partitions = new ArrayList<>(partitionCount); + String longTail = String.join("", Collections.nCopies(64 * 1024, "x")); + for (int index = 0; index < partitionCount; index++) { + String value = tailSkew && index % 997 == 0 ? longTail : String.valueOf(index); + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < 4; field++) { + typedSpec.put("partition_key_" + field, value + '_' + field); + } + String displayName = "partition_key_0=" + value; + partitions.add(new PartitionPayload( + displayName, new ArrayList<>(typedSpec.values()), index)); + } + } + } + + private static Fixture newFixture(int fieldCount, int partitionCount) throws Exception { + List fields = new ArrayList<>(fieldCount + 2); + fields.add(new DataField(0, "partition_key", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new VarCharType())); + } + List nestedFields = new ArrayList<>(); + for (int index = 0; index < 8; index++) { + nestedFields.add(DataTypes.FIELD(fieldCount + index + 1, + "nested_field_" + index, DataTypes.STRING())); + } + fields.add(new DataField(fieldCount + 9, "nested_payload", new RowType(nestedFields))); + TableSchema schema = new TableSchema( + 0L, fields, fieldCount + 9, Collections.singletonList("partition_key"), + Collections.emptyList(), Collections.emptyMap(), null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/doris-paimon-cache-size-benchmark"), + schema, CatalogEnvironment.empty()); + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 7L, schema.id(), 1L); + HashMap partitions = new HashMap<>(partitionCount); + long retainedPartitionPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=" + index; + String value = String.valueOf(index); + partitions.put(name, new Partition(Collections.singletonMap("partition_key", value), + 10L + index, 1024L + index, 1L, 1_700_000_000_000L + index, 1, true, + 1_700_000_000_000L, "benchmark", 1_700_000_000_000L + index, "benchmark", + Collections.singletonMap("source", "benchmark"))); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(name)); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes("partition_key")); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + new PaimonPartitionInfo(Collections.emptyMap(), partitions, retainedPartitionPayloadBytes), + new PaimonSnapshot(7L, schema.id(), table)); + return new Fixture(key, value); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private long buildPartitionInfo(PartitionPayloadState state, boolean countPayload) { + HashMap partitions = new HashMap<>(state.partitionCount); + long retainedPayloadBytes = 0L; + for (PartitionPayload payload : state.partitions) { + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < payload.values.size(); field++) { + String fieldName = "partition_key_" + field; + String fieldValue = payload.values.get(field); + typedSpec.put(fieldName, fieldValue); + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldName); + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldValue); + } + } + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, payload.displayName); + } + int index = payload.index; + partitions.put(payload.displayName, new Partition( + typedSpec, 10L + index, 1024L + index, 1L, + 1_700_000_000_000L + index, 1, false)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo( + Collections.emptyMap(), partitions, retainedPayloadBytes); + return MetaCacheWeightUtils.saturatedAdd( + partitionInfo.getNameToPartition().size(), partitionInfo.getRetainedPayloadBytes()); + } + + private static class PartitionPayload { + private final String displayName; + private final List values; + private final int index; + + private PartitionPayload( + String displayName, List values, int index) { + this.displayName = displayName; + this.values = values; + this.index = index; + } + } + + private static class Fixture { + private final PaimonSnapshotEntryKey key; + private final PaimonSnapshotCacheValue value; + + private Fixture(PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + this.key = key; + this.value = value; + } + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4e535950aae0e3..c3e95e2d4ff24a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2462,6 +2462,11 @@ public class Config extends ConfigBase { }) public static long external_cache_refresh_time_minutes = 10; // 10 mins + @ConfField(mutable = false, masterOnly = false, + description = {"FE-wide maximum weight for managed external metadata caches. Supports byte units " + + "or a percentage of the JVM max heap; 0 disables the global quota."}) + public static String external_meta_cache_max_weight = "0"; + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. @ConfField(mutable = true, masterOnly = false, description = {"Whether external meta cache uses manual miss load instead of Caffeine sync load."}) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index a880c5ffad4db4..93ceeb49b58144 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -851,6 +851,11 @@ under the License. mockito-inline test + + org.openjdk.jol + jol-core + test + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index cb4ef35eb002fd..81160fb4e382ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -635,6 +635,18 @@ public class SchemaTable extends Table { .column("LAST_LOAD_SUCCESS_TIME", ScalarType.createStringType()) .column("LAST_LOAD_FAILURE_TIME", ScalarType.createStringType()) .column("LAST_ERROR", ScalarType.createStringType()) + .column("WEIGHT_BOUNDED", ScalarType.createType(PrimitiveType.BOOLEAN)) + .column("MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ESTIMATED_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("EVICTION_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("WEIGHT_REJECT_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_ESTIMATED_WEIGHT", + ScalarType.createType(PrimitiveType.BIGINT)) + .column("GLOBAL_MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("GLOBAL_ESTIMATED_WEIGHT", + ScalarType.createType(PrimitiveType.BIGINT)) + .column("LAST_WEIGHT_REJECT_REASON", ScalarType.createStringType()) .build()) ) .put("backend_kerberos_ticket_cache", diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java index 674bf0aa39cd5b..7ad55174e303e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import com.github.benmanes.caffeine.cache.Weigher; import org.jetbrains.annotations.NotNull; import java.time.Duration; @@ -49,7 +50,10 @@ public class CacheFactory { private OptionalLong expireAfterAccessSec; private OptionalLong refreshAfterWriteSec; private long maxSize; + private OptionalLong maxWeight; + private Weigher weigher; private boolean enableStats; + private boolean softValues; // Ticker is used to provide a time source for the cache. // Only used for test, to provide a fake time source. // If not provided, the system time is used. @@ -61,11 +65,34 @@ public CacheFactory( long maxSize, boolean enableStats, Ticker ticker) { + this(expireAfterAccessSec, refreshAfterWriteSec, maxSize, OptionalLong.empty(), null, enableStats, ticker); + } + + @SuppressWarnings("unchecked") + public CacheFactory( + OptionalLong expireAfterAccessSec, + OptionalLong refreshAfterWriteSec, + long maxSize, + OptionalLong maxWeight, + Weigher weigher, + boolean enableStats, + Ticker ticker) { this.expireAfterAccessSec = expireAfterAccessSec; this.refreshAfterWriteSec = refreshAfterWriteSec; this.maxSize = maxSize; + this.maxWeight = maxWeight; + this.weigher = (Weigher) weigher; this.enableStats = enableStats; this.ticker = ticker; + if (maxWeight.isPresent() && this.weigher == null) { + throw new IllegalArgumentException("maximumWeight requires a weigher"); + } + } + + /** Configure values as soft references so unused cache entries may be reclaimed under heap pressure. */ + public CacheFactory withSoftValues() { + softValues = true; + return this; } // Build a loading cache, without executor, it will use fork-join pool for refresh @@ -116,7 +143,11 @@ public AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cac @NotNull private Caffeine buildWithParams() { Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); + if (maxWeight.isPresent()) { + builder.maximumWeight(maxWeight.getAsLong()).weigher(weigher); + } else { + builder.maximumSize(maxSize); + } if (expireAfterAccessSec.isPresent()) { builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); @@ -129,6 +160,10 @@ private Caffeine buildWithParams() { builder.recordStats(); } + if (softValues) { + builder.softValues(); + } + if (ticker != null) { builder.ticker(ticker); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index b44cb735f2ac61..1ab521987e08f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -539,6 +539,7 @@ private void createCatalogInternal(CatalogIf catalog, boolean isReplay) throws D try { if (!isReplay && catalog instanceof ExternalCatalog) { ((ExternalCatalog) catalog).checkProperties(); + validateSuppliedCacheProperties((ExternalCatalog) catalog, catalog.getProperties()); } Map props = catalog.getProperties(); if (props.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { @@ -623,6 +624,40 @@ public List listCatalogsWithCheckPriv(UserIdentity userIdentity) { } + /** CREATE: every supplied external meta cache property is validated strictly. */ + private static void validateSuppliedCacheProperties(ExternalCatalog catalog, Map properties) + throws DdlException { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + return; + } + try { + cacheMgr.validateCatalogCacheProperties(catalog, properties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + } + + /** + * ALTER: newly supplied external meta cache properties are validated strictly, persisted + * ones only as runtime honors them, so a legacy key cannot block an unrelated update. + */ + private static void validateSuppliedCacheProperties(ExternalCatalog catalog, + Map persistedProperties, Map updatedProperties) + throws DdlException { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + return; + } + try { + cacheMgr.validateCatalogCachePropertyUpdate(catalog, persistedProperties, updatedProperties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + } + /** * Reply for alter catalog props event. */ @@ -637,6 +672,7 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope boolean tentativelyMutated = false; try { ExternalCatalog externalCatalog = (ExternalCatalog) catalog; + validateSuppliedCacheProperties(externalCatalog, oldProperties, newProps); boolean validatedWithoutMutation = externalCatalog.validatePropertiesBeforeUpdate( oldProperties, newProps); if (!validatedWithoutMutation) { @@ -648,7 +684,15 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope // Only legacy validators publish a tentative candidate. Detached validators // leave the live CatalogProperty untouched while concurrent initialization runs. if (oldProperties != null && tentativelyMutated) { - ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null + ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + } else { + cacheMgr.rollbackCatalogProperties( + (ExternalCatalog) catalog, oldProperties); + } } if (validationException instanceof DdlException) { throw (DdlException) validationException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 481c1fcb3aca5f..bc1e361a4a2c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase; import org.apache.doris.datasource.lance.LanceExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; @@ -446,6 +447,22 @@ protected void checkProperties(CatalogProperty property) throws DdlException { } } + try { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr extMetaCacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (extMetaCacheMgr == null) { + // This fallback is only for isolated construction tests before Env is initialized. + ExternalMetaCacheBudgetManager.fromConfig().validateCatalogMaxWeight(properties); + } else { + // Validate what runtime will honor. Newly supplied keys are validated strictly by + // CatalogMgr for CREATE and ALTER; persisted legacy keys that initialization + // ignores must not reject an unrelated later ALTER. + extMetaCacheMgr.validateEffectiveCatalogCacheProperties(this, properties); + } + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + // check schema.cache.ttl-second parameter String schemaCacheTtlSecond = property.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL) @@ -1367,9 +1384,31 @@ public int hashCode() { public void notifyPropertiesUpdated(Map updatedProps) { CatalogIf.super.notifyPropertiesUpdated(updatedProps); String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); - if (java.util.Objects.nonNull(schemaCacheTtl)) { - ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + if (java.util.Objects.nonNull(schemaCacheTtl) + || updatedProps.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { extMetaCacheMgr.removeCatalog(id); + return; + } + for (String key : updatedProps.keySet()) { + if (key == null || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + continue; + } + String engine = remainder.substring(0, separator); + try { + extMetaCacheMgr.removeCatalogByEngine(id, engine); + } catch (IllegalArgumentException e) { + // New DDL is validated before it reaches this notification. A persisted key with + // an unknown or legacy engine namespace (edit-log replay, image load) has no cache + // group to retire and must not abort the replay; runtime sanitization ignores it. + LOG.warn("Ignoring external meta cache property '{}' with unknown engine namespace '{}' " + + "for catalog {}: {}", key, engine, id, e.getMessage()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 007e850e54e24e..316e3d3a9fd007 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalMetaCache; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.ExternalMetaCacheRegistry; import org.apache.doris.datasource.metacache.ExternalMetaCacheRouteResolver; import org.apache.doris.datasource.metacache.LegacyMetaCacheFactory; @@ -38,17 +39,22 @@ import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Striped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.Lock; import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -95,6 +101,10 @@ public class ExternalMetaCacheMgr { private final ExternalMetaCacheRegistry cacheRegistry; private final ExternalMetaCacheRouteResolver routeResolver; private final LegacyMetaCacheFactory legacyMetaCacheFactory; + private final ExternalMetaCacheBudgetManager budgetManager; + // Catalog property publication and cache-group replacement share this striped lifecycle fence. + // Initialized lookups retain the lock-free fast path above it. + private final Striped catalogLifecycleLocks = Striped.lock(64); // all catalogs could share the same fsCache. private FileSystemCache fsCache; @@ -102,6 +112,7 @@ public class ExternalMetaCacheMgr { private ExternalRowCountCache rowCountCache; public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { + budgetManager = ExternalMetaCacheBudgetManager.fromConfig(); rowCountRefreshExecutor = newThreadPool(isCheckpointCatalog, Config.max_external_cache_loader_thread_pool_size, Config.max_external_cache_loader_thread_pool_size * 1000, @@ -191,28 +202,180 @@ public DorisExternalMetaCache doris(long catalogId) { } public void prepareCatalog(long catalogId) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalog"); - return; + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalog"); + return; + } + Map runtimeProperties = sanitizeCatalogCachePropertiesForRuntime( + catalogId, catalogProperties); + routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, runtimeProperties)); + } finally { + lifecycleLock.unlock(); } - routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); } public void prepareCatalogByEngine(long catalogId, String engine) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { return; } - prepareCatalogByEngine(catalogId, engine, catalogProperties); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } public void prepareCatalogByEngine(long catalogId, String engine, Map catalogProperties) { + ExternalMetaCache targetCache = this.engine(engine); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } + } + + private void prepareCatalogByEngineLocked( + long catalogId, ExternalMetaCache targetCache, Map catalogProperties) { Map safeCatalogProperties = catalogProperties == null ? Maps.newHashMap() : Maps.newHashMap(catalogProperties); - routeSpecifiedEngine(engine, cache -> cache.initCatalog(catalogId, safeCatalogProperties)); + safeCatalogProperties = sanitizeCatalogCachePropertiesForRuntime(catalogId, safeCatalogProperties); + targetCache.initCatalog(catalogId, safeCatalogProperties); + } + + public void validateCatalogCacheProperties(Map catalogProperties) { + budgetManager.validateCatalogMaxWeight(catalogProperties); + validateCatalogCachePropertyNamespaces(catalogProperties); + cacheRegistry.allCaches().forEach(cache -> cache.validateCatalogProperties(catalogProperties)); + } + + private Map sanitizeCatalogCachePropertiesForRuntime( + long catalogId, Map catalogProperties) { + Map sanitized = Maps.newHashMap(catalogProperties); + try { + budgetManager.parseCatalogMaxWeight(sanitized); + } catch (IllegalArgumentException e) { + sanitized.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' for catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, catalogId, e.getMessage()); + } + return sanitized; + } + + private void validateCatalogCachePropertyNamespaces(Map catalogProperties) { + String globalKey = ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY; + String prefix = "meta.cache."; + for (String key : catalogProperties.keySet()) { + if (key == null || globalKey.equals(key) || !key.startsWith(prefix)) { + continue; + } + String remainder = key.substring(prefix.length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String configuredEngine = remainder.substring(0, separator); + ExternalMetaCache resolved = cacheRegistry.resolve(configuredEngine); + if (!resolved.engine().equals(configuredEngine)) { + throw new IllegalArgumentException("External meta cache properties must use canonical engine '" + + resolved.engine() + "' instead of alias '" + configuredEngine + "': " + key); + } + } + } + + /** Strict DDL validation also rejects a valid engine namespace not routed by the catalog type. */ + public void validateCatalogCacheProperties(CatalogIf catalog, Map catalogProperties) { + validateCatalogCacheProperties(catalogProperties); + Set routedEngines = routeResolver.resolveCatalogCaches(catalog.getId(), catalog).stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + for (String key : catalogProperties.keySet()) { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + String configuredEngine = separator < 0 ? remainder : remainder.substring(0, separator); + if (!routedEngines.contains(configuredEngine)) { + throw new IllegalArgumentException("External meta cache engine '" + configuredEngine + + "' is not supported by catalog type " + catalog.getClass().getSimpleName() + ": " + key); + } + } + } + + /** + * Strict validation of an ALTER: every newly supplied cache key must be valid, while keys + * already persisted before the update are reduced to what runtime initialization honors + * (image/replay may carry legacy or obsolete keys that lazy initialization ignores) so an + * old unknown key cannot lock the catalog out of unrelated updates. The hierarchy is + * validated over the merged runtime view. + */ + public void validateCatalogCachePropertyUpdate( + CatalogIf catalog, Map persistedProperties, + Map updatedProperties) { + validateCatalogCacheProperties(catalog, updatedProperties); + Map effective = runtimeEffectiveCacheProperties( + catalog, persistedProperties == null ? Collections.emptyMap() : persistedProperties); + effective.putAll(updatedProperties); + validateRuntimeCacheProperties(catalog, effective); + } + + /** + * Validate persisted properties as runtime will apply them: unknown engine namespaces and + * options no engine honors are dropped, then the remaining set is validated with the + * semantics initialization uses. + */ + public void validateEffectiveCatalogCacheProperties( + CatalogIf catalog, Map catalogProperties) { + validateRuntimeCacheProperties(catalog, runtimeEffectiveCacheProperties(catalog, catalogProperties)); + } + + private void validateRuntimeCacheProperties(CatalogIf catalog, Map effective) { + budgetManager.parseCatalogMaxWeight(effective); + for (ExternalMetaCache cache : routeResolver.resolveCatalogCaches(catalog.getId(), catalog)) { + cache.validateCatalogPropertiesForRuntime(effective); + } + } + + private Map runtimeEffectiveCacheProperties( + CatalogIf catalog, Map catalogProperties) { + Map effective = sanitizeCatalogCachePropertiesForRuntime( + catalog.getId(), catalogProperties); + List routedCaches = routeResolver.resolveCatalogCaches(catalog.getId(), catalog); + Set routedEngines = routedCaches.stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + effective.keySet().removeIf(key -> { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + return false; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + return separator <= 0 || !routedEngines.contains(remainder.substring(0, separator)); + }); + for (ExternalMetaCache cache : routedCaches) { + effective = cache.sanitizeCatalogPropertiesForRuntime(effective); + } + return effective; } public void invalidateCatalog(long catalogId) { @@ -228,15 +391,42 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { } public void removeCatalog(long catalogId) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "removeCatalog", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalog", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + } + + /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ + public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { + long catalogId = catalog.getId(); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + catalog.rollBackCatalogProps(oldProperties); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void removeCatalogByEngine(long catalogId, String engine) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "removeCatalogByEngine", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "removeCatalogByEngine", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void invalidateDb(long catalogId, String dbName) { @@ -299,16 +489,54 @@ public MetaCacheEntryStats getEntryStats() { private void initEngineCaches() { registerBuiltinEngineCaches(); + bindCatalogPreparers(); + } + + private void bindCatalogPreparers() { + for (ExternalMetaCache cache : cacheRegistry.allCaches()) { + String engine = cache.engine(); + cache.bindCatalogPreparer(catalogId -> tryPrepareCatalogByEngine(catalogId, engine)); + } + } + + /** + * Re-prepare a catalog whose group was retired between a caller's preparation and its lookup. + * Lookups may run inside a cache loader, and retirement holds the lifecycle lock while it + * closes groups, so this never blocks: when the fence is contended the lookup keeps its + * pre-existing failure and the next call prepares under the new policy. + */ + private void tryPrepareCatalogByEngine(long catalogId, String engine) { + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + if (!lifecycleLock.tryLock()) { + return; + } + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "tryPrepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } private void registerBuiltinEngineCaches() { - cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor)); - cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor)); - cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor)); + cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor, budgetManager)); + cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor, budgetManager)); } private void routeCatalogEngines(long catalogId, Consumer action) { @@ -413,6 +641,7 @@ public static Map getCacheStats(CacheStats cacheStats, long esti void replaceEngineCachesForTest(List caches) { cacheRegistry.resetForTest(caches); + bindCatalogPreparers(); } /** @@ -428,8 +657,9 @@ void replaceEngineCachesForTest(List caches) { * loading/invalidation. No engine-specific metadata (partitions/files/snapshots) is cached. */ private static class DefaultExternalMetaCache extends AbstractExternalMetaCache { - DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor) { - super(engine, refreshExecutor); + DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(engine, refreshExecutor, budgetManager); registerEntry(MetaCacheEntryDef.of( ENTRY_SCHEMA, SchemaCacheKey.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java index d14ba5645bf269..e7487c065b5ce3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -72,7 +73,11 @@ public class DorisExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public DorisExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public DorisExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); backendsEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_BACKENDS, String.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e2d73fd7a16edf..883a38780149d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -28,10 +28,8 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.hudi.HudiExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.iceberg.IcebergUtils; -import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; import org.apache.doris.fs.FileSystemProvider; @@ -218,10 +216,6 @@ public void notifyPropertiesUpdated(Map updatedProps) { if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); } - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java new file mode 100644 index 00000000000000..0d5f8a8343415f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +/** Constant-time retained-weight formula for Hive partition-value cache entries. */ +final class HiveCacheSizeEstimator { + // Calibrated against complete 4.1 object graphs. The payload reserve covers the partition + // name plus derived value/literal strings and therefore remains skew-sensitive. + private static final long ENTRY_BASE_BYTES = objectBytes(2L * 1024L); + private static final long PARTITION_BASE_BYTES = objectBytes(896L); + private static final long PARTITION_COLUMN_BYTES = objectBytes(256L); + // One copy is retained as the partition name and another in the decoded partition values. + private static final long PARTITION_NAME_PAYLOAD_COPIES = 2L; + + private HiveCacheSizeEstimator() { + } + + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + + static MetaCacheSizeEstimate estimatePartitionValuesEntry( + PartitionValueCacheKey key, HivePartitionValues value) { + if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { + return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } + long partitionCount = value.getIdToPartitionItem() == null + ? 0L : value.getIdToPartitionItem().size(); + long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( + PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionColumnCount(), PARTITION_COLUMN_BYTES)); + long bytes = MetaCacheWeightUtils.saturatedAdd( + ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionNamePayloadBytes(), PARTITION_NAME_PAYLOAD_COPIES)); + return MetaCacheSizeEstimate.complete(bytes); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 73986138c51cb0..1e48fc0604c82f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -42,8 +42,12 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.FileSystemDirectoryLister; @@ -59,10 +63,12 @@ import com.google.common.base.Strings; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import lombok.Data; +import lombok.Getter; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Partition; @@ -108,6 +114,7 @@ */ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private static final Logger LOG = LogManager.getLogger(HiveExternalMetaCache.class); + private static final int PARTITION_EVENT_REPLACE_MAX_RETRIES = 8; public static final String ENGINE = "hive"; public static final String ENTRY_SCHEMA = "schema"; @@ -127,7 +134,12 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private final PartitionCacheCoordinator partitionCacheCoordinator = new PartitionCacheCoordinator(); public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, fileListingExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); this.fileListingExecutor = fileListingExecutor; schemaEntry = registerEntry(MetaCacheEntryDef.of( @@ -144,7 +156,8 @@ public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fi CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, - Config.max_hive_partition_table_cache_num))); + Config.max_hive_partition_table_cache_num)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); partitionEntry = registerEntry(MetaCacheEntryDef.of( ENTRY_PARTITION, PartitionCacheKey.class, @@ -292,9 +305,13 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); + long partitionNamePayloadBytes = 0L; String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); for (String partitionName : partitionNames) { + partitionNamePayloadBytes = MetaCacheWeightUtils.saturatedAdd( + partitionNamePayloadBytes, + MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); idToPartitionItem.put(partitionId, listPartitionItem); @@ -302,7 +319,15 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap); + HivePartitionValues partitionValues = + new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + partitionNamePayloadBytes, key.types == null ? 0 : key.types.size()); + preparePartitionValuesForPublication(partitionValues); + return partitionValues; + } + + private void preparePartitionValuesForPublication(HivePartitionValues partitionValues) { + partitionValues.rebuildSortedPartitionRangesForPublication(); } private ListPartitionItem toListPartitionItem(String partitionName, List types, String catalogName) { @@ -635,7 +660,7 @@ private void invalidatePartitionCache(NameMapping nameMapping, String partitionN List values = HiveUtil.toPartitionValues(partitionName); PartitionCacheKey partKey = new PartitionCacheKey(nameMapping, values); - HivePartition partition = partitionEntry.getIfPresent(partKey); + HivePartition partition = partitionEntry.peekIfPresent(partKey); if (partition == null) { // Partition metadata cache miss: the exact FileCacheKey cannot be rebuilt here because it // needs the partition path and input format carried by HivePartition. Invalidate this @@ -715,41 +740,64 @@ private void addPartitionsCache(NameMapping nameMapping, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, partitionColumnTypes); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; - } - - HivePartitionValues copy = partitionValues.copy(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItem = new HashMap<>(); - HMSExternalCatalog catalog = hmsCatalog(catalogId); String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); - for (String partitionName : partitionNames) { - if (partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", - partitionName, localTblName); - continue; + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; } - long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); - ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); - idToPartitionItemBefore.put(partitionId, listPartitionItem); - idToPartitionItem.put(partitionId, listPartitionItem); - partitionNameToIdMapBefore.put(partitionName, partitionId); - } - Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); - Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - partitionValuesMapBefore.putAll(partitionValuesMap); - copy.rebuildSortedPartitionRanges(); - - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); + HivePartitionValues copy = current.mutableCopy(); + Map allItems = copy.getIdToPartitionItem(); + Map allNames = copy.getPartitionNameToIdMap(); + Map addedItems = new HashMap<>(); + for (String partitionName : partitionNames) { + if (allNames.containsKey(partitionName)) { + if (attempt == 0) { + LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", + partitionName, localTblName); + } + continue; + } + long partitionId = Util.genIdByName( + catalog.getName(), localDbName, localTblName, partitionName); + ListPartitionItem item = toListPartitionItem(partitionName, key.types, catalog.getName()); + allItems.put(partitionId, item); + addedItems.put(partitionId, item); + allNames.put(partitionName, partitionId); + copy.addPartitionNamePayload(partitionName); + } + if (addedItems.isEmpty()) { + // Even a replay/no-op event must fence a refresh that started before the event. + // Otherwise that refresh could replace this already-correct graph with stale HMS data. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } + continue; + } + copy.getPartitionValuesMap().putAll( + ListPartitionPrunerV2.getPartitionValuesMap(addedItems)); + preparePartitionValuesForPublication(copy); + + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after add event was rejected: {}", key); + return; + } } + // Repeated conflicts mean we cannot prove the cached graph contains this event. Force + // the next reader to rebuild it from HMS rather than retaining a possibly stale value. + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated add-event conflicts: {}", key); } private void dropPartitionsCache(ExternalTable dorisTable, @@ -765,41 +813,63 @@ private void dropPartitionsCache(ExternalTable dorisTable, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, null); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; + if (invalidPartitionCache) { + for (String partitionName : partitionNames) { + invalidatePartitionCache(nameMapping, partitionName); + } } - HivePartitionValues copy = partitionValues.copy(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map> partitionValuesMap = copy.getPartitionValuesMap(); - - for (String partitionName : partitionNames) { - if (!partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", - partitionName, nameMapping.getFullLocalName()); + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; + } + HivePartitionValues copy = current.mutableCopy(); + Map allNames = copy.getPartitionNameToIdMap(); + Map allItems = copy.getIdToPartitionItem(); + Map> allValues = copy.getPartitionValuesMap(); + boolean changed = false; + for (String partitionName : partitionNames) { + Long partitionId = allNames.remove(partitionName); + if (partitionId == null) { + LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", + partitionName, nameMapping.getFullLocalName()); + continue; + } + allItems.remove(partitionId); + allValues.remove(partitionId); + copy.removePartitionNamePayload(partitionName); + changed = true; + } + if (!changed) { + // See the add-event no-op path: event ordering still has to win over an older refresh. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } continue; } - Long partitionId = partitionNameToIdMapBefore.remove(partitionName); - idToPartitionItemBefore.remove(partitionId); - partitionValuesMap.remove(partitionId); - - if (invalidPartitionCache) { - invalidatePartitionCache(nameMapping, partitionName); + preparePartitionValuesForPublication(copy); + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after drop event was rejected: {}", key); + return; } } - - copy.rebuildSortedPartitionRanges(); - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); - } + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated drop-event conflicts: {}", key); } } @VisibleForTesting public void putPartitionValuesCacheForTest(PartitionValueCacheKey key, HivePartitionValues values) { + preparePartitionValuesForPublication(values); partitionValuesEntry.get(key.getNameMapping().getCtlId()).put(key, values); } @@ -842,15 +912,15 @@ public List getFilesByTransaction(List partitions /** * The key of hive partition values cache. */ - @Data + @Getter public static class PartitionValueCacheKey { - private NameMapping nameMapping; + private final NameMapping nameMapping; // Not part of cache identity. - private List types; + private final List types; public PartitionValueCacheKey(NameMapping nameMapping, List types) { this.nameMapping = nameMapping; - this.types = types; + this.types = types == null ? null : ImmutableList.copyOf(types); } @Override @@ -1037,7 +1107,7 @@ public static class HiveFileStatus { AcidInfo acidInfo; } - @Data + @Getter public static class HivePartitionValues { private BiMap partitionNameToIdMap; private Map idToPartitionItem; @@ -1045,6 +1115,12 @@ public static class HivePartitionValues { // Sorted partition ranges for binary search filtering. private SortedPartitionRanges sortedPartitionRanges; + // Prepared once after construction/update; the cache weigher only reads this value. + private transient volatile MetaCacheSizeEstimate sizeEstimate; + // Maintained while the metadata is already being loaded or updated. Admission only reads it. + private long partitionNamePayloadBytes; + private int partitionColumnCount; + private transient boolean sortedPartitionRangesPrepared; public HivePartitionValues() { } @@ -1052,22 +1128,101 @@ public HivePartitionValues() { public HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap) { + this(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + countPartitionNamePayloadBytes(partitionNameToIdMap), + inferPartitionColumnCount(partitionValuesMap)); + } + + HivePartitionValues(Map idToPartitionItem, + BiMap partitionNameToIdMap, + Map> partitionValuesMap, + long partitionNamePayloadBytes, + int partitionColumnCount) { this.idToPartitionItem = idToPartitionItem; this.partitionNameToIdMap = partitionNameToIdMap; this.partitionValuesMap = partitionValuesMap; - this.sortedPartitionRanges = buildSortedPartitionRanges(); + this.partitionNamePayloadBytes = partitionNamePayloadBytes; + this.partitionColumnCount = partitionColumnCount; } - public HivePartitionValues copy() { + HivePartitionValues mutableCopy() { HivePartitionValues copy = new HivePartitionValues(); - copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); - copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); - copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); + copy.partitionNameToIdMap = partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap); + copy.idToPartitionItem = idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem); + copy.partitionValuesMap = partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap); + copy.partitionNamePayloadBytes = partitionNamePayloadBytes; + copy.partitionColumnCount = partitionColumnCount; return copy; } - public void rebuildSortedPartitionRanges() { - this.sortedPartitionRanges = buildSortedPartitionRanges(); + /** Compatibility hook for tests and benchmarks; publication uses copy-on-write updates. */ + void sealForPublication() { + if (!sortedPartitionRangesPrepared) { + rebuildSortedPartitionRangesForPublication(); + } + } + + void rebuildSortedPartitionRangesForPublication() { + sortedPartitionRanges = buildSortedPartitionRanges(); + sortedPartitionRangesPrepared = true; + } + + MetaCacheSizeEstimate prepareForCachePublication(PartitionValueCacheKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely( + "hive_partition_values_preparation_failed", () -> { + prepareSizeEstimate(key); + return getSizeEstimate(); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + MetaCacheSizeEstimate result = sizeEstimate; + return result == null ? MetaCacheSizeEstimate.incomplete("estimate_not_prepared") : result; + } + + void prepareSizeEstimate(PartitionValueCacheKey key) { + sizeEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, this); + } + + long getPartitionNamePayloadBytes() { + return partitionNamePayloadBytes; + } + + int getPartitionColumnCount() { + return partitionColumnCount; + } + + private void addPartitionNamePayload(String partitionName) { + partitionNamePayloadBytes = MetaCacheWeightUtils.saturatedAdd( + partitionNamePayloadBytes, + MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); + } + + private void removePartitionNamePayload(String partitionName) { + partitionNamePayloadBytes = Math.max(0L, partitionNamePayloadBytes + - MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); + } + + private static long countPartitionNamePayloadBytes(BiMap names) { + long payloadBytes = 0L; + if (names != null) { + for (String name : names.keySet()) { + payloadBytes = MetaCacheWeightUtils.saturatedAdd( + payloadBytes, MetaCacheWeightUtils.estimatedStringPayloadBytes(name)); + } + } + return payloadBytes; + } + + private static int inferPartitionColumnCount(Map> values) { + if (values == null || values.isEmpty()) { + return 0; + } + List first = values.values().iterator().next(); + return first == null ? 0 : first.size(); } public java.util.Optional> getSortedPartitionRanges() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 74d2aa99900340..a83777b1e53e68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -83,7 +84,11 @@ public class HudiExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public HudiExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_PARTITION, HudiPartitionCacheKey.class, TablePartitionValues.class, this::loadPartitionValuesCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiPartitionCacheKey::getNameMapping))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..94283ca6b25bb3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,1230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.iceberg.BlobMetadata; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.UnknownTransform; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Publication-time retained-weight formulas for Iceberg cache entries. */ +final class IcebergCacheSizeEstimator { + // Calibrated against JOL retained-graph deltas in IcebergExternalMetaCacheTest. + // Every metadata element visited (field, type, snapshot, summary entry, ...) costs a few + // reads; the bound only guards against pathological metadata and is far above real tables + // (a 10,000-snapshot history with 15 summary keys each is 160,000 elements). Exceeding it + // rejects weighted admission, so it must not be reachable by ordinary long-lived tables. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L; + // Total name characters the estimator may lower-case while reserving case-insensitive indexes. + private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // TableMetadata-side share of one schema version: schemas list slot and schemasById entry, + // including the growth of both from their singleton to their regular immutable shapes. + private static final long SCHEMA_VERSION_BYTES = objectBytes(128L); + private static final long PARTITION_SPEC_BYTES = objectBytes(256L); + // Exact active-layout sizes of the Iceberg/Guava objects that lazy partition, sort and + // schema state allocates. Iceberg 1.10.1 field layouts are pinned by ICEBERG_LAZY_LAYOUT_SUPPORTED. + private static final long PARTITION_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L); + private static final long SORT_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + // Identity/Bucket/Truncate transforms are allocated per parsed field; time transforms are enums. + private static final long TRANSFORM_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long NESTED_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L); + private static final long STRUCT_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L); + private static final long SCHEMA_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L); + private static final long IMMUTABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long IMMUTABLE_MAP_KEY_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SINGLETON_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long REGULAR_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long LONG_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + // Literals.BaseLiteral: value plus the transient serialized-buffer slot. + private static final long LITERAL_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + // JDK 17 HeapByteBuffer: Buffer header fields, address, segment, hb and offset. + private static final long BYTE_BUFFER_BYTES = objectBytes(56L); + private static final long BIG_DECIMAL_BYTES = objectBytes(104L); + private static final long BOXED_DEFAULT_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 16L); + private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; + // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). + private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L), + MetaCacheWeightUtils.estimatedIntArrayBytes(1L)); + private static final long LIST_MULTIMAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L); + private static final long CAPTURING_SUPPLIER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long POSITION_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L); + // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor. Required ancestors + // collapse into a single Position2/3Accessor that replaces the inner accessor, which retains + // less than this per-level reservation. + private static final long WRAPPED_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L); + private static final long LIST_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L); + private static final long DECIMAL_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final long FIXED_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long GEOMETRY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long GEOGRAPHY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SORT_ORDER_BYTES = objectBytes(256L); + private static final long TABLE_PROPERTY_BYTES = objectBytes(40L); + private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L); + private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L); + private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L); + private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L); + private static final long SNAPSHOT_REF_BYTES = objectBytes(128L); + private static final long STATISTICS_FILE_BYTES = objectBytes(512L); + private static final long BLOB_METADATA_BYTES = objectBytes(128L); + private static final long BLOB_FIELD_BYTES = objectBytes(32L); + private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); + private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); + // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a + // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. + private static final long PARTITION_BYTES = objectBytes(680L); + // Outer map entry and table share of one merged-overlap group; the alias set itself and its + // contents are charged by IcebergPartitionInfo per enclosed partition name. + private static final long PARTITION_ALIAS_BYTES = objectBytes(144L); + // One name-mapping field: map node, boxed id and list object; alias arrays and Strings are + // charged by IcebergSnapshotCacheValue when the mapping is copied. + private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); + private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); + private static final long DATA_FILE_BYTES = objectBytes(896L); + private static final long DELETE_FILE_BYTES = objectBytes(1024L); + private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L); + private static final String BASE_SNAPSHOT_CLASS_NAME = "org.apache.iceberg.BaseSnapshot"; + private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS = + loadBaseSnapshotRetainedCacheFields(); + // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog supplier + // (REST snapshot-loading-mode=refs). Publication must not perform that IO. + private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD = + loadTableMetadataField("snapshotsLoaded", boolean.class); + private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD = + loadTableMetadataField("snapshotsSupplier", null); + // The formulas above are built on the Iceberg 1.10.1 instance-field layouts of the classes a + // cached table retains. Every non-static field is pinned, not only the transient lazy ones: a + // library upgrade that adds a retained reference makes weighted admission fail closed. + private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED = checkIcebergLayout(); + + private IcebergCacheSizeEstimator() { + } + + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } + Table table = value.getRetainedIcebergTable(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } + long bytes = KEY_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation())); + + IcebergPartitionInfo partitionInfo = value.getPartitionInfo(); + bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); + bytes = addCount(bytes, value.getNameMapping().map(Map::size).orElse(0), + NAME_MAPPING_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedNameMappingPayloadBytes()); + + if (value.getRetainedIcebergTable().isPresent()) { + // The projection keeps its own reference to the frozen table generation. That graph + // is charged here as well as by the table entry that produced it: the two entries have + // independent lifetimes (TTL, weight eviction, soft collection) and either may outlive + // the other, so each must be able to carry the graph on its own. Budgets should be + // sized for the table metadata being counted once per dependent entry. + Table table = value.getRetainedIcebergTable().get(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + } + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateManifestEntry( + IcebergManifestEntryKey key, ManifestCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } + if (!value.isAccountingComplete()) { + return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + MANIFEST_ENTRY_BASE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); + bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_BYTES); + bytes = addCount(bytes, value.getDeleteFiles().size(), DELETE_FILE_BYTES); + bytes = addCount(bytes, value.getDataFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + private static MetaCacheSizeEstimate checkJvmObjectLayout() { + return MetaCacheWeightUtils.isSupportedJvmObjectLayout() + ? MetaCacheSizeEstimate.complete(1L) + : MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } + + private static MetaCacheSizeEstimate checkSupportedTable(Table table) { + if (!ICEBERG_LAZY_LAYOUT_SUPPORTED) { + return MetaCacheSizeEstimate.incomplete("unsupported_iceberg_lazy_layout"); + } + if (table == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); + } + if (!(table instanceof HasTableOperations)) { + return MetaCacheSizeEstimate.incomplete( + "unsupported_iceberg_table:" + table.getClass().getName()); + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata"); + } + if (!areSnapshotsLoaded(metadata)) { + return MetaCacheSizeEstimate.incomplete("iceberg_snapshots_not_loaded"); + } + if (metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location"); + } + return MetaCacheSizeEstimate.complete(1L); + } + + /** Reads only metadata collection sizes and a constant number of strings; no FileIO is used. */ + private static long estimateTable(Table table) { + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + long bytes = MetaCacheWeightUtils.saturatedAdd( + TABLE_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.location())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.metadataFileLocation())); + + bytes = addCount(bytes, metadata.properties().size(), TABLE_PROPERTY_BYTES); + if (metadata.currentSnapshot() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_BYTES); + } + return bytes; + } + + /** + * Fully accounts variable payload with a bounded amount of publication-time work. Only + * already-parsed metadata is read; the SDK state it touches on the way (StructType, ListType + * and MapType fieldList copies, the identifier field set) is small, accounted and O(N). + */ + static long retainedTablePayloadBytes(Table table) { + if (!(table instanceof HasTableOperations)) { + return 0L; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return 0L; + } + if (!areSnapshotsLoaded(metadata)) { + // snapshots()/refs() would call the catalog's lazy snapshot supplier: fail closed. + throw new IllegalStateException("Iceberg table snapshots are not loaded"); + } + + long bytes = 0L; + AccountingBudget budget = new AccountingBudget( + MAX_TABLE_ACCOUNTING_ELEMENTS, MAX_TABLE_ACCOUNTING_CHARACTERS); + for (PartitionSpec spec : metadata.specs()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionSpecBytes(spec, budget)); + } + for (SortOrder sortOrder : metadata.sortOrders()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, sortOrderBytes(sortOrder, budget)); + } + for (Schema schema : metadata.schemas()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaBytes(schema, budget)); + } + budget.chargeElements(metadata.properties().size()); + for (Map.Entry property : metadata.properties().entrySet()) { + bytes = addString(bytes, property.getKey()); + bytes = addString(bytes, property.getValue()); + } + for (Snapshot snapshot : metadata.snapshots()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, snapshotBytes(snapshot, budget)); + } + budget.chargeElements(metadata.snapshotLog().size()); + bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + budget.chargeElements(metadata.previousFiles().size()); + for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); + bytes = addString(bytes, previousFile.file()); + } + budget.chargeElements(metadata.refs().size()); + bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); + for (String refName : metadata.refs().keySet()) { + bytes = addString(bytes, refName); + } + budget.chargeElements(metadata.statisticsFiles().size()); + for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + for (BlobMetadata blob : statisticsFile.blobMetadata()) { + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, + MetaCacheWeightUtils.saturatedAdd( + blob.fields().size(), blob.properties().size()))); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, BLOB_METADATA_BYTES); + bytes = addString(bytes, blob.type()); + bytes = addCount(bytes, blob.fields().size(), BLOB_FIELD_BYTES); + bytes = addStringMap(bytes, blob.properties(), TABLE_PROPERTY_BYTES); + } + } + budget.chargeElements(metadata.partitionStatisticsFiles().size()); + for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + } + budget.chargeElements(metadata.encryptionKeys().size()); + for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { + budget.chargeElements(encryptedKey.properties().size()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); + bytes = addString(bytes, encryptedKey.keyId()); + bytes = addString(bytes, encryptedKey.encryptedById()); + bytes = addBufferPayload(bytes, encryptedKey.encryptedKeyMetadata()); + bytes = addStringMap(bytes, encryptedKey.properties(), TABLE_PROPERTY_BYTES); + } + bytes = addString(bytes, metadata.uuid()); + return bytes; + } + + /** + * Account a PartitionSpec together with the lazy state that a normal scan materializes after + * admission: fieldList, javaClasses, partitionType() with its StructType indexes, the secondary + * Schema/Binder graph behind partitionType().asSchema() and fieldsBySourceId. Iceberg 1.10.1 + * allocates one Object[fieldCount] per distinct source id inside fieldsBySourceId, so that + * retained graph is O(distinctSourceIds * fieldCount); it is reserved here in O(fieldCount) + * publication work without materializing any of it. + */ + private static long partitionSpecBytes(PartitionSpec spec, AccountingBudget budget) { + List fields = spec.fields(); + long fieldCount = fields.size(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); + long bytes = PARTITION_SPEC_BYTES; + if (fieldCount == 0L) { + return bytes; + } + Set distinctSourceIds = new HashSet<>(); + long uncachedSourceIds = 0L; + long uncachedFieldIds = 0L; + long lowerCaseNameBytes = 0L; + for (PartitionField field : fields) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_FIELD_BYTES); + bytes = addTransformPayload(bytes, field.transform()); + bytes = addString(bytes, field.name()); + lowerCaseNameBytes = MetaCacheWeightUtils.saturatedAdd( + lowerCaseNameBytes, generatedLowerCaseNameBytes(field.name(), budget)); + if (isUncachedInteger(field.fieldId())) { + uncachedFieldIds++; + } + if (distinctSourceIds.add(field.sourceId()) && isUncachedInteger(field.sourceId())) { + uncachedSourceIds++; + } + } + // Eager PartitionField[] plus lazy fieldList and javaClasses. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + // partitionType(): the StructType itself also exists for an unpartitioned spec. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + structTypeIndexBytes(fieldCount, uncachedFieldIds, lowerCaseNameBytes)); + if (!spec.schema().idsToOriginal().isEmpty()) { + // rawPartitionType() rebuilds the struct with original ids. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); + bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); + } + // A partition filter binds against partitionType().asSchema(). + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes( + SchemaShape.flat(fieldCount, uncachedFieldIds, lowerCaseNameBytes))); + // fieldsBySourceId: HashMap. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LIST_MULTIMAP_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CAPTURING_SUPPLIER_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(distinctSourceIds.size(), uncachedSourceIds)); + return addCount(bytes, distinctSourceIds.size(), + MetaCacheWeightUtils.saturatedAdd(ARRAY_LIST_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount))); + } + + /** Account a SortOrder: SortField[] with per-field transforms plus the lazy fieldList copy. */ + private static long sortOrderBytes(SortOrder sortOrder, AccountingBudget budget) { + long fieldCount = sortOrder.fields().size(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); + long bytes = SORT_ORDER_BYTES; + if (fieldCount == 0L) { + return bytes; + } + for (SortField field : sortOrder.fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_FIELD_BYTES); + bytes = addTransformPayload(bytes, field.transform()); + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + return MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); + } + + /** Transform instance plus the payload only some transforms retain. */ + private static long addTransformPayload(long bytes, Transform transform) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TRANSFORM_BYTES); + if (transform instanceof UnknownTransform) { + return addString(bytes, transform.toString()); + } + if (transform.toString().startsWith(TRUNCATE_TRANSFORM_PREFIX)) { + // Truncate is package-private; its serialized name is the SPI contract. + return MetaCacheWeightUtils.saturatedAdd(bytes, TRUNCATE_WIDTH_BYTES); + } + return bytes; + } + + /** Lazy StructType indexes: fieldList, fieldsByName, fieldsByLowerCaseName and fieldsById. */ + private static long structTypeIndexBytes( + long fieldCount, long uncachedFieldIds, long lowerCaseNameBytes) { + long bytes = immutableListBytes(fieldCount); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, 0L, 0L)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, 0L, lowerCaseNameBytes)); + return MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, uncachedFieldIds, 0L)); + } + + /** + * The Schema created by StructType.asSchema(): its constructor materializes idToName and two + * empty id maps; Binder and projection paths add nameToId, lowerCaseNameToId, idToField and + * idToAccessor; its own StructType copy grows the same lazy indexes as the root struct. + */ + private static long secondarySchemaBytes(SchemaShape shape) { + long bytes = schemaObjectBytes(shape); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); + return MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( + shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, + shape.topLevelLowerCaseStringBytes)); + } + + /** Schema object, empty identifier int[], the two empty id maps and the eager idToName keySet. */ + private static long schemaObjectBytes(SchemaShape shape) { + long bytes = MetaCacheWeightUtils.saturatedAdd( + SCHEMA_BYTES, MetaCacheWeightUtils.estimatedIntArrayBytes(0L)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); + if (shape.fieldCount == 1L) { + return MetaCacheWeightUtils.saturatedAdd(bytes, SINGLETON_IMMUTABLE_SET_BYTES); + } + return shape.fieldCount > 1L + ? MetaCacheWeightUtils.saturatedAdd(bytes, IMMUTABLE_MAP_KEY_SET_BYTES) : bytes; + } + + /** + * idToName (eager in the constructor), nameToId and idToField. Every map boxes uncached ids + * itself; idToName and nameToId each retain their own copy of every nested canonical name and + * nameToId also retains the short aliases. + */ + private static long schemaLookupBytes(SchemaShape shape) { + long bytes = immutableNameMapBytes( + shape.fieldCount, shape.uncachedFieldIdCount, shape.pathStringBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableNameMapBytes( + shape.nameEntryCount, shape.uncachedNameIdCount, + MetaCacheWeightUtils.saturatedAdd( + shape.pathStringBytes, shape.aliasStringBytes))); + return MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(shape.fieldCount, shape.uncachedFieldIdCount)); + } + + /** lowerCaseNameToId and idToAccessor, materialized by case-insensitive lookups and Binder. */ + private static long schemaLazyIndexBytes(SchemaShape shape) { + long bytes = immutableNameMapBytes( + shape.nameEntryCount, shape.uncachedNameIdCount, shape.lowerCaseStringBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(shape.accessorFieldCount, shape.uncachedAccessorIdCount)); + bytes = addCount(bytes, shape.accessorFieldCount, POSITION_ACCESSOR_BYTES); + return addCount(bytes, shape.wrappedAccessorCount, WRAPPED_ACCESSOR_BYTES); + } + + /** + * One table schema version with every index a normal scan can materialize afterwards. Only + * metadata already parsed is read; nothing lazy is touched, and each field is visited once. + */ + private static long schemaBytes(Schema schema, AccountingBudget budget) { + budget.chargeElements(1L); + SchemaShape shape = new SchemaShape(); + long bytes = SCHEMA_VERSION_BYTES; + for (Types.NestedField field : schema.columns()) { + bytes = addFieldPayload( + bytes, field, PathState.ROOT, FieldKind.STRUCT_FIELD, budget, shape); + } + Set identifierFieldIds = schema.identifierFieldIds(); + budget.chargeElements(identifierFieldIds.size()); + bytes = addIdentifierFieldPayload(bytes, identifierFieldIds); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, shape.typeObjectBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaObjectBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); + if (shape.fieldCount == 0L) { + // Nothing can be looked up in an empty schema; its indexes stay shared singletons. + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); + // Future lazy growth: main lookups, root struct indexes and the asSchema() secondary graph. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( + shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, + shape.topLevelLowerCaseStringBytes)); + return MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes(shape)); + } + + /** ImmutableList.copyOf(array): shared empty, singleton, or a list object plus copied array. */ + private static long immutableListBytes(long elementCount) { + if (elementCount <= 0L) { + return 0L; + } + if (elementCount == 1L) { + return IMMUTABLE_LIST_BYTES; + } + return MetaCacheWeightUtils.saturatedAdd(IMMUTABLE_LIST_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount)); + } + + /** Growth of a reference array that replaces an empty array retained by the empty shape. */ + private static long objectArrayGrowthBytes(long elementCount) { + long populated = MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount); + long empty = MetaCacheWeightUtils.estimatedObjectArrayBytes(0L); + return populated == Long.MAX_VALUE ? populated : populated - empty; + } + + /** Boxed Integer keys outside the JVM Integer cache are retained per lookup map. */ + private static boolean isUncachedInteger(int value) { + return value < -128 || value > 127; + } + + /** + * Retained bytes of one lower-cased copy of a name, or 0 when the name is already lower case + * and the index reuses it. Every case-insensitive index (partition StructType, secondary + * Schema and secondary StructType) allocates its own copy, so callers add this per index. + */ + private static long generatedLowerCaseNameBytes(String name, AccountingBudget budget) { + budget.chargeCharacters(name.length()); + String lowerName = name.toLowerCase(Locale.ROOT); + if (lowerName.equals(name)) { + return 0L; + } + return MetaCacheWeightUtils.estimatedGeneratedStringBytes( + lowerName.length(), MetaCacheWeightUtils.isLatin1String(lowerName)); + } + + private static long hashIdMapBytes(long entryCount, long uncachedIds) { + long bytes = HASH_MAP_BYTES; + if (entryCount <= 0L) { + // HashMap allocates its table on the first put. + return bytes; + } + bytes = addCount(bytes, entryCount, HASH_MAP_NODE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + hashMapCapacity(entryCount))); + return addCount(bytes, uncachedIds, INTEGER_BYTES); + } + + private static long immutableNameMapBytes( + long entryCount, long uncachedIds, long generatedStringBytes) { + long bytes = 0L; + if (entryCount == 1L) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(8L, 0L)); + } else if (entryCount > 1L) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 4L)); + bytes = addCount(bytes, entryCount, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(entryCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + immutableMapTableCapacity(entryCount))); + } + bytes = addCount(bytes, uncachedIds, INTEGER_BYTES); + return MetaCacheWeightUtils.saturatedAdd(bytes, generatedStringBytes); + } + + private static long snapshotBytes(Snapshot snapshot, AccountingBudget budget) { + rejectMaterializedSnapshotPayload(snapshot); + Map summary = snapshot.summary(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( + 1L, summary == null ? 0L : summary.size())); + long bytes = HISTORICAL_SNAPSHOT_BYTES; + // The parsed parent id is never inside the Long cache; the row-id fields are boxed too + // and only tiny values would share a cached instance, so each present field is charged. + bytes = addBoxedLong(bytes, snapshot.parentId()); + bytes = addBoxedLong(bytes, snapshot.firstRowId()); + bytes = addBoxedLong(bytes, snapshot.addedRows()); + bytes = addString(bytes, snapshot.operation()); + String manifestListLocation = snapshot.manifestListLocation(); + if (manifestListLocation == null) { + // A snapshot serialized with an inline "manifests" array (legacy writers) retains a + // String[] of manifest locations that is only exposed through ManifestFile wrappers. + // Reject weighted admission instead of doing IO or admitting an underestimate. + throw new IllegalStateException( + "Iceberg snapshot with inline manifest list is unsupported"); + } + bytes = addString(bytes, manifestListLocation); + bytes = addString(bytes, snapshot.keyId()); + return addStringMap(bytes, summary, TABLE_PROPERTY_BYTES); + } + + private static void rejectMaterializedSnapshotPayload(Snapshot snapshot) { + if (!BASE_SNAPSHOT_CLASS_NAME.equals(snapshot.getClass().getName())) { + throw new IllegalStateException( + "Unsupported Iceberg snapshot implementation: " + + snapshot.getClass().getName()); + } + if (BASE_SNAPSHOT_RETAINED_CACHE_FIELDS == null) { + throw new IllegalStateException( + "Iceberg BaseSnapshot retained-cache inspection is unavailable"); + } + try { + // The field list is resolved once per process. Publication only performs a bounded + // number of O(1) reads and never walks a retained manifest/file graph. + for (Field retainedCacheField : BASE_SNAPSHOT_RETAINED_CACHE_FIELDS) { + if (retainedCacheField.get(snapshot) != null) { + throw new IllegalStateException( + "Iceberg snapshot has materialized retained payload: " + + retainedCacheField.getName()); + } + } + } catch (IllegalAccessException e) { + throw new IllegalStateException( + "Cannot inspect Iceberg BaseSnapshot retained payload", e); + } + } + + /** Iceberg marks snapshots loaded at construction unless a lazy supplier was configured. */ + private static boolean areSnapshotsLoaded(TableMetadata metadata) { + if (TABLE_METADATA_SNAPSHOTS_LOADED_FIELD == null + || TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD == null) { + return false; + } + try { + return TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD.get(metadata) == null + || TABLE_METADATA_SNAPSHOTS_LOADED_FIELD.getBoolean(metadata); + } catch (IllegalAccessException | RuntimeException e) { + return false; + } + } + + private static Field loadTableMetadataField(String name, Class expectedType) { + try { + Field field = TableMetadata.class.getDeclaredField(name); + if ((expectedType != null && field.getType() != expectedType) + || Modifier.isStatic(field.getModifiers())) { + return null; + } + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException | RuntimeException e) { + return null; + } + } + + private static Field[] loadBaseSnapshotRetainedCacheFields() { + try { + Class snapshotClass = Class.forName( + BASE_SNAPSHOT_CLASS_NAME, false, Snapshot.class.getClassLoader()); + List retainedCacheFields = new ArrayList<>(); + for (Field field : snapshotClass.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (Modifier.isTransient(modifiers) && !Modifier.isStatic(modifiers) + && !field.getType().isPrimitive()) { + field.setAccessible(true); + retainedCacheFields.add(field); + } + } + return retainedCacheFields.isEmpty() + ? null : retainedCacheFields.toArray(new Field[0]); + } catch (ReflectiveOperationException | RuntimeException e) { + return null; + } + } + + private static boolean checkIcebergLayout() { + ClassLoader loader = Snapshot.class.getClassLoader(); + return MetaCacheWeightUtils.hasExpectedInstanceFields(Schema.class, + "struct:StructType", "schemaId:int", "identifierFieldIds:int[]", + "highestFieldId:int", "aliasToId:BiMap", "idToField:Map", "nameToId:Map", + "lowerCaseNameToId:Map", "idToAccessor:Map", "idToName:Map", + "identifierFieldIdSet:Set", "idsToReassigned:Map", "idsToOriginal:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionSpec.class, + "schema:Schema", "specId:int", "fields:PartitionField[]", + "fieldsBySourceId:ListMultimap", "lazyJavaClasses:Class[]", + "lazyPartitionType:StructType", "lazyRawPartitionType:StructType", + "fieldList:List", "lastAssignedFieldId:int") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionField.class, + "sourceId:int", "fieldId:int", "name:String", "transform:Transform") + && MetaCacheWeightUtils.hasExpectedInstanceFields(SortOrder.class, + "schema:Schema", "orderId:int", "fields:SortField[]", "fieldList:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(SortField.class, + "transform:Transform", "sourceId:int", "direction:SortDirection", + "nullOrder:NullOrder") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.StructType.class, + "fields:NestedField[]", "schema:Schema", "fieldList:List", + "fieldsByName:Map", "fieldsByLowerCaseName:Map", "fieldsById:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.ListType.class, + "elementField:NestedField", "fields:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.MapType.class, + "keyField:NestedField", "valueField:NestedField", "fields:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.NestedField.class, + "isOptional:boolean", "id:int", "name:String", "type:Type", + "doc:String", "initialDefault:Literal", "writeDefault:Literal") + && MetaCacheWeightUtils.hasExpectedInstanceFields(TableMetadata.class, + "metadataFileLocation:String", "formatVersion:int", "uuid:String", + "location:String", "lastSequenceNumber:long", "lastUpdatedMillis:long", + "lastColumnId:int", "currentSchemaId:int", "schemas:List", + "defaultSpecId:int", "specs:List", "lastAssignedPartitionId:int", + "defaultSortOrderId:int", "sortOrders:List", "properties:Map", + "currentSnapshotId:long", "schemasById:Map", "specsById:Map", + "sortOrdersById:Map", "snapshotLog:List", "previousFiles:List", + "statisticsFiles:List", "partitionStatisticsFiles:List", "changes:List", + "nextRowId:long", "encryptionKeys:List", + "snapshotsSupplier:SerializableSupplier", "snapshots:List", + "snapshotsById:Map", "refs:Map", "snapshotsLoaded:boolean") + && MetaCacheWeightUtils.hasExpectedInstanceFields(BASE_SNAPSHOT_CLASS_NAME, loader, + "snapshotId:long", "parentId:Long", "sequenceNumber:long", + "timestampMillis:long", "manifestListLocation:String", + "operation:String", "summary:Map", "schemaId:Integer", + "v1ManifestLocations:String[]", "firstRowId:Long", "addedRows:Long", + "keyId:String", "allManifests:List", "dataManifests:List", + "deleteManifests:List", "addedDataFiles:List", "removedDataFiles:List", + "addedDeleteFiles:List", "removedDeleteFiles:List"); + } + + private static long addBoxedLong(long bytes, Long value) { + return value == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, LONG_BYTES); + } + + private static long addBufferPayload(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + + /** + * Account one NestedField, its owned strings and its type subtree, and record the shape data + * the lookup-map formulas need. Canonical and short names follow Iceberg's IndexByName: a + * nested name joins its ancestors with '.', a struct-typed list element or map value is left + * out of its children's short names (which then become aliases), and every lower-case index + * lower-cases each entry. + */ + private static long addFieldPayload( + long bytes, Types.NestedField field, PathState ancestors, FieldKind kind, + AccountingBudget budget, SchemaShape shape) { + budget.chargeElements(1L); + budget.chargeCharacters(field.name().length()); + String name = field.name(); + String lowerName = name.toLowerCase(Locale.ROOT); + boolean nameLatin1 = MetaCacheWeightUtils.isLatin1String(name); + boolean lowerLatin1 = MetaCacheWeightUtils.isLatin1String(lowerName); + shape.addField(field.fieldId(), ancestors, name, nameLatin1, lowerName, lowerLatin1); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + if (kind == FieldKind.STRUCT_FIELD) { + // List element and map key/value fields are named by shared "element"/"key"/"value" + // literals inside Iceberg's type constructors. + bytes = addString(bytes, name); + } + bytes = addString(bytes, field.doc()); + bytes = addDefaultPayload(bytes, field.initialDefaultLiteral()); + bytes = addDefaultPayload(bytes, field.writeDefaultLiteral()); + boolean pushShortName = kind == FieldKind.STRUCT_FIELD || kind == FieldKind.MAP_KEY + || !field.type().isStructType(); + // Only fields nested through a chain of struct fields get accessors; anything below a + // list or map does not. + boolean structChildren = kind == FieldKind.STRUCT_FIELD && field.type().isStructType(); + PathState children = ancestors.push(name.length(), nameLatin1, lowerName.length(), + lowerLatin1, pushShortName, structChildren); + return addTypePayload(bytes, field.type(), children, budget, shape); + } + + private static long addTypePayload( + long bytes, Type type, PathState ancestors, AccountingBudget budget, + SchemaShape shape) { + if (ancestors.typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException( + "Iceberg cache accounting type depth exceeded"); + } + budget.chargeElements(1L); + if (type.isStructType()) { + List fields = type.asStructType().fields(); + // A nested struct's fieldList is materialized by every visitor. Its own name/id + // lookup indexes and asSchema() are not reserved: read paths resolve nested names + // through the root Schema maps and Binder binds only root and partition structs; + // nested-column DDL runs against a freshly loaded live table, not a cached one. + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(STRUCT_TYPE_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(fields.size())), + immutableListBytes(fields.size()))); + for (Types.NestedField field : fields) { + bytes = addFieldPayload( + bytes, field, ancestors, FieldKind.STRUCT_FIELD, budget, shape); + } + } else if (type.isListType()) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + LIST_TYPE_BYTES, IMMUTABLE_LIST_BYTES)); + bytes = addFieldPayload(bytes, type.asListType().fields().get(0), + ancestors, FieldKind.LIST_ELEMENT, budget, shape); + } else if (type.isMapType()) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(MAP_TYPE_BYTES, IMMUTABLE_LIST_BYTES), + MetaCacheWeightUtils.estimatedObjectArrayBytes(2L))); + bytes = addFieldPayload(bytes, type.asMapType().fields().get(0), + ancestors, FieldKind.MAP_KEY, budget, shape); + bytes = addFieldPayload(bytes, type.asMapType().fields().get(1), + ancestors, FieldKind.MAP_VALUE, budget, shape); + } else if (type instanceof Types.DecimalType) { + shape.addTypeObject(DECIMAL_TYPE_BYTES); + } else if (type instanceof Types.FixedType) { + shape.addTypeObject(FIXED_TYPE_BYTES); + } else if (type instanceof Types.GeometryType) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOMETRY_TYPE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes( + ((Types.GeometryType) type).crs()))); + } else if (type instanceof Types.GeographyType) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOGRAPHY_TYPE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes( + ((Types.GeographyType) type).crs()))); + } + // Other primitive types are shared singletons. + return bytes; + } + + /** Account the int[] and lazy ImmutableSet retained by Schema.identifierFieldIds(). */ + private static long addIdentifierFieldPayload(long bytes, Set fieldIds) { + long count = fieldIds.size(); + if (count == 0L) { + return bytes; + } + long uncachedIds = 0L; + for (int fieldId : fieldIds) { + if (isUncachedInteger(fieldId)) { + uncachedIds++; + } + } + // The int[] grows from the empty array of a schema without identifier fields. + long additions = MetaCacheWeightUtils.estimatedIntArrayPayloadBytes(count); + additions = addCount(additions, uncachedIds, INTEGER_BYTES); + if (count == 1L) { + // ImmutableSet.copyOf(one element) is a SingletonImmutableSet. + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedAdd(additions, SINGLETON_IMMUTABLE_SET_BYTES)); + } + // RegularImmutableSet: the set object, its dense elements array and open-addressing table. + // The shared empty set of an identifier-free schema stays reachable through other schemas, + // so nothing is subtracted for it. + additions = MetaCacheWeightUtils.saturatedAdd(additions, REGULAR_IMMUTABLE_SET_BYTES); + additions = MetaCacheWeightUtils.saturatedAdd( + additions, MetaCacheWeightUtils.estimatedObjectArrayBytes(count)); + additions = MetaCacheWeightUtils.saturatedAdd(additions, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + immutableSetTableCapacity(count))); + return MetaCacheWeightUtils.saturatedAdd(bytes, additions); + } + + private static long immutableSetTableCapacity(long size) { + long capacity = 2L; + while (MetaCacheWeightUtils.saturatedMultiply(size, 10L) + > MetaCacheWeightUtils.saturatedMultiply(capacity, 7L)) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long hashMapCapacity(long size) { + long capacity = 16L; + while (size > capacity - capacity / 4L) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long immutableMapTableCapacity(long size) { + long capacity = Long.highestOneBit(size); + return MetaCacheWeightUtils.saturatedMultiply(size, 5L) + > MetaCacheWeightUtils.saturatedMultiply(capacity, 6L) + ? MetaCacheWeightUtils.saturatedMultiply(capacity, 2L) : capacity; + } + + /** + * A v3 field default is retained as an Iceberg Literal wrapper (value plus a transient + * ByteBuffer slot) around its boxed or buffer value. + */ + private static long addDefaultPayload(long bytes, Literal literal) { + if (literal == null) { + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LITERAL_BYTES); + Object value = literal.value(); + if (value instanceof CharSequence) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedAdd( + BYTE_BUFFER_BYTES, + MetaCacheWeightUtils.estimatedByteArrayBytes(((ByteBuffer) value).capacity()))); + } else if (value instanceof byte[]) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedByteArrayBytes(((byte[]) value).length)); + } else if (value instanceof BigDecimal) { + return MetaCacheWeightUtils.saturatedAdd(bytes, BIG_DECIMAL_BYTES); + } else if (value == null || value instanceof Boolean) { + return bytes; + } + // Boxed numbers, UUIDs and other small immutable values. + return MetaCacheWeightUtils.saturatedAdd(bytes, BOXED_DEFAULT_BYTES); + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addStringMap(long bytes, Map values, long entryBytes) { + if (values == null) { + return bytes; + } + bytes = addCount(bytes, values.size(), entryBytes); + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } + + /** + * Hard bound on publication-time estimator work. Elements bound the number of metadata + * objects visited; characters bound the String scanning (lower-casing) performed for + * lazy-index reservations. Exceeding either throws, which estimateSafely turns into an + * incomplete estimate: weighted admission is rejected but the load itself succeeds. + */ + private static final class AccountingBudget { + private long remainingElements; + private long remainingCharacters; + + private AccountingBudget(long elements, long characters) { + this.remainingElements = elements; + this.remainingCharacters = characters; + } + + private void chargeElements(long elements) { + if (elements < 0L || elements > remainingElements) { + throw new IllegalStateException("Iceberg cache accounting work budget exceeded"); + } + remainingElements -= elements; + } + + private void chargeCharacters(long characters) { + if (characters < 0L || characters > remainingCharacters) { + throw new IllegalStateException( + "Iceberg cache accounting character budget exceeded"); + } + remainingCharacters -= characters; + } + } + + private enum FieldKind { + STRUCT_FIELD, LIST_ELEMENT, MAP_KEY, MAP_VALUE + } + + /** + * Immutable name-stack state of a field's ancestors: character counts and Latin-1 coders of + * the joined canonical path, the short-alias path and both lower-cased forms. + */ + private static final class PathState { + private static final PathState ROOT = new PathState( + -1L, true, -1L, true, -1L, true, -1L, true, 0, 0); + + private final long pathCharacters; + private final boolean pathLatin1; + private final long shortPathCharacters; + private final boolean shortPathLatin1; + private final long lowerPathCharacters; + private final boolean lowerPathLatin1; + private final long shortLowerPathCharacters; + private final boolean shortLowerPathLatin1; + // Struct-field ancestors of the next field, or -1 inside a list or map (no accessors). + private final int structDepth; + private final int typeDepth; + + private PathState(long pathCharacters, boolean pathLatin1, long shortPathCharacters, + boolean shortPathLatin1, long lowerPathCharacters, boolean lowerPathLatin1, + long shortLowerPathCharacters, boolean shortLowerPathLatin1, int structDepth, + int typeDepth) { + this.pathCharacters = pathCharacters; + this.pathLatin1 = pathLatin1; + this.shortPathCharacters = shortPathCharacters; + this.shortPathLatin1 = shortPathLatin1; + this.lowerPathCharacters = lowerPathCharacters; + this.lowerPathLatin1 = lowerPathLatin1; + this.shortLowerPathCharacters = shortLowerPathCharacters; + this.shortLowerPathLatin1 = shortLowerPathLatin1; + this.structDepth = structDepth; + this.typeDepth = typeDepth; + } + + private boolean isRoot() { + return pathCharacters < 0L; + } + + private boolean shortPathDiffers() { + return shortPathCharacters != pathCharacters; + } + + /** + * Push a field name for its children; the short name is pushed only when requested and + * accessor depth continues only for the children of a struct-typed struct field. + */ + private PathState push(long nameCharacters, boolean nameLatin1, long lowerCharacters, + boolean lowerLatin1, boolean pushShortName, boolean structChildren) { + return new PathState( + join(pathCharacters, nameCharacters), pathLatin1 && nameLatin1, + pushShortName ? join(shortPathCharacters, nameCharacters) : shortPathCharacters, + pushShortName ? shortPathLatin1 && nameLatin1 : shortPathLatin1, + join(lowerPathCharacters, lowerCharacters), lowerPathLatin1 && lowerLatin1, + pushShortName ? join(shortLowerPathCharacters, lowerCharacters) + : shortLowerPathCharacters, + pushShortName ? shortLowerPathLatin1 && lowerLatin1 : shortLowerPathLatin1, + structChildren && structDepth >= 0 ? structDepth + 1 : -1, typeDepth + 1); + } + + private static long join(long parentCharacters, long nameCharacters) { + return parentCharacters < 0L ? nameCharacters + : MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(parentCharacters, 1L), + nameCharacters); + } + } + + /** Cardinalities and generated-String bytes that size a schema's lookup indexes. */ + private static final class SchemaShape { + private long fieldCount; + private long topLevelFieldCount; + private long uncachedFieldIdCount; + private long uncachedTopLevelFieldIdCount; + private long nameEntryCount; + private long uncachedNameIdCount; + // One copy each; the formulas add a copy per index that retains it. + private long pathStringBytes; + private long aliasStringBytes; + private long lowerCaseStringBytes; + private long topLevelLowerCaseStringBytes; + private long accessorFieldCount; + private long uncachedAccessorIdCount; + private long wrappedAccessorCount; + private long typeObjectBytes; + + /** A flat struct of {@code fieldCount} top-level fields, as used by partition types. */ + private static SchemaShape flat( + long fieldCount, long uncachedFieldIds, long lowerCaseStringBytes) { + SchemaShape shape = new SchemaShape(); + shape.fieldCount = fieldCount; + shape.topLevelFieldCount = fieldCount; + shape.uncachedFieldIdCount = uncachedFieldIds; + shape.uncachedTopLevelFieldIdCount = uncachedFieldIds; + shape.nameEntryCount = fieldCount; + shape.uncachedNameIdCount = uncachedFieldIds; + shape.lowerCaseStringBytes = lowerCaseStringBytes; + shape.topLevelLowerCaseStringBytes = lowerCaseStringBytes; + shape.accessorFieldCount = fieldCount; + shape.uncachedAccessorIdCount = uncachedFieldIds; + return shape; + } + + private void addField(int fieldId, PathState ancestors, String name, boolean nameLatin1, + String lowerName, boolean lowerLatin1) { + boolean uncached = isUncachedInteger(fieldId); + fieldCount++; + nameEntryCount++; + if (uncached) { + uncachedFieldIdCount++; + uncachedNameIdCount++; + } + if (ancestors.structDepth >= 0) { + accessorFieldCount++; + wrappedAccessorCount = MetaCacheWeightUtils.saturatedAdd( + wrappedAccessorCount, ancestors.structDepth); + if (uncached) { + uncachedAccessorIdCount++; + } + } + if (ancestors.isRoot()) { + topLevelFieldCount++; + if (uncached) { + uncachedTopLevelFieldIdCount++; + } + if (!name.equals(lowerName)) { + // Lower-case indexes only allocate when toLowerCase() changes the name. + long lowerBytes = MetaCacheWeightUtils.estimatedGeneratedStringBytes( + lowerName.length(), lowerLatin1); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( + lowerCaseStringBytes, lowerBytes); + topLevelLowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( + topLevelLowerCaseStringBytes, lowerBytes); + } + return; + } + // Nested: IndexByName joins a new canonical String, and the lower-case index keeps + // either that joined String or its lower-cased copy. + pathStringBytes = MetaCacheWeightUtils.saturatedAdd(pathStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.pathCharacters, name.length()), + ancestors.pathLatin1 && nameLatin1)); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.lowerPathCharacters, lowerName.length()), + ancestors.lowerPathLatin1 && lowerLatin1)); + if (ancestors.shortPathDiffers()) { + // A short alias exists whenever an ancestor was left out of the short path. + // Iceberg drops an alias that collides with a canonical name; counting the rare + // collision is conservative and avoids building name sets at publication. + nameEntryCount++; + if (uncached) { + uncachedNameIdCount++; + } + aliasStringBytes = MetaCacheWeightUtils.saturatedAdd(aliasStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.shortPathCharacters, name.length()), + ancestors.shortPathLatin1 && nameLatin1)); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join( + ancestors.shortLowerPathCharacters, lowerName.length()), + ancestors.shortLowerPathLatin1 && lowerLatin1)); + } + } + + private void addTypeObject(long bytes) { + typeObjectBytes = MetaCacheWeightUtils.saturatedAdd(typeObjectBytes, bytes); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 8407a29d8908a9..e8b24299d53df2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; @@ -29,9 +28,12 @@ import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -47,16 +49,19 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import javax.annotation.Nullable; /** * Iceberg engine implementation of {@link AbstractExternalMetaCache}. * *

Registered entries: *

    - *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping, each - * memoizing its latest snapshot runtime projection
    • + *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping
    • + *
  • {@code snapshot}: immutable snapshot projections keyed by a stable metadata generation
    • *
  • {@code view}: loaded Iceberg {@link View} instances
    • *
  • {@code manifest}: parsed manifest payload ({@link ManifestCacheValue}) keyed by * manifest path and content type
    • @@ -69,7 +74,7 @@ *

    Invalidation behavior: *

      *
    • catalog invalidation clears all entries and drops Iceberg {@link ManifestFiles} IO cache
      • - *
    • db/table invalidation clears table/view/schema entries, while keeping manifest entries
      • + *
    • db/table invalidation clears table/snapshot/view/schema entries, while keeping manifest entries
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ @@ -78,26 +83,41 @@ public class IcebergExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "iceberg"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_VIEW = "view"; public static final String ENTRY_MANIFEST = "manifest"; public static final String ENTRY_SCHEMA = "schema"; private static final long DEFAULT_MANIFEST_CACHE_CAPACITY = 100_000L; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle viewEntry; private final EntryHandle manifestEntry; private final EntryHandle schemaEntry; public IcebergExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withSizeEstimator(this::prepareTableForCachePublication) + .withReplacementListener(this::retireTableGeneration)); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); viewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_VIEW, NameMapping.class, View.class, this::loadView, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); manifestEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_MANIFEST, IcebergManifestEntryKey.class, ManifestCacheValue.class, - CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY))); + CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimator.estimateSafely( + "iceberg_manifest_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateManifestEntry(key, value)))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSchemaCacheKey::getNameMapping))); @@ -108,13 +128,87 @@ public Table getIcebergTable(ExternalTable dorisTable) { return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getIcebergTable(); } + public Table getWritableIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + + " when loading a writable Iceberg table"); + } + IcebergMetadataOps ops = resolveMetadataOps(catalog); + // DDL/actions must start from the live catalog generation. DML that was planned against a + // retained read generation wraps this live table separately in IcebergTransaction. + return executeAuthenticated(catalog, () -> ops.loadTable( + nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + } + + Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + MetaCacheEntry entry = + tableEntry.get(nameMapping.getCtlId()); + IcebergTableCacheValue tableValue = + entry.get(nameMapping); + return createQueryTable(nameMapping, tableValue); + } + + private Table createQueryTable( + NameMapping nameMapping, IcebergTableCacheValue tableValue) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || snapshotEntry.get(nameMapping.getCtlId()).isWeightBounded(); + if (!isolateForQueries) { + return tableValue.getIcebergTable(); + } + Table queryTable = tableValue.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(queryTable); + return queryTable; + } + public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + IcebergTableCacheValue tableValue = + tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + Table retainedTable = tableValue.getRetainedIcebergTable(); + java.util.Optional optionalKey = + IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!optionalKey.isPresent()) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared(); + return executeAuthenticated(nameMapping.getCtlId(), + () -> loadSnapshotProjection( + dorisTable, + isolateForQueries ? tableValue.newQueryScopedTable() + : tableValue.getIcebergTable(), + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)); + } + IcebergSnapshotEntryKey key = optionalKey.get(); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || entry.isWeightBounded(); + IcebergSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + Table projectionTable = isolateForQueries + ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); + IcebergSnapshotCacheValue value = loadSnapshotProjection( + dorisTable, projectionTable, + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + if (entry.isWeightBounded()) { + value.prepareForCachePublication(key); + } + return value; + })); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && !tableValue.isSamePhysicalGeneration(currentTable)) { + // A query may have captured the previous table immediately before refresh publication. + // It can use that immutable value, but must not republish an unreachable old projection. + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public List getSnapshotList(ExternalTable dorisTable) { - Table icebergTable = getIcebergTable(dorisTable); + Table icebergTable = getQueryScopedIcebergTable(dorisTable); List snapshots = com.google.common.collect.Lists.newArrayList(); com.google.common.collect.Iterables.addAll(snapshots, icebergTable.snapshots()); return snapshots; @@ -126,8 +220,31 @@ public View getIcebergView(ExternalTable dorisTable) { } public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { - IcebergSchemaCacheKey key = new IcebergSchemaCacheKey(nameMapping, schemaId); - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()).get(key); + IcebergTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable()); + } + + IcebergSchemaCacheValue getIcebergSchemaCacheValue( + NameMapping nameMapping, long schemaId, Table retainedTable) { + Optional generation = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!generation.isPresent()) { + return (IcebergSchemaCacheValue) loadSchemaCacheValue( + new IcebergSchemaCacheKey(nameMapping, schemaId), retainedTable); + } + IcebergSchemaCacheKey key = new IcebergSchemaCacheKey( + nameMapping, generation.get().getTableUuid(), schemaId); + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry + .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null) { + Optional currentGeneration = IcebergSnapshotEntryKey.tryCreate( + nameMapping, currentTable.getRetainedIcebergTable()); + if (!currentGeneration.isPresent() + || !currentGeneration.get().getTableUuid().equals(generation.get().getTableUuid())) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } + } return (IcebergSchemaCacheValue) schemaCacheValue; } @@ -139,11 +256,13 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, MetaCacheEntry manifestEntry = this.manifestEntry.get(nameMapping.getCtlId()); IcebergManifestEntryKey key = IcebergManifestEntryKey.of(manifest); - boolean hit = manifestEntry.getIfPresent(key) != null; + boolean hit = manifestEntry.peekIfPresent(key) != null; if (cacheHitRecorder != null) { cacheHitRecorder.accept(hit); } - return manifestEntry.get(key, ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); + return manifestEntry.get(key, + ignored -> loadManifestCacheValue( + manifest, icebergTable, key.getContent(), manifestEntry.isWeightBounded())); } @Override @@ -159,25 +278,32 @@ public void invalidateCatalogEntries(long catalogId) { } private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { throw new RuntimeException(String.format("Cannot find catalog %d when loading table %s/%s.", nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); } IcebergMetadataOps ops = resolveMetadataOps(catalog); - try { - Table table = ((ExternalCatalog) catalog).getExecutionAuthenticator() - .execute(() -> ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - ExternalTable dorisTable = findExternalTable(nameMapping, ENGINE); - return new IcebergTableCacheValue(table, () -> loadSnapshotProjection(dorisTable, table)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } + return executeAuthenticated(catalog, () -> { + Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + IcebergTableCacheValue value = new IcebergTableCacheValue(table); + MetaCacheEntry currentEntry = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + if (currentEntry != null && currentEntry.isWeightBounded()) { + prepareTableForCachePublication(nameMapping, value); + } + return value; + }); + } + + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + return value.prepareForCachePublication(nameMapping); } private View loadView(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (!(catalog instanceof IcebergExternalCatalog)) { return null; } @@ -191,7 +317,7 @@ private View loadView(NameMapping nameMapping) { } private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFile manifest, Table icebergTable, - ManifestContent content) { + ManifestContent content, boolean accountRetainedSize) { if (manifest == null || icebergTable == null) { String manifestPath = manifest == null ? "null" : manifest.path(); throw new CacheException("Manifest cache loader context is missing for %s", @@ -199,10 +325,9 @@ private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFil } try { if (content == ManifestContent.DELETES) { - return ManifestCacheValue.forDeleteFiles( - loadDeleteFiles(manifest, icebergTable)); + return loadDeleteFiles(manifest, icebergTable, accountRetainedSize); } - return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, icebergTable)); + return loadDataFiles(manifest, icebergTable, accountRetainedSize); } catch (IOException e) { throw new CacheException("Failed to read manifest %s", e, manifest.path()); } @@ -216,27 +341,64 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } - private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table icebergTable) { + private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + dorisTable.setUpdateTime(System.currentTimeMillis()); + boolean isView = dorisTable instanceof IcebergExternalTable + && ((IcebergExternalTable) dorisTable).isView(); + return IcebergUtils.loadSchemaCacheValue( + dorisTable, key.getSchemaId(), isView, retainedTable).orElseThrow(() -> + new CacheException("failed to load iceberg schema cache value for: %s.%s.%s, schemaId: %s", + null, key.getNameMapping().getCtlId(), key.getNameMapping().getLocalDbName(), + key.getNameMapping().getLocalTblName(), key.getSchemaId())); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable IcebergTableCacheValue previousValue, IcebergTableCacheValue currentValue) { + if (previousValue != null && previousValue.isSamePhysicalGeneration(currentValue)) { + return; + } + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.belongsTo(currentValue)); + } + Optional currentUuid = currentValue.getTableUuid(); + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.getTableUuid().equals(currentUuid)); + } + } + + private IcebergSnapshotCacheValue loadSnapshotProjection( + ExternalTable dorisTable, Table projectionTable, Table retainedTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); } try { - // Freeze before deriving snapshot, partitions, and aliases; BaseTable accessors share - // refreshable operations and otherwise could mix two concurrent metadata generations. - Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; - IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(retainedTable); + IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(projectionTable); IcebergPartitionInfo icebergPartitionInfo; if (!table.isValidRelatedTable()) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { - icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, retainedTable, + icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue( - icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(retainedTable), - retainedTable); + Optional>> nameMapping = + IcebergUtils.getNameMapping(projectionTable); + return isolateForQueries + ? new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable, retainedCurrentSnapshotJson) + : new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } @@ -251,32 +413,58 @@ private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) { throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); } + private T executeAuthenticated(long catalogId, Callable task) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + catalogId + " when loading Iceberg metadata."); + } + return executeAuthenticated(catalog, task); + } + + private T executeAuthenticated(CatalogIf catalog, Callable task) { + if (!(catalog instanceof ExternalCatalog)) { + throw new RuntimeException("Iceberg metadata cache requires an external catalog"); + } + try { + return ((ExternalCatalog) catalog).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.iceberg.table.enable", "meta.cache.iceberg.snapshot.enable"); + compatibility.put("meta.cache.iceberg.table.ttl-second", "meta.cache.iceberg.snapshot.ttl-second"); + compatibility.put("meta.cache.iceberg.table.capacity", "meta.cache.iceberg.snapshot.capacity"); + return compatibility; } - private List loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDataFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - List dataFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.read(manifest, table.io())) { for (org.apache.iceberg.DataFile dataFile : reader) { - dataFiles.add(dataFile.copy()); + builder.addDataFile(dataFile.copy()); } } - return dataFiles; + return builder.build(); } - private List loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDeleteFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - List deleteFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { for (org.apache.iceberg.DeleteFile deleteFile : reader) { - deleteFiles.add(deleteFile.copy()); + builder.addDeleteFile(deleteFile.copy()); } } - return deleteFiles; + return builder.build(); } private void dropManifestFileIoCacheForCatalog(long catalogId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 5d59440a5a62b1..9c9ee6d53b6416 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -148,6 +148,10 @@ public Table getIcebergTable() { return IcebergUtils.getIcebergTable(this); } + public Table getWritableIcebergTable() { + return IcebergUtils.getWritableIcebergTable(this); + } + @Override public String getComment() { return properties().getOrDefault(TABLE_COMMENT_PROP, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index bcbe03de6e6d01..d996f80754a37f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -483,7 +483,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); BranchOptions branchOptions = branchInfo.getBranchOptions(); Long snapshotId = branchOptions.getSnapshotId() @@ -571,7 +571,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); TagOptions tagOptions = tagInfo.getTagOptions(); Long snapshotId = tagOptions.getSnapshotId() .orElse( @@ -623,7 +623,7 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(tagName); if (snapshotRef != null || !ifExists) { @@ -644,7 +644,7 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(branchName); if (snapshotRef != null || !ifExists) { @@ -747,7 +747,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -778,7 +778,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); if (!parentPath.getType().isStructType()) { @@ -808,7 +808,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); for (Column column : columns) { validateAddColumnMetadata(column, true); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); @@ -831,7 +831,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -851,7 +851,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -868,7 +868,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, oldName, "rename"); validateRowLineageColumnMutation(icebergTable, newName, "rename to"); Schema schema = icebergTable.schema(); @@ -893,7 +893,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), @@ -955,7 +955,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() .caseInsensitiveField(columnPath.getTopLevelName()); @@ -1024,7 +1024,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCollectionPseudoFieldComment( @@ -1075,7 +1075,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); if (!columnPath.isNested()) { validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); } @@ -1642,7 +1642,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String columnName : newOrder) { @@ -1709,7 +1709,7 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); String transformName = clause.getTransformName(); @@ -1738,7 +1738,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); if (clause.getPartitionFieldName() != null) { @@ -1765,7 +1765,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); // remove old partition field diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java index cccc6244a0d0cc..96ed0a0dcc1d8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import java.util.List; public class IcebergPartition { @@ -29,10 +31,19 @@ public class IcebergPartition { private final long lastUpdateTime; private final long lastSnapshotId; private final List transforms; + private final long retainedPayloadBytes; public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, long lastUpdateTime, long lastSnapshotId, List partitionValues, List transforms) { + this(partitionName, specId, recordCount, fileSizeInBytes, fileCount, lastUpdateTime, + lastSnapshotId, partitionValues, transforms, + estimateRetainedPayloadBytes(partitionName, partitionValues, transforms)); + } + + public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, + long lastUpdateTime, long lastSnapshotId, List partitionValues, + List transforms, long retainedPayloadBytes) { this.partitionName = partitionName; this.specId = specId; this.recordCount = recordCount; @@ -42,6 +53,7 @@ public IcebergPartition(String partitionName, int specId, long recordCount, long this.lastSnapshotId = lastSnapshotId; this.partitionValues = partitionValues; this.transforms = transforms; + this.retainedPayloadBytes = retainedPayloadBytes; } public String getPartitionName() { @@ -79,4 +91,26 @@ public List getPartitionValues() { public List getTransforms() { return transforms; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long estimateRetainedPayloadBytes( + String partitionName, List partitionValues, List transforms) { + long bytes = MetaCacheWeightUtils.estimatedStringBytes(partitionName); + bytes = addStrings(bytes, partitionValues); + return addStrings(bytes, transforms); + } + + private static long addStrings(long bytes, List values) { + if (values == null) { + return bytes; + } + for (String value : values) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(value)); + } + return bytes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index de36f0855ddd14..05f469dfdc5229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -18,31 +18,54 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; -import com.google.common.collect.Maps; - +import java.util.Collections; import java.util.Map; import java.util.Set; public class IcebergPartitionInfo { + // Each RangePartitionItem endpoint holds one LiteralExpr per partition column beyond the + // first (createPartitionKey fills the vacancy with an infinity literal): literal, its lazy + // supplier, children list and array. Calibrated against JOL in IcebergExternalMetaCacheTest. + private static final long RANGE_KEY_EXTRA_COLUMN_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(208L); + private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; + // A merged-overlap alias group is a HashSet of the enclosed physical partition names; the + // names themselves are shared with the partition maps. + private static final long HASH_SET_BYTES = MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private final Map nameToPartitionItem; private final Map nameToIcebergPartition; private final Map> nameToIcebergPartitionNames; + private final long retainedPayloadBytes; private static final IcebergPartitionInfo EMPTY = new IcebergPartitionInfo(); private IcebergPartitionInfo() { - this.nameToPartitionItem = Maps.newHashMap(); - this.nameToIcebergPartition = Maps.newHashMap(); - this.nameToIcebergPartitionNames = Maps.newHashMap(); + this.nameToPartitionItem = Collections.emptyMap(); + this.nameToIcebergPartition = Collections.emptyMap(); + this.nameToIcebergPartitionNames = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public IcebergPartitionInfo(Map nameToPartitionItem, Map nameToIcebergPartition, Map> nameToIcebergPartitionNames) { + this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, + MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes(nameToIcebergPartition), + partitionAliasBytes(nameToIcebergPartitionNames))); + } + + public IcebergPartitionInfo(Map nameToPartitionItem, + Map nameToIcebergPartition, + Map> nameToIcebergPartitionNames, + long retainedPayloadBytes) { this.nameToPartitionItem = nameToPartitionItem; this.nameToIcebergPartition = nameToIcebergPartition; this.nameToIcebergPartitionNames = nameToIcebergPartitionNames; + this.retainedPayloadBytes = retainedPayloadBytes; } static IcebergPartitionInfo empty() { @@ -57,6 +80,65 @@ public Map getNameToIcebergPartition() { return nameToIcebergPartition; } + Map> getNameToIcebergPartitionNames() { + return nameToIcebergPartitionNames; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (IcebergPartition partition : partitions.values()) { + if (partition != null) { + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, partition.getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionItemColumnBytes( + partition.getPartitionValues() == null + ? 0 : partition.getPartitionValues().size())); + } + } + return bytes; + } + + /** + * Retained bytes of the merged-overlap alias sets: every group keeps one HashSet with one + * node per enclosed physical partition name (the estimator's per-group constant covers only + * the outer map entry and the empty set object). + */ + static long partitionAliasBytes(Map> nameToIcebergPartitionNames) { + if (nameToIcebergPartitionNames == null) { + return 0L; + } + long bytes = 0L; + for (Set aliases : nameToIcebergPartitionNames.values()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_SET_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedHashMapBytes(aliases == null ? 0L : aliases.size())); + } + return bytes; + } + + /** + * Structural bytes a partition item retains for every partition column beyond the first; + * the fixed per-partition constants of the estimator cover a single column. The width is + * taken from the loaded metadata generation, so a spec that grew after the related-table + * check was cached is still charged for its full width. + */ + static long partitionItemColumnBytes(long partitionColumnCount) { + if (partitionColumnCount <= 1L) { + return 0L; + } + return MetaCacheWeightUtils.saturatedMultiply( + MetaCacheWeightUtils.saturatedMultiply( + partitionColumnCount - 1L, RANGE_ENDPOINTS_PER_ITEM), + RANGE_KEY_EXTRA_COLUMN_BYTES); + } + public long getLatestSnapshotId(String partitionName) { Set icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName); if (icebergPartitionNames == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java index 7c2d09511a2c93..9916d0afbb17e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java @@ -22,14 +22,26 @@ import com.google.common.base.Objects; +import java.util.Optional; + public class IcebergSchemaCacheKey extends SchemaCacheKey { + private final String tableUuid; private final long schemaId; public IcebergSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, "", schemaId); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId) { super(nameMapping); + this.tableUuid = java.util.Objects.requireNonNull(tableUuid, "tableUuid can not be null"); this.schemaId = schemaId; } + public Optional getTableUuid() { + return tableUuid.isEmpty() ? Optional.empty() : Optional.of(tableUuid); + } + public long getSchemaId() { return schemaId; } @@ -46,11 +58,11 @@ public boolean equals(Object o) { return false; } IcebergSchemaCacheKey that = (IcebergSchemaCacheKey) o; - return schemaId == that.schemaId; + return schemaId == that.schemaId && tableUuid.equals(that.tableUuid); } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableUuid, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 30cf64fcfc6bfa..f7dbed2effd183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,8 +17,17 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; import org.apache.iceberg.BaseTable; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; @@ -27,7 +36,6 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -40,36 +48,70 @@ public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; private final Optional>> nameMapping; - private final Optional icebergTable; + private Optional
    icebergTable; + private final long retainedNameMappingPayloadBytes; + private String retainedCurrentSnapshotJson; + private boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { - this(partitionInfo, snapshot, Optional.empty(), Optional.empty()); + this(partitionInfo, snapshot, Optional.empty(), Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping) { - this(partitionInfo, snapshot, nameMapping, Optional.empty()); + this(partitionInfo, snapshot, nameMapping, Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping, Table icebergTable) { - this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable)); + this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable), null, false); + } + + IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping, Table retainedTable, + String retainedCurrentSnapshotJson) { + this(partitionInfo, snapshot, nameMapping, Optional.of(retainedTable), + retainedCurrentSnapshotJson, true); } private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, - Optional>> nameMapping, Optional
    icebergTable) { + Optional>> nameMapping, Optional
    icebergTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; // A cached BaseTable shares live TableOperations; retain a metadata-only generation so a // later commit through that same Table cannot move an already bound statement forward. this.icebergTable = icebergTable.map(IcebergSnapshotCacheValue::retainTableGeneration); - this.nameMapping = nameMapping.map(mapping -> { + this.retainedCurrentSnapshotJson = retainedCurrentSnapshotJson; + if (isolateForQueries) { + this.icebergTable = this.icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + this.queryIsolationPrepared = true; + } + if (nameMapping.isPresent()) { Map> copy = new HashMap<>(); - // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. - mapping.forEach((id, names) -> copy.put(id, - Collections.unmodifiableList(new ArrayList<>(names)))); - return Collections.unmodifiableMap(copy); - }); + long payloadBytes = 0L; + for (Map.Entry> entry : nameMapping.get().entrySet()) { + List names = ImmutableList.copyOf(entry.getValue()); + copy.put(entry.getKey(), names); + if (names.size() > 1) { + // A field with several historical names keeps an element array per name. + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(names.size())); + } + for (String name : names) { + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(name)); + } + } + this.nameMapping = Optional.of(Collections.unmodifiableMap(copy)); + this.retainedNameMappingPayloadBytes = payloadBytes; + } else { + this.nameMapping = Optional.empty(); + this.retainedNameMappingPayloadBytes = 0L; + } } public IcebergPartitionInfo getPartitionInfo() { @@ -85,6 +127,53 @@ public Optional>> getNameMapping() { } public Optional
    getIcebergTable() { + return queryIsolationPrepared + ? icebergTable.map(table -> createQueryScopedTable( + table, retainedCurrentSnapshotJson)) + : icebergTable; + } + + MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", + () -> { + // Account before serializing the current snapshot: v1 snapshot JSON + // materializes the transient manifest list that accounting rejects. + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); + if (retainedCurrentSnapshotJson == null) { + retainedCurrentSnapshotJson = icebergTable + .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); + } + return IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); + if (sizeEstimate.isComplete()) { + icebergTable = icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + long getRetainedNameMappingPayloadBytes() { + return retainedNameMappingPayloadBytes; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return retainedSnapshotJsonBytes(retainedCurrentSnapshotJson); + } + + Optional
    getRetainedIcebergTable() { return icebergTable; } @@ -92,33 +181,115 @@ static Table retainTableGeneration(Table table) { if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { return table; } + if (table instanceof QueryScopedTable) { + // Already fixed to one admitted metadata generation and isolating its snapshot + // copies from the cached BaseSnapshot instances. Rebuilding it as a plain BaseTable + // would hand historical scans the shared snapshots, whose lazily materialized + // manifest lists would then grow the cached generation past its admitted weight. + return table; + } TableOperations operations = ((HasTableOperations) table).operations(); // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. - TableOperations frozenOperations = new FrozenTableOperations(operations, operations.current()); + TableOperations frozenOperations = new FrozenTableOperations( + operations, operations.current(), false); return tableWithOperations(table, frozenOperations); } + static Table retainNonGrowingGeneration(Table table) { + if (!isFrozenGeneration(table) || isNonGrowingGeneration(table)) { + return table; + } + TableOperations retainedOperations = ((HasTableOperations) table).operations(); + // Do not rebuild parsed metadata with Iceberg's write-side Builder. Builder validation and + // ID reuse rules are intentionally stricter than metadata parsing and can reject legal + // upgraded tables or renumber sparse/equivalent schema histories. The frozen metadata is + // never exposed after query isolation; each caller receives exact query-local operations. + return tableWithOperations(table, new FrozenTableOperations( + retainedOperations, retainedOperations.current(), true)); + } + + static String retainCurrentSnapshotJson(Table table) { + if (!(table instanceof HasTableOperations)) { + return null; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + Snapshot snapshot = metadata == null ? null : metadata.currentSnapshot(); + return snapshot == null ? null : SnapshotParser.toJson(snapshot, false); + } + + static long retainedSnapshotJsonBytes(String snapshotJson) { + return MetaCacheWeightUtils.estimatedStringBytes(snapshotJson); + } + + static Table createQueryScopedTable(Table retainedTable, String currentSnapshotJson) { + if (!isFrozenGeneration(retainedTable)) { + return retainedTable; + } + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + if (retainedTable instanceof BaseTable) { + return new QueryScopedTable(retainedOperations, retainedTable.name(), + ((BaseTable) retainedTable).reporter(), currentSnapshotJson); + } + return new QueryScopedTable(retainedOperations, retainedTable.name(), null, + currentSnapshotJson); + } + + static void loadQueryMetadataForStatement(Table table) { + if (table instanceof QueryScopedTable) { + ((QueryScopedTable) table).queryMetadata(); + } + } + static boolean isFrozenGeneration(Table table) { return table instanceof HasTableOperations && ((HasTableOperations) table).operations() instanceof FrozenTableOperations; } + /** + * True for any table bound to a retained metadata generation that cannot commit by itself: + * a frozen generation, or a query-scoped view over one. Writers must re-base such tables onto + * live operations through {@link #createWritableTable}. + */ + static boolean isRetainedGeneration(Table table) { + if (!(table instanceof HasTableOperations)) { + return false; + } + TableOperations operations = ((HasTableOperations) table).operations(); + return operations instanceof FrozenTableOperations + || operations instanceof QueryScopedTableOperations; + } + + static TableOperations unwrapRetainedTableOperations(TableOperations operations) { + TableOperations current = Objects.requireNonNull(operations, "operations can not be null"); + while (current instanceof RetainedTableOperations) { + current = ((RetainedTableOperations) current).delegate; + } + return current; + } + static Table createWritableTable(Table retainedTable, Table liveTable) { - if (!isFrozenGeneration(retainedTable)) { + if (!isRetainedGeneration(retainedTable)) { return retainedTable; } if (!(liveTable instanceof HasTableOperations) - || isFrozenGeneration(liveTable)) { + || isRetainedGeneration(liveTable)) { throw new IllegalArgumentException( "Iceberg commit table must provide writable table operations"); } - TableMetadata retainedMetadata = ((HasTableOperations) retainedTable).operations().current(); - TableOperations liveOperations = ((HasTableOperations) liveTable).operations(); + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + TableMetadata retainedMetadata = retainedOperations.current(); + TableOperations liveOperations = unwrapRetainedTableOperations( + ((HasTableOperations) liveTable).operations()); return tableWithOperations(retainedTable, new WritableTableOperations(liveOperations, retainedMetadata)); } + static boolean isNonGrowingGeneration(Table table) { + return isFrozenGeneration(table) + && ((FrozenTableOperations) ((HasTableOperations) table).operations()).nonGrowing; + } + private static Table tableWithOperations(Table table, TableOperations operations) { if (table instanceof BaseTable) { return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); @@ -166,15 +337,63 @@ public LocationProvider locationProvider() { } } - private static class FrozenTableOperations extends RetainedTableOperations { - private FrozenTableOperations(TableOperations delegate, TableMetadata metadata) { - super(delegate, metadata); + private static class FrozenTableOperations implements TableOperations { + private final TableMetadata metadata; + private final FileIO fileIO; + private final EncryptionManager encryptionManager; + private final LocationProvider locationProvider; + private final boolean nonGrowing; + + private FrozenTableOperations(TableOperations source, TableMetadata metadata, + boolean nonGrowing) { + this.metadata = metadata; + this.fileIO = source.io(); + this.encryptionManager = source.encryption(); + this.locationProvider = source.locationProvider(); + this.nonGrowing = nonGrowing; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; } @Override public void commit(TableMetadata base, TableMetadata newMetadata) { throw new UnsupportedOperationException("Frozen Iceberg table generation is read-only"); } + + @Override + public FileIO io() { + return fileIO; + } + + @Override + public EncryptionManager encryption() { + return encryptionManager; + } + + @Override + public String metadataFileLocation(String fileName) { + String metadataLocation = metadata.metadataFileLocation(); + if (metadataLocation == null) { + throw new UnsupportedOperationException( + "Frozen Iceberg table has no metadata directory"); + } + int separator = metadataLocation.lastIndexOf('/'); + return separator < 0 ? fileName + : metadataLocation.substring(0, separator + 1) + fileName; + } + + @Override + public LocationProvider locationProvider() { + return locationProvider; + } } private static class WritableTableOperations extends RetainedTableOperations { @@ -199,8 +418,8 @@ public TableMetadata refresh() { // fail instead of silently committing files produced for another metadata generation. if (!isWriterCompatible(refreshedMetadata)) { throw new CommitFailedException( - "Cannot retry Iceberg commit after schema, spec, sort order, location, " - + "format version, or table properties changed"); + "Cannot retry Iceberg commit after the table UUID, schema, spec, sort " + + "order, location, format version, or table properties changed"); } currentMetadata = refreshedMetadata; return refreshedMetadata; @@ -208,12 +427,16 @@ public TableMetadata refresh() { @Override public void commit(TableMetadata base, TableMetadata newMetadata) { - delegate.commit(base, newMetadata); - currentMetadata = newMetadata; + TableMetadata delegateBase = prepareDelegateCommit(delegate, base, currentMetadata); + delegate.commit(delegateBase, newMetadata); + currentMetadata = delegate.current(); } private boolean isWriterCompatible(TableMetadata refreshedMetadata) { - return retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() + // A dropped and recreated table can restart schema/spec/order ids at the same + // location; only the same table UUID may absorb a retried commit. + return Objects.equals(retainedMetadata.uuid(), refreshedMetadata.uuid()) + && retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() && retainedMetadata.currentSchemaId() == refreshedMetadata.currentSchemaId() && retainedMetadata.defaultSpecId() == refreshedMetadata.defaultSpecId() && retainedMetadata.defaultSortOrderId() == refreshedMetadata.defaultSortOrderId() @@ -221,4 +444,136 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { && Objects.equals(retainedMetadata.properties(), refreshedMetadata.properties()); } } + + /** + * Query-local read-only operations over the exact retained metadata. Only snapshot state is + * isolated per query (see QueryScopedTable); TableMetadata, Schema, StructType and + * PartitionSpec are shared with the cached generation, and their lazy indexes grow inside the + * cache value. IcebergCacheSizeEstimator reserves that growth at publication. + */ + private static final class QueryScopedTableOperations extends RetainedTableOperations { + private QueryScopedTableOperations(TableOperations retainedOperations) { + super(retainedOperations, retainedOperations.current()); + } + + @Override + public void commit(TableMetadata base, TableMetadata metadata) { + throw new UnsupportedOperationException("Query-scoped Iceberg table is read-only"); + } + } + + /** + * A per-caller view whose Iceberg lazy snapshot state (manifest lists, manifests, files) is + * never written into the cache value. It does not isolate schema/spec lazy indexes. + */ + private static final class QueryScopedTable extends BaseTable { + private final QueryScopedTableOperations queryOperations; + private final Snapshot currentSnapshot; + private final Map querySnapshots = new HashMap<>(); + + private QueryScopedTable(TableOperations retainedOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + this(new QueryScopedTableOperations(retainedOperations), name, reporter, currentSnapshotJson); + } + + private QueryScopedTable(QueryScopedTableOperations queryOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + super(queryOperations, name, reporter == null + ? org.apache.iceberg.metrics.LoggingMetricsReporter.instance() : reporter); + this.queryOperations = queryOperations; + this.currentSnapshot = currentSnapshotJson == null + ? null : SnapshotParser.fromJson(currentSnapshotJson); + if (currentSnapshot != null) { + querySnapshots.put(currentSnapshot.snapshotId(), currentSnapshot); + } + } + + @Override + public Snapshot currentSnapshot() { + return currentSnapshot; + } + + @Override + public Snapshot snapshot(long snapshotId) { + if (currentSnapshot != null && currentSnapshot.snapshotId() == snapshotId) { + return currentSnapshot; + } + return copyForQuery(queryMetadata().snapshot(snapshotId)); + } + + @Override + public Iterable snapshots() { + ImmutableList.Builder snapshots = ImmutableList.builder(); + for (Snapshot snapshot : queryMetadata().snapshots()) { + snapshots.add(copyForQuery(snapshot)); + } + return snapshots.build(); + } + + @Override + public List history() { + return queryMetadata().snapshotLog(); + } + + @Override + public Map refs() { + return queryMetadata().refs(); + } + + @Override + public List statisticsFiles() { + return queryMetadata().statisticsFiles(); + } + + @Override + public List partitionStatisticsFiles() { + return queryMetadata().partitionStatisticsFiles(); + } + + private synchronized TableMetadata queryMetadata() { + return queryOperations.current(); + } + + private synchronized Snapshot copyForQuery(Snapshot snapshot) { + if (snapshot == null) { + return null; + } + return querySnapshots.computeIfAbsent(snapshot.snapshotId(), ignored -> + SnapshotParser.fromJson(SnapshotParser.toJson(snapshot, false))); + } + } + + private static TableMetadata prepareDelegateCommit(TableOperations delegate, + TableMetadata base, TableMetadata wrapperCurrent) { + if (base != wrapperCurrent) { + throw new CommitFailedException("Cannot commit from a stale Iceberg table view"); + } + TableMetadata delegateCurrent = delegate.current(); + if (!isSameGeneration(base, delegateCurrent)) { + throw new CommitFailedException("Cannot commit from a stale Iceberg metadata generation"); + } + return delegateCurrent; + } + + private static boolean isSameGeneration(TableMetadata retained, TableMetadata live) { + if (retained == live) { + return true; + } + if (retained == null || live == null) { + return false; + } + if (!Objects.equals(retained.uuid(), live.uuid())) { + return false; + } + if (retained.metadataFileLocation() != null || live.metadataFileLocation() != null) { + return Objects.equals(retained.metadataFileLocation(), live.metadataFileLocation()); + } + return retained.lastUpdatedMillis() == live.lastUpdatedMillis() + && retained.lastSequenceNumber() == live.lastSequenceNumber() + && retained.currentSchemaId() == live.currentSchemaId() + && retained.defaultSpecId() == live.defaultSpecId() + && retained.defaultSortOrderId() == live.defaultSortOrderId() + && Objects.equals(retained.location(), live.location()) + && Objects.equals(retained.properties(), live.properties()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java new file mode 100644 index 00000000000000..13db520e2d755d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; + +import java.util.Objects; +import java.util.Optional; + +/** Stable identity for an Iceberg snapshot projection built from one frozen metadata generation. */ +public final class IcebergSnapshotEntryKey { + private final NameMapping nameMapping; + private final String tableUuid; + private final String metadataFileLocation; + private final long snapshotId; + private final int schemaId; + private final int defaultSpecId; + + private IcebergSnapshotEntryKey(NameMapping nameMapping, String tableUuid, String metadataFileLocation, + long snapshotId, int schemaId, int defaultSpecId) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.tableUuid = Objects.requireNonNull(tableUuid, "tableUuid can not be null"); + this.metadataFileLocation = Objects.requireNonNull( + metadataFileLocation, "metadataFileLocation can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.defaultSpecId = defaultSpecId; + } + + /** + * Build a key from the same retained table generation that will be used by the value loader. + * Tables without a stable metadata location intentionally bypass the snapshot cache. + */ + public static Optional tryCreate(NameMapping nameMapping, Table retainedTable) { + if (!(retainedTable instanceof HasTableOperations)) { + return Optional.empty(); + } + TableMetadata metadata = ((HasTableOperations) retainedTable).operations().current(); + if (metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + || metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return Optional.empty(); + } + Snapshot snapshot = metadata.currentSnapshot(); + long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); + return Optional.of(new IcebergSnapshotEntryKey(nameMapping, metadata.uuid(), metadata.metadataFileLocation(), + snapshotId, metadata.currentSchemaId(), metadata.defaultSpecId())); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public String getMetadataFileLocation() { + return metadataFileLocation; + } + + public String getTableUuid() { + return tableUuid; + } + + public long getSnapshotId() { + return snapshotId; + } + + public int getSchemaId() { + return schemaId; + } + + public int getDefaultSpecId() { + return defaultSpecId; + } + + boolean belongsTo(IcebergTableCacheValue tableValue) { + Optional generation = tryCreate( + nameMapping, tableValue.getRetainedIcebergTable()); + return generation.isPresent() + && tableUuid.equals(generation.get().tableUuid) + && metadataFileLocation.equals(generation.get().metadataFileLocation); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof IcebergSnapshotEntryKey)) { + return false; + } + IcebergSnapshotEntryKey that = (IcebergSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && defaultSpecId == that.defaultSpecId + && nameMapping.equals(that.nameMapping) + && tableUuid.equals(that.tableUuid) + && metadataFileLocation.equals(that.metadataFileLocation); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, tableUuid, metadataFileLocation, snapshotId, schemaId, defaultSpecId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index 28e45b47acd250..f38d2689dfbbf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheKey; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.systable.SysTable; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; @@ -32,6 +33,7 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.Table; @@ -44,9 +46,6 @@ public class IcebergSysExternalTable extends ExternalTable { private final IcebergExternalTable sourceTable; private final String sysTableType; - private volatile Table sysIcebergTable; - private volatile List fullSchema; - private volatile SchemaCacheValue schemaCacheValue; public IcebergSysExternalTable(IcebergExternalTable sourceTable, String sysTableType) { super(generateSysTableId(sourceTable.getId(), sysTableType), @@ -100,24 +99,43 @@ public boolean supportsSnapshotSelection() { } public Table getSysIcebergTable() { - if (sysIcebergTable == null) { - synchronized (this) { - if (sysIcebergTable == null) { - Table baseTable = sourceTable.getIcebergTable(); - MetadataTableType tableType = MetadataTableType.from(sysTableType); - if (tableType == null) { - throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); - } - sysIcebergTable = MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); - } + MetadataTableType tableType = MetadataTableType.from(sysTableType); + if (tableType == null) { + throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); + } + // Metadata tables capture their base operations. Keep them statement-local so exact + // previousFiles/history state and stale-generation retry never leak into this table object. + return MetadataTableUtils.createMetadataTableInstance(resolveBaseTable(), tableType); + } + + /** + * The base generation this statement binds the metadata table to. Snapshot-selectable + * metadata tables derive both their scan and their schema from the source relation's frozen + * snapshot (the same generation IcebergScanNode scans), so analysis and execution cannot see + * different partition specs or schemas when the table entry refreshes mid-statement. Static + * metadata tables and statements without a bound snapshot read the latest generation. + * + *

    The statement snapshot is looked up by source table (like the scan node's fallback); + * a statement that time-travels the same table under several relations resolves the default + * or, if ambiguous, the latest generation for the schema. + */ + @VisibleForTesting + Table resolveBaseTable() { + if (supportsSnapshotSelection()) { + Optional

    frozenTable = MvccUtil.getSnapshotFromContext(sourceTable) + .filter(IcebergMvccSnapshot.class::isInstance) + .map(IcebergMvccSnapshot.class::cast) + .flatMap(snapshot -> snapshot.getSnapshotCacheValue().getIcebergTable()); + if (frozenTable.isPresent()) { + return frozenTable.get(); } } - return sysIcebergTable; + return IcebergUtils.getQueryScopedIcebergTable(sourceTable); } @Override public List getFullSchema() { - return getOrCreateSchemaCacheValue().getSchema(); + return loadSchemaCacheValue().getSchema(); } @Override @@ -156,12 +174,12 @@ public long fetchRowCount() { @Override public Optional initSchema(SchemaCacheKey key) { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override public Optional getSchemaCacheValue() { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override @@ -178,19 +196,12 @@ private static long generateSysTableId(long sourceTableId, String sysTableType) return sourceTableId ^ (sysTableType.hashCode() * 31L); } - private SchemaCacheValue getOrCreateSchemaCacheValue() { - if (schemaCacheValue == null) { - synchronized (this) { - if (schemaCacheValue == null) { - if (fullSchema == null) { - fullSchema = IcebergUtils.parseSchema(getSysIcebergTable().schema(), - getCatalog().getEnableMappingVarbinary(), - getCatalog().getEnableMappingTimestampTz()); - } - schemaCacheValue = new SchemaCacheValue(fullSchema); - } - } - } - return schemaCacheValue; + private SchemaCacheValue loadSchemaCacheValue() { + // Metadata-table schemas may change after source schema or partition-spec evolution. + // Resolve the schema from the statement's bound generation instead of permanently pairing + // this long-lived system-table object with its first observed generation. + return new SchemaCacheValue(IcebergUtils.parseSchema(getSysIcebergTable().schema(), + getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index cdef77346ade27..0b6bd23c634381 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -17,25 +17,124 @@ package org.apache.doris.datasource.iceberg; -import com.google.common.base.Suppliers; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; -import java.util.function.Supplier; +import java.util.Objects; +import java.util.Optional; public class IcebergTableCacheValue { - private final Table icebergTable; - private final Supplier latestSnapshotCacheValue; + private volatile Table icebergTable; + private String retainedCurrentSnapshotJson; + private volatile boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; - public IcebergTableCacheValue(Table icebergTable, Supplier latestSnapshotCacheValue) { - this.icebergTable = icebergTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + public IcebergTableCacheValue(Table icebergTable) { + this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); } public Table getIcebergTable() { + Table retainedTable = icebergTable; + return queryIsolationPrepared || IcebergSnapshotCacheValue.isNonGrowingGeneration(retainedTable) + ? IcebergSnapshotCacheValue.createQueryScopedTable( + retainedTable, retainedCurrentSnapshotJson) + : retainedTable; + } + + public Table getWritableIcebergTable(Table liveTable) { + Table retainedTable = icebergTable; + return IcebergSnapshotCacheValue.createWritableTable(retainedTable, liveTable); + } + + synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", + () -> { + // Order matters: serializing a v1 snapshot materializes its transient + // manifest list, which the payload accounting rejects, so account first. + retainedTablePayloadBytes = + IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + return IcebergCacheSizeEstimator.estimateTableEntry(key, this); + }); + if (sizeEstimate.isComplete()) { + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + Table getRetainedIcebergTable() { return icebergTable; } - public IcebergSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + synchronized Table newQueryScopedTable() { + if (!queryIsolationPrepared) { + // A failed optional size preparation must only reject weighted cache admission. Do not + // repeat the same unsupported metadata access on the query path and turn it into a + // table-load failure; this value is not retained by the weighted cache in that case. + if (sizeEstimate != null && !sizeEstimate.isComplete()) { + return icebergTable; + } + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + return IcebergSnapshotCacheValue.createQueryScopedTable( + icebergTable, retainedCurrentSnapshotJson); + } + + String getRetainedCurrentSnapshotJson() { + return retainedCurrentSnapshotJson; + } + + boolean isQueryIsolationPrepared() { + return queryIsolationPrepared; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return IcebergSnapshotCacheValue.retainedSnapshotJsonBytes( + retainedCurrentSnapshotJson); + } + + Optional getTableUuid() { + TableMetadata metadata = retainedMetadata(); + return metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + ? Optional.empty() : Optional.of(metadata.uuid()); + } + + boolean isSamePhysicalGeneration(IcebergTableCacheValue other) { + if (other == null) { + return false; + } + TableMetadata left = retainedMetadata(); + TableMetadata right = other.retainedMetadata(); + return left != null && right != null + && Objects.equals(left.uuid(), right.uuid()) + && Objects.equals(left.metadataFileLocation(), right.metadataFileLocation()); + } + + private TableMetadata retainedMetadata() { + Table retainedTable = icebergTable; + return retainedTable instanceof HasTableOperations + ? ((HasTableOperations) retainedTable).operations().current() : null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 71935cbd88157b..50d245ab0ec6d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -293,13 +293,13 @@ public void beginDelete(ExternalTable dorisTable, Table targetTable) throws User } private Table createTransactionTable(ExternalTable dorisTable, Table retainedTable) { - if (!IcebergSnapshotCacheValue.isFrozenGeneration(retainedTable)) { + if (!IcebergSnapshotCacheValue.isRetainedGeneration(retainedTable)) { return retainedTable; } // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, // while writer-contract changes still invalidate files produced for the retained metadata. return IcebergSnapshotCacheValue.createWritableTable( - retainedTable, IcebergUtils.getIcebergTable(dorisTable)); + retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable)); } /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 0b375c70d6791e..ef408a81e20636 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -56,6 +56,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.metastore.HMSBaseProperties; @@ -1057,6 +1058,14 @@ public static Table getIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTable(dorisTable); } + public static Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getQueryScopedIcebergTable(dorisTable); + } + + public static Table getWritableIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); + } + private static IcebergExternalMetaCache icebergExternalMetaCache(ExternalCatalog catalog) { Preconditions.checkNotNull(catalog, "catalog can not be null"); return Env.getCurrentEnv().getExtMetaCacheMgr().iceberg(catalog.getId()); @@ -1716,6 +1725,12 @@ public static IcebergSchemaCacheValue getSchemaCacheValue(ExternalTable dorisTab .getIcebergSchemaCacheValue(dorisTable.getOrBuildNameMapping(), schemaId); } + static IcebergSchemaCacheValue getSchemaCacheValue( + ExternalTable dorisTable, long schemaId, Table retainedTable) { + return icebergExternalMetaCache(dorisTable).getIcebergSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), schemaId, retainedTable); + } + public static IcebergSnapshot getLatestIcebergSnapshot(Table table) { Snapshot snapshot = table.currentSnapshot(); long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); @@ -1751,10 +1766,18 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T } Map nameToPartition = Maps.newHashMap(); Map nameToPartitionItem = Maps.newHashMap(); + long retainedPayloadBytes = 0L; - List partitionColumns = IcebergUtils.getSchemaCacheValue(dorisTable, schemaId).getPartitionColumns(); + List partitionColumns = IcebergUtils.getSchemaCacheValue( + dorisTable, schemaId, table).getPartitionColumns(); + long partitionItemColumnBytes = IcebergPartitionInfo.partitionItemColumnBytes( + partitionColumns.size()); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partitionItemColumnBytes); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); @@ -1762,7 +1785,10 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); - return new IcebergPartitionInfo(nameToPartitionItem, nameToPartition, partitionNameMap); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, IcebergPartitionInfo.partitionAliasBytes(partitionNameMap)); + return new IcebergPartitionInfo( + nameToPartitionItem, nameToPartition, partitionNameMap, retainedPayloadBytes); } private static List loadIcebergPartition(Table table, long snapshotId) { @@ -1802,6 +1828,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike StringBuilder sb = new StringBuilder(); List partitionValues = Lists.newArrayList(); List transforms = Lists.newArrayList(); + long retainedPayloadBytes = 0L; for (int i = 0; i < partitionSpec.fields().size(); ++i) { PartitionField partitionField = partitionSpec.fields().get(i); Class fieldClass = partitionSpec.javaClasses()[i]; @@ -1817,12 +1844,19 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike sb.append(fieldValue); sb.append("/"); partitionValues.add(fieldValue); - transforms.add(partitionField.transform().toString()); + String transform = partitionField.transform().toString(); + transforms.add(transform); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(fieldValue)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(transform)); } if (sb.length() > 0) { sb.delete(sb.length() - 1, sb.length()); } String partitionName = sb.toString(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(partitionName)); long recordCount = row.get(2, Long.class); long fileCount = row.get(3, Integer.class); long fileSizeInBytes = row.get(4, Long.class); @@ -1841,7 +1875,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike lastUpdateSnapShotId = UNKNOWN_SNAPSHOT_ID; } return new IcebergPartition(partitionName, specId, recordCount, fileSizeInBytes, fileCount, - lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms); + lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms, retainedPayloadBytes); } @VisibleForTesting @@ -1979,7 +2013,10 @@ public int compare(Map.Entry p1, Map.Entry retainedTable = sv.getRetainedIcebergTable(); + return retainedTable.isPresent() + ? getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId(), retainedTable.get()) + : getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId()); } public static IcebergSnapshotCacheValue getLatestSnapshotCacheValue(ExternalTable dorisTable) { @@ -2000,7 +2037,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( Optional scanParams) { if (tableSnapshot.isPresent() || IcebergUtils.isIcebergBranchOrTag(scanParams)) { // If a snapshot is specified, use the specified snapshot and the corresponding schema (not latest). - Table icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(getIcebergTable(dorisTable)); + IcebergExternalMetaCache metaCache = icebergExternalMetaCache(dorisTable); + Table icebergTable = metaCache.getQueryScopedIcebergTable(dorisTable); IcebergTableQueryInfo info; try { info = getQuerySpecSnapshot(icebergTable, tableSnapshot, scanParams); @@ -2010,8 +2048,7 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), - getNameMapping(icebergTable), - icebergTable); + getNameMapping(icebergTable), icebergTable); } return getLatestSnapshotCacheValue(dorisTable); } @@ -2027,11 +2064,12 @@ public static List getIcebergSchema(ExternalTable dorisTable, Optional getIcebergPartitionColumns(Optional snapshot, ExternalTable dorisTable) { IcebergSnapshotCacheValue snapshotValue = getSnapshotCacheValue(snapshot, dorisTable); - if (snapshotValue.getIcebergTable().isPresent()) { + Optional
    snapshotTable = snapshotValue.getIcebergTable(); + if (snapshotTable.isPresent()) { // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep // the same schema and snapshot IDs while changing spec(), so derive both from T0. return buildTableSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), - snapshotValue.getIcebergTable().get()).getPartitionColumns(); + snapshotTable.get()).getPartitionColumns(); } return getSchemaCacheValue(dorisTable, snapshotValue).getPartitionColumns(); } @@ -2047,9 +2085,16 @@ public static View getIcebergView(ExternalTable dorisTable) { public static Optional loadSchemaCacheValue( ExternalTable dorisTable, long schemaId, boolean isView) { + return loadSchemaCacheValue(dorisTable, schemaId, isView, null); + } + + public static Optional loadSchemaCacheValue( + ExternalTable dorisTable, long schemaId, boolean isView, Table retainedTable) { return isView ? loadViewSchemaCacheValue(dorisTable, schemaId) - : loadTableSchemaCacheValue(dorisTable, schemaId); + : retainedTable == null + ? loadTableSchemaCacheValue(dorisTable, schemaId) + : Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, retainedTable)); } private static Optional loadViewSchemaCacheValue(ExternalTable dorisTable, long schemaId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java index e303f0e9111486..94924514a58a48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long sourceSnapshotId = namedArguments.getLong(SNAPSHOT_ID); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java index 0937af8ba4cac4..82a93022354067 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java @@ -149,7 +149,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); // Parse parameters String olderThan = namedArguments.getString(OLDER_THAN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java index a5560db65520e4..cd746a7dbe6959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String sourceBranch = namedArguments.getString(BRANCH); String desBranch = namedArguments.getString(TO); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java index e1bf8cbdad4472..bf3f116d1cba81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java @@ -66,7 +66,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String targetWapId = namedArguments.getString(WAP_ID); // Find the target WAP snapshot diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java index 430e9fe9d5e22d..dce45c2729693b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { try { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot current = icebergTable.currentSnapshot(); if (current == null) { // No current snapshot means the table is empty, no manifests to rewrite diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java index 8d6b3842a9dc80..a5609f83439d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long targetSnapshotId = namedArguments.getLong(SNAPSHOT_ID); Snapshot targetSnapshot = icebergTable.snapshot(targetSnapshotId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java index 6957c563512657..de7e2a680791c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java @@ -96,7 +96,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String timestampStr = namedArguments.getString(TIMESTAMP); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java index 44df40f8f492b9..5b2c5bd220eb7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java @@ -87,7 +87,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot previousSnapshot = icebergTable.currentSnapshot(); Long previousSnapshotId = previousSnapshot != null ? previousSnapshot.snapshotId() : null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index e98ca6b2fb2808..cb8923a5bf2e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -17,30 +17,120 @@ package org.apache.doris.datasource.iceberg.cache; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.types.Types; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; /** * Cached manifest payload containing parsed files. */ public class ManifestCacheValue { + private static final long AUXILIARY_LIST_ENTRY_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(32L); + // A copied PartitionData instance shares its partition type, Avro schema, and serialized + // schema with the other files produced by one manifest reader. These constants are calibrated + // against the production reuseContainers()+file.copy() graph in IcebergExternalMetaCacheTest. + private static final long SHARED_PARTITION_BASE_BYTES = 1320L; + private static final long SHARED_PARTITION_FIELD_BYTES = 904L; + private static final long PARTITION_FIELD_NAME_RETENTION_COPIES = 2L; + // Bound variable-payload traversal (bound entries plus partition values) independently of + // manifest size. Values beyond this limit are rejected from a weighted cache instead of being + // admitted with an underestimate, so it sits well above ordinary wide manifests: 10,000 files + // with 200 lower/upper bounds each is 4,000,000 elements. + private static final long MAX_DEEP_ACCOUNTING_ELEMENTS = 8_000_000L; + // The per-file constants in IcebergCacheSizeEstimator describe the Iceberg 1.10.1 copies that + // ManifestReader + ContentFile.copy() produce. Only those implementations, with their pinned + // instance-field layouts, are accounted; anything else fails closed at build time. + private static final String GENERIC_DATA_FILE_CLASS_NAME = "org.apache.iceberg.GenericDataFile"; + private static final String GENERIC_DELETE_FILE_CLASS_NAME = + "org.apache.iceberg.GenericDeleteFile"; + private static final boolean CONTENT_FILE_LAYOUT_SUPPORTED = checkContentFileLayout(); + private final List dataFiles; private final List deleteFiles; + private final long dataFileMetricEntryCount; + private final long deleteFileMetricEntryCount; + private final long retainedPayloadBytes; + private final boolean accountingComplete; - private ManifestCacheValue(List dataFiles, List deleteFiles) { - this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; - this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + private ManifestCacheValue(List dataFiles, List deleteFiles, + long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes, + boolean accountingComplete) { + this.dataFiles = ImmutableList.copyOf(dataFiles); + this.deleteFiles = ImmutableList.copyOf(deleteFiles); + this.dataFileMetricEntryCount = dataFileMetricEntryCount; + this.deleteFileMetricEntryCount = deleteFileMetricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; + this.accountingComplete = accountingComplete; } public static ManifestCacheValue forDataFiles(List dataFiles) { - return new ManifestCacheValue(dataFiles, Collections.emptyList()); + Builder builder = dataFilesBuilder(); + if (dataFiles != null) { + dataFiles.forEach(builder::addDataFile); + } + return builder.build(); } public static ManifestCacheValue forDeleteFiles(List deleteFiles) { - return new ManifestCacheValue(Collections.emptyList(), deleteFiles); + Builder builder = deleteFilesBuilder(); + if (deleteFiles != null) { + deleteFiles.forEach(builder::addDeleteFile); + } + return builder.build(); + } + + public static Builder dataFilesBuilder() { + return dataFilesBuilder(true); + } + + public static Builder dataFilesBuilder(boolean accountRetainedSize) { + return new Builder(true, accountRetainedSize); + } + + public static Builder deleteFilesBuilder() { + return deleteFilesBuilder(true); + } + + public static Builder deleteFilesBuilder(boolean accountRetainedSize) { + return new Builder(false, accountRetainedSize); + } + + private static boolean checkContentFileLayout() { + ClassLoader loader = ContentFile.class.getClassLoader(); + return MetaCacheWeightUtils.hasExpectedInstanceFields(GENERIC_DATA_FILE_CLASS_NAME, loader) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + GENERIC_DELETE_FILE_CLASS_NAME, loader) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.iceberg.BaseFile", loader, + "partitionType:StructType", "fileOrdinal:Long", "manifestLocation:String", + "partitionSpecId:int", "content:FileContent", "filePath:String", + "format:FileFormat", "partitionData:PartitionData", "recordCount:Long", + "fileSizeInBytes:long", "dataSequenceNumber:Long", + "fileSequenceNumber:Long", "columnSizes:Map", "valueCounts:Map", + "nullValueCounts:Map", "nanValueCounts:Map", "lowerBounds:Map", + "upperBounds:Map", "splitOffsets:long[]", "equalityIds:int[]", + "keyMetadata:byte[]", "sortOrderId:Integer", "firstRowId:Long", + "referencedDataFile:String", "contentOffset:Long", + "contentSizeInBytes:Long", "avroSchema:Schema") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.iceberg.avro.SupportsIndexProjection", loader, + "fromProjectionPos:int[]") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionData.class, + "partitionType:StructType", "size:int", "data:Object[]", + "stringSchema:String", "schema:Schema"); } public List getDataFiles() { @@ -50,4 +140,281 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + public long getDataFileMetricEntryCount() { + return dataFileMetricEntryCount; + } + + public long getDeleteFileMetricEntryCount() { + return deleteFileMetricEntryCount; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + public boolean isAccountingComplete() { + return accountingComplete; + } + + /** Accounts retained payload while the manifest reader builds the cached lists. */ + public static final class Builder { + private final boolean dataContent; + private final boolean accountRetainedSize; + private final List dataFiles = new ArrayList<>(); + private final List deleteFiles = new ArrayList<>(); + private long metricEntryCount; + private long retainedPayloadBytes; + private long deepAccountingElements; + private boolean accountingComplete; + private final IdentityHashMap> + accountedPartitionSchemas = new IdentityHashMap<>(); + + private Builder(boolean dataContent, boolean accountRetainedSize) { + this.dataContent = dataContent; + this.accountRetainedSize = accountRetainedSize; + this.accountingComplete = accountRetainedSize; + } + + public void addDataFile(DataFile file) { + if (!dataContent) { + throw new IllegalStateException("delete manifest builder cannot accept a data file"); + } + dataFiles.add(file); + recordAccounting(file); + } + + public void addDeleteFile(DeleteFile file) { + if (dataContent) { + throw new IllegalStateException("data manifest builder cannot accept a delete file"); + } + deleteFiles.add(file); + recordAccounting(file); + } + + public ManifestCacheValue build() { + return new ManifestCacheValue(dataFiles, deleteFiles, + dataContent ? metricEntryCount : 0L, + dataContent ? 0L : metricEntryCount, + retainedPayloadBytes, accountingComplete); + } + + private void recordAccounting(ContentFile file) { + if (!accountRetainedSize || !accountingComplete) { + return; + } + try { + requireSupportedContentFile(file); + StructLike partition = file.partition(); + long nextDeepElements = MetaCacheWeightUtils.saturatedAdd( + deepAccountingElements, deepAccountingElements(file, partition)); + if (nextDeepElements > MAX_DEEP_ACCOUNTING_ELEMENTS) { + rejectAccounting(); + return; + } + deepAccountingElements = nextDeepElements; + addAccounting(account(file, partition)); + accountPartitionOwnership(partition); + } catch (RuntimeException | LinkageError e) { + // A new or third-party ContentFile implementation must not turn optional cache + // accounting into a manifest-read failure. Keep the files for the current query + // and mark the value incomplete so weighted admission rejects it. + rejectAccounting(); + } + } + + private void requireSupportedContentFile(ContentFile file) { + if (!CONTENT_FILE_LAYOUT_SUPPORTED) { + throw new IllegalStateException("unsupported Iceberg content file layout"); + } + String expectedClassName = dataContent + ? GENERIC_DATA_FILE_CLASS_NAME : GENERIC_DELETE_FILE_CLASS_NAME; + if (file == null || !expectedClassName.equals(file.getClass().getName())) { + throw new IllegalStateException("unsupported Iceberg content file implementation: " + + (file == null ? "null" : file.getClass().getName())); + } + } + + private void addAccounting(FileAccounting accounting) { + metricEntryCount = MetaCacheWeightUtils.saturatedAdd( + metricEntryCount, accounting.metricEntryCount); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, accounting.retainedPayloadBytes); + } + + private void rejectAccounting() { + metricEntryCount = 0L; + retainedPayloadBytes = 0L; + deepAccountingElements = 0L; + accountingComplete = false; + } + + private void accountPartitionOwnership(StructLike partition) { + if (partition == null || partition.size() == 0) { + return; + } + if (!(partition instanceof PartitionData)) { + throw new IllegalArgumentException( + "unsupported Iceberg partition container: " + + partition.getClass().getName()); + } + PartitionData partitionData = (PartitionData) partition; + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partitionInstanceBytes(partitionData)); + Object partitionTypeIdentity = partitionData.getPartitionType(); + // A reader-produced PartitionData carries its Avro schema; only a Java-deserialized + // instance would rebuild it here (CPU only, no IO). + Object schemaIdentity = partitionData.getSchema(); + IdentityHashMap schemas = accountedPartitionSchemas.computeIfAbsent( + partitionTypeIdentity, ignored -> new IdentityHashMap<>()); + if (schemas.put(schemaIdentity, Boolean.TRUE) == null) { + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, + sharedPartitionBytes(partitionData.getPartitionType())); + } + } + } + + private static FileAccounting account(ContentFile file, StructLike partition) { + return new FileAccounting( + metricEntryCount(file), retainedPayloadBytes(file, partition)); + } + + private static long deepAccountingElements(ContentFile file, StructLike partition) { + long elements = MetaCacheWeightUtils.saturatedAdd( + mapSize(file.lowerBounds()), mapSize(file.upperBounds())); + if (partition != null) { + if (partition.size() < 0) { + throw new IllegalArgumentException("negative Iceberg partition size"); + } + elements = MetaCacheWeightUtils.saturatedAdd(elements, partition.size()); + } + return elements; + } + + private static final class FileAccounting { + private final long metricEntryCount; + private final long retainedPayloadBytes; + + private FileAccounting(long metricEntryCount, long retainedPayloadBytes) { + this.metricEntryCount = metricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; + } + } + + private static long metricEntryCount(ContentFile file) { + long count = mapSize(file.columnSizes()); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.valueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nullValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nanValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.lowerBounds())); + return MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.upperBounds())); + } + + private static long retainedPayloadBytes(ContentFile file, StructLike partition) { + long bytes = MetaCacheWeightUtils.estimatedCharSequenceBytes(file.path()); + bytes = addBuffer(bytes, file.keyMetadata()); + bytes = addBuffers(bytes, file.lowerBounds()); + bytes = addBuffers(bytes, file.upperBounds()); + bytes = addListEntries(bytes, file.splitOffsets()); + bytes = addListEntries(bytes, file.equalityFieldIds()); + if (file instanceof DeleteFile) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes( + ((DeleteFile) file).referencedDataFile())); + } + return addPartitionPayload(bytes, partition); + } + + private static long addBuffers(long bytes, Map buffers) { + if (buffers == null) { + return bytes; + } + for (ByteBuffer buffer : buffers.values()) { + bytes = addBuffer(bytes, buffer); + } + return bytes; + } + + private static long addBuffer(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + + private static long addListEntries(long bytes, List values) { + if (values == null) { + return bytes; + } + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + values.size(), AUXILIARY_LIST_ENTRY_BYTES)); + } + + private static long addPartitionPayload(long bytes, StructLike partition) { + if (partition == null) { + return bytes; + } + for (int index = 0; index < partition.size(); index++) { + Object value = partition.get(index, Object.class); + if (value instanceof CharSequence) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + bytes = addByteArray(bytes, ((ByteBuffer) value).capacity()); + } else if (value instanceof byte[]) { + bytes = addByteArray(bytes, ((byte[]) value).length); + } else if (value instanceof java.math.BigDecimal) { + int bits = ((java.math.BigDecimal) value).unscaledValue().bitLength(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(96L)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedIntArrayPayloadBytes( + (bits + 31L) / 32L)); + } else if (value instanceof java.util.UUID) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(32L)); + } else if (value instanceof Long || value instanceof Double) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(24L)); + } else if (value instanceof Number || value instanceof Boolean + || value instanceof Character) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(16L)); + } else if (value != null) { + throw new IllegalArgumentException( + "unsupported Iceberg partition value: " + value.getClass().getName()); + } + } + return bytes; + } + + private static long addByteArray(long bytes, long payloadBytes) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedByteArrayBytes(payloadBytes)); + } + + private static long partitionInstanceBytes(PartitionData partition) { + return MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectBytes(32L), + MetaCacheWeightUtils.estimatedObjectArrayBytes(partition.size())); + } + + private static long sharedPartitionBytes(Types.StructType partitionType) { + long rawBytes = MetaCacheWeightUtils.saturatedAdd( + SHARED_PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + partitionType.fields().size(), SHARED_PARTITION_FIELD_BYTES)); + long bytes = MetaCacheWeightUtils.estimatedObjectBytes(rawBytes); + for (Types.NestedField field : partitionType.fields()) { + long payloadBytes = MetaCacheWeightUtils.estimatedStringPayloadBytes(field.name()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + payloadBytes, PARTITION_FIELD_NAME_RETENTION_COPIES)); + } + return bytes; + } + + private static int mapSize(Map map) { + return map == null ? 0 : map.size(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 957ab6ed55e193..570652f245ff71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -781,8 +781,9 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); - if (cacheValue.getIcebergTable().isPresent()) { - Table frozenBaseTable = cacheValue.getIcebergTable().get(); + Optional
    frozenTable = cacheValue.getIcebergTable(); + if (frozenTable.isPresent()) { + Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); if (systemTable.supportsSnapshotSelection()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java index 46e58f1e380081..da2980f7185ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -52,7 +53,12 @@ public class MaxComputeExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public MaxComputeExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public MaxComputeExternalMetaCache(ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionValuesEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_PARTITION_VALUES, NameMapping.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index a3a44151e45e2f..7dd629685b12a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -27,14 +27,19 @@ import org.apache.doris.datasource.SchemaCacheValue; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.LongConsumer; import java.util.function.Predicate; /** @@ -46,6 +51,8 @@ * to initialize a catalog explicitly before accessing entries. */ public abstract class AbstractExternalMetaCache implements ExternalMetaCache { + private static final Logger LOG = LogManager.getLogger(AbstractExternalMetaCache.class); + protected static CacheSpec defaultEntryCacheSpec() { return CacheSpec.of( true, @@ -62,12 +69,21 @@ protected static CacheSpec defaultSchemaCacheSpec() { private final String engine; private final ExecutorService refreshExecutor; + private final ExternalMetaCacheBudgetManager budgetManager; private final Map catalogEntries = Maps.newConcurrentMap(); private final Map> metaCacheEntryDefs = Maps.newConcurrentMap(); protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor) { + this(engine, refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.empty())); + } + + private volatile LongConsumer catalogPreparer; + + protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { this.engine = engine; this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); } @Override @@ -81,10 +97,97 @@ public Collection aliases() { } @Override - public void initCatalog(long catalogId, Map catalogProperties) { + public void validateCatalogProperties(Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, id -> buildCatalogEntryGroup(safeCatalogProperties)); + validateMappedCatalogProperties(safeCatalogProperties, true); + } + + @Override + public Map sanitizeCatalogPropertiesForRuntime(Map catalogProperties) { + return sanitizeCatalogPropertiesForRuntime(catalogProperties, warning -> LOG.debug(warning)); + } + + @Override + public void validateCatalogPropertiesForRuntime(Map catalogProperties) { + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + validateMappedCatalogProperties(safeCatalogProperties, false); + } + + /** + * Exactly what initCatalog keeps: mapped legacy keys, only known entries/options in this + * engine's namespace, a parsable catalog max-weight, and entry max-weights within it. + */ + private Map sanitizeCatalogPropertiesForRuntime( + Map catalogProperties, Consumer warningConsumer) { + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( + safeCatalogProperties, engine, metaCacheEntryDefs, warningConsumer); + try { + budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY + "': " + e.getMessage()); + } + OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { + if (entryDef.getSizeEstimator() == null) { + continue; + } + String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) + + entryDef.getName() + ".max-weight"; + if (!safeCatalogProperties.containsKey(maxWeightKey)) { + continue; + } + CacheSpec cacheSpec = CacheSpec.fromProperties( + safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); + try { + budgetManager.validateCatalogEntryHierarchy( + runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(maxWeightKey); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + maxWeightKey + "': " + e.getMessage()); + } + } + return safeCatalogProperties; + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + synchronized (this) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + Map safeCatalogProperties = sanitizeCatalogPropertiesForRuntime( + catalogProperties, + warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); + validateMappedCatalogProperties(safeCatalogProperties, false); + catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); + } + } + + private void validateMappedCatalogProperties( + Map catalogProperties, boolean validateAgainstLocalGlobalLimit) { + CacheSpec.validateEngineProperties(catalogProperties, engine, metaCacheEntryDefs); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + metaCacheEntryDefs.values().stream() + .filter(entryDef -> entryDef.getSizeEstimator() != null) + .map(entryDef -> CacheSpec.fromProperties( + catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec())) + .forEach(cacheSpec -> { + if (validateAgainstLocalGlobalLimit) { + budgetManager.validateHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } else { + budgetManager.validateCatalogEntryHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } + }); } @Override @@ -114,6 +217,7 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class MetaCacheEntryDef def = requireMetaCacheEntryDef(entryName); ensureTypeCompatible(def, keyType, valueType); + beforeCatalogEntryLookupForTest(catalogId, entryName); MetaCacheEntry cacheEntry = group.get(entryName); if (cacheEntry == null) { throw new IllegalStateException(String.format( @@ -124,10 +228,10 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class } @Override - public void invalidateCatalog(long catalogId) { + public synchronized void invalidateCatalog(long catalogId) { CatalogEntryGroup removed = catalogEntries.remove(catalogId); if (removed != null) { - removed.invalidateAll(); + removed.close(); } } @@ -162,8 +266,8 @@ public Map stats(long catalogId) { } @Override - public void close() { - catalogEntries.values().forEach(CatalogEntryGroup::invalidateAll); + public synchronized void close() { + catalogEntries.values().forEach(CatalogEntryGroup::close); catalogEntries.clear(); } @@ -191,6 +295,10 @@ protected final MetaCacheEntry entry(long catalogId, MetaCacheEntry return entry(catalogId, entryDef.getName(), entryDef.getKeyType(), entryDef.getValueType()); } + // Let tests pause after capturing a group and before looking up its entry. + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + } + protected final String metaCacheTtlKey(String entryName) { return "meta.cache." + engine + "." + entryName + ".ttl-second"; } @@ -224,6 +332,13 @@ protected final ExternalTable findExternalTable(NameMapping nameMapping, String private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { CatalogEntryGroup group = catalogEntries.get(catalogId); + if (group == null && catalogPreparer != null) { + // The caller prepared the catalog before capturing this engine, but a cache-policy + // ALTER retired the group in between. Re-prepare once under the lifecycle fence so + // the lookup observes the new policy instead of failing a valid catalog. + catalogPreparer.accept(catalogId); + group = catalogEntries.get(catalogId); + } if (group == null) { throw new IllegalStateException(String.format( "Catalog %d is not initialized for engine '%s'.", @@ -232,6 +347,11 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { return group; } + @Override + public void bindCatalogPreparer(LongConsumer catalogPreparer) { + this.catalogPreparer = catalogPreparer; + } + protected CatalogIf getCatalog(long catalogId) { if (Env.getCurrentEnv() == null || Env.getCurrentEnv().getCatalogMgr() == null) { return null; @@ -283,23 +403,52 @@ private void invalidateEntryIfMatched(CatalogEntryGroup group, MetaCacheE } } - private CatalogEntryGroup buildCatalogEntryGroup(Map catalogProperties) { + private CatalogEntryGroup buildCatalogEntryGroup(long catalogId, Map catalogProperties) { CatalogEntryGroup group = new CatalogEntryGroup(); - metaCacheEntryDefs.values() - .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(entryDef, catalogProperties))); - return group; + try { + metaCacheEntryDefs.values().forEach(entryDef -> group.put( + entryDef.getName(), newMetaCacheEntry(catalogId, entryDef, catalogProperties))); + return group; + } catch (RuntimeException | Error e) { + group.close(); + throw e; + } } @SuppressWarnings("unchecked") private MetaCacheEntry newMetaCacheEntry( - MetaCacheEntryDef rawEntryDef, Map catalogProperties) { + long catalogId, MetaCacheEntryDef rawEntryDef, Map catalogProperties) { MetaCacheEntryDef entryDef = (MetaCacheEntryDef) rawEntryDef; CacheSpec cacheSpec = CacheSpec.fromProperties( catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); - return new MetaCacheEntry<>(entryDef.getName(), - wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), - cacheSpec, - refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly()); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + if (cacheSpec.isWeightBounded() && entryDef.getSizeEstimator() == null) { + throw new IllegalArgumentException(String.format( + "Entry '%s' for engine '%s' configures max-weight but has no estimator.", + entryDef.getName(), engine)); + } + boolean enableWeight = entryDef.getSizeEstimator() != null + && (cacheSpec.isWeightBounded() + || catalogMaxWeight.isPresent() + || budgetManager.getGlobalMaxWeight().isPresent()); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = null; + if (enableWeight) { + entryBudget = budgetManager.createEntryBudget( + catalogId, engine, entryDef.getName(), catalogMaxWeight, cacheSpec.getMaxWeight()); + cacheSpec = cacheSpec.withMaxWeight(entryBudget.getEffectiveMaxWeight()); + } + try { + return new MetaCacheEntry<>(entryDef.getName(), + wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), + cacheSpec, + refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), + entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener()); + } catch (RuntimeException | Error e) { + if (entryBudget != null) { + entryBudget.close(); + } + throw e; + } } private Function wrapSchemaValidator(Function loader, Class valueType) { @@ -327,8 +476,12 @@ public MetaCacheEntry get(long catalogId) { return entry(catalogId, entryDef); } + @SuppressWarnings("unchecked") public MetaCacheEntry getIfInitialized(long catalogId) { - return isCatalogInitialized(catalogId) ? get(catalogId) : null; + // Read the group once. A concurrent invalidation may close that captured entry, which + // is safe; looking the group up a second time could instead throw after the first check. + CatalogEntryGroup group = catalogEntries.get(catalogId); + return group == null ? null : (MetaCacheEntry) group.get(entryDef.getName()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 0bb640ad0d753c..34ccf41c718b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -21,10 +21,18 @@ import org.apache.commons.lang3.math.NumberUtils; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -33,7 +41,8 @@ *
      *
    • enable=false disables cache
      • *
    • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
      • - *
    • capacity=0 disables cache; capacity is count-based
      • + *
    • capacity=0 disables cache; otherwise capacity is the count limit only when max-weight is absent
      • + *
    • when max-weight is present, Caffeine uses the weight limit instead of the positive capacity
      • *
    */ public final class CacheSpec { @@ -43,19 +52,36 @@ public final class CacheSpec { private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_VOLUME_PATTERN = Pattern.compile("^([0-9]+)\\s*(B|KB|MB|GB|TB|PB)?$", + Pattern.CASE_INSENSITIVE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + if (maxWeight < 0) { + throw new IllegalArgumentException("maxWeight can not be negative: " + maxWeight); + } + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); + } + + public CacheSpec withMaxWeight(long effectiveMaxWeight) { + return ofWeight(enable, ttlSecond, capacity, effectiveMaxWeight); } public static PropertySpec.Builder propertySpecBuilder() { @@ -77,7 +103,8 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getWeightProperty(properties, propertySpec.getMaxWeightKey()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** @@ -95,6 +122,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT) .build(); } @@ -151,6 +179,68 @@ public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capaci return enable && ttlSecond != 0 && capacity != 0; } + /** + * Parse an exact byte value with an optional binary unit. Percentages are accepted only + * when {@code allowPercent} is true and are resolved against {@code maxHeapBytes}. + */ + public static long parseWeight(String value, String key, boolean allowPercent, long maxHeapBytes) { + String normalized = Objects.requireNonNull(value, "value").trim(); + if (normalized.isEmpty()) { + throw invalidWeight(key, value); + } + if (normalized.endsWith("%")) { + if (!allowPercent || maxHeapBytes <= 0) { + throw invalidWeight(key, value); + } + String percentageText = normalized.substring(0, normalized.length() - 1).trim(); + try { + BigDecimal percentage = new BigDecimal(percentageText); + if (percentage.signum() < 0 || percentage.compareTo(BigDecimal.valueOf(100L)) > 0) { + throw invalidWeight(key, value); + } + BigInteger bytes = BigDecimal.valueOf(maxHeapBytes) + .multiply(percentage) + .divide(BigDecimal.valueOf(100L)) + .toBigInteger(); + return checkedLong(bytes, key, value); + } catch (NumberFormatException e) { + throw invalidWeight(key, value); + } + } + + Matcher matcher = DATA_VOLUME_PATTERN.matcher(normalized); + if (!matcher.matches()) { + throw invalidWeight(key, value); + } + BigInteger amount = new BigInteger(matcher.group(1)); + String rawUnit = matcher.group(2); + String unit = rawUnit == null ? "B" : rawUnit.toUpperCase(Locale.ROOT); + int power; + switch (unit) { + case "B": + power = 0; + break; + case "KB": + power = 1; + break; + case "MB": + power = 2; + break; + case "GB": + power = 3; + break; + case "TB": + power = 4; + break; + case "PB": + power = 5; + break; + default: + throw invalidWeight(key, value); + } + return checkedLong(amount.multiply(BigInteger.valueOf(1024L).pow(power)), key, value); + } + /** * Build standard external meta cache key prefix for one engine. * Example: {@code meta.cache.iceberg.} @@ -166,6 +256,110 @@ public static boolean isMetaCacheKeyForEngine(String key, String engine) { return key != null && engine != null && key.startsWith(metaCacheKeyPrefix(engine)); } + /** + * Strictly validate one engine namespace so misspelled entries/options cannot be silently ignored. + * The catalog-wide {@code meta.cache.max-weight} key is validated by the budget manager. + */ + static void validateEngineProperties(Map properties, String engine, + Map> entryDefs) { + Set weightedEntries = new java.util.HashSet<>(); + for (MetaCacheEntryDef entryDef : entryDefs.values()) { + if (entryDef.getSizeEstimator() != null) { + weightedEntries.add(entryDef.getName()); + } + } + validateEngineProperties(properties, engine, entryDefs.keySet(), weightedEntries); + } + + /** + * Ignore invalid persisted cache options during image/replay initialization. + * New CREATE/ALTER statements still use {@link #validateEngineProperties} and fail strictly. + */ + static Map sanitizeEnginePropertiesForRuntime( + Map properties, String engine, + Map> entryDefs, Consumer warningConsumer) { + Map sanitized = new HashMap<>(properties); + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + try { + validateEngineProperties(Collections.singletonMap(key, property.getValue()), engine, entryDefs); + } catch (IllegalArgumentException e) { + sanitized.remove(key); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + key + "': " + e.getMessage()); + } + } + return sanitized; + } + + public static void validateEngineProperties(Map properties, String engine, + Set entryNames, Set weightedEntryNames) { + if (properties == null || properties.isEmpty()) { + return; + } + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + String remainder = key.substring(enginePrefix.length()); + int optionSeparator = remainder.lastIndexOf('.'); + if (optionSeparator <= 0 || optionSeparator == remainder.length() - 1) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String entryName = remainder.substring(0, optionSeparator); + String option = remainder.substring(optionSeparator + 1); + if (!entryNames.contains(entryName)) { + throw new IllegalArgumentException("Unknown external meta cache entry property: " + key); + } + String value = property.getValue(); + switch (option) { + case "enable": + requireStrictBoolean(key, value); + break; + case "ttl-second": + requireLongAtLeast(key, value, CACHE_NO_TTL); + break; + case "capacity": + requireLongAtLeast(key, value, 0L); + break; + case "max-weight": + if (!weightedEntryNames.contains(entryName)) { + throw new IllegalArgumentException( + "External meta cache entry does not support max-weight: " + key); + } + parseWeight(value, key, false, 0L); + break; + default: + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + } + } + + private static void requireStrictBoolean(String key, String value) { + if (value == null || (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value))) { + throw new IllegalArgumentException("Invalid boolean cache property '" + key + "': " + value); + } + } + + private static void requireLongAtLeast(String key, String value, long minimum) { + final long parsed; + try { + parsed = Long.parseLong(value); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid integer cache property '" + key + "': " + value, e); + } + if (parsed < minimum) { + throw new IllegalArgumentException("Cache property '" + key + "' must be >= " + minimum + + ", but was " + value); + } + } + /** * Convert ttlSecond to OptionalLong for CacheFactory. * ttlSecond=-1 means no expiration; ttlSecond=0 disables cache. @@ -193,6 +387,27 @@ private static long getLongProperty(Map properties, String key, return NumberUtils.toLong(value, defaultValue); } + private static OptionalLong getWeightProperty(Map properties, String key) { + if (key == null) { + return OptionalLong.empty(); + } + String value = properties.get(key); + return value == null + ? OptionalLong.empty() + : OptionalLong.of(parseWeight(value, key, false, 0L)); + } + + private static long checkedLong(BigInteger value, String key, String rawValue) { + if (value.signum() < 0 || value.compareTo(LONG_MAX) > 0) { + throw invalidWeight(key, rawValue); + } + return value.longValue(); + } + + private static IllegalArgumentException invalidWeight(String key, String value) { + return new IllegalArgumentException("Invalid cache weight for '" + key + "': " + value); + } + public boolean isEnable() { return enable; } @@ -205,6 +420,19 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return isCacheEnabled(enable, ttlSecond, capacity) + && (!maxWeight.isPresent() || maxWeight.getAsLong() != 0L); + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -212,15 +440,17 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, String maxWeightKey) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; } public String getEnableKey() { @@ -247,6 +477,10 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -254,6 +488,7 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -273,6 +508,11 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key) { + this.maxWeightKey = Objects.requireNonNull(key, "key"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -280,7 +520,8 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java index c195087f415bfc..d37e91a8922019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; import java.util.Objects; @@ -27,6 +29,8 @@ * Catalog scoped entry container. */ public class CatalogEntryGroup { + private static final Logger LOG = LogManager.getLogger(CatalogEntryGroup.class); + private final Map> entries = new ConcurrentHashMap<>(); public MetaCacheEntry get(String entryName) { @@ -46,4 +50,19 @@ public Map stats() { public void invalidateAll() { entries.values().forEach(MetaCacheEntry::invalidateAll); } + + public void close() { + entries.forEach((name, entry) -> { + try { + entry.close(); + } catch (RuntimeException e) { + LOG.error("Failed to close external metadata cache entry {}; continuing group retirement", + name, e); + } + }); + // Keep the closed entries reachable from this retired group. A query may have captured the + // group immediately before its catalog is removed; returning a closed entry lets that query + // serve an uncached load instead of spuriously observing an uninitialized entry. The group + // is already absent from the owner map and is reclaimed with the last concurrent reader. + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 1a067726ec9136..47e623b219c19a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.LongConsumer; /** * Engine-level abstraction for external metadata cache. @@ -41,6 +42,34 @@ public interface ExternalMetaCache { */ Collection aliases(); + /** Validate cache properties in this engine's canonical namespace. */ + default void validateCatalogProperties(Map catalogProperties) { + } + + /** + * Drop the properties in this engine's namespace that runtime initialization would ignore + * (unknown entries, obsolete or unparsable options), returning what the engine will honor. + */ + default Map sanitizeCatalogPropertiesForRuntime(Map catalogProperties) { + return catalogProperties; + } + + /** + * Validate cache properties with the semantics initialization applies to persisted state: + * entry weights must fit their catalog bound, but the catalog bound is not compared with this + * FE's local global bound (runtime clamps it instead). + */ + default void validateCatalogPropertiesForRuntime(Map catalogProperties) { + } + + /** + * Bind the callback that (re)prepares a catalog group under the manager's lifecycle fence. + * A lookup that finds no group (the catalog was retired by a concurrent cache-policy ALTER + * after the caller prepared it) uses it once before failing. + */ + default void bindCatalogPreparer(LongConsumer catalogPreparer) { + } + /** * Initialize all registered entries for one catalog under current engine. * Entry instances are created eagerly at this stage. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java new file mode 100644 index 00000000000000..904a3fb0ad8f75 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -0,0 +1,587 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongUnaryOperator; +import java.util.stream.Collectors; + +/** + * FE-wide admission accounting for managed external metadata caches. + * + *

    All changes are serialized by one short critical section. Cache loads and + * estimators run outside it, so the lock only protects a few arithmetic and map + * operations while making global/catalog/entry reservation atomic. + */ +public final class ExternalMetaCacheBudgetManager { + private static final Logger LOG = LogManager.getLogger(ExternalMetaCacheBudgetManager.class); + private static final ExecutorService PEER_RECLAIM_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-peer-reclaim"); + thread.setDaemon(true); + return thread; + }); + + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; + + private final Object lock = new Object(); + private final OptionalLong globalMaxWeight; + private final Map catalogBuckets = new HashMap<>(); + private final Map entryBuckets = new HashMap<>(); + private final Map entryBudgets = new HashMap<>(); + private long globalUsedWeight; + private final AtomicLong globalRejectedCount = new AtomicLong(); + + public ExternalMetaCacheBudgetManager(OptionalLong globalMaxWeight) { + this.globalMaxWeight = Objects.requireNonNull(globalMaxWeight, "globalMaxWeight"); + if (globalMaxWeight.isPresent() && globalMaxWeight.getAsLong() <= 0) { + throw new IllegalArgumentException("global max weight must be positive when enabled"); + } + } + + public static ExternalMetaCacheBudgetManager fromConfig() { + String configured = Config.external_meta_cache_max_weight; + long parsed = CacheSpec.parseWeight( + configured, + "external_meta_cache_max_weight", + true, + Runtime.getRuntime().maxMemory()); + if (configured.trim().endsWith("%") && parsed == 0L) { + throw new IllegalArgumentException( + "external_meta_cache_max_weight percentage must be greater than 0%"); + } + return new ExternalMetaCacheBudgetManager(parsed == 0L ? OptionalLong.empty() : OptionalLong.of(parsed)); + } + + public OptionalLong parseCatalogMaxWeight(Map catalogProperties) { + String configured = catalogProperties.get(CATALOG_MAX_WEIGHT_PROPERTY); + if (configured == null) { + return OptionalLong.empty(); + } + long parsed = CacheSpec.parseWeight(configured, CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } + + /** Validate a catalog limit at DDL time against this FE's configured global bound. */ + public OptionalLong validateCatalogMaxWeight(Map catalogProperties) { + OptionalLong catalogMaxWeight = parseCatalogMaxWeight(catalogProperties); + validateHierarchy(catalogMaxWeight, OptionalLong.empty()); + return catalogMaxWeight; + } + + /** + * Create the budget handle used by one physical per-catalog cache entry. + */ + public EntryBudget createEntryBudget(long catalogId, String engine, String entryName, + OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + Objects.requireNonNull(engine, "engine"); + Objects.requireNonNull(entryName, "entryName"); + Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight"); + Objects.requireNonNull(entryMaxWeight, "entryMaxWeight"); + validateCatalogEntryHierarchy(catalogMaxWeight, entryMaxWeight); + + OptionalLong effectiveMax = minimumPresent(globalMaxWeight, catalogMaxWeight, entryMaxWeight); + if (!effectiveMax.isPresent()) { + throw new IllegalArgumentException("entry budget requires at least one configured weight bound"); + } + + EntryScope scope = new EntryScope(catalogId, engine, entryName); + synchronized (lock) { + Bucket catalogBucket = catalogBuckets.get(catalogId); + long catalogLimit = minimumLimit(globalMaxWeight, catalogMaxWeight); + if (catalogBucket == null) { + catalogBucket = new Bucket(catalogLimit); + catalogBuckets.put(catalogId, catalogBucket); + } else if (catalogBucket.maxWeight != catalogLimit) { + throw new IllegalStateException("Conflicting catalog cache max weight for catalog " + catalogId); + } + + if (entryBuckets.containsKey(scope)) { + throw new IllegalStateException("Duplicated external meta cache budget: " + scope); + } + Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + EntryBudget entryBudget = new EntryBudget( + this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); + entryBuckets.put(scope, entryBucket); + entryBudgets.put(scope, entryBudget); + return entryBudget; + } + } + + public OptionalLong getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalUsedWeight() { + synchronized (lock) { + return globalUsedWeight; + } + } + + public long getGlobalRejectedCount() { + return globalRejectedCount.get(); + } + + public void validateHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + if (globalMaxWeight.isPresent() && catalogMaxWeight.isPresent() + && catalogMaxWeight.getAsLong() > globalMaxWeight.getAsLong()) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " can not exceed FE global max weight"); + } + OptionalLong parent = catalogMaxWeight.isPresent() ? catalogMaxWeight : globalMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + /** + * Validate persisted catalog-to-entry hierarchy without comparing it with this FE's local + * global bound. Catalog properties are validated on the master, while the global percentage + * is resolved independently from each FE's heap. Runtime admission therefore clamps to the + * local global limit instead of rejecting a catalog accepted on a larger master. + */ + public void validateCatalogEntryHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + OptionalLong parent = catalogMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + private Optional tryReserve(EntryBudget entryBudget, long bytes) { + checkWeight(bytes); + synchronized (lock) { + if (entryBudget.closed) { + return Optional.empty(); + } + if (!fits(limitOf(globalMaxWeight), globalUsedWeight, bytes) + || !fits(entryBudget.catalogBucket.maxWeight, entryBudget.catalogBucket.usedWeight, bytes) + || !fits(entryBudget.entryBucket.maxWeight, entryBudget.entryBucket.usedWeight, bytes)) { + entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return Optional.empty(); + } + addUsed(entryBudget, bytes); + return Optional.of(new AdmissionReservation(this, entryBudget, bytes)); + } + } + + private boolean resize(AdmissionReservation reservation, long newBytes) { + checkWeight(newBytes); + synchronized (lock) { + if (!reservation.active || reservation.entryBudget.closed) { + return false; + } + long delta = newBytes - reservation.bytes; + if (delta > 0 && (!fits(limitOf(globalMaxWeight), globalUsedWeight, delta) + || !fits(reservation.entryBudget.catalogBucket.maxWeight, + reservation.entryBudget.catalogBucket.usedWeight, delta) + || !fits(reservation.entryBudget.entryBucket.maxWeight, + reservation.entryBudget.entryBucket.usedWeight, delta))) { + reservation.entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return false; + } + if (delta >= 0) { + addUsed(reservation.entryBudget, delta); + } else { + subtractUsed(reservation.entryBudget, -delta); + } + reservation.bytes = newBytes; + return true; + } + } + + private void release(AdmissionReservation reservation) { + synchronized (lock) { + if (!reservation.active) { + return; + } + if (reservation.entryBudget.closed) { + reservation.bytes = 0L; + reservation.active = false; + return; + } + subtractUsed(reservation.entryBudget, reservation.bytes); + reservation.bytes = 0L; + reservation.active = false; + } + } + + private void close(EntryBudget entryBudget) { + synchronized (lock) { + if (entryBudget.closed) { + return; + } + if (entryBudget.entryBucket.usedWeight != 0L) { + long leakedWeight = entryBudget.entryBucket.usedWeight; + LOG.error("Force-closing external metadata cache budget {} with {} bytes still reserved", + entryBudget.scope, leakedWeight); + if (leakedWeight <= globalUsedWeight + && leakedWeight <= entryBudget.catalogBucket.usedWeight) { + globalUsedWeight -= leakedWeight; + entryBudget.catalogBucket.usedWeight -= leakedWeight; + entryBudget.entryBucket.usedWeight = 0L; + } else { + LOG.error("External metadata cache accounting is inconsistent while closing {}; " + + "globalUsed={}, catalogUsed={}, entryUsed={}", + entryBudget.scope, globalUsedWeight, + entryBudget.catalogBucket.usedWeight, leakedWeight); + globalUsedWeight = Math.max(0L, globalUsedWeight - leakedWeight); + entryBudget.catalogBucket.usedWeight = Math.max( + 0L, entryBudget.catalogBucket.usedWeight - leakedWeight); + entryBudget.entryBucket.usedWeight = 0L; + } + } + entryBudget.closed = true; + entryBudget.reclaimer = null; + entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); + entryBudgets.remove(entryBudget.scope, entryBudget); + Bucket catalogBucket = entryBudget.catalogBucket; + boolean catalogStillReferenced = entryBuckets.keySet().stream() + .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); + if (!catalogStillReferenced && catalogBucket.usedWeight == 0L) { + catalogBuckets.remove(entryBudget.scope.catalogId, catalogBucket); + } + } + } + + private void addUsed(EntryBudget entryBudget, long bytes) { + globalUsedWeight += bytes; + entryBudget.catalogBucket.usedWeight += bytes; + entryBudget.entryBucket.usedWeight += bytes; + } + + private void subtractUsed(EntryBudget entryBudget, long bytes) { + if (bytes > globalUsedWeight + || bytes > entryBudget.catalogBucket.usedWeight + || bytes > entryBudget.entryBucket.usedWeight) { + throw new IllegalStateException("external meta cache budget accounting underflow"); + } + globalUsedWeight -= bytes; + entryBudget.catalogBucket.usedWeight -= bytes; + entryBudget.entryBucket.usedWeight -= bytes; + } + + private void requestPeerReclaim(EntryBudget requester, long additionalBytes) { + if (additionalBytes <= 0L || requester.closed) { + return; + } + long reclaimBytes; + synchronized (lock) { + if (requester.closed) { + return; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + reclaimBytes = Math.max(globalDeficit, catalogDeficit); + } + if (reclaimBytes <= 0L) { + return; + } + // Rejected values are returned uncached; there is no queue of pending admissions to fund. + // Coalesce concurrent misses to the largest single admission instead of summing identical + // deficits and evicting an entire peer cache during a miss burst. + requester.requestedAdmissionBytes.accumulateAndGet(additionalBytes, Math::max); + schedulePeerReclaim(requester); + } + + private void schedulePeerReclaim(EntryBudget requester) { + if (!requester.reclaimScheduled.compareAndSet(false, true)) { + return; + } + try { + PEER_RECLAIM_EXECUTOR.execute(() -> drainPeerReclaim(requester)); + } catch (RejectedExecutionException e) { + requester.reclaimScheduled.set(false); + LOG.warn("Failed to schedule peer reclamation for external metadata cache budget {}", + requester.scope, e); + } + } + + private void drainPeerReclaim(EntryBudget requester) { + try { + long requestedAdmissionBytes = requester.requestedAdmissionBytes.getAndSet(0L); + if (requestedAdmissionBytes <= 0L || requester.closed) { + return; + } + List candidates; + synchronized (lock) { + candidates = entryBudgets.values().stream() + .filter(candidate -> candidate != requester && !candidate.closed) + .filter(candidate -> candidate.reclaimer != null) + .filter(candidate -> candidate.entryBucket.usedWeight > 0L) + .sorted((left, right) -> { + boolean leftSibling = left.scope.catalogId == requester.scope.catalogId; + boolean rightSibling = right.scope.catalogId == requester.scope.catalogId; + if (leftSibling != rightSibling) { + return leftSibling ? -1 : 1; + } + return Long.compare( + right.entryBucket.usedWeight, left.entryBucket.usedWeight); + }) + .collect(Collectors.toList()); + } + long remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + for (EntryBudget candidate : candidates) { + boolean sibling = candidate.scope.catalogId == requester.scope.catalogId; + if (!sibling && currentCatalogDeficit(requester, requestedAdmissionBytes) > 0L) { + // Another catalog cannot create headroom under the requester's catalog limit. + continue; + } + LongUnaryOperator reclaimer = candidate.reclaimer; + if (reclaimer == null || candidate.closed) { + continue; + } + try { + reclaimer.applyAsLong(remaining); + remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + } catch (RuntimeException e) { + LOG.warn("Failed to reclaim external metadata cache budget from peer {}", + candidate.scope, e); + } + if (remaining == 0L) { + break; + } + } + } finally { + requester.reclaimScheduled.set(false); + if (!requester.closed && requester.requestedAdmissionBytes.get() > 0L) { + schedulePeerReclaim(requester); + } + } + } + + private long currentReclaimDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + if (requester.closed) { + return 0L; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + return Math.max(globalDeficit, catalogDeficit); + } + } + + private long currentCatalogDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit( + requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes); + } + } + + private static long deficit(long maxWeight, long usedWeight, long additionalBytes) { + if (maxWeight == Long.MAX_VALUE || additionalBytes <= maxWeight - Math.min(usedWeight, maxWeight)) { + return 0L; + } + return MetaCacheWeightUtils.saturatedAdd(usedWeight, additionalBytes) - maxWeight; + } + + private static boolean fits(long maxWeight, long usedWeight, long delta) { + return delta >= 0 && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; + } + + private static long limitOf(OptionalLong configured) { + return configured.isPresent() ? configured.getAsLong() : Long.MAX_VALUE; + } + + private static long minimumLimit(OptionalLong first, OptionalLong second) { + return Math.min(limitOf(first), limitOf(second)); + } + + private static OptionalLong minimumPresent(OptionalLong first, OptionalLong second, OptionalLong third) { + if (!first.isPresent() && !second.isPresent() && !third.isPresent()) { + return OptionalLong.empty(); + } + long minimum = Math.min(limitOf(first), Math.min(limitOf(second), limitOf(third))); + return OptionalLong.of(minimum); + } + + private static void checkWeight(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache reservation can not be negative: " + bytes); + } + } + + private static final class Bucket { + private final long maxWeight; + private long usedWeight; + + private Bucket(long maxWeight) { + this.maxWeight = maxWeight; + } + } + + private static final class EntryScope { + private final long catalogId; + private final String engine; + private final String entryName; + + private EntryScope(long catalogId, String engine, String entryName) { + this.catalogId = catalogId; + this.engine = engine; + this.entryName = entryName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryScope)) { + return false; + } + EntryScope that = (EntryScope) other; + return catalogId == that.catalogId && engine.equals(that.engine) && entryName.equals(that.entryName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, engine, entryName); + } + + @Override + public String toString() { + return catalogId + "/" + engine + "/" + entryName; + } + } + + public static final class EntryBudget { + private final ExternalMetaCacheBudgetManager manager; + private final EntryScope scope; + private final Bucket catalogBucket; + private final Bucket entryBucket; + private final long effectiveMaxWeight; + private final AtomicLong rejectedCount = new AtomicLong(); + private final AtomicLong requestedAdmissionBytes = new AtomicLong(); + private final AtomicBoolean reclaimScheduled = new AtomicBoolean(); + private volatile LongUnaryOperator reclaimer; + // Mutated under manager.lock and read by asynchronous reclamation workers. + private volatile boolean closed; + + private EntryBudget(ExternalMetaCacheBudgetManager manager, EntryScope scope, + Bucket catalogBucket, Bucket entryBucket, long effectiveMaxWeight) { + this.manager = manager; + this.scope = scope; + this.catalogBucket = catalogBucket; + this.entryBucket = entryBucket; + this.effectiveMaxWeight = effectiveMaxWeight; + } + + public Optional tryReserve(long bytes) { + return manager.tryReserve(this, bytes); + } + + void setReclaimer(LongUnaryOperator reclaimer) { + this.reclaimer = Objects.requireNonNull(reclaimer, "reclaimer"); + } + + void requestPeerReclaim(long additionalBytes) { + manager.requestPeerReclaim(this, additionalBytes); + } + + public long getEffectiveMaxWeight() { + return effectiveMaxWeight; + } + + public long getUsedWeight() { + synchronized (manager.lock) { + return entryBucket.usedWeight; + } + } + + public long getCatalogUsedWeight() { + synchronized (manager.lock) { + return catalogBucket.usedWeight; + } + } + + public long getCatalogMaxWeight() { + return catalogBucket.maxWeight == Long.MAX_VALUE ? -1L : catalogBucket.maxWeight; + } + + public long getRejectedCount() { + return rejectedCount.get(); + } + + public long getGlobalUsedWeight() { + return manager.getGlobalUsedWeight(); + } + + public long getGlobalMaxWeight() { + return manager.globalMaxWeight.isPresent() ? manager.globalMaxWeight.getAsLong() : -1L; + } + + public void close() { + manager.close(this); + } + } + + public static final class AdmissionReservation { + private final ExternalMetaCacheBudgetManager manager; + private final EntryBudget entryBudget; + private long bytes; + private boolean active = true; + + private AdmissionReservation(ExternalMetaCacheBudgetManager manager, EntryBudget entryBudget, long bytes) { + this.manager = manager; + this.entryBudget = entryBudget; + this.bytes = bytes; + } + + public boolean tryResize(long newBytes) { + return manager.resize(this, newBytes); + } + + public void release() { + manager.release(this); + } + + public long getBytes() { + synchronized (manager.lock) { + return bytes; + } + } + + public boolean isActive() { + synchronized (manager.lock) { + return active; + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 30668163539d3b..e76bfc0540f6d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -19,14 +19,29 @@ import org.apache.doris.common.CacheFactory; import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.Weigher; import com.github.benmanes.caffeine.cache.stats.CacheStats; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -40,8 +55,23 @@ * key/predicate/full invalidation, and lightweight runtime stats. */ public class MetaCacheEntry { + private static final Logger LOG = LogManager.getLogger(MetaCacheEntry.class); // Use striped locks to deduplicate slow external loads without managing per-key lock lifecycle. private static final int LOAD_LOCK_STRIPES = 128; + private static final int LOCAL_EVICTION_BATCH_SIZE = 16; + private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; + private static final long WEIGHT_REJECT_LOG_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1L); + // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced + // generation map per physical entry after callbacks return; cleanup tasks never capture values. + private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-removal-cleanup"); + thread.setDaemon(true); + return thread; + }); + // Conservative retained cost outside the estimator-owned key/value graph: Caffeine's data + // node and policy links plus the reservation ConcurrentHashMap node, record and token. This + // deliberately overestimates common compressed-oops layouts; calibrate downward only with JOL. + static final long FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES = 512L; private final String name; @Nullable @@ -49,15 +79,42 @@ public class MetaCacheEntry { private final CacheSpec cacheSpec; private final boolean effectiveEnabled; private final boolean autoRefresh; + private final ExecutorService refreshExecutor; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final EntryBudget entryBudget; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; + private final boolean weightBounded; + // Entries with publication-time work use the same generation-fenced refresh protocol even + // before a max-weight is configured. This keeps estimation and dependency retirement on every + // load/refresh path instead of letting Caffeine publish values behind those hooks. + private final boolean generationFencedRefresh; // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. private final LoadingCache loadingData; // Use the plain cache view for manual miss load so slow I/O does not happen in Caffeine's sync load path. private final Cache data; // Protect one key stripe at a time to deduplicate concurrent miss loads with bounded lock count. private final Object[] loadLocks = new Object[LOAD_LOCK_STRIPES]; + // Serialize weighted cache mutation with reservation ownership changes. + private final Object admissionLock = new Object(); + // Ownership records deliberately contain no V reference. A weighted cache's Caffeine soft + // reference must be the only cache-owned path to its value, while generation fencing keeps + // delayed removal callbacks from releasing a replacement reservation. + private final Map reservations = new ConcurrentHashMap<>(); + private final Map refreshRecords = new ConcurrentHashMap<>(); + private final Map pendingRemovalGenerations = new ConcurrentHashMap<>(); + private final AtomicBoolean removalCleanupScheduled = new AtomicBoolean(false); + private final Map refreshesInFlight = new ConcurrentHashMap<>(); + // A state exists only while a miss/refresh for the key is in flight. Mutations advance that + // state's epoch, fencing stale publication without retaining every key ever observed. + private final Map keyMutationStates = new ConcurrentHashMap<>(); private final AtomicLong invalidateCount = new AtomicLong(0); - // Bump generation before invalidation so in-flight manual loads do not repopulate stale values. - private final AtomicLong invalidateGeneration = new AtomicLong(0); + // Full invalidation is the only cross-key fence. Ordinary mutations use the per-key state. + private final AtomicLong fullInvalidationGeneration = new AtomicLong(0); + // Primitive owner id lets queued refresh work fence a reservation without retaining its value. + private final AtomicLong reservationGeneration = new AtomicLong(0); // Track load statistics outside Caffeine because manual miss loads bypass the built-in load counters. private final AtomicLong loadSuccessCount = new AtomicLong(0); private final AtomicLong loadFailureCount = new AtomicLong(0); @@ -65,6 +122,17 @@ public class MetaCacheEntry { private final AtomicLong lastLoadSuccessTimeMs = new AtomicLong(-1L); private final AtomicLong lastLoadFailureTimeMs = new AtomicLong(-1L); private final AtomicReference lastError = new AtomicReference<>(""); + private final AtomicLong weightAdmissionRejectedCount = new AtomicLong(0L); + private final AtomicLong localEvictionCount = new AtomicLong(0L); + private final AtomicLong localEvictionWeight = new AtomicLong(0L); + // Exact byte weight released by Caffeine-driven evictions (size, expiry, soft collection). + // Caffeine's own eviction weight is clamped to the int weigher, so it is not used for stats. + private final AtomicLong automaticEvictionWeight = new AtomicLong(0L); + // Generation reported evicted for a key, consumed by the fenced cleanup of that generation. + private final Map pendingEvictionGenerations = new ConcurrentHashMap<>(); + private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final AtomicLong lastWeightRejectLogTimeMs = new AtomicLong(0L); + private final AtomicBoolean closed = new AtomicBoolean(false); public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { this(name, loader, cacheSpec, refreshExecutor, true, false); @@ -77,6 +145,20 @@ public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, E public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, null, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -91,23 +173,48 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); this.autoRefresh = autoRefresh; - Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); + this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.sizeEstimator = sizeEstimator; + this.entryBudget = entryBudget; + this.replacementListener = replacementListener; + this.weightBounded = this.cacheSpec.isWeightBounded(); + this.generationFencedRefresh = autoRefresh + && (sizeEstimator != null || replacementListener != null); + if (weightBounded && (sizeEstimator == null || entryBudget == null)) { + throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); + } + if (weightBounded) { + entryBudget.setReclaimer(this::reclaimForPeer); + } + this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); OptionalLong refreshAfterWriteSec = - effectiveEnabled && autoRefresh + effectiveEnabled && autoRefresh && !weightBounded && !generationFencedRefresh ? OptionalLong.of(Config.external_cache_refresh_time_minutes * 60) : OptionalLong.empty(); long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; + Weigher cacheWeigher = weightBounded ? this::weigh : null; CacheFactory cacheFactory = new CacheFactory( expireAfterAccessSec, refreshAfterWriteSec, maxSize, + weightBounded ? OptionalLong.of(effectiveEnabled ? this.cacheSpec.getMaxWeight().getAsLong() : 0L) + : OptionalLong.empty(), + cacheWeigher, true, null); - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + if (weightBounded) { + cacheFactory.withSoftValues(); + } + if (weightBounded || generationFencedRefresh) { + // Direct notification avoids queuing REPLACED values. The listener itself is lock-free + // and delegates only current-owner cleanup, so it is safe under Caffeine's eviction lock. + this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener( + this::loadFromDefaultLoader, this::onRemoval); + } else { + this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + } this.data = loadingData; // Initialize striped locks eagerly to keep the hot path allocation-free. for (int i = 0; i < loadLocks.length; i++) { @@ -120,6 +227,9 @@ public String name() { } public V get(K key) { + if (closed.get()) { + return loadAndTrack(key, this::applyDefaultLoader); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key); } @@ -128,6 +238,9 @@ public V get(K key) { public V get(K key, Function missLoader) { Function loadFunction = Objects.requireNonNull(missLoader, "missLoader can not be null"); + if (closed.get()) { + return loadAndTrack(key, loadFunction); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key, typedKey -> loadAndTrack(typedKey, loadFunction)); } @@ -135,42 +248,237 @@ public V get(K key, Function missLoader) { } public V getIfPresent(K key) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return null; } - return data.getIfPresent(key); + V value = data.getIfPresent(key); + if (value != null) { + maybeRefreshManagedValue(key, value); + } + return value; + } + + /** Return the current value without recording a user-visible cache request. */ + public V peekIfPresent(K key) { + if (!effectiveEnabled || closed.get()) { + return null; + } + return data.asMap().get(key); + } + + /** + * Fence loads and refreshes that started before an event, but only while the expected value is + * still current. Publication-managed entries retain the known-good value and advance the key's + * mutation epoch. Other count-based entries must invalidate because their legacy + * Caffeine-managed refresh path does not participate in that generation protocol. + */ + public boolean fenceInFlightLoadIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + synchronized (admissionLock) { + if (!effectiveEnabled || closed.get() || data.asMap().get(key) != expectedCurrent) { + return false; + } + advanceKeyMutation(key); + if (!weightBounded && !generationFencedRefresh) { + if (!data.asMap().remove(key, expectedCurrent)) { + return false; + } + invalidateCount.incrementAndGet(); + } + return true; + } } public void put(K key, V value) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return; } - data.put(key, value); + if (weightBounded) { + admitWeightedValue(key, value, null, false, null, -1L, true); + } else { + synchronized (admissionLock) { + if (!closed.get()) { + advanceKeyMutation(key); + putNonWeightedValue(key, value); + } + } + } } - public void invalidateKey(K key) { - invalidateGeneration.incrementAndGet(); - if (data.asMap().remove(key) != null) { + /** Result of an atomic compare-and-replace operation. */ + public enum ReplaceResult { + REPLACED, + NOT_CURRENT, + REJECTED, + DISABLED + } + + /** + * Replace one cached value only when it is still the expected identity. + * + *

    Weighted entries perform the identity check, budget resize and Caffeine write under the + * same admission lock. Callers can therefore distinguish a concurrent update from admission + * rejection and avoid retaining a value they already know is stale. + */ + public ReplaceResult tryReplace(K key, V expectedCurrent, V newValue) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + Objects.requireNonNull(newValue, "newValue can not be null"); + if (!effectiveEnabled || closed.get()) { + return ReplaceResult.DISABLED; + } + if (weightBounded) { + return toReplaceResult(admitWeightedValue( + key, newValue, expectedCurrent, true, null, -1L, true)); + } + synchronized (admissionLock) { + AtomicReference result = new AtomicReference<>(ReplaceResult.NOT_CURRENT); + AtomicReference published = new AtomicReference<>(); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (closed.get()) { + result.set(ReplaceResult.DISABLED); + return current; + } + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + published.set(publishRefreshRecord(key)); + result.set(ReplaceResult.REPLACED); + return newValue; + }); + RefreshRecord record = published.get(); + if (record != null && refreshRecords.get(key) == record + && data.asMap().get(key) == newValue) { + record.published = true; + } + if (result.get() == ReplaceResult.REPLACED && data.asMap().get(key) == newValue) { + notifyReplacement(key, expectedCurrent, newValue); + } + return result.get(); + } + } + + /** Remove a key only if it still maps to the expected value identity. */ + public boolean invalidateKeyIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + if (!weightBounded) { + synchronized (admissionLock) { + AtomicBoolean removed = new AtomicBoolean(false); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + invalidateCount.incrementAndGet(); + refreshRecords.remove(key); + removed.set(true); + return null; + }); + return removed.get(); + } + } + synchronized (admissionLock) { + V current = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (current != expectedCurrent || record == null || !record.published) { + return false; + } + advanceKeyMutation(key); + if (!data.asMap().remove(key, current)) { + return false; + } + releaseReservation(key, record.generation); invalidateCount.incrementAndGet(); + return true; + } + } + + public void invalidateKey(K key) { + synchronized (admissionLock) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + if (removed != null) { + refreshRecords.remove(key); + invalidateCount.incrementAndGet(); + } + } } } public void invalidateIf(Predicate predicate) { - invalidateGeneration.incrementAndGet(); - data.asMap().keySet().removeIf(key -> { - if (predicate.test(key)) { - invalidateCount.incrementAndGet(); - return true; + synchronized (admissionLock) { + Set candidates = new HashSet<>(data.asMap().keySet()); + candidates.addAll(keyMutationStates.keySet()); + if (weightBounded) { + candidates.addAll(reservations.keySet()); + } else if (generationFencedRefresh) { + candidates.addAll(refreshRecords.keySet()); } - return false; - }); + for (K key : candidates) { + if (predicate.test(key)) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + refreshRecords.remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + } + } + } + } } public void invalidateAll() { - invalidateGeneration.incrementAndGet(); - long size = data.estimatedSize(); - data.invalidateAll(); - invalidateCount.addAndGet(size); + synchronized (admissionLock) { + fullInvalidationGeneration.incrementAndGet(); + if (weightBounded) { + long size = data.estimatedSize(); + beforeWeightedInvalidateAllForTest(); + data.invalidateAll(); + reservations.values().forEach(record -> record.reservation.release()); + reservations.clear(); + pendingRemovalGenerations.clear(); + pendingEvictionGenerations.clear(); + invalidateCount.addAndGet(size); + } else { + long size = data.estimatedSize(); + data.invalidateAll(); + refreshRecords.clear(); + pendingRemovalGenerations.clear(); + pendingEvictionGenerations.clear(); + invalidateCount.addAndGet(size); + } + } + } + + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + invalidateAll(); + if (entryBudget != null) { + entryBudget.close(); + } } public void forEach(BiConsumer consumer) { @@ -198,62 +506,651 @@ public MetaCacheEntryStats stats() { failureCount, totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - cacheStats.evictionCount(), + MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionCount(), localEvictionCount.get()), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), - lastError.get()); + lastError.get(), + weightBounded, + weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, + weightBounded ? entryBudget.getUsedWeight() : -1L, + weightBounded ? MetaCacheWeightUtils.saturatedAdd( + automaticEvictionWeight.get(), localEvictionWeight.get()) : -1L, + weightBounded ? weightAdmissionRejectedCount.get() : -1L, + weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, + weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, + weightBounded ? entryBudget.getGlobalMaxWeight() : -1L, + weightBounded ? entryBudget.getGlobalUsedWeight() : -1L, + weightBounded ? lastWeightRejectReason.get() : ""); + } + + public boolean isWeightBounded() { + return weightBounded; + } + + private AdmissionResult admitWeightedValue( + K key, V value, @Nullable V expectedCurrent, boolean requireExpected, + @Nullable KeyMutationToken expectedMutation, long expectedReservationGeneration, + boolean advanceMutationOnAdmission) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + MetaCacheSizeEstimate estimate; + try { + estimate = Objects.requireNonNull(sizeEstimator.estimate(key, value), "size estimate"); + } catch (IllegalArgumentException e) { + throw e; + } catch (RuntimeException e) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + if (!estimate.isComplete()) { + rejectWeight(estimate.getIncompleteReason()); + return AdmissionResult.REJECTED; + } + + long estimatedPayloadBytes = estimate.getBytes(); + // A retained non-null key/value plus Caffeine node can never consume zero bytes. Treat a + // complete zero as an estimator contract violation so an omitted formula cannot bypass + // every quota and admit an unbounded number of zero-weight entries. + if (estimatedPayloadBytes == 0L) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + long newWeight = MetaCacheWeightUtils.saturatedAdd( + estimatedPayloadBytes, FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES); + synchronized (admissionLock) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + if (expectedMutation != null && !isKeyMutationCurrent(key, expectedMutation)) { + return AdmissionResult.NOT_CURRENT; + } + V oldValue = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (expectedReservationGeneration >= 0L + && (record == null || record.generation != expectedReservationGeneration)) { + return AdmissionResult.NOT_CURRENT; + } + if (requireExpected && oldValue != expectedCurrent) { + return AdmissionResult.NOT_CURRENT; + } + if (oldValue == null && record != null) { + // The previous generation was already removed by Caffeine; if that removal was + // an eviction whose asynchronous cleanup has not run yet, account it here. + if (pendingEvictionGenerations.remove(key, record.generation)) { + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } + reservations.remove(key, record); + record.reservation.release(); + record = null; + } + if (oldValue != null && (record == null || !record.published)) { + rejectWeight("missing_reservation"); + return AdmissionResult.REJECTED; + } + + if (record == null) { + Optional reservation = reserveWithLocalEviction(key, newWeight); + if (!reservation.isPresent()) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, reservation.get(), nextReservationGeneration()); + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + if (reservations.get(key) == newRecord && data.asMap().get(key) == value) { + newRecord.published = true; + notifyReplacement(key, null, value); + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + reservations.remove(key, newRecord); + newRecord.reservation.release(); + throw e; + } + } + + ReservationRecord previousRecord = record; + long reservedWeight = Math.max(previousRecord.weight, newWeight); + if (!resizeWithLocalEviction(key, previousRecord.reservation, reservedWeight)) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, previousRecord.reservation, nextReservationGeneration()); + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + boolean retained = reservations.get(key) == newRecord && data.asMap().get(key) == value; + if (retained) { + newRecord.published = true; + } + if (retained && reservedWeight != newWeight && !newRecord.reservation.tryResize(newWeight)) { + throw new IllegalStateException("failed to release cache replacement reservation delta"); + } + if (retained) { + notifyReplacement(key, oldValue, value); + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + if (reservations.replace(key, newRecord, previousRecord)) { + if (data.asMap().get(key) == null) { + reservations.remove(key, previousRecord); + previousRecord.reservation.release(); + } else if (!previousRecord.reservation.tryResize(previousRecord.weight)) { + throw new IllegalStateException("failed to roll back cache replacement reservation", e); + } + } + throw e; + } + } + } + + private Optional reserveWithLocalEviction(K incomingKey, long bytes) { + if (bytes > entryBudget.getEffectiveMaxWeight()) { + return Optional.empty(); + } + Optional reservation = entryBudget.tryReserve(bytes); + while (!reservation.isPresent()) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + entryBudget.requestPeerReclaim(bytes); + break; + } + reservation = entryBudget.tryReserve(bytes); + } + return reservation; + } + + private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation reservation, long newBytes) { + if (newBytes > entryBudget.getEffectiveMaxWeight()) { + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + while (true) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + entryBudget.requestPeerReclaim(Math.max(0L, newBytes - reservation.getBytes())); + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + } + } + + private int evictLocalColdest(K incomingKey, int limit) { + if (!data.policy().eviction().isPresent()) { + return 0; + } + Map coldest = data.policy().eviction().get().coldest(limit); + int evicted = 0; + for (Map.Entry candidate : coldest.entrySet()) { + if (Objects.equals(candidate.getKey(), incomingKey)) { + continue; + } + V current = data.asMap().get(candidate.getKey()); + ReservationRecord record = reservations.get(candidate.getKey()); + long evictedWeight = record != null && record.published && current != null ? record.weight : 0L; + if (current == candidate.getValue() && data.asMap().remove(candidate.getKey(), current)) { + if (record != null) { + releaseReservation(candidate.getKey(), record.generation); + } + localEvictionCount.incrementAndGet(); + localEvictionWeight.accumulateAndGet(evictedWeight, MetaCacheWeightUtils::saturatedAdd); + evicted++; + } + } + return evicted; + } + + private long reclaimForPeer(long targetBytes) { + if (targetBytes <= 0L || closed.get()) { + return 0L; + } + synchronized (admissionLock) { + long before = entryBudget.getUsedWeight(); + long reclaimed = 0L; + while (reclaimed < targetBytes + && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE) > 0) { + reclaimed = Math.max(0L, before - entryBudget.getUsedWeight()); + } + return reclaimed; + } + } + + private int weigh(K key, V value) { + ReservationRecord record = reservations.get(key); + // Every supported write path installs the reservation record before calling data.put. + // Missing ownership is an invariant violation, so fail closed without invoking an O(n) + // estimator from Caffeine's hot weigher callback. + long weight = record == null ? Integer.MAX_VALUE : record.weight; + return weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight; + } + + private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { + if (key == null) { + return; + } + if (!weightBounded && !generationFencedRefresh) { + return; + } + if (closed.get()) { + return; + } + // Replacement transfers the existing reservation to the newly published generation. A + // soft-value collection instead reports a null value with COLLECTED and must release it. + if (cause == RemovalCause.REPLACED) { + return; + } + if (Thread.holdsLock(admissionLock)) { + // Other removals have already removed the Caffeine mapping and can release their owner + // inline. A stale callback cannot release a replacement while its mapping is visible. + if (data.asMap().get(key) == null) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null) { + if (cause.wasEvicted()) { + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } + releaseReservation(key, record.generation); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null) { + releaseRefreshRecord(key, record.generation); + } + } + } + return; + } + beforeRemovalOwnerSnapshotForTest(key); + long ownerGeneration = currentOwnerGeneration(key); + if (ownerGeneration >= 0L) { + beforeRemovalReleaseForTest(key); + if (closed.get()) { + return; + } + if (cause.wasEvicted()) { + pendingEvictionGenerations.merge(key, ownerGeneration, Math::max); + } + pendingRemovalGenerations.merge(key, ownerGeneration, Math::max); + if (closed.get()) { + pendingRemovalGenerations.remove(key, ownerGeneration); + pendingEvictionGenerations.remove(key, ownerGeneration); + return; + } + scheduleRemovalCleanup(); + } + } + + private void scheduleRemovalCleanup() { + if (closed.get()) { + return; + } + if (removalCleanupScheduled.compareAndSet(false, true)) { + try { + REMOVAL_CLEANUP_EXECUTOR.execute(this::drainRemovalCleanups); + } catch (RejectedExecutionException e) { + removalCleanupScheduled.set(false); + LOG.warn("Failed to schedule removal cleanup for external metadata cache entry {}", name, e); + } + } + } + + private void drainRemovalCleanups() { + try { + int processed = 0; + for (Map.Entry cleanup : pendingRemovalGenerations.entrySet()) { + if (processed++ >= REMOVAL_CLEANUP_BATCH_SIZE) { + break; + } + K key = cleanup.getKey(); + long generation = cleanup.getValue(); + // Claim before cleanup. If the same generation is reported again while cleanup + // runs, its notification creates a new pending item instead of being lost when + // this worker finishes. + if (!pendingRemovalGenerations.remove(key, generation)) { + continue; + } + boolean evicted = pendingEvictionGenerations.remove(key, generation); + if (!evicted) { + // A newer generation superseded the evicted one; its eviction can no longer + // be attributed, so drop the stale marker instead of retaining the key. + pendingEvictionGenerations.remove(key); + } + try { + cleanupRemovedReservation(key, generation, evicted); + } catch (RuntimeException e) { + // Restore the generation for retry. The finally block requeues one bounded + // drain instead of permanently wedging this entry's scheduled flag. + pendingRemovalGenerations.merge(key, generation, Math::max); + if (evicted) { + pendingEvictionGenerations.merge(key, generation, Math::max); + } + LOG.warn("Failed to clean a removal reservation for external metadata cache entry {}", + name, e); + } + } + } finally { + removalCleanupScheduled.set(false); + if (!closed.get() && !pendingRemovalGenerations.isEmpty()) { + // One bounded task per turn prevents a hot entry from monopolizing the process-wide + // cleanup executor; a later task is queued behind already scheduled catalogs. + scheduleRemovalCleanup(); + } + } + } + + private void cleanupRemovedReservation(K key, long expectedReservationGeneration, boolean evicted) { + beforeRemovalCleanupLockForTest(key); + synchronized (admissionLock) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null + && reservations.remove(key, record)) { + if (evicted) { + // The exact reservation, not Caffeine's int-clamped weigher value. + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } + record.reservation.release(); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null) { + refreshRecords.remove(key, record); + } + } + } + afterRemovalCleanupForTest(key); + } + + private boolean releaseReservation(K key, long expectedGeneration) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedGeneration && reservations.remove(key, record)) { + record.reservation.release(); + return true; + } + return false; + } + + private void releaseRefreshRecord(K key, long expectedGeneration) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedGeneration) { + refreshRecords.remove(key, record); + } + } + + private long currentOwnerGeneration(K key) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + return record == null ? -1L : record.generation; + } + RefreshRecord record = refreshRecords.get(key); + return record == null ? -1L : record.generation; + } + + private void putNonWeightedValue(K key, V value) { + V previousValue = data.asMap().get(key); + RefreshRecord previous = refreshRecords.get(key); + RefreshRecord next = publishRefreshRecord(key); + try { + beforeNonWeightedCachePutForTest(key, value); + data.put(key, value); + if (next != null && refreshRecords.get(key) == next && data.asMap().get(key) == value) { + next.published = true; + } + if (data.asMap().get(key) == value) { + notifyReplacement(key, previousValue, value); + } + } catch (RuntimeException | Error e) { + if (next != null) { + if (previous == null) { + refreshRecords.remove(key, next); + } else { + refreshRecords.replace(key, next, previous); + } + } + throw e; + } + } + + @Nullable + private RefreshRecord publishRefreshRecord(K key) { + if (!generationFencedRefresh) { + return null; + } + RefreshRecord record = new RefreshRecord(nextReservationGeneration()); + refreshRecords.put(key, record); + return record; + } + + private void notifyReplacement(K key, @Nullable V previousValue, V currentValue) { + if (replacementListener == null || previousValue == currentValue) { + return; + } + try { + replacementListener.onReplacement(key, previousValue, currentValue); + } catch (RuntimeException e) { + LOG.warn("Failed to retire dependencies after replacing external metadata cache entry {}", name, e); + } + } + + private long nextReservationGeneration() { + return reservationGeneration.incrementAndGet(); + } + + private void rejectWeight(String reason) { + weightAdmissionRejectedCount.incrementAndGet(); + String normalizedReason = reason == null || reason.isEmpty() ? "unknown" : reason; + lastWeightRejectReason.set(normalizedReason); + long now = System.currentTimeMillis(); + long previous = lastWeightRejectLogTimeMs.get(); + if (now - previous >= WEIGHT_REJECT_LOG_INTERVAL_MS + && lastWeightRejectLogTimeMs.compareAndSet(previous, now)) { + LOG.warn("Rejected external metadata cache admission for entry {}: reason={}, entryUsed={}, " + + "entryMax={}, catalogUsed={}, catalogMax={}, globalUsed={}, globalMax={}", + name, normalizedReason, entryBudget.getUsedWeight(), entryBudget.getEffectiveMaxWeight(), + entryBudget.getCatalogUsedWeight(), entryBudget.getCatalogMaxWeight(), + entryBudget.getGlobalUsedWeight(), entryBudget.getGlobalMaxWeight()); + } + } + + private void maybeRefreshManagedValue(K key, V currentValue) { + if (closed.get() || !generationFencedRefresh || loader == null) { + return; + } + if (!weightBounded) { + maybeRefreshNonWeightedValue(key, currentValue); + return; + } + ReservationRecord record = reservations.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void maybeRefreshNonWeightedValue(K key, V currentValue) { + RefreshRecord record = refreshRecords.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void submitNonWeightedRefresh( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed == null) { + return; + } + synchronized (admissionLock) { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + advanceKeyMutation(key); + putNonWeightedValue(key, refreshed); + } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isRefreshRecordCurrent( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + RefreshRecord record = refreshRecords.get(key); + return record != null && record.generation == expectedRefreshGeneration + && record.published && data.asMap().get(key) != null; + } + + private void submitWeightedRefresh( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isReservationCurrent(key, expectedReservationGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { + admitWeightedValue( + key, refreshed, null, false, expectedMutation, + expectedReservationGeneration, true); + // Admission rejection leaves the already reserved, known-good generation + // in place. A larger refresh must not turn a transient quota shortage into + // a forced cache miss for every subsequent reader. + } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isReservationCurrent( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + ReservationRecord record = reservations.get(key); + return record != null && record.generation == expectedReservationGeneration + && record.published && data.asMap().get(key) != null; } // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { - return Config.enable_external_meta_cache_manual_miss_load; + return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; } // Execute slow miss loads outside Caffeine's sync load path and suppress stale write-back after invalidation. private V getWithManualLoad(K key, Function loadFunction) { - if (!effectiveEnabled) { - // Bypass cache entirely when the entry is disabled so manual miss load does not relax disable semantics. + if (!effectiveEnabled || closed.get()) { + // Disabled and closed entries may still serve the caller, but can not retain the loaded value. return loadAndTrack(key, loadFunction); } V value = data.getIfPresent(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } synchronized (loadLock(key)) { + if (!effectiveEnabled || closed.get()) { + return loadAndTrack(key, loadFunction); + } value = data.asMap().get(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } - long generation = invalidateGeneration.get(); - V loaded = loadAndTrack(key, loadFunction); - if (generation != invalidateGeneration.get()) { - return loaded; - } + KeyMutationToken mutation = beginKeyMutation(key); + try { + V loaded = loadAndTrack(key, loadFunction); + if (!isKeyMutationCurrent(key, mutation)) { + return loaded; + } - // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. - if (loaded == null) { - return null; - } + // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. + if (loaded == null) { + return null; + } - // Leave a narrow hook for tests to pause exactly before the cache put race window. - beforeManualCachePutForTest(key, loaded); - data.put(key, loaded); - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, loaded); + // Leave a narrow hook for tests to pause exactly before the cache put race window. + beforeManualCachePutForTest(key, loaded); + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + if (weightBounded) { + admitWeightedValue(key, loaded, null, false, mutation, -1L, false); + } else { + synchronized (admissionLock) { + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + beforeNonWeightedManualCachePutForTest(key, loaded); + putNonWeightedValue(key, loaded); + } + } + return loaded; + } finally { + endKeyMutation(key, mutation); } - return loaded; } } - // Remove only the value loaded by the current request and keep newer replacements intact. - private void removeLoadedValue(K key, V loaded) { - data.asMap().computeIfPresent(key, (ignored, currentValue) -> currentValue == loaded ? null : currentValue); - } - // Map keys to a fixed lock stripe set to bound memory usage while keeping same-key deduplication. private Object loadLock(K key) { int hash = key == null ? 0 : key.hashCode(); @@ -264,6 +1161,67 @@ private Object loadLock(K key) { void beforeManualCachePutForTest(K key, V loaded) { } + // Let tests pause after the final generation check while holding the admission lock. + void beforeNonWeightedManualCachePutForTest(K key, V loaded) { + } + + // Called inside Caffeine's direct removal callback; tests use it to force the eviction-lock race. + void beforeRemovalReleaseForTest(K key) { + } + + // Let tests pause a callback after Caffeine removal but before reservation-owner lookup. + void beforeRemovalOwnerSnapshotForTest(K key) { + } + + // Called after reservation ownership is published and before Caffeine receives the value. + void beforeWeightedCachePutForTest(K key, V value) { + } + + // Called after refresh ownership is published and before Caffeine receives a count-bounded value. + void beforeNonWeightedCachePutForTest(K key, V value) { + } + + // Called after one asynchronous generation-conditional removal cleanup has finished. + void afterRemovalCleanupForTest(K key) { + } + + // Called immediately before the asynchronous drain tries to acquire admissionLock. + void beforeRemovalCleanupLockForTest(K key) { + } + + // Let tests establish admissionLock -> Caffeine eviction-lock ordering deterministically. + void beforeWeightedInvalidateAllForTest() { + } + + // Invoke the direct-listener branch while holding the mutation lock. + void notifyRemovalUnderAdmissionLockForTest(K key, V value, RemovalCause cause) { + synchronized (admissionLock) { + onRemoval(key, value, cause); + } + } + + // Enqueue the same generation-only refresh task without waiting for the production interval. + void triggerRefreshForTest(K key) { + V current = data.asMap().get(key); + if (current == null || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.published) { + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } else if (generationFencedRefresh) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.published) { + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } + refreshesInFlight.remove(key); + } + private V loadFromDefaultLoader(K key) { return loadAndTrack(key, this::applyDefaultLoader); } @@ -294,4 +1252,100 @@ private V loadAndTrack(K key, Function loadFunction) { throw e; } } + + private KeyMutationToken beginKeyMutation(K key) { + synchronized (admissionLock) { + KeyMutationState state = keyMutationStates.computeIfAbsent(key, ignored -> new KeyMutationState()); + state.inFlight++; + return new KeyMutationToken(state, state.generation, fullInvalidationGeneration.get()); + } + } + + private void advanceKeyMutation(K key) { + // Callers already serialize cache mutation with admissionLock. Keeping the helper lock-free + // prevents accidental deadlock if it is used by a listener reached under that same lock. + KeyMutationState state = keyMutationStates.get(key); + if (state != null) { + state.generation++; + } + } + + private boolean isKeyMutationCurrent(K key, KeyMutationToken token) { + return token.fullInvalidationGeneration == fullInvalidationGeneration.get() + && token.state == keyMutationStates.get(key) + && token.generation == token.state.generation; + } + + private void endKeyMutation(K key, KeyMutationToken token) { + synchronized (admissionLock) { + if (--token.state.inFlight == 0) { + keyMutationStates.remove(key, token.state); + } + } + } + + private static final class ReservationRecord { + private final long weight; + private final long writeNanos; + private final AdmissionReservation reservation; + private final long generation; + private volatile boolean published; + + private ReservationRecord(long weight, AdmissionReservation reservation, long generation) { + this.weight = weight; + this.reservation = reservation; + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class RefreshRecord { + private final long writeNanos; + private final long generation; + private volatile boolean published; + + private RefreshRecord(long generation) { + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class KeyMutationState { + private volatile long generation; + private int inFlight; + } + + private static final class KeyMutationToken { + private final KeyMutationState state; + private final long generation; + private final long fullInvalidationGeneration; + + private KeyMutationToken( + KeyMutationState state, long generation, long fullInvalidationGeneration) { + this.state = state; + this.generation = generation; + this.fullInvalidationGeneration = fullInvalidationGeneration; + } + } + + private static ReplaceResult toReplaceResult(AdmissionResult result) { + switch (result) { + case ADMITTED: + return ReplaceResult.REPLACED; + case NOT_CURRENT: + return ReplaceResult.NOT_CURRENT; + case REJECTED: + return ReplaceResult.REJECTED; + case DISABLED: + default: + return ReplaceResult.DISABLED; + } + } + + private enum AdmissionResult { + ADMITTED, + NOT_CURRENT, + REJECTED, + DISABLED + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 1f48057a44fc40..689d3b6dc7ca99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -101,10 +101,15 @@ public final class MetaCacheEntryDef { private final boolean autoRefresh; private final boolean contextualOnly; private final MetaCacheEntryInvalidation invalidation; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -123,6 +128,8 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.autoRefresh = autoRefresh; this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); + this.sizeEstimator = sizeEstimator; + this.replacementListener = replacementListener; } /** @@ -142,7 +149,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation); + invalidation, null, null); } /** @@ -164,7 +171,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation); + invalidation, null, null); } /** @@ -179,7 +186,22 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation); + invalidation, null, null); + } + + /** Return a definition with a publication-time size estimator. */ + public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, + Objects.requireNonNull(estimator, "estimator"), replacementListener); + } + + /** Return a definition that synchronously retires dependencies after a value replacement. */ + public MetaCacheEntryDef withReplacementListener( + MetaCacheEntryReplacementListener listener) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, + Objects.requireNonNull(listener, "listener")); } /** @@ -232,4 +254,14 @@ public boolean isContextualOnly() { public MetaCacheEntryInvalidation getInvalidation() { return invalidation; } + + @Nullable + public MetaCacheSizeEstimator getSizeEstimator() { + return sizeEstimator; + } + + @Nullable + public MetaCacheEntryReplacementListener getReplacementListener() { + return replacementListener; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java new file mode 100644 index 00000000000000..1bfb0cf3990962 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import javax.annotation.Nullable; + +/** Receives a successfully published value while the entry mutation is still serialized. */ +@FunctionalInterface +public interface MetaCacheEntryReplacementListener { + void onReplacement(K key, @Nullable V previousValue, V currentValue); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 495fd011083bb0..433cc027515873 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -51,6 +51,16 @@ public final class MetaCacheEntryStats { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final boolean weightBounded; + private final long maxWeight; + private final long estimatedWeight; + private final long evictionWeight; + private final long weightAdmissionRejectedCount; + private final long catalogMaxWeight; + private final long catalogEstimatedWeight; + private final long globalMaxWeight; + private final long globalEstimatedWeight; + private final String lastWeightRejectReason; /** * Build an immutable stats snapshot. @@ -74,7 +84,17 @@ public MetaCacheEntryStats( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + boolean weightBounded, + long maxWeight, + long estimatedWeight, + long evictionWeight, + long weightAdmissionRejectedCount, + long catalogMaxWeight, + long catalogEstimatedWeight, + long globalMaxWeight, + long globalEstimatedWeight, + String lastWeightRejectReason) { this.configEnabled = configEnabled; this.effectiveEnabled = effectiveEnabled; this.autoRefresh = autoRefresh; @@ -94,6 +114,16 @@ public MetaCacheEntryStats( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = Objects.requireNonNull(lastError, "lastError"); + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.evictionWeight = evictionWeight; + this.weightAdmissionRejectedCount = weightAdmissionRejectedCount; + this.catalogMaxWeight = catalogMaxWeight; + this.catalogEstimatedWeight = catalogEstimatedWeight; + this.globalMaxWeight = globalMaxWeight; + this.globalEstimatedWeight = globalEstimatedWeight; + this.lastWeightRejectReason = Objects.requireNonNull(lastWeightRejectReason, "lastWeightRejectReason"); } public boolean isConfigEnabled() { @@ -186,4 +216,44 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public boolean isWeightBounded() { + return weightBounded; + } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getEvictionWeight() { + return evictionWeight; + } + + public long getWeightAdmissionRejectedCount() { + return weightAdmissionRejectedCount; + } + + public long getCatalogMaxWeight() { + return catalogMaxWeight; + } + + public long getCatalogEstimatedWeight() { + return catalogEstimatedWeight; + } + + public long getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalEstimatedWeight() { + return globalEstimatedWeight; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java new file mode 100644 index 00000000000000..d44702ce559212 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; + +/** + * Immutable result value returned by {@link MetaCacheSizeEstimator}; this class is not an + * estimator implementation. An incomplete result carries no usable byte count and must fail + * cache admission closed. + */ +public final class MetaCacheSizeEstimate { + private final long bytes; + private final boolean complete; + private final String incompleteReason; + + private MetaCacheSizeEstimate(long bytes, boolean complete, String incompleteReason) { + this.bytes = bytes; + this.complete = complete; + this.incompleteReason = Objects.requireNonNull(incompleteReason, "incompleteReason"); + } + + public static MetaCacheSizeEstimate complete(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache size estimate can not be negative: " + bytes); + } + return new MetaCacheSizeEstimate(bytes, true, ""); + } + + public static MetaCacheSizeEstimate incomplete(String reason) { + String safeReason = Objects.requireNonNull(reason, "reason").trim(); + if (safeReason.isEmpty()) { + throw new IllegalArgumentException("incomplete cache size estimate requires a reason"); + } + return new MetaCacheSizeEstimate(0L, false, safeReason); + } + + public long getBytes() { + return bytes; + } + + public boolean isComplete() { + return complete; + } + + public String getIncompleteReason() { + return incompleteReason; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..e74c6e631b69c1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Supplies the admission weight of one key/value pair. + * + *

    The callback runs once after load and before admission. Implementations may linearly count + * loader-owned collections needed to cover skewed payloads, but must not recursively reflect over + * arbitrary object graphs, perform additional IO, materialize lazy SDK state, or copy payloads + * solely to estimate weight. Caffeine's weigher reads only the admitted reservation record, so + * cache hits and eviction remain O(1). + */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + MetaCacheSizeEstimate estimate(K key, V value); + + /** Convert preparation failures into fail-closed incomplete estimates. */ + static MetaCacheSizeEstimate estimateSafely( + String failureReason, Supplier estimation) { + Objects.requireNonNull(failureReason, "failureReason"); + Objects.requireNonNull(estimation, "estimation"); + try { + return Objects.requireNonNull(estimation.get(), "size estimate"); + } catch (RuntimeException | LinkageError e) { + // A missing or incompatible SDK class must reject weighted admission, not the load. + return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java new file mode 100644 index 00000000000000..8e32a989588677 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -0,0 +1,368 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.NameMapping; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.management.ManagementFactory; +import java.lang.management.PlatformManagedObject; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Overflow-safe helpers for conservative external metadata cache weights. */ +public final class MetaCacheWeightUtils { + private static final long NAME_MAPPING_BASE_BYTES = 64L; + private static final MethodHandle STRING_VALUE_GETTER; + private static final long STRING_VALUE_OFFSET; + private static final int OBJECT_ALIGNMENT_BYTES; + private static final int OBJECT_REFERENCE_BYTES; + private static final int OBJECT_HEADER_BYTES; + private static final int OBJECT_ARRAY_BASE_BYTES; + private static final int BYTE_ARRAY_BASE_BYTES; + private static final int CHAR_ARRAY_BASE_BYTES; + private static final int INT_ARRAY_BASE_BYTES; + private static final boolean SUPPORTED_OBJECT_LAYOUT; + private static final long OBJECT_LAYOUT_PERCENT; + + static { + MethodHandle stringValueGetter = null; + long stringValueOffset = -1L; + int referenceBytes = Long.BYTES; + // 24B is the safe fallback for an uncompressed class pointer. Unsafe replaces these + // values with the exact active-VM layout when access is available. + int objectArrayBaseBytes = 24; + int byteArrayBaseBytes = 24; + int charArrayBaseBytes = 24; + int intArrayBaseBytes = 24; + try { + Class unsafeClass = Class.forName("sun.misc.Unsafe"); + Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + Object unsafe = unsafeField.get(null); + stringValueOffset = (long) unsafeClass + .getMethod("objectFieldOffset", Field.class) + .invoke(unsafe, String.class.getDeclaredField("value")); + stringValueGetter = MethodHandles.lookup() + .unreflect(unsafeClass.getMethod("getObject", Object.class, long.class)) + .bindTo(unsafe) + .asType(MethodType.methodType(Object.class, Object.class, long.class)); + referenceBytes = (int) unsafeClass + .getMethod("arrayIndexScale", Class.class) + .invoke(unsafe, Object[].class); + objectArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, Object[].class); + byteArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, byte[].class); + charArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, char[].class); + intArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, int[].class); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // A conservative UTF-16 fallback is used when the VM hides String storage. + } + STRING_VALUE_GETTER = stringValueGetter; + STRING_VALUE_OFFSET = stringValueOffset; + OBJECT_REFERENCE_BYTES = referenceBytes; + OBJECT_ARRAY_BASE_BYTES = objectArrayBaseBytes; + BYTE_ARRAY_BASE_BYTES = byteArrayBaseBytes; + CHAR_ARRAY_BASE_BYTES = charArrayBaseBytes; + INT_ARRAY_BASE_BYTES = intArrayBaseBytes; + String alignmentOption = readVmOption("ObjectAlignmentInBytes"); + int objectAlignmentBytes = parseObjectAlignment(alignmentOption); + OBJECT_ALIGNMENT_BYTES = objectAlignmentBytes; + SUPPORTED_OBJECT_LAYOUT = alignmentOption != null + && (objectAlignmentBytes == 8 || objectAlignmentBytes == 16); + boolean compressedClassPointers = readBooleanVmOption( + "UseCompressedClassPointers", false); + OBJECT_HEADER_BYTES = Long.BYTES + + (compressedClassPointers ? Integer.BYTES : Long.BYTES); + long referencePercent = referenceBytes <= Integer.BYTES ? 100L : 145L; + long classPointerPercent = compressedClassPointers ? 100L : 140L; + long alignmentPercent = objectAlignmentBytes <= 8 ? 100L : 120L; + OBJECT_LAYOUT_PERCENT = (referencePercent * classPointerPercent * alignmentPercent + + 9_999L) / 10_000L; + } + + private MetaCacheWeightUtils() { + } + + public static long estimatedStringBytes(String value) { + if (value == null) { + return 0L; + } + Object storage = stringStorage(value); + long backingArrayBytes; + if (storage instanceof byte[]) { + backingArrayBytes = estimatedByteArrayBytes(((byte[]) storage).length); + } else if (storage instanceof char[]) { + backingArrayBytes = alignedArrayBytes( + CHAR_ARRAY_BASE_BYTES, ((char[]) storage).length, Character.BYTES); + } else { + backingArrayBytes = alignedArrayBytes( + CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES); + } + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), backingArrayBytes); + } + + /** Estimate retained character data without materializing a String copy. */ + public static long estimatedCharSequenceBytes(CharSequence value) { + if (value == null) { + return 0L; + } + if (value instanceof String) { + return estimatedStringBytes((String) value); + } + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), + alignedArrayBytes(CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES)); + } + + /** Whether calibrated formulas support the active VM object alignment. */ + public static boolean isSupportedJvmObjectLayout() { + return SUPPORTED_OBJECT_LAYOUT; + } + + /** Adjust a default compressed-reference object-graph constant to the active VM layout. */ + public static long estimatedObjectBytes(long compressedReferenceBytes) { + long product = saturatedMultiply(compressedReferenceBytes, OBJECT_LAYOUT_PERCENT); + if (product == Long.MAX_VALUE) { + return product; + } + long roundedProduct = saturatedAdd(product, 99L); + return roundedProduct == Long.MAX_VALUE ? roundedProduct : roundedProduct / 100L; + } + + /** Returns the actual backing-array payload in O(1), or a conservative UTF-16 fallback. */ + public static long estimatedStringPayloadBytes(String value) { + if (value == null) { + return 0L; + } + Object storage = stringStorage(value); + if (storage instanceof byte[]) { + return alignPayload(((byte[]) storage).length); + } + if (storage instanceof char[]) { + return alignPayload(saturatedMultiply( + ((char[]) storage).length, Character.BYTES)); + } + return alignPayload(saturatedMultiply(value.length(), Character.BYTES)); + } + + /** Estimate a generated String whose encoded width is derived from its source components. */ + public static long estimatedGeneratedStringBytes(long characterCount, boolean latin1) { + long payloadBytes = saturatedMultiply(characterCount, latin1 ? 1L : Character.BYTES); + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), + estimatedByteArrayBytes(payloadBytes)); + } + + /** Whether this VM stores the String with one byte per character. */ + public static boolean isLatin1String(String value) { + if (value == null || value.isEmpty()) { + return true; + } + Object storage = stringStorage(value); + return storage instanceof byte[] && ((byte[]) storage).length == value.length(); + } + + /** VM-layout size of a retained byte array, conservatively if VM introspection is hidden. */ + public static long estimatedByteArrayBytes(long length) { + return alignedArrayBytes(BYTE_ARRAY_BASE_BYTES, length, Byte.BYTES); + } + + /** VM-layout size of an object-reference array, conservatively if introspection is hidden. */ + public static long estimatedObjectArrayBytes(long length) { + return alignedArrayBytes(OBJECT_ARRAY_BASE_BYTES, length, OBJECT_REFERENCE_BYTES); + } + + /** + * A java.util.HashMap holding {@code entries} mappings: the map object, its power-of-two + * table (allocated on the first put) and one node per entry; keys and values are separate. + */ + public static long estimatedHashMapBytes(long entries) { + long bytes = estimatedObjectLayoutBytes(4L, 16L); + if (entries <= 0L) { + return bytes; + } + long capacity = 16L; + while (entries > capacity - capacity / 4L) { + capacity = saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + break; + } + } + bytes = saturatedAdd(bytes, estimatedObjectArrayBytes(capacity)); + return saturatedAdd(bytes, saturatedMultiply(entries, estimatedObjectLayoutBytes(3L, 4L))); + } + + /** Size of an object with a known field layout on the active VM. */ + public static long estimatedObjectLayoutBytes(long referenceFields, long primitiveBytes) { + if (referenceFields < 0L || primitiveBytes < 0L) { + return Long.MAX_VALUE; + } + long bytes = saturatedAdd( + OBJECT_HEADER_BYTES, + saturatedMultiply(referenceFields, OBJECT_REFERENCE_BYTES)); + return alignPayload(saturatedAdd(bytes, primitiveBytes)); + } + + /** VM-layout size of a retained int array, conservatively if introspection is hidden. */ + public static long estimatedIntArrayBytes(long length) { + return alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); + } + + /** Incremental VM-layout payload of an int array whose header is accounted elsewhere. */ + public static long estimatedIntArrayPayloadBytes(long length) { + long populated = alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); + long empty = alignPayload(INT_ARRAY_BASE_BYTES); + return populated == Long.MAX_VALUE ? populated : populated - empty; + } + + /** Estimate the fixed set of names retained by a cache key. */ + public static long estimatedNameMappingBytes(NameMapping nameMapping) { + if (nameMapping == null) { + return 0L; + } + long bytes = estimatedObjectBytes(NAME_MAPPING_BASE_BYTES); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); + return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); + } + + /** + * Whether {@code type} itself declares exactly the expected non-static instance fields, each + * written as {@code name:SimpleTypeName}. Estimator formulas are calibrated against pinned SDK + * layouts; callers fail closed when a library upgrade adds, removes or retypes a field so a + * new retained reference cannot be silently undercounted. Superclasses are pinned separately. + */ + public static boolean hasExpectedInstanceFields(Class type, String... expectedFields) { + if (type == null) { + return false; + } + Set expected = new HashSet<>(Arrays.asList(expectedFields)); + Set actual = new HashSet<>(); + try { + for (Field field : type.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) || field.isSynthetic()) { + continue; + } + actual.add(field.getName() + ":" + field.getType().getSimpleName()); + } + } catch (RuntimeException | LinkageError e) { + return false; + } + return actual.equals(expected); + } + + /** Same as {@link #hasExpectedInstanceFields(Class, String...)} for a class resolved by name. */ + public static boolean hasExpectedInstanceFields( + String className, ClassLoader loader, String... expectedFields) { + try { + return hasExpectedInstanceFields( + Class.forName(className, false, loader), expectedFields); + } catch (ReflectiveOperationException | RuntimeException | LinkageError e) { + return false; + } + } + + public static long saturatedAdd(long left, long right) { + if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left < 0L || right < 0L || (left != 0L && right > Long.MAX_VALUE / left)) { + return Long.MAX_VALUE; + } + return left * right; + } + + private static boolean readBooleanVmOption(String option, boolean fallback) { + String value = readVmOption(option); + return value == null ? fallback : Boolean.parseBoolean(value); + } + + private static String readVmOption(String option) { + try { + @SuppressWarnings("unchecked") + Class beanClass = + (Class) + Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); + Object bean = ManagementFactory.getPlatformMXBean(beanClass); + Method getVmOption = beanClass.getMethod("getVMOption", String.class); + Object vmOption = getVmOption.invoke(bean, option); + Method getValue = vmOption.getClass().getMethod("getValue"); + return (String) getValue.invoke(vmOption); + } catch (ReflectiveOperationException | RuntimeException ignored) { + return null; + } + } + + private static long alignPayload(long bytes) { + if (bytes == Long.MAX_VALUE) { + return bytes; + } + long remainder = bytes % OBJECT_ALIGNMENT_BYTES; + return remainder == 0L ? bytes + : saturatedAdd(bytes, OBJECT_ALIGNMENT_BYTES - remainder); + } + + private static long alignedArrayBytes(long baseBytes, long length, long elementBytes) { + if (length < 0L) { + return Long.MAX_VALUE; + } + return alignPayload(saturatedAdd( + baseBytes, saturatedMultiply(length, elementBytes))); + } + + private static Object stringStorage(String value) { + if (STRING_VALUE_GETTER != null && STRING_VALUE_OFFSET >= 0L) { + try { + return (Object) STRING_VALUE_GETTER.invokeExact( + (Object) value, STRING_VALUE_OFFSET); + } catch (Throwable ignored) { + // Return null so callers use the conservative UTF-16 fallback. + } + } + return null; + } + + private static int parseObjectAlignment(String value) { + if (value == null) { + return 16; + } + try { + int alignment = Integer.parseInt(value); + return alignment > 0 ? alignment : 16; + } catch (NumberFormatException ignored) { + return 16; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java index b6f36b803b24cc..d3088942ace261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java @@ -44,7 +44,8 @@ public final class PaimonLatestSnapshotProjectionLoader { @FunctionalInterface public interface SchemaValueLoader { - PaimonSchemaCacheValue load(NameMapping nameMapping, long schemaId); + PaimonSchemaCacheValue load( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable); } private final PaimonPartitionInfoLoader partitionInfoLoader; @@ -59,7 +60,8 @@ public PaimonLatestSnapshotProjectionLoader(PaimonPartitionInfoLoader partitionI public PaimonSnapshotCacheValue load(NameMapping nameMapping, Table paimonTable) { try { PaimonSnapshot latestSnapshot = resolveLatestSnapshot(paimonTable, true); - List partitionColumns = schemaValueLoader.load(nameMapping, latestSnapshot.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, latestSnapshot.getSchemaId(), 0L, latestSnapshot.getTable()) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, latestSnapshot.getTable(), partitionColumns); @@ -85,11 +87,21 @@ public PaimonSnapshotCacheValue loadFence(NameMapping nameMapping, Table paimonT } public PaimonSnapshotCacheValue loadAtFence(NameMapping nameMapping, PaimonSnapshot fence) { - return loadEffectiveAtFence(nameMapping, fence.getTable(), fence); + return loadAtFence(nameMapping, fence, 0L); + } + + public PaimonSnapshotCacheValue loadAtFence( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return loadEffectiveAtFence(nameMapping, fence.getTable(), fence, tableGeneration); } public PaimonSnapshotCacheValue loadEffectiveAtFence( NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence) { + return loadEffectiveAtFence(nameMapping, effectiveTable, fence, 0L); + } + + public PaimonSnapshotCacheValue loadEffectiveAtFence( + NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence, long tableGeneration) { try { // The fence owns both version and table generation. Reopening the catalog here can pair // the old snapshot id with a newer schema or branch after invalidation. @@ -102,12 +114,14 @@ public PaimonSnapshotCacheValue loadEffectiveAtFence( latestSchemaTable.copyWithoutTimeTravel( PaimonScanParams.isolateSnapshotRead(fence.getSnapshotId()))); } - List partitionColumns = schemaValueLoader.load(nameMapping, fence.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, fence.getSchemaId(), tableGeneration, effectiveTable) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, snapshotTable, partitionColumns); return new PaimonSnapshotCacheValue(partitionInfo, - new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable)); + new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable), + false, tableGeneration); } catch (Exception e) { throw new CacheException("failed to load paimon snapshot at fence %s.%s.%s: %s", e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java index 0a134cfd7d7d32..fc9fbeaa755752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java @@ -25,6 +25,7 @@ import org.apache.paimon.table.Table; import java.io.IOException; +import java.util.concurrent.Callable; /** * Loads the base Paimon table handle used by cache entries and runtime projections. @@ -45,4 +46,14 @@ public PaimonExternalCatalog catalog(NameMapping nameMapping) throws IOException return (PaimonExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalogOrException(nameMapping.getCtlId(), id -> new IOException("Catalog not found: " + id)); } + + public T executeAuthenticated(NameMapping nameMapping, Callable task) { + try { + return catalog(nameMapping).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new CacheException("failed to load authenticated paimon metadata %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), + e.getMessage()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java new file mode 100644 index 00000000000000..4011e6a797b77e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -0,0 +1,592 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableMap; +import org.apache.paimon.privilege.PrivilegedFileStoreTable; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobType; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.SmallIntType; +import org.apache.paimon.types.TimeType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; +import org.apache.paimon.types.VarBinaryType; +import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VariantType; +import org.apache.paimon.types.VectorType; + +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** Publication-time retained-weight formulas for Paimon table handles and snapshot projections. */ +final class PaimonCacheSizeEstimator { + // Calibrated against JOL retained-graph deltas in PaimonExternalMetaCacheTest. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long SNAPSHOT_BASE_BYTES = objectBytes(4L * 1024L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // PaimonTableCacheValue: table ref, generation, payload bytes, estimate ref (+ estimate object). + private static final long TABLE_VALUE_BASE_BYTES = objectBytes(96L); + // A top-level DataField, its list slot and shared per-field overhead; the DataType instance + // is accounted separately by addTypePayload. + private static final long TABLE_FIELD_BYTES = objectBytes(40L); + private static final long TABLE_OPTION_BYTES = objectBytes(44L); + private static final long TABLE_KEY_BYTES = objectBytes(128L); + // Exact Paimon 1.4.2 layouts, pinned by PAIMON_TYPE_LAYOUT_SUPPORTED. + private static final long DATA_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 4L); + private static final long ARRAY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); + private static final long VECTOR_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 5L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 1L); + private static final long MULTISET_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); + // RowType plus Collections.unmodifiableList(new ArrayList<>(fields)). + private static final long ROW_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 1L); + private static final long UNMODIFIABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final int ROW_TYPE_LAZY_MAP_COUNT = 4; + // Accepted leaf DataType implementations and the int fields each adds to DataType's nullable + // flag and type root. Any other class, including a future or third-party type, rejects + // weighted admission instead of being counted as an arbitrary primitive. + private static final String[] NO_LEAF_FIELDS = {}; + private static final String[] LENGTH_LEAF_FIELDS = {"length:int"}; + private static final String[] PRECISION_LEAF_FIELDS = {"precision:int"}; + private static final Map, String[]> LEAF_TYPE_FIELDS = + ImmutableMap., String[]>builder() + .put(CharType.class, LENGTH_LEAF_FIELDS) + .put(VarCharType.class, LENGTH_LEAF_FIELDS) + .put(BooleanType.class, NO_LEAF_FIELDS) + .put(BinaryType.class, LENGTH_LEAF_FIELDS) + .put(VarBinaryType.class, LENGTH_LEAF_FIELDS) + .put(DecimalType.class, new String[] {"precision:int", "scale:int"}) + .put(TinyIntType.class, NO_LEAF_FIELDS) + .put(SmallIntType.class, NO_LEAF_FIELDS) + .put(IntType.class, NO_LEAF_FIELDS) + .put(BigIntType.class, NO_LEAF_FIELDS) + .put(FloatType.class, NO_LEAF_FIELDS) + .put(DoubleType.class, NO_LEAF_FIELDS) + .put(DateType.class, NO_LEAF_FIELDS) + .put(TimeType.class, PRECISION_LEAF_FIELDS) + .put(TimestampType.class, PRECISION_LEAF_FIELDS) + .put(LocalZonedTimestampType.class, PRECISION_LEAF_FIELDS) + .put(VariantType.class, NO_LEAF_FIELDS) + .put(BlobType.class, NO_LEAF_FIELDS) + .build(); + private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED = checkPaimonTypeLayout(); + private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); + // One Paimon Partition record with its single-column LinkedHashMap spec plus map entry; extra + // columns are charged by PaimonPartitionInfo. + // FileStoreTable.lazyStore: the store object, its CoreOptions/Options copy, SchemaManager, + // and the partition/bucket-key/row/key/value RowTypes it derives from the TableSchema. It is + // created by the partition projection before publication or by scan planning afterwards. + private static final long STORE_BASE_BYTES = objectBytes(1_536L); + private static final long STORE_OPTION_BYTES = objectBytes(48L); + // KeyValueFileStore keeps prefixed key-field copies and shares the value fields; the + // AppendOnlyFileStore deep copy of the whole type tree is reserved by + // retainedTablePayloadBytes, which already walks that tree. + private static final long STORE_KEY_FIELD_BYTES = objectBytes(112L); + private static final long STORE_LIST_SLOT_BYTES = objectBytes(8L); + private static final String APPEND_ONLY_TABLE_CLASS_NAME = + "org.apache.paimon.table.AppendOnlyFileStoreTable"; + private static final String PRIMARY_KEY_TABLE_CLASS_NAME = + "org.apache.paimon.table.PrimaryKeyFileStoreTable"; + private static final String MERGE_ENGINE_OPTION = "merge-engine"; + private static final long PARTITION_BYTES = objectBytes(272L); + private static final long PARTITION_ITEM_BYTES = objectBytes(640L); + private static final long WRAPPER_BYTES = objectBytes(512L); + + private PaimonCacheSizeEstimator() { + } + + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + + /** DataType: typeRoot reference plus the isNullable flag, then the subclass int fields. */ + private static long leafTypeBytes(String[] intFields) { + return MetaCacheWeightUtils.estimatedObjectLayoutBytes( + 1L, 1L + (long) Integer.BYTES * intFields.length); + } + + /** Pin the Paimon 1.4.2 DataType/DataField/RowType layouts the formulas above are built on. */ + private static boolean checkPaimonTypeLayout() { + boolean supported = MetaCacheWeightUtils.hasExpectedInstanceFields( + DataType.class, "isNullable:boolean", "typeRoot:DataTypeRoot") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + DataField.class, "id:int", "name:String", "type:DataType", + "description:String", "defaultValue:String") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + RowType.class, "fields:List", "laziedNameToField:Map", + "laziedNameToIndex:Map", "laziedFieldIdToField:Map", + "laziedFieldIdToIndex:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + ArrayType.class, "elementType:DataType") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + VectorType.class, "elementType:DataType", "length:int") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + MapType.class, "keyType:DataType", "valueType:DataType") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + MultisetType.class, "elementType:DataType"); + for (Map.Entry, String[]> leaf : LEAF_TYPE_FIELDS.entrySet()) { + supported &= MetaCacheWeightUtils.hasExpectedInstanceFields( + leaf.getKey(), leaf.getValue()); + } + return supported; + } + + /** Pin TableSchema and the two accepted FileStoreTable implementations. */ + private static boolean checkPaimonTableLayout() { + ClassLoader loader = FileStoreTable.class.getClassLoader(); + String[] abstractTableFields = { + "fileIO:FileIO", "path:Path", "tableSchema:TableSchema", + "catalogEnvironment:CatalogEnvironment", "manifestCache:SegmentsCache", + "snapshotCache:Cache", "statsCache:Cache", "dvmetaCache:DVMetaCache"}; + return MetaCacheWeightUtils.hasExpectedInstanceFields( + TableSchema.class, "version:int", "id:long", "fields:List", + "highestFieldId:int", "partitionKeys:List", "primaryKeys:List", + "bucketKeys:List", "numBucket:int", "options:Map", "comment:String", + "timeMillis:long") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.AbstractFileStoreTable", loader, + abstractTableFields) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.AppendOnlyFileStoreTable", loader, + "lazyStore:AppendOnlyFileStore") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.PrimaryKeyFileStoreTable", loader, + "lazyStore:KeyValueFileStore"); + } + + /** + * Retained weight of the base table entry. The table handle is owned independently of the + * snapshot projections that reference it (they may pin an older generation), so the same + * table graph is charged to both owners rather than shared. + */ + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, PaimonTableCacheValue value) { + String unsupported = unsupportedReason(value.getPaimonTable()); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(value.getPaimonTable()))); + } + + private static String unsupportedReason(Table table) { + if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { + return "unsupported_jvm_object_alignment"; + } + if (!PAIMON_TYPE_LAYOUT_SUPPORTED || !PAIMON_TABLE_LAYOUT_SUPPORTED) { + return "unsupported_paimon_layout"; + } + if (!isSupportedTable(table)) { + return "unsupported_paimon_table:" + (table == null ? "null" : table.getClass().getName()); + } + return null; + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + String unsupported = unsupportedReason(table); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getPartitionInfo().getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); + } + + private static boolean isSupportedTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return isSupportedTable(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return isSupportedTable(fallback.wrapped()) && isSupportedTable(fallback.other()); + } + if (!(table instanceof FileStoreTable)) { + return false; + } + String className = table.getClass().getName(); + return APPEND_ONLY_TABLE_CLASS_NAME.equals(className) + || PRIMARY_KEY_TABLE_CLASS_NAME.equals(className); + } + + /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ + private static long estimateTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, + estimateTable(((PrivilegedFileStoreTable) table).wrapped())); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + long bytes = MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, estimateTable(fallback.wrapped())); + return MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(fallback.other())); + } + + FileStoreTable fileStoreTable = (FileStoreTable) table; + TableSchema schema = fileStoreTable.schema(); + long bytes = TABLE_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(fileStoreTable.location().toString())); + bytes = addCount(bytes, schema.fields().size(), TABLE_FIELD_BYTES); + bytes = addCount(bytes, schema.options().size(), TABLE_OPTION_BYTES); + bytes = addCount(bytes, schema.partitionKeys().size(), TABLE_KEY_BYTES); + bytes = addCount(bytes, schema.primaryKeys().size(), TABLE_KEY_BYTES); + bytes = addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); + return MetaCacheWeightUtils.saturatedAdd(bytes, storeGraphBytes(fileStoreTable, schema)); + } + + /** + * Reserve the store graph the table materializes without opening it: TableSchema + * cardinalities decide its size, and every RowType it derives can grow the four lazy lookup + * maps after admission exactly like nested RowTypes. + */ + private static long storeGraphBytes(FileStoreTable table, TableSchema schema) { + long bytes = STORE_BASE_BYTES; + bytes = addCount(bytes, schema.options().size(), STORE_OPTION_BYTES); + List fields = schema.fields(); + long fieldCount = fields.size(); + long uncachedFieldIds = 0L; + for (DataField field : fields) { + if (isUncachedInteger(field.id())) { + uncachedFieldIds++; + } + } + long partitionKeys = schema.partitionKeys().size(); + long bucketKeys = schema.bucketKeys().size(); + // Partition and bucket key RowTypes reference a subset of the fields; ids beyond the + // Integer cache are counted as if all of them were uncached, which is conservative. + long uncachedPartitionKeys = Math.min(partitionKeys, uncachedFieldIds); + long uncachedBucketKeys = Math.min(bucketKeys, uncachedFieldIds); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(partitionKeys, uncachedPartitionKeys)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(bucketKeys, uncachedBucketKeys)); + if (PRIMARY_KEY_TABLE_CLASS_NAME.equals(table.getClass().getName())) { + long primaryKeys = schema.primaryKeys().size(); + bytes = addCount(bytes, primaryKeys, STORE_KEY_FIELD_BYTES); + for (String primaryKey : schema.primaryKeys()) { + // Each trimmed key field gets a fresh "_KEY_" + name string. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(primaryKey)); + } + bytes = addCount(bytes, fieldCount, STORE_LIST_SLOT_BYTES); + // Key fields are re-numbered above the Integer cache; the value type shares fields. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(primaryKeys, primaryKeys)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); + if (retainsMergeFunctionRowType(schema)) { + // partial-update / aggregation merge factories keep a second logical RowType and + // option-derived per-field maps (aggregation also copies CoreOptions). + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); + bytes = addCount(bytes, schema.options().size(), STORE_OPTION_BYTES); + } + return bytes; + } + // The copied row type of an append-only store (fields, types and nested lookup maps) is + // reserved by retainedTablePayloadBytes; the top-level RowType wrapper is charged here. + return MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); + } + + private static boolean retainsMergeFunctionRowType(TableSchema schema) { + String mergeEngine = schema.options().get(MERGE_ENGINE_OPTION); + if (mergeEngine == null) { + return false; + } + String normalized = mergeEngine.trim().toLowerCase(Locale.ROOT).replace('_', '-'); + return "partial-update".equals(normalized) || "aggregation".equals(normalized); + } + + private static boolean isUncachedInteger(int value) { + return value < -128 || value > 127; + } + + /** + * Captures skew-sensitive schema text once when the snapshot cache value is constructed. + * All collections are already materialized in TableSchema; this never opens the table store. + */ + static long retainedTablePayloadBytes(Table table) { + return retainedTablePayloadBytes( + table, new AccountingBudget(MAX_TABLE_ACCOUNTING_ELEMENTS)); + } + + private static long retainedTablePayloadBytes(Table table, AccountingBudget budget) { + budget.charge(1L); + if (table instanceof PrivilegedFileStoreTable) { + return retainedTablePayloadBytes( + ((PrivilegedFileStoreTable) table).wrapped(), budget); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return MetaCacheWeightUtils.saturatedAdd( + retainedTablePayloadBytes(fallback.wrapped(), budget), + retainedTablePayloadBytes(fallback.other(), budget)); + } + if (!(table instanceof FileStoreTable)) { + return 0L; + } + + TableSchema schema = ((FileStoreTable) table).schema(); + if (schema == null) { + return 0L; + } + long bytes = addString(0L, schema.comment()); + TypeTreeStructure structure = new TypeTreeStructure(); + for (DataField field : schema.fields()) { + bytes = addFieldPayload(bytes, field, false, budget, 0, structure); + } + if (isAppendOnlyTable(table)) { + // AppendOnlyFileStore keeps logicalRowType().notNull(): a deep copy of every field + // and type (names and descriptions are shared), including nested RowTypes with their + // own lazy lookup maps. Reserve that copy without creating the store. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structure.bytes); + } + budget.charge(schema.options().size()); + for (Map.Entry option : schema.options().entrySet()) { + bytes = addString(bytes, option.getKey()); + bytes = addString(bytes, option.getValue()); + } + bytes = addStrings(bytes, schema.partitionKeys(), budget); + bytes = addStrings(bytes, schema.primaryKeys(), budget); + return addStrings(bytes, schema.bucketKeys(), budget); + } + + private static long addStrings( + long bytes, List values, AccountingBudget budget) { + budget.charge(values.size()); + for (String value : values) { + bytes = addString(bytes, value); + } + return bytes; + } + + private static long addFieldPayload( + long bytes, DataField field, boolean nested, AccountingBudget budget, + int typeDepth, TypeTreeStructure structure) { + budget.charge(1L); + // Top-level DataFields are covered by TABLE_FIELD_BYTES; a copied tree owns them all. + structure.add(DATA_FIELD_BYTES); + if (nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, DATA_FIELD_BYTES); + } + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.description()); + bytes = addString(bytes, field.defaultValue()); + return addTypePayload(bytes, field.type(), budget, typeDepth, structure); + } + + /** + * Account one DataType instance and its owned children. Every accepted implementation is + * matched explicitly; an unknown class throws so estimateSafely rejects weighted admission + * instead of counting a future composite type as a small primitive. + */ + private static long addTypePayload( + long bytes, DataType type, AccountingBudget budget, int typeDepth, + TypeTreeStructure structure) { + if (typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException( + "Paimon cache accounting type depth exceeded"); + } + budget.charge(1L); + if (type == null) { + throw new IllegalStateException("Paimon field type is missing"); + } + Class typeClass = type.getClass(); + if (typeClass == RowType.class) { + RowType rowType = (RowType) type; + List fields = rowType.getFields(); + long rowBytes = rowTypeBytes(fields); + structure.add(rowBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowBytes); + for (DataField field : fields) { + bytes = addFieldPayload(bytes, field, true, budget, typeDepth + 1, structure); + } + return bytes; + } + if (typeClass == ArrayType.class) { + structure.add(ARRAY_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_TYPE_BYTES); + return addTypePayload( + bytes, ((ArrayType) type).getElementType(), budget, typeDepth + 1, structure); + } + if (typeClass == VectorType.class) { + structure.add(VECTOR_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, VECTOR_TYPE_BYTES); + return addTypePayload( + bytes, ((VectorType) type).getElementType(), budget, typeDepth + 1, structure); + } + if (typeClass == MapType.class) { + structure.add(MAP_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MAP_TYPE_BYTES); + bytes = addTypePayload( + bytes, ((MapType) type).getKeyType(), budget, typeDepth + 1, structure); + return addTypePayload( + bytes, ((MapType) type).getValueType(), budget, typeDepth + 1, structure); + } + if (typeClass == MultisetType.class) { + structure.add(MULTISET_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MULTISET_TYPE_BYTES); + // MultisetType.copy() shares its element type, so a deep copy stops here. + return addTypePayload(bytes, ((MultisetType) type).getElementType(), budget, + typeDepth + 1, new TypeTreeStructure()); + } + String[] leafFields = LEAF_TYPE_FIELDS.get(typeClass); + if (leafFields == null) { + throw new IllegalStateException( + "Unsupported Paimon data type: " + typeClass.getName()); + } + long leafBytes = leafTypeBytes(leafFields); + structure.add(leafBytes); + return MetaCacheWeightUtils.saturatedAdd(bytes, leafBytes); + } + + private static boolean isAppendOnlyTable(Table table) { + return APPEND_ONLY_TABLE_CLASS_NAME.equals(table.getClass().getName()); + } + + /** Non-String bytes of a schema type tree, i.e. what a deep DataType copy allocates again. */ + private static final class TypeTreeStructure { + private long bytes; + + private void add(long structuralBytes) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structuralBytes); + } + } + + /** + * RowType, its unmodifiable ArrayList copy of the fields, and the four lazy lookup maps that + * getField/getFieldIndex materialize after admission. The maps are reserved up front in O(N) + * so a query cannot grow the retained graph past the admitted weight; nothing is materialized. + */ + private static long rowTypeBytes(List fields) { + long uncachedFieldIds = 0L; + for (DataField field : fields) { + if (isUncachedInteger(field.id())) { + uncachedFieldIds++; + } + } + return rowTypeBytes(fields.size(), uncachedFieldIds); + } + + private static long rowTypeBytes(long fieldCount, long uncachedFieldIds) { + long bytes = MetaCacheWeightUtils.saturatedAdd(ROW_TYPE_BYTES, UNMODIFIABLE_LIST_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_LIST_BYTES); + if (fieldCount == 0L) { + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); + long uncachedIndexes = fieldCount > 128L ? fieldCount - 128L : 0L; + long mapBytes = MetaCacheWeightUtils.saturatedAdd(HASH_MAP_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(hashMapCapacity(fieldCount))); + mapBytes = addCount(mapBytes, fieldCount, HASH_MAP_NODE_BYTES); + bytes = addCount(bytes, ROW_TYPE_LAZY_MAP_COUNT, mapBytes); + // Boxed keys/values outside the Integer cache: nameToIndex values, fieldIdToField keys, + // and fieldIdToIndex boxes both again. + bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); + bytes = addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); + return addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + } + + private static long hashMapCapacity(long size) { + long capacity = 16L; + while (size > capacity - capacity / 4L) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } + + private static final class AccountingBudget { + private long remaining; + + private AccountingBudget(long remaining) { + this.remaining = remaining; + } + + private void charge(long elements) { + if (elements < 0L || elements > remaining) { + throw new IllegalStateException( + "Paimon cache accounting work budget exceeded"); + } + remaining -= elements; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index cde2bbb31efd18..ec5436b98e174d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -23,6 +23,8 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; @@ -33,6 +35,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; +import javax.annotation.Nullable; /** * Paimon engine implementation of {@link AbstractExternalMetaCache}. @@ -40,36 +43,50 @@ *

    Registered entries: *

      *
    • {@code table}: loaded Paimon table handle per table mapping
      • + *
    • {@code snapshot}: immutable partition projection keyed by a captured snapshot/schema fence
      • *
    • {@code schema}: schema cache keyed by table identity + schema id
      • *
    * - *

    Latest snapshot metadata is modeled as a runtime projection memoized inside the table cache - * value instead of as an independent cache entry. + *

    The latest main-branch snapshot is captured once as a fence and loaded through an independent + * contextual entry. Branch/tag/options projections remain statement-local and are not aliased to + * this main-snapshot key. * *

    Invalidation behavior: *

      - *
    • db/table invalidation clears table and schema entries by matching local names
      • + *
    • db/table invalidation clears table, snapshot and schema entries by matching local names
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ public class PaimonExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "paimon"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_SCHEMA = "schema"; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle schemaEntry; private final PaimonTableLoader tableLoader; private final PaimonLatestSnapshotProjectionLoader latestSnapshotProjectionLoader; public PaimonExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableLoader = new PaimonTableLoader(); latestSnapshotProjectionLoader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key)) + .withReplacementListener(this::retireTableGeneration)); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, PaimonSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSchemaCacheKey::getNameMapping))); @@ -86,41 +103,76 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()).getSnapshot(); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + nameMapping, fence, tableValue.getGeneration()); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + PaimonSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration()))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableValue.getGeneration()) { + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { - return latestSnapshotProjectionLoader.load(dorisTable.getOrBuildNameMapping(), effectiveTable); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.load(nameMapping, effectiveTable)); } public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - Table table = tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getPaimonTable(); - return latestSnapshotProjectionLoader.loadFence(nameMapping, table); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, PaimonSnapshot fence) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, Table effectiveTable, PaimonSnapshot fence) { - return latestSnapshotProjectionLoader.loadEffectiveAtFence( - dorisTable.getOrBuildNameMapping(), effectiveTable, fence); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadEffectiveAtFence( + nameMapping, effectiveTable, fence)); } public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId) { - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()) - .get(new PaimonSchemaCacheKey(nameMapping, schemaId)); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getPaimonSchemaCacheValue( + nameMapping, schemaId, tableValue.getGeneration(), tableValue.getPaimonTable()); + } + + PaimonSchemaCacheValue getPaimonSchemaCacheValue( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { + PaimonSchemaCacheKey key = new PaimonSchemaCacheKey(nameMapping, tableGeneration, schemaId); + if (tableGeneration <= 0L) { + return (PaimonSchemaCacheValue) executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable)); + } + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableGeneration) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } return (PaimonSchemaCacheValue) schemaCacheValue; } private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - Table paimonTable = tableLoader.load(nameMapping); - return new PaimonTableCacheValue(paimonTable, - () -> latestSnapshotProjectionLoader.load(nameMapping, paimonTable)); + return new PaimonTableCacheValue(tableLoader.load(nameMapping)); } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -131,8 +183,47 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } + private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + if (!(dorisTable instanceof PaimonExternalTable)) { + return loadSchemaCacheValue(key); + } + dorisTable.setUpdateTime(System.currentTimeMillis()); + return ((PaimonExternalTable) dorisTable).loadSchemaForCache(retainedTable, key.getSchemaId()); + } + + private PaimonSnapshotCacheValue loadLatestSnapshotFence(NameMapping nameMapping, Table retainedTable) { + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadFence(nameMapping, retainedTable)); + } + + private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent.Callable task) { + return tableLoader.executeAuthenticated(nameMapping, task); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable PaimonTableCacheValue previousValue, PaimonTableCacheValue currentValue) { + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.paimon.table.enable", "meta.cache.paimon.snapshot.enable"); + compatibility.put("meta.cache.paimon.table.ttl-second", "meta.cache.paimon.snapshot.ttl-second"); + compatibility.put("meta.cache.paimon.table.capacity", "meta.cache.paimon.snapshot.capacity"); + return compatibility; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index d0a3c858f4b847..defa3dd4d00ec5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -490,6 +490,14 @@ private PaimonSchemaCacheValue loadSchema(DataTable table, long schemaId) { return loadSchema(table.schemaManager().schema(schemaId)); } + PaimonSchemaCacheValue loadSchemaForCache(Table retainedTable, long schemaId) { + if (!(retainedTable instanceof DataTable)) { + throw new CacheException("retained paimon table does not expose schema history: %s", + null, retainedTable == null ? "null" : retainedTable.getClass().getName()); + } + return loadSchema((DataTable) retainedTable, schemaId); + } + private PaimonSchemaCacheValue loadSchema(TableSchema tableSchema) { List columns = tableSchema.fields(); List dorisColumns = Lists.newArrayListWithCapacity(columns.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 207810b66f5a68..f4570dccf57f6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.paimon.partition.Partition; @@ -43,24 +44,38 @@ public enum PruningStatus { UNPRUNABLE } + // Each ListPartitionItem key holds one LiteralExpr (with lazy supplier, children list and + // array) and the Paimon Partition spec one map entry per partition column beyond the first; + // the fixed per-partition constants cover a single column. Calibrated against JOL. + private static final long PARTITION_EXTRA_COLUMN_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(216L); + public static final PaimonPartitionInfo EMPTY = new PaimonPartitionInfo(PruningStatus.PRUNABLE); public static final PaimonPartitionInfo UNPRUNABLE = new PaimonPartitionInfo(PruningStatus.UNPRUNABLE); private final PruningStatus pruningStatus; private final Map nameToPartitionItem; private final Map nameToPartition; + private final long retainedPayloadBytes; private PaimonPartitionInfo(PruningStatus pruningStatus) { this.pruningStatus = pruningStatus; this.nameToPartitionItem = Collections.emptyMap(); this.nameToPartition = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public PaimonPartitionInfo(Map nameToPartitionItem, Map nameToPartition) { + this(nameToPartitionItem, nameToPartition, retainedPayloadBytes(nameToPartition)); + } + + public PaimonPartitionInfo(Map nameToPartitionItem, + Map nameToPartition, long retainedPayloadBytes) { this.pruningStatus = PruningStatus.PRUNABLE; this.nameToPartitionItem = nameToPartitionItem; this.nameToPartition = nameToPartition; + this.retainedPayloadBytes = retainedPayloadBytes; } public Map getNameToPartitionItem() { @@ -74,4 +89,58 @@ public Map getNameToPartition() { public PruningStatus getPruningStatus() { return pruningStatus; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + static long addRetainedStringPayload(long bytes, String value) { + return addString(bytes, value); + } + + /** Structural bytes one partition retains for every partition column beyond the first. */ + static long partitionColumnBytes(long partitionColumnCount) { + if (partitionColumnCount <= 1L) { + return 0L; + } + return MetaCacheWeightUtils.saturatedMultiply( + partitionColumnCount - 1L, PARTITION_EXTRA_COLUMN_BYTES); + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (Map.Entry entry : partitions.entrySet()) { + bytes = addString(bytes, entry.getKey()); + Partition partition = entry.getValue(); + if (partition == null) { + continue; + } + bytes = addStrings(bytes, partition.spec()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionColumnBytes( + partition.spec() == null ? 0 : partition.spec().size())); + bytes = addString(bytes, partition.createdBy()); + bytes = addString(bytes, partition.updatedBy()); + bytes = addStrings(bytes, partition.options()); + } + return bytes; + } + + private static long addStrings(long bytes, Map values) { + if (values == null) { + return bytes; + } + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java index 4eccb269c2fe56..49d5847e0e0469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java @@ -23,13 +23,23 @@ import com.google.common.base.Objects; public class PaimonSchemaCacheKey extends SchemaCacheKey { + private final long tableGeneration; private final long schemaId; public PaimonSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, 0L, schemaId); + } + + public PaimonSchemaCacheKey(NameMapping nameMapping, long tableGeneration, long schemaId) { super(nameMapping); + this.tableGeneration = tableGeneration; this.schemaId = schemaId; } + public long getTableGeneration() { + return tableGeneration; + } + public long getSchemaId() { return schemaId; } @@ -46,11 +56,11 @@ public boolean equals(Object o) { return false; } PaimonSchemaCacheKey that = (PaimonSchemaCacheKey) o; - return schemaId == that.schemaId; + return tableGeneration == that.tableGeneration && schemaId == that.schemaId; } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableGeneration, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index 37be7c6a5f3585..e6b37d4c020b72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -17,21 +17,33 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; private final boolean schemaFromSnapshotTable; + private final long tableGeneration; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { - this(partitionInfo, snapshot, false); + this(partitionInfo, snapshot, false, 0L); } public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, boolean schemaFromSnapshotTable) { + this(partitionInfo, snapshot, schemaFromSnapshotTable, 0L); + } + + public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, + boolean schemaFromSnapshotTable, long tableGeneration) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; this.schemaFromSnapshotTable = schemaFromSnapshotTable; + this.tableGeneration = tableGeneration; } public PaimonPartitionInfo getPartitionInfo() { @@ -45,4 +57,29 @@ public PaimonSnapshot getSnapshot() { public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } + + public long getTableGeneration() { + return tableGeneration; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + MetaCacheSizeEstimate prepareForCachePublication(PaimonSnapshotEntryKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_snapshot_preparation_failed", + () -> { + retainedTablePayloadBytes = + PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); + return PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java new file mode 100644 index 00000000000000..d820d3c15e992b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; + +import java.util.Objects; + +/** Stable identity for a Paimon projection hydrated from one captured snapshot/schema fence. */ +public final class PaimonSnapshotEntryKey { + private final NameMapping nameMapping; + private final long snapshotId; + private final long schemaId; + private final long tableGeneration; + + public PaimonSnapshotEntryKey( + NameMapping nameMapping, long snapshotId, long schemaId, long tableGeneration) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.tableGeneration = tableGeneration; + } + + public static PaimonSnapshotEntryKey of( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return new PaimonSnapshotEntryKey( + nameMapping, fence.getSnapshotId(), fence.getSchemaId(), tableGeneration); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public long getSnapshotId() { + return snapshotId; + } + + public long getSchemaId() { + return schemaId; + } + + public long getTableGeneration() { + return tableGeneration; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PaimonSnapshotEntryKey)) { + return false; + } + PaimonSnapshotEntryKey that = (PaimonSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && tableGeneration == that.tableGeneration + && nameMapping.equals(that.nameMapping); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, snapshotId, schemaId, tableGeneration); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index 7539f28d770bf6..9e82df8f8f209c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -17,28 +17,68 @@ package org.apache.doris.datasource.paimon; -import com.google.common.base.Suppliers; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + import org.apache.paimon.table.Table; -import java.util.function.Supplier; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; /** - * Cache value for Paimon table metadata and its latest runtime snapshot projection. + * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry; the + * only post-admission growth of this value is the lazily built store graph and RowType lookup + * maps of the table itself, which the publication estimate reserves up front. */ public class PaimonTableCacheValue { + private static final AtomicLong NEXT_GENERATION = new AtomicLong(); + private final Table paimonTable; - private final Supplier latestSnapshotCacheValue; + private final long generation; + private volatile long retainedTablePayloadBytes; + private volatile MetaCacheSizeEstimate sizeEstimate; - public PaimonTableCacheValue(Table paimonTable, Supplier latestSnapshotCacheValue) { + public PaimonTableCacheValue(Table paimonTable) { this.paimonTable = paimonTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + this.generation = NEXT_GENERATION.incrementAndGet(); + } + + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue ignoredFence) { + this(paimonTable); + Objects.requireNonNull(ignoredFence, "latestSnapshotFence can not be null"); } public Table getPaimonTable() { return paimonTable; } - public PaimonSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + public long getGeneration() { + return generation; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + /** + * Compute the retained weight once before the value is published to a weight-bounded cache. + * Never opens the table store; failures fail closed as an incomplete estimate. + */ + synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_table_preparation_failed", + () -> { + retainedTablePayloadBytes = + PaimonCacheSizeEstimator.retainedTablePayloadBytes(paimonTable); + return PaimonCacheSizeEstimator.estimateTableEntry(key, this); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index eaa59b28c2f4c1..66511038e7f4c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -31,6 +31,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TArrayField; @@ -176,6 +177,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List candidates = Lists.newArrayListWithExpectedSize(partitionEntries.size()); Map> displayNameToTypedSpec = Maps.newHashMap(); + long retainedPayloadBytes = 0L; for (PartitionEntry partitionEntry : partitionEntries) { Map typedSpec = getPartitionInfoMap( @@ -186,6 +188,8 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List partitionValues = Lists.newArrayListWithExpectedSize(partitionColumns.size()); LinkedHashMap orderedTypedSpec = new LinkedHashMap<>(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + PaimonPartitionInfo.partitionColumnBytes(partitionColumns.size())); for (Column partitionColumn : partitionColumns) { String partitionColumnName = partitionColumn.getName(); Preconditions.checkState(typedSpec.containsKey(partitionColumnName), @@ -193,6 +197,10 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List previousTypedSpec = displayNameToTypedSpec.putIfAbsent( displayName, orderedTypedSpec); if (previousTypedSpec != null) { @@ -247,7 +257,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List 0L) { + return paimonExternalMetaCache(dorisTable).getPaimonSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), snapshotValue.getSnapshot().getSchemaId(), + snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable()); + } return getSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6217d06b587b46..d4c3c4a0d21261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -1775,6 +1775,24 @@ private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaT trow.addToColumnValue(new TCell().setStringVal( formatMetaCacheTime(entryStats.getLastLoadFailureTimeMs(), timeZone))); trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastError())); // LAST_ERROR + // Memory governance: -1 for count-bounded entries without a weight budget. + trow.addToColumnValue(new TCell().setBoolVal(entryStats.isWeightBounded())); // WEIGHT_BOUNDED + trow.addToColumnValue(new TCell().setLongVal(entryStats.getMaxWeight())); // MAX_WEIGHT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getEstimatedWeight())); // ESTIMATED_WEIGHT + trow.addToColumnValue(new TCell().setLongVal(entryStats.getEvictionWeight())); // EVICTION_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getWeightAdmissionRejectedCount())); // WEIGHT_REJECT_COUNT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getCatalogMaxWeight())); // CATALOG_MAX_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getCatalogEstimatedWeight())); // CATALOG_ESTIMATED_WEIGHT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getGlobalMaxWeight())); // GLOBAL_MAX_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getGlobalEstimatedWeight())); // GLOBAL_ESTIMATED_WEIGHT + trow.addToColumnValue(new TCell().setStringVal( + entryStats.getLastWeightRejectReason())); // LAST_WEIGHT_REJECT_REASON dataBatch.add(trow); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index b9e84c076905d5..111a8173257c3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -33,6 +33,7 @@ import mockit.MockUp; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.HashMap; @@ -40,6 +41,12 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class ExternalMetaCacheRouteResolverTest { @@ -51,6 +58,124 @@ public void testEngineAliasCompatibility() { Assert.assertEquals("maxcompute", metaCacheMgr.engine("max_compute").engine()); } + @Test + public void testCatalogCachePropertiesRejectUnknownEngineEntryAndAliasNamespace() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = new HashMap<>(); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); + + properties.put("meta.cache.hvie.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + properties.clear(); + + properties.put("meta.cache.hms.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + properties.clear(); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + } + + @Test + public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = Collections.singletonMap( + "meta.cache.hive.partition_values.capacity", "10"); + PaimonExternalCatalog catalog = new PaimonExternalCatalog( + 1L, "paimon", null, Collections.emptyMap(), ""); + + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); + } + + @Test + public void testCatalogCachePropertyUpdateIgnoresPersistedLegacyKeys() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); + Map persisted = new HashMap<>(); + // Legacy keys admitted by image/replay: a typo, an unknown engine namespace and an + // option that is valid but stale relative to the update below. + persisted.put("meta.cache.hive.partiton_values.capacity", "10"); + persisted.put("meta.cache.hvie.partition_values.capacity", "10"); + persisted.put("meta.cache.max-weight", "64MB"); + persisted.put("meta.cache.hive.partition_values.max-weight", "16MB"); + + // The persisted map is not valid as a whole ... + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, persisted)); + // ... but runtime only honors the sane subset, so an unrelated ALTER passes ... + metaCacheMgr.validateEffectiveCatalogCacheProperties(catalog, persisted); + metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partition_values.capacity", "20")); + // ... while newly supplied keys stay strict, alone and against the honored hierarchy. + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partiton_values.enable", "true"))); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partition_values.max-weight", "128MB"))); + } + + @Test + public void testPropertyNotificationToleratesUnknownEngineNamespace() { + long catalogId = 15L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + Map replayedProperties = new HashMap<>(); + replayedProperties.put("meta.cache.hvie.partition_values.capacity", "10"); + replayedProperties.put("meta.cache.hive.partition_values.capacity", "10"); + + // Edit-log replay publishes persisted properties without DDL validation; an unknown + // legacy namespace must be ignored instead of aborting the replay. + catalog.notifyPropertiesUpdated(replayedProperties); + } + + @Test + public void testLookupRepreparesCatalogRetiredByConcurrentPolicyChange() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + long catalogId = 16L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + ExternalMetaCache hive = metaCacheMgr.hive(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + + // A cache-policy ALTER retires the group after the caller captured the engine ... + metaCacheMgr.removeCatalog(catalogId); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + // ... and the pending lookup re-prepares the catalog instead of failing. + hive.checkCatalogInitialized(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + metaCacheMgr.removeCatalog(catalogId); + + // A catalog that was really dropped is not re-created by a stale lookup. + mockCurrentCatalog(catalogId, null); + Assert.assertThrows(IllegalStateException.class, () -> hive.checkCatalogInitialized(catalogId)); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + } + + @Test + public void testRuntimePreparationIgnoresInvalidPersistedCacheProperties() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = new HashMap<>(); + properties.put("meta.cache.max-weight", "1.5GB"); + properties.put("meta.cache.hive.partition_values.enable", "1"); + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + + metaCacheMgr.prepareCatalogByEngine(101L, "hive", properties); + + Assert.assertFalse(metaCacheMgr.getCatalogCacheStats(101L).isEmpty()); + metaCacheMgr.removeCatalog(101L); + } + @Test public void testRouteByCatalogType() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); @@ -110,6 +235,83 @@ public void testPrepareCatalogByEngineSkipsMissingCatalog() throws Exception { Assert.assertEquals(0, hive.initCatalogCalls); } + @Test + public void testPreparedEngineUsesLockFreeFastPath() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> true); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + long catalogId = 12L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + + Assert.assertEquals(1, hive.initCatalogCalls); + } + + @Test + public void testCatalogRemovalFencesInFlightFirstInitialization() throws Exception { + CountDownLatch initializationEntered = new CountDownLatch(1); + CountDownLatch releaseInitialization = new CountDownLatch(1); + BlockingRecordingExternalMetaCache hive = new BlockingRecordingExternalMetaCache( + "hive", initializationEntered, releaseInitialization); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + ExecutorService workers = Executors.newFixedThreadPool(2); + long catalogId = 13L; + Map oldProperties = Collections.singletonMap("generation", "old"); + Map newProperties = Collections.singletonMap("generation", "new"); + try { + Future initialization = workers.submit( + () -> metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", oldProperties)); + Assert.assertTrue(initializationEntered.await(3L, TimeUnit.SECONDS)); + + CountDownLatch removalStarted = new CountDownLatch(1); + Future removal = workers.submit(() -> { + removalStarted.countDown(); + metaCacheMgr.removeCatalogByEngine(catalogId, "hive"); + }); + Assert.assertTrue(removalStarted.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("removal must wait for the property snapshot publication", removal.isDone()); + + releaseInitialization.countDown(); + initialization.get(3L, TimeUnit.SECONDS); + removal.get(3L, TimeUnit.SECONDS); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", newProperties); + Assert.assertEquals("new", hive.lastCatalogProperties.get("generation")); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + } finally { + releaseInitialization.countDown(); + workers.shutdownNow(); + } + } + + @Test + public void testRollbackRetiresGroupInitializedFromRejectedCandidate() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + long catalogId = 14L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + Map oldProperties = Collections.singletonMap("generation", "old"); + + metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties); + + Mockito.verify(catalog).rollBackCatalogProps(oldProperties); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + Assert.assertEquals(1, hive.invalidateCatalogCalls); + } + @Test public void testGetSchemaCacheValueReturnsEmptyWhenCatalogMissing() throws Exception { MissingCatalogSchemaExternalMetaCache schemaCache = new MissingCatalogSchemaExternalMetaCache("default"); @@ -373,4 +575,35 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class throw new IllegalStateException("catalog " + catalogId + " is not initialized"); } } + + private static final class BlockingRecordingExternalMetaCache extends RecordingExternalMetaCache { + private final CountDownLatch initializationEntered; + private final CountDownLatch releaseInitialization; + private final AtomicBoolean blockNextInitialization = new AtomicBoolean(true); + private Map lastCatalogProperties = Collections.emptyMap(); + + private BlockingRecordingExternalMetaCache(String engine, + CountDownLatch initializationEntered, CountDownLatch releaseInitialization) { + super(engine, Collections.emptyList(), catalog -> true); + this.initializationEntered = initializationEntered; + this.releaseInitialization = releaseInitialization; + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (blockNextInitialization.compareAndSet(true, false)) { + initializationEntered.countDown(); + try { + if (!releaseInitialization.await(3L, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to publish catalog initialization"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + lastCatalogProperties = new HashMap<>(catalogProperties); + super.initCatalog(catalogId, catalogProperties); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 151d46252d9084..df24c0c754d6a2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -17,23 +17,49 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import com.google.common.collect.HashBiMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; public class HiveMetaStoreCacheTest { + @Test + public void testPartitionValueWeightScalesLinearlyToOneHundredThousandPartitions() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.singletonList(Type.STRING)); + long base = partitionValueWeight(key, 0); + long oneThousand = partitionValueWeight(key, 1_000); + long tenThousand = partitionValueWeight(key, 10_000); + long oneHundredThousand = partitionValueWeight(key, 100_000); + + long oneThousandPayload = oneThousand - base; + Assertions.assertTrue(oneThousandPayload > 0L); + Assertions.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assertions.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + @Test public void testInvalidateTableCache() { ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( @@ -144,6 +170,103 @@ public void testInvalidatePartitionCacheClearsStaleFileCacheOnPartitionMiss() { } } + @Test + public void testPartitionValuesEstimateIsPreparedAgainAfterCopy() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.emptyList()); + PartitionKey partitionKey = new PartitionKey(); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + partitionItem.setDefaultPartition(true); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("p", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, Collections.emptyList()); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + Assertions.assertTrue(values.getSizeEstimate().isComplete()); + Assertions.assertTrue(values.getSizeEstimate().getBytes() > 0L); + + HiveExternalMetaCache.HivePartitionValues copy = values.mutableCopy(); + Assertions.assertFalse(copy.getSizeEstimate().isComplete()); + copy.sealForPublication(); + copy.prepareSizeEstimate(new HiveExternalMetaCache.PartitionValueCacheKey( + key.getNameMapping(), null)); + Assertions.assertTrue(copy.getSizeEstimate().isComplete()); + Assertions.assertTrue(copy.getSizeEstimate().getBytes() > 0L); + ListPartitionItem publishedItem = (ListPartitionItem) values.getIdToPartitionItem().get(1L); + Assertions.assertSame(partitionItem, publishedItem, + "cache publication must not rewrite common catalog partition objects"); + Assertions.assertSame(partitionKey, publishedItem.getItems().get(0)); + } + + @Test + public void testPartitionValuesEstimateSupportsRealLiteralGraph() throws Exception { + List types = java.util.Arrays.asList(Type.STRING, Type.INT, Type.DATEV2, Type.DECIMALV2); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + java.util.Arrays.asList( + new PartitionValue("tail-value"), + new PartitionValue("42"), + new PartitionValue("2026-08-12"), + new PartitionValue("123456789.0123")), + types, true); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("s=tail-value/i=42/d=2026-08-12/n=123456789.0123", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, java.util.Arrays.asList( + "tail-value", "42", "2026-08-12", "123456789.0123")); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + + Assertions.assertTrue(values.getSizeEstimate().isComplete(), + values.getSizeEstimate().getIncompleteReason()); + long estimatedBytes = values.getSizeEstimate().getBytes(); + PartitionKey publishedKey = ((ListPartitionItem) values.getIdToPartitionItem().get(1L)).getItems().get(0); + StringLiteral publishedString = (StringLiteral) publishedKey.getKeys().get(0); + // Exercise normal read-only lazy paths after publication. Their bounded memoized state is + // covered by estimator headroom without changing or cloning common expression classes. + publishedString.getExprName(); + values.getSortedPartitionRanges().orElseThrow(AssertionError::new).sortedPartitions + .forEach(partition -> partition.range.toString()); + values.prepareSizeEstimate(key); + Assertions.assertEquals(estimatedBytes, values.getSizeEstimate().getBytes()); + Assertions.assertSame(partitionKey, publishedKey, + "cache publication must not rewrite common catalog partition objects"); + } + + @Test + public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { + List types = Collections.singletonList(Type.STRING); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + HiveExternalMetaCache.HivePartitionValues empty = realPartitionValues(types, 0, 16); + HiveExternalMetaCache.HivePartitionValues populated = realPartitionValues(types, 32, 16); + HiveExternalMetaCache.HivePartitionValues shortTail = realPartitionValues(types, 1, 16); + HiveExternalMetaCache.HivePartitionValues longTail = realPartitionValues(types, 1, 4096); + + long emptyEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, empty).getBytes(); + long populatedEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, populated).getBytes(); + long shortTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, shortTail).getBytes(); + long longTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive partition values", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, @@ -178,4 +301,42 @@ private long entrySize(MetaCacheEntry entry) { entry.forEach((k, v) -> count.incrementAndGet()); return count.get(); } + + private long partitionValueWeight( + HiveExternalMetaCache.PartitionValueCacheKey key, int partitionCount) { + Map items = sizeOnlyMap(partitionCount); + HiveExternalMetaCache.HivePartitionValues values = + new HiveExternalMetaCache.HivePartitionValues( + items, null, null, partitionCount * 16L, 1); + MetaCacheSizeEstimate estimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, values); + Assertions.assertTrue(estimate.isComplete(), estimate.getIncompleteReason()); + return estimate.getBytes(); + } + + private HiveExternalMetaCache.HivePartitionValues realPartitionValues( + List types, int partitionCount, int valueLength) throws Exception { + Map items = new HashMap<>(); + HashBiMap names = HashBiMap.create(); + Map> values = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + String.join("", Collections.nCopies(valueLength, "x")); + long id = index + 1L; + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + Collections.singletonList(new PartitionValue(value)), types, true); + items.put(id, new ListPartitionItem(Collections.singletonList(partitionKey))); + names.put("p=" + value, id); + values.put(id, Collections.singletonList(value)); + } + HiveExternalMetaCache.HivePartitionValues result = + new HiveExternalMetaCache.HivePartitionValues(items, names, values); + result.sealForPublication(); + return result; + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index af3be4475f71ac..9992238170c2c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -225,6 +225,8 @@ protected void runBeforeAll() throws Exception { } return invocation.callRealMethod(); }); + icebergUtilsMock.when(() -> IcebergUtils.getWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class))).thenReturn(mockedIcebergTable); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 00dde9f4cc0a74..419bb991f05de2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -17,28 +17,1693 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; +import java.lang.reflect.Field; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + // U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) lower-cases to two characters in Locale.ROOT. + private static final String DOTTED_CAPITAL_I = String.valueOf((char) 0x0130); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testSnapshotAndManifestWeightsScaleLinearlyToOneHundredThousandItems() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, tableWithMetadataLocation("/metadata/linear-v1.json")).get(); + long snapshotBase = snapshotWeight(snapshotKey, 0); + long snapshotOneThousand = snapshotWeight(snapshotKey, 1_000); + assertLinearScale(snapshotBase, snapshotOneThousand, + snapshotWeight(snapshotKey, 10_000), snapshotWeight(snapshotKey, 100_000)); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey( + "/manifest/linear.avro", ManifestContent.DATA); + long manifestBase = manifestWeight(manifestKey, 0); + long manifestOneThousand = manifestWeight(manifestKey, 1_000); + assertLinearScale(manifestBase, manifestOneThousand, + manifestWeight(manifestKey, 10_000), manifestWeight(manifestKey, 100_000)); + } + + @Test + public void testWeightedEntriesAreRegistered() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.max-weight", "4MB"); + properties.put("meta.cache.iceberg.snapshot.max-weight", "8MB"); + properties.put("meta.cache.iceberg.manifest.enable", "true"); + properties.put("meta.cache.iceberg.manifest.max-weight", "16MB"); + cache.initCatalog(1L, properties); + + Map stats = cache.stats(1L); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_TABLE).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.enable", "false"); + properties.put("meta.cache.iceberg.table.ttl-second", "17"); + properties.put("meta.cache.iceberg.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(IcebergExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeyIncludesMetadataGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table first = tableWithMetadataLocation("/metadata/v1.json"); + Table second = tableWithMetadataLocation("/metadata/v2.json"); + Table recreated = tableWithMetadataLocation("/metadata/v1.json"); + + IcebergSnapshotEntryKey firstKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey sameKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey secondKey = IcebergSnapshotEntryKey.tryCreate(mapping, second).get(); + IcebergSnapshotEntryKey recreatedKey = IcebergSnapshotEntryKey.tryCreate(mapping, recreated).get(); + + Assert.assertEquals(firstKey, sameKey); + Assert.assertNotEquals(firstKey, secondKey); + Assert.assertNotEquals("drop/recreate may reuse HadoopCatalog's v1 path", firstKey, recreatedKey); + Assert.assertEquals("/metadata/v1.json", firstKey.getMetadataFileLocation()); + Assert.assertNotEquals(firstKey.getTableUuid(), recreatedKey.getTableUuid()); + Assert.assertFalse(IcebergSnapshotEntryKey.tryCreate(mapping, + newInterfaceProxy(Table.class)).isPresent()); + } + + @Test + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v1.json")); + IcebergTableCacheValue second = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v2.json")); + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + tables.put(mapping, first); + IcebergSnapshotEntryKey oldSnapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, first.getRetainedIcebergTable()).get(); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); + IcebergSchemaCacheKey oldSchemaKey = new IcebergSchemaCacheKey( + mapping, first.getTableUuid().get(), 0L); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue oldTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-old.json")); + IcebergTableCacheValue newTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-new.json")); + cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, newTable); + IcebergSchemaCacheKey staleKey = new IcebergSchemaCacheKey( + mapping, oldTable.getTableUuid().get(), 0L); + IcebergSchemaCacheValue staleValue = new IcebergSchemaCacheValue( + Collections.emptyList(), Collections.emptyList()); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(staleKey, staleValue); + + Assert.assertSame(staleValue, cache.getIcebergSchemaCacheValue( + mapping, 0L, oldTable.getRetainedIcebergTable())); + Assert.assertNull(schemas.peekIfPresent(staleKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testFrozenGenerationPreservesSparseEquivalentSchemaIds() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/sparse.json", "{" + + "\"format-version\":2,\"table-uuid\":\"sparse-schema-table\"," + + "\"location\":\"file:/warehouse/sparse\",\"last-sequence-number\":0," + + "\"last-updated-ms\":1,\"last-column-id\":2,\"current-schema-id\":2," + + "\"schemas\":[" + + "{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}," + + "{\"type\":\"struct\",\"schema-id\":1,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}," + + "{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]}," + + "{\"type\":\"struct\",\"schema-id\":2,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":-1,\"refs\":{},\"snapshots\":[]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[],\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + TableMetadata retainedMetadata = ((HasTableOperations) retained).operations().current(); + + Assert.assertEquals(2, retainedMetadata.currentSchemaId()); + Assert.assertEquals(java.util.Arrays.asList(0, 1, 2), retainedMetadata.schemas().stream() + .map(Schema::schemaId).collect(Collectors.toList())); + Assert.assertEquals(1, retainedMetadata.schemas().stream() + .filter(schema -> schema.schemaId() == 2).findFirst().get().columns().size()); + } + + @Test + public void testFrozenGenerationAcceptsSnapshotCreatedBeforeV3Upgrade() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/upgraded-v3.json", "{" + + "\"format-version\":3,\"table-uuid\":\"upgraded-v3-table\"," + + "\"location\":\"file:/warehouse/v3\",\"last-sequence-number\":1," + + "\"last-updated-ms\":2,\"last-column-id\":1,\"current-schema-id\":0," + + "\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":7,\"next-row-id\":0," + + "\"refs\":{\"main\":{\"snapshot-id\":7,\"type\":\"branch\"}}," + + "\"snapshots\":[{\"sequence-number\":0,\"snapshot-id\":7," + + "\"timestamp-ms\":1,\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[],\"schema-id\":0}]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[{\"timestamp-ms\":1,\"snapshot-id\":7}]," + + "\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + + Assert.assertEquals(7L, retained.currentSnapshot().snapshotId()); + Assert.assertEquals(3, ((HasTableOperations) retained).operations().current().formatVersion()); + } + + @Test + public void testTableSnapshotAndManifestEstimatesArePrecomputed() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table table = tableWithMetadataLocation("/metadata/v1.json"); + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(table); + tableValue.prepareForCachePublication(mapping); + Assert.assertTrue(tableValue.getSizeEstimate().isComplete()); + Assert.assertTrue(tableValue.getSizeEstimate().getBytes() > 0L); + + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), table); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Assert.assertTrue(snapshotValue.getSizeEstimate().getBytes() > 0L); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey("/manifest/a.avro", ManifestContent.DATA); + ManifestCacheValue manifestValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/a.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + MetaCacheSizeEstimate manifestEstimate = + IcebergCacheSizeEstimator.estimateManifestEntry(manifestKey, manifestValue); + Assert.assertTrue(manifestEstimate.getIncompleteReason(), manifestEstimate.isComplete()); + Assert.assertTrue(manifestEstimate.getBytes() > 0L); + + Table unsupportedTable = newInterfaceProxy(Table.class); + MetaCacheSizeEstimate unsupported = IcebergCacheSizeEstimator.estimateTableEntry( + mapping, new IcebergTableCacheValue(unsupportedTable)); + Assert.assertFalse(unsupported.isComplete()); + Assert.assertTrue(unsupported.getIncompleteReason().startsWith("unsupported_iceberg_table:")); + } + + @Test + public void testIcebergPreparationFailureIsFailClosed() { + TableMetadata brokenMetadata = Mockito.mock(TableMetadata.class); + Mockito.when(brokenMetadata.currentSnapshot()) + .thenThrow(new IllegalStateException("unsupported snapshot state")); + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(brokenMetadata); + Table brokenTable = new BaseTable(operations, "db.tbl"); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(brokenTable); + MetaCacheSizeEstimate tableEstimate = tableValue.prepareForCachePublication(mapping); + + Assert.assertFalse(tableEstimate.isComplete()); + Assert.assertTrue(tableEstimate.getIncompleteReason() + .startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(tableValue.getRetainedIcebergTable(), tableValue.newQueryScopedTable()); + + Table healthyTable = tableWithMetadataLocation("/metadata/fail-closed-key.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, healthyTable).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), brokenTable); + MetaCacheSizeEstimate snapshotEstimate = snapshotValue.prepareForCachePublication(key); + + Assert.assertFalse(snapshotEstimate.isComplete()); + Assert.assertTrue(snapshotEstimate.getIncompleteReason() + .startsWith("iceberg_snapshot_preparation_failed:")); + } + + @Test + public void testManifestAccountingAcceptsOnlyGenericContentFileCopies() { + DataFile copied = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/copied.parquet").withFileSizeInBytes(10L).withRecordCount(1L) + .build().copy(); + ManifestCacheValue supported = ManifestCacheValue.forDataFiles( + Collections.singletonList(copied)); + Assert.assertTrue(supported.isAccountingComplete()); + Assert.assertTrue(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/copied.avro", ManifestContent.DATA), + supported).isComplete()); + + // A proxy, mock or third-party ContentFile has an unknown retained layout: keep the file + // for the current query but reject weighted admission. + DataFile proxy = newInterfaceProxy(DataFile.class); + ManifestCacheValue unsupported = ManifestCacheValue.forDataFiles( + Collections.singletonList(proxy)); + Assert.assertEquals(Collections.singletonList(proxy), unsupported.getDataFiles()); + Assert.assertFalse(unsupported.isAccountingComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", + IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/proxy.avro", ManifestContent.DATA), + unsupported).getIncompleteReason()); + + // A data-file implementation inside a delete manifest is equally unsupported. + ManifestCacheValue.Builder deleteBuilder = ManifestCacheValue.deleteFilesBuilder(); + deleteBuilder.addDeleteFile(newInterfaceProxy(DeleteFile.class)); + Assert.assertFalse(deleteBuilder.build().isAccountingComplete()); + } + + @Test + @SuppressWarnings("unchecked") + public void testManifestAccountingFailureKeepsFilesAndRejectsWeightedAdmission() { + Map brokenColumnSizes = Mockito.mock(Map.class); + Mockito.when(brokenColumnSizes.size()) + .thenThrow(new IllegalStateException("new metrics representation")); + DataFile file = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/broken-metrics.parquet").withFileSizeInBytes(10L) + .withMetrics(new Metrics(1L, brokenColumnSizes, null, null, null)) + .build(); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles(Collections.singletonList(file)); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/fail-closed.avro", ManifestContent.DATA), value); + + Assert.assertEquals(Collections.singletonList(file), value.getDataFiles()); + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(estimate.isComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", estimate.getIncompleteReason()); + } + + @Test + public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { + String largePayload = repeatedCharacter('x', 64 * 1024); + Table smallTable = tableWithNestedSchemaAndProperty("x", "x"); + Table largeTable = tableWithNestedSchemaAndProperty(largePayload, largePayload); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(smallTable); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue(largeTable); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + long expectedPayloadDelta = (MetaCacheWeightUtils.estimatedStringBytes(largePayload) + - MetaCacheWeightUtils.estimatedStringBytes("x")) * 2L; + Assert.assertTrue(largeValue.getSizeEstimate().getBytes() + - smallValue.getSizeEstimate().getBytes() >= expectedPayloadDelta); + } + + @Test + public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { + List fields = IntStream.range(0, 100) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema largeSchema = new Schema(0, fields); + Schema smallSchema = new Schema(1, fields.get(0)); + TableMetadata schemaHistory = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + schemaHistory = TableMetadata.buildFrom(schemaHistory) + .addSchema(smallSchema) + .setCurrentSchema(smallSchema.schemaId()) + .discardChanges() + .build(); + TableMetadata smallSchemaOnly = TableMetadata.newTableMetadata( + smallSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(largeSchema).withSpecId(0); + SortOrder.Builder sortBuilder = SortOrder.builderFor(largeSchema).withOrderId(1); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + sortBuilder.asc(field.name()); + } + PartitionSpec populatedSpec = specBuilder.build(); + SortOrder populatedSortOrder = sortBuilder.build(); + TableMetadata fieldHistory = TableMetadata.newTableMetadata( + largeSchema, populatedSpec, populatedSortOrder, + "file:/warehouse/field-history", Collections.emptyMap()); + fieldHistory = TableMetadata.buildFrom(fieldHistory) + .setDefaultPartitionSpec( + PartitionSpec.builderFor(largeSchema).withSpecId(1).build()) + .setDefaultSortOrder(SortOrder.unsorted()) + .discardChanges() + .build(); + TableMetadata emptyFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + TableMetadata partitionFields = TableMetadata.newTableMetadata( + largeSchema, populatedSpec, SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + TableMetadata sortFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), populatedSortOrder, + "file:/warehouse/field-history", Collections.emptyMap()); + + // Cache values come from Iceberg's parser. Round-trip builder fixtures so JOL measures + // the same canonical ownership graph used in production instead of write-side builders. + // Metadata locations of compared fixtures have equal length: that String is not part of + // retainedTablePayloadBytes and must not leak into the JOL delta. + schemaHistory = roundTripMetadata(schemaHistory, "/metadata/jol-schema-large.json"); + smallSchemaOnly = roundTripMetadata(smallSchemaOnly, "/metadata/jol-schema-small.json"); + fieldHistory = roundTripMetadata(fieldHistory, "/metadata/jol-fields-both.json"); + emptyFields = roundTripMetadata(emptyFields, "/metadata/jol-fields-none.json"); + partitionFields = roundTripMetadata(partitionFields, "/metadata/jol-fields-spec.json"); + sortFields = roundTripMetadata(sortFields, "/metadata/jol-fields-sort.json"); + + long schemaDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(schemaHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(smallSchemaOnly)); + long specAndSortDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(fieldHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + long partitionFieldDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(partitionFields)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + long sortFieldDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(sortFields)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + + materializeAllLazyState(schemaHistory); + materializeAllLazyState(smallSchemaOnly); + materializeAllLazyState(fieldHistory); + materializeAllLazyState(emptyFields); + materializeAllLazyState(partitionFields); + materializeAllLazyState(sortFields); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg schema history", 0L, schemaDelta, + smallSchemaOnly, schemaHistory); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg partition fields", 0L, partitionFieldDelta, + emptyFields, partitionFields); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg sort fields", 0L, sortFieldDelta, + emptyFields, sortFields); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg spec and sort fields", 0L, specAndSortDelta, + emptyFields, fieldHistory); + } + + @Test + public void testSchemaLookupFormulaScalesWithSchemaWidth() { + for (int fieldCount : new int[] {3, 32, 100, 1000}) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + assertSchemaLookupFormula( + new Schema(0, fields), "schema lookup width " + fieldCount); + } + // Single-column schemas cannot grow; check their partition-spec graph instead. + assertPartitionSpecFormula(new Schema(0, Types.NestedField.optional( + 1, "field_0", Types.StringType.get())), "partition spec width 1"); + } + + @Test + public void testSchemaLookupFormulaCountsListAndMapSyntheticFields() { + List listFields = new ArrayList<>(); + listFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + listFields.add(Types.NestedField.optional( + index + 2, "list_" + index, + Types.ListType.ofOptional(10_000 + index, Types.StringType.get()))); + } + assertSchemaLookupFormula(new Schema(0, listFields), "schema lookup list synthetic fields"); + + List mapFields = new ArrayList<>(); + mapFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + mapFields.add(Types.NestedField.optional( + index + 2, "map_" + index, + Types.MapType.ofOptional( + 10_000 + index * 2, 10_001 + index * 2, + Types.StringType.get(), Types.LongType.get()))); + } + assertSchemaLookupFormula(new Schema(0, mapFields), "schema lookup map synthetic fields"); + } + + @Test + public void testSchemaLookupFormulaCountsNestedStructShortAliases() { + List listFields = new ArrayList<>(); + listFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + Types.StructType elementType = Types.StructType.of(Types.NestedField.optional( + 20_000 + index, "leaf", Types.StringType.get())); + listFields.add(Types.NestedField.optional( + index + 2, "list_" + index, + Types.ListType.ofOptional(10_000 + index, elementType))); + } + assertSchemaLookupFormula( + new Schema(0, listFields), "list struct short aliases"); + + List mapFields = new ArrayList<>(); + mapFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + Types.StructType valueType = Types.StructType.of(Types.NestedField.optional( + 30_000 + index, "leaf", Types.LongType.get())); + mapFields.add(Types.NestedField.optional( + index + 2, "map_" + index, + Types.MapType.ofOptional( + 10_000 + index * 2, 10_001 + index * 2, + Types.StringType.get(), valueType))); + } + assertSchemaLookupFormula( + new Schema(0, mapFields), "map struct short aliases"); + } + + @Test + public void testSchemaIdentifierFieldFormulaScalesWithWidth() { + for (int fieldCount : new int[] {1, 32, 100, 1000}) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.required( + index + 1, "identifier_" + index, Types.LongType.get())) + .collect(Collectors.toList()); + Set identifierIds = fields.stream() + .map(Types.NestedField::fieldId) + .collect(Collectors.toSet()); + Schema withoutIdentifiers = new Schema(0, fields); + Schema withIdentifiers = new Schema(0, fields, identifierIds); + TableMetadata empty = roundTripMetadata(TableMetadata.newTableMetadata( + withoutIdentifiers, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-identifiers", Collections.emptyMap()), + "/metadata/jol-schema-identifiers-none-" + fieldCount + ".json"); + TableMetadata populated = roundTripMetadata(TableMetadata.newTableMetadata( + withIdentifiers, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-identifiers", Collections.emptyMap()), + "/metadata/jol-schema-identifiers-with-" + fieldCount + ".json"); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(empty)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populated)); + materializeAllLazyState(empty); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg identifier fields " + fieldCount, + emptyEstimate, populatedEstimate, empty, populated); + } + } + + @Test + public void testUnicodeLowerCaseSchemaFormulaAgainstJolOwnedGraph() { + // U+0130 lower-cases to "i" plus U+0307 in Locale.ROOT: the generated lower-case index + // keys are longer than their sources and switch the String coder to UTF-16. + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithSchemaAndProperties( + unicodeNestedSchema(1), Collections.emptyMap()); + IcebergTableCacheValue populated = tableValueWithSchemaAndProperties( + unicodeNestedSchema(33), Collections.emptyMap()); + + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(small); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg unicode lower-case nested fields", + smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testUnicodeLowerCasePartitionNameFormulaAgainstJolOwnedGraph() { + List fields = new ArrayList<>(); + fields.add(Types.NestedField.optional(1, DOTTED_CAPITAL_I + "dentity_Key", Types.StringType.get())); + for (int index = 0; index < 8; index++) { + fields.add(Types.NestedField.optional(index + 2, DOTTED_CAPITAL_I + "_field_" + index, + Types.StringType.get())); + } + // The identity partition name is lower-cased three times: by the partition StructType, + // by the secondary Schema and by the secondary StructType. + assertPartitionSpecFormula(new Schema(0, fields), "unicode partition name"); + } + + @Test + public void testTableAccountingCharacterBudgetFailsClosedWithoutFailingLoad() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + String hugeName = repeatedCharacter('x', 1 << 20); + List fields = IntStream.range(0, 5) + .mapToObj(index -> Types.NestedField.optional( + index + 1, hugeName + index, Types.StringType.get())) + .collect(Collectors.toList()); + IcebergTableCacheValue value = tableValueWithSchemaAndProperties( + new Schema(0, fields), Collections.emptyMap()); + + IllegalStateException budgetFailure = Assert.assertThrows(IllegalStateException.class, + () -> IcebergCacheSizeEstimator.retainedTablePayloadBytes( + value.getRetainedIcebergTable())); + Assert.assertTrue(budgetFailure.getMessage(), + budgetFailure.getMessage().contains("character budget")); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(mapping); + + Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), + estimate.getIncompleteReason().startsWith("iceberg_table_preparation_failed:")); + Assert.assertNotNull(value.getRetainedIcebergTable()); + Assert.assertEquals(5, value.getRetainedIcebergTable().schema().columns().size()); + Assert.assertSame(value.getRetainedIcebergTable(), value.newQueryScopedTable()); + } + + @Test + public void testTableAccountingElementBudgetFailsClosedWithoutFailingLoad() { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + @SuppressWarnings("unchecked") + Map oversizedProperties = Mockito.mock(Map.class); + Mockito.when(oversizedProperties.size()).thenReturn(2_000_001); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(oversizedProperties); + Table table = tableWithMetadata(metadata); + + IllegalStateException budgetFailure = Assert.assertThrows(IllegalStateException.class, + () -> IcebergCacheSizeEstimator.retainedTablePayloadBytes(table)); + Assert.assertTrue(budgetFailure.getMessage(), + budgetFailure.getMessage().contains("work budget")); + Mockito.verify(oversizedProperties, Mockito.never()).entrySet(); + + IcebergTableCacheValue value = new IcebergTableCacheValue(table); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), + estimate.getIncompleteReason().startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(value.getRetainedIcebergTable(), value.newQueryScopedTable()); + } + + @Test + public void testAdmittedTableEstimateCoversFullyMaterializedRetainedGraph() { + // Whole-entry oracle: the admitted weight must cover the complete retained graph after + // every lazy Schema/StructType/PartitionSpec index a scan can create has materialized, + // including the O(distinctSources * fields) fieldsBySourceId graph. Component deltas can + // miss a shared baseline; this compares absolute sizes. + // The tight bound applies once the variable payload dominates the fixed per-table base + // (TABLE_BASE_BYTES); small tables are deliberately covered by that base. + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + for (int width : new int[] {1, 100, 1000}) { + assertAdmittedEstimateCoversRetainedGraph( + "flat identity-partitioned " + width, mapping, + tableValueWithIdentityPartitionedFields(width), width >= 100); + assertAdmittedEstimateCoversRetainedGraph( + "nested mixed-case " + width, mapping, + tableValueWithNestedMixedCaseFields(width), width >= 1000); + } + } + + private void assertAdmittedEstimateCoversRetainedGraph( + String fixture, NameMapping mapping, IcebergTableCacheValue value, + boolean requireTightBound) { + long estimate = value.prepareForCachePublication(mapping).getBytes(); + long before = EstimatorCalibrationAssertions.graphSize(value); + materializeAllLazyState(value); + long after = EstimatorCalibrationAssertions.graphSize(value); + Assert.assertTrue(fixture + " lazy state must grow the retained graph", after > before); + Assert.assertTrue(fixture + " underestimates the materialized entry: estimate=" + + estimate + ", retained=" + after, estimate >= after); + if (requireTightBound) { + Assert.assertTrue(fixture + " is excessively conservative: estimate=" + estimate + + ", retained=" + after, estimate <= Math.ceil(after * 1.10D)); + } + } + + @Test + public void testSchemaFormulaCountsBoxedIdsOfUncachedFieldIds() { + // TableMetadata.newTableMetadata() reassigns fresh ids from 1, which the JVM Integer + // cache serves for free. Add the schema to existing metadata instead so ids above 127 + // survive and every lookup map really boxes its keys and values. + List oneFlat = Collections.singletonList( + Types.NestedField.optional(10_000, "Field_0", Types.StringType.get())); + List manyFlat = IntStream.range(0, 32) + .mapToObj(index -> Types.NestedField.optional( + 10_000 + index, "Field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + assertRetainedPayloadDelta("uncached flat field ids", + metadataWithAddedSchema(new Schema(1, oneFlat)), + metadataWithAddedSchema(new Schema(1, manyFlat)), "jol-uncached-ids"); + + List nestedFields = IntStream.range(0, 32) + .mapToObj(index -> Types.NestedField.optional( + 20_000 + index, "Nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema oneNested = new Schema(1, Types.NestedField.optional(10_000, "payload", + Types.StructType.of(nestedFields.get(0)))); + Schema manyNested = new Schema(1, Types.NestedField.optional(10_000, "payload", + Types.StructType.of(nestedFields))); + assertRetainedPayloadDelta("uncached nested field ids", + metadataWithAddedSchema(oneNested), metadataWithAddedSchema(manyNested), + "jol-uncached-ids"); + } + + private TableMetadata metadataWithAddedSchema(Schema schema) { + return metadataWithAddedSchema(schema, 2); + } + + private TableMetadata metadataWithAddedSchema(Schema schema, int formatVersion) { + Schema base = new Schema(0, Types.NestedField.optional(1, "base", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata( + base, PartitionSpec.unpartitioned(), SortOrder.unsorted(), "file:/warehouse/uncached-ids", + Collections.singletonMap(TableProperties.FORMAT_VERSION, Integer.toString(formatVersion))); + return TableMetadata.buildFrom(metadata).addSchema(schema) + .setCurrentSchema(schema.schemaId()).discardChanges().build(); + } + + @Test + public void testSchemaFormulaCountsFieldDefaultLiterals() { + // v3 field defaults are retained as Literal wrappers around boxed or String values. + assertRetainedPayloadDelta("defaulted fields", + metadataWithAddedSchema(new Schema(1, defaultedFields(1)), 3), + metadataWithAddedSchema(new Schema(1, defaultedFields(32)), 3), "jol-defaults"); + } + + private List defaultedFields(int fieldCount) { + List fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional("text_" + index).withId(10_000 + index) + .ofType(Types.StringType.get()) + .withInitialDefault(Expressions.lit("initial_" + index)) + .withWriteDefault(Expressions.lit("write_" + index)) + .build()); + fields.add(Types.NestedField.optional("number_" + index).withId(20_000 + index) + .ofType(Types.LongType.get()) + .withInitialDefault(Expressions.lit(100_000L + index)) + .withWriteDefault(Expressions.lit(200_000L + index)) + .build()); + } + return fields; + } + + @Test + public void testTablePayloadAccountsForRetainedHistoricalMetadata() { + String largePayload = repeatedCharacter('x', 64 * 1024); + long smallBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload("x", 32))); + long largeBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload(largePayload, 64 * 1024))); + + Assert.assertTrue(largeBytes - smallBytes >= 64L * 1024L - 32L); + } + + @Test + public void testStatisticsBlobFormulaAgainstJolOwnedGraph() { + GenericStatisticsFile empty = new GenericStatisticsFile( + 1L, "/stats/file.puffin", 1L, 1L, Collections.emptyList()); + List blobs = IntStream.range(0, 32) + .mapToObj(index -> new GenericBlobMetadata( + "blob-type-" + index, + 1L, + 1L, + java.util.Arrays.asList(10_000 + index, 20_000 + index), + Collections.singletonMap( + "property-" + index, "value-" + index))) + .collect(Collectors.toList()); + GenericStatisticsFile populated = new GenericStatisticsFile( + 1L, "/stats/file.puffin", 1L, 1L, blobs); + TableMetadata emptyMetadata = metadataWithStatisticsFile(empty); + TableMetadata populatedMetadata = metadataWithStatisticsFile(populated); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyMetadata)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populatedMetadata)); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg statistics blobs", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testTableEstimateAccountsForRetainedBranchHistory() { + TableMetadata oneCommit = metadataWithSnapshotSequence(1L); + TableMetadata tenThousandCommits = metadataWithSnapshotSequence(10_000L); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(tableWithMetadata(oneCommit)); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue( + tableWithMetadata(tenThousandCommits)); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + Assert.assertTrue(smallValue.getSizeEstimate().getIncompleteReason(), + smallValue.getSizeEstimate().isComplete()); + Assert.assertTrue(largeValue.getSizeEstimate().getIncompleteReason(), + largeValue.getSizeEstimate().isComplete()); + Assert.assertEquals(smallValue.getSizeEstimate().getBytes(), + largeValue.getSizeEstimate().getBytes()); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshots(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshots(); + Mockito.verify(oneCommit, Mockito.never()).lastSequenceNumber(); + Mockito.verify(tenThousandCommits, Mockito.never()).lastSequenceNumber(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).refs(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).refs(); + } + + @Test + public void testWeightedTablePreparationRunsInsideCatalogAuthenticator() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + AtomicBoolean authenticated = new AtomicBoolean(); + AtomicBoolean firstPreparation = new AtomicBoolean(true); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Table table = tableWithMetadataLocation("/metadata/authenticated-v1.json"); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenReturn(table); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + Assert.assertTrue(authenticated.compareAndSet(false, true)); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + if (firstPreparation.compareAndSet(true, false)) { + Assert.assertTrue("publication preparation must retain Kerberos scope", + authenticated.get()); + } + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = new NameMapping( + 1L, "db", "tbl", "remote_db", "remote_tbl"); + + IcebergTableCacheValue value = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + IcebergTableCacheValue cached = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertSame(value, cached); + Mockito.verify(metadataOps, Mockito.times(1)).loadTable("remote_db", "remote_tbl"); + Assert.assertFalse(authenticated.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testQueryScopedMetadataReusesFrozenGenerationWithoutFileIo() throws Exception { + String tableLocation = temporaryFolder.newFolder("authenticated-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + AtomicInteger metadataReads = new AtomicInteger(); + FileIO trackingFileIO = Mockito.mock(FileIO.class); + Mockito.when(trackingFileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + metadataReads.incrementAndGet(); + return liveTable.io().newInputFile((String) invocation.getArgument(0)); + }); + TableMetadata metadata = ((HasTableOperations) liveTable).operations().current(); + Table trackedTable = new BaseTable( + new StaticTableOperations(metadata, trackingFileIO), liveTable.name()); + IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable); + countValue.getWritableIcebergTable(liveTable); + Assert.assertEquals("count-based writes must not add metadata FileIO", 0, metadataReads.get()); + IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable); + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Table statementTable = value.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(statementTable); + IcebergSnapshotCacheValue statementValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), statementTable); + Assert.assertEquals(statementTable.schema().asStruct(), + statementValue.getIcebergTable().get().schema().asStruct()); + com.google.common.collect.Lists.newArrayList( + statementValue.getIcebergTable().get().snapshots()); + Assert.assertEquals("statement handoff must reuse frozen metadata", 0, metadataReads.get()); + value.getWritableIcebergTable(liveTable); + + Assert.assertEquals(0, metadataReads.get()); + } + + @Test + public void testCountModeTimeTravelDoesNotEnableQueryIsolation() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/count-v1.json")); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, value); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertSame(value.getRetainedIcebergTable(), queryTable); + Assert.assertFalse(value.isQueryIsolationPrepared()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws Exception { + String firstLocation = temporaryFolder.newFolder("bundle-first").toURI().toString(); + String secondLocation = temporaryFolder.newFolder("bundle-second").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table firstTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), firstLocation); + firstTable.newAppend().appendFile(DataFiles.builder(firstTable.spec()) + .withPath(firstLocation + "/data/a.parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()).commit(); + Table secondTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), secondLocation); + secondTable.newAppend().appendFile(DataFiles.builder(secondTable.spec()) + .withPath(secondLocation + "/data/b.parquet") + .withFileSizeInBytes(20L).withRecordCount(2L).build()).commit(); + long firstSnapshotId = firstTable.currentSnapshot().snapshotId(); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + MetaCacheEntry entry = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + entry.put(mapping, new IcebergTableCacheValue(firstTable)); + ExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(dorisTable); + entry.put(mapping, new IcebergTableCacheValue(secondTable)); + + Assert.assertEquals(firstSnapshotId, + queryTable.currentSnapshot().snapshotId()); + Assert.assertEquals(firstTable.location(), + queryTable.location()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testPinnedGenerationSurvivesMetadataFileRetirement() throws Exception { + String staleLocation = temporaryFolder.newFolder("stale-metadata").toURI().toString(); + String freshLocation = temporaryFolder.newFolder("fresh-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table staleTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), staleLocation); + Table freshTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), freshLocation); + String staleMetadataLocation = ((HasTableOperations) staleTable) + .operations().current().metadataFileLocation(); + IcebergTableCacheValue staleValue = new IcebergTableCacheValue(staleTable); + staleValue.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + staleTable.io().deleteFile(staleMetadataLocation); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + Mockito.when(metadataOps.loadTable("db", "tbl")).thenReturn(freshTable); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, staleValue); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertEquals(staleTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.never()).loadTable("db", "tbl"); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testWeightedV1TablePublicationFailsClosedWithoutIo() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot currentSnapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/current-a.avro\"," + + "\"/manifest/current-b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .setBranchSnapshot(currentSnapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); + Table liveTable = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(mapping); + + Assert.assertFalse(value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); + Mockito.verifyNoInteractions(fileIO); + retained.currentSnapshot().allManifests(fileIO); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/current-a.avro"); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/current-b.avro"); + } + + @Test + public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Exception { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + String tableLocation = temporaryFolder.newFolder("v2-table").toURI().toString(); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + liveTable.newAppend().appendFile( + DataFiles.builder(liveTable.spec()) + .withPath(tableLocation + "/data/a.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()).commit(); + Assert.assertNotNull(liveTable.currentSnapshot().manifestListLocation()); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Table firstQuery = value.getIcebergTable(); + Table secondQuery = value.getIcebergTable(); + List firstManifests = + firstQuery.currentSnapshot().dataManifests(firstQuery.io()); + List secondManifests = + secondQuery.currentSnapshot().dataManifests(secondQuery.io()); + Assert.assertEquals(1, firstManifests.size()); + Assert.assertEquals(1, secondManifests.size()); + Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); + Assert.assertNotSame(firstManifests, secondManifests); + Assert.assertNotSame(retained.currentSnapshot(), firstQuery.currentSnapshot()); + + // A time-travel projection built from the query-scoped view keeps that isolation: a + // historical manifest-list read must not touch the cached generation's snapshots. + long snapshotId = liveTable.currentSnapshot().snapshotId(); + IcebergSnapshotCacheValue historical = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(snapshotId, 0L), + Optional.empty(), value.getIcebergTable()); + Table historicalTable = historical.getIcebergTable().get(); + Assert.assertEquals(1, historicalTable.snapshot(snapshotId).dataManifests(historicalTable.io()).size()); + Assert.assertNotSame(retained.snapshot(snapshotId), historicalTable.snapshot(snapshotId)); + Snapshot cachedSnapshot = retained.snapshot(snapshotId); + for (Field retainedField : cachedSnapshot.getClass().getDeclaredFields()) { + if (java.lang.reflect.Modifier.isTransient(retainedField.getModifiers()) + && !retainedField.getType().isPrimitive()) { + retainedField.setAccessible(true); + Assert.assertNull(retainedField.getName() + " must stay unmaterialized in the cache", + retainedField.get(cachedSnapshot)); + } + } + } + + @Test + public void testTablePublicationDoesNotReadHistoricalManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot historical = SnapshotParser.fromJson("{\"snapshot-id\":6,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/history.avro\",\"schema-id\":0}"); + Snapshot current = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/current.avro\",\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .addSnapshot(historical) + .setBranchSnapshot(current, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v2.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + IcebergTableCacheValue value = new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl")); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Mockito.verify(fileIO, Mockito.never()).newInputFile("/manifest-list/history.avro"); + } + + @Test + public void testManifestEstimateScalesWithFileCount() { + ManifestCacheValue oneFile = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/one.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + ManifestCacheValue twoFiles = ManifestCacheValue.forDataFiles(java.util.Arrays.asList( + oneFile.getDataFiles().get(0), oneFile.getDataFiles().get(0))); + IcebergManifestEntryKey key = new IcebergManifestEntryKey("/manifest/data.avro", ManifestContent.DATA); + + long oneFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, oneFile).getBytes(); + long twoFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, twoFiles).getBytes(); + + Assert.assertTrue(twoFileBytes > oneFileBytes); + } + + @Test + public void testManifestFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol.avro", ManifestContent.DATA); + ManifestCacheValue empty = ManifestCacheValue.forDataFiles(Collections.emptyList()); + ManifestCacheValue populated = ManifestCacheValue.forDataFiles( + IntStream.range(0, 32).mapToObj(this::dataFileWithMetrics) + .collect(Collectors.toList())); + ManifestCacheValue shortTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(16))); + ManifestCacheValue longTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(4096))); + ManifestCacheValue emptyDeletes = ManifestCacheValue.forDeleteFiles(Collections.emptyList()); + ManifestCacheValue populatedDeletes = ManifestCacheValue.forDeleteFiles( + IntStream.range(0, 32).mapToObj(this::deleteFileWithMetrics) + .collect(Collectors.toList())); + + long emptyEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, empty).getBytes(); + long populatedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, populated).getBytes(); + long shortTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, shortTail).getBytes(); + long longTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, longTail).getBytes(); + IcebergManifestEntryKey deleteKey = new IcebergManifestEntryKey( + "/manifest/jol-delete.avro", ManifestContent.DELETES); + long emptyDeleteEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + deleteKey, emptyDeletes).getBytes(); + long populatedDeleteEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + deleteKey, populatedDeletes).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest files", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg long-tail path", shortTailEstimate, longTailEstimate, shortTail, longTail); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest delete files", emptyDeleteEstimate, populatedDeleteEstimate, + emptyDeletes, populatedDeletes); + } + + @Test + public void testManifestPartitionDataFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol-partitioned.avro", ManifestContent.DATA); + for (int fieldCount : new int[] {1, 8, 32, 100}) { + ManifestCacheValue unpartitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, 0)); + ManifestCacheValue partitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, fieldCount)); + long unpartitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, unpartitioned).getBytes(); + long partitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, partitioned).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest partition fields " + fieldCount, + unpartitionedEstimate, partitionedEstimate, + unpartitioned, partitioned); + } + } + + @Test + public void testManifestVariablePartitionValuesAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol-variable-partition.avro", ManifestContent.DATA); + ManifestCacheValue unpartitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, 0)); + long unpartitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, unpartitioned).getBytes(); + + ManifestCacheValue strings = ManifestCacheValue.forDataFiles( + manifestFilesWithPartitions(32, 8, Types.StringType.get(), + (fileIndex, fieldIndex) -> repeatedCharacter('s', 64) + + fileIndex + "_" + fieldIndex)); + long stringEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, strings).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest string partitions", + unpartitionedEstimate, stringEstimate, unpartitioned, strings); + + ManifestCacheValue binary = ManifestCacheValue.forDataFiles( + manifestFilesWithPartitions(32, 8, Types.BinaryType.get(), + (fileIndex, fieldIndex) -> ByteBuffer.allocate(64))); + long binaryEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, binary).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest binary partitions", + unpartitionedEstimate, binaryEstimate, unpartitioned, binary); + } + + @Test + public void testManifestAccountingFindsPathTailAtAnyPosition() { + List baselineFiles = new ArrayList<>(); + List tailFiles = new ArrayList<>(); + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + for (int index = 0; index < 101; index++) { + baselineFiles.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/file-" + index + ".parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()); + tailFiles.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(index == 57 ? largePath : "/data/file-" + index + ".parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()); + } + ManifestCacheValue baseline = ManifestCacheValue.forDataFiles(baselineFiles); + ManifestCacheValue withTail = ManifestCacheValue.forDataFiles(tailFiles); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-tail.avro", ManifestContent.DATA); + + long baselineBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, baseline).getBytes(); + long tailBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, withTail).getBytes(); + + Assert.assertTrue(tailBytes - baselineBytes >= largePath.length() - 32L); + Assert.assertTrue(tailBytes - baselineBytes < largePath.length() * 2L); + } + + @Test + @SuppressWarnings("unchecked") + public void testManifestAccountingFailsClosedBeyondWorkBudget() { + Map oversizedBounds = Mockito.mock(Map.class); + Mockito.when(oversizedBounds.size()).thenReturn(8_000_001); + // GenericDataFile wraps the bounds map without copying it, so the oversized size is + // observed by accounting without allocating the entries. + DataFile file = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/oversized-bounds.parquet").withFileSizeInBytes(10L) + .withMetrics(new Metrics(1L, null, null, null, null, oversizedBounds, null)) + .build(); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles( + Collections.singletonList(file)); + + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/oversized.avro", ManifestContent.DATA), + value).isComplete()); + } + + @Test + public void testManifestAccountingFailsClosedForUnreadablePartition() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("id").build(); + DataFile file = DataFiles.builder(spec) + .withPath("/data/unreadable-partition.parquet").withFileSizeInBytes(10L) + .withRecordCount(1L).withPartitionPath("id=1").build(); + // A partition value of a class the accounting does not know cannot be sized. + ((PartitionData) file.partition()).set(0, new Object()); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles( + Collections.singletonList(file)); + + Assert.assertFalse(value.isAccountingComplete()); + } + + @Test + public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + // Compare two non-empty schemas: an empty schema never materializes any lookup index, + // so it is not a fair baseline for the per-field formula. + IcebergTableCacheValue emptyTable = tableValueWithFields(1); + IcebergTableCacheValue populatedTable = tableValueWithFields(32); + long emptyTableEstimate = emptyTable.prepareForCachePublication(mapping).getBytes(); + long populatedTableEstimate = populatedTable.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(emptyTable); + materializeAllLazyState(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg table fields", emptyTableEstimate, populatedTableEstimate, + emptyTable, populatedTable); + + Table keyTable = tableWithMetadataLocation("/metadata/jol-snapshot-v1.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, keyTable).get(); + IcebergSnapshotCacheValue emptySnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(0), new IcebergSnapshot(-1L, 0L)); + IcebergSnapshotCacheValue populatedSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32), new IcebergSnapshot(-1L, 0L)); + long emptySnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, emptySnapshot).getBytes(); + long populatedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, populatedSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot partitions", emptySnapshotEstimate, populatedSnapshotEstimate, + emptySnapshot, populatedSnapshot); + + // A spec that widened after the related-table check retains one more literal per range + // endpoint and one more value/transform per partition; the projection charges the width + // it actually loaded instead of the single field the check assumed. + IcebergSnapshotCacheValue wideSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 3), new IcebergSnapshot(-1L, 0L)); + long wideSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, wideSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg wide snapshot partitions", populatedSnapshotEstimate, wideSnapshotEstimate, + populatedSnapshot, wideSnapshot); + + // Overlapping physical partitions merge into one Doris partition that keeps every + // enclosed name in a HashSet; the weight follows the set cardinality, not the group count. + IcebergSnapshotCacheValue aliasedSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 1, true), new IcebergSnapshot(-1L, 0L)); + long aliasedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, aliasedSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg partition aliases", populatedSnapshotEstimate, aliasedSnapshotEstimate, + populatedSnapshot, aliasedSnapshot); + + // A name mapping retains an element array per field once it has several historical names. + IcebergSnapshotCacheValue singleNames = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.of(nameMappingWithAliases(32, 1))); + IcebergSnapshotCacheValue manyNames = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.of(nameMappingWithAliases(32, 8))); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg name mapping aliases", + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, singleNames).getBytes(), + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, manyNames).getBytes(), + singleNames, manyNames); + } + + private Map> nameMappingWithAliases(int fieldCount, int aliasesPerField) { + Map> mapping = new java.util.HashMap<>(); + for (int field = 0; field < fieldCount; field++) { + List names = new ArrayList<>(); + for (int alias = 0; alias < aliasesPerField; alias++) { + names.add("field_" + field + "_v" + alias); + } + mapping.put(1000 + field, names); + } + return mapping; + } + + @Test + public void testNestedSchemaFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithNestedFields(1); + IcebergTableCacheValue populated = tableValueWithNestedFields(33); + + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(small); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg nested fields", smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testTablePropertyFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue empty = tableValueWithProperties(0); + IcebergTableCacheValue populated = tableValueWithProperties(32); + + long emptyEstimate = empty.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg table properties", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testSnapshotHistoryFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithSnapshotHistory(1, false); + IcebergTableCacheValue populated = tableValueWithSnapshotHistory(33, false); + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot history", smallEstimate, populatedEstimate, small, populated); + + IcebergTableCacheValue withoutLog = tableValueWithSnapshotHistory(33, false); + IcebergTableCacheValue withLog = tableValueWithSnapshotHistory(33, true); + long withoutLogEstimate = withoutLog.prepareForCachePublication(mapping).getBytes(); + long withLogEstimate = withLog.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot log", withoutLogEstimate, withLogEstimate, withoutLog, withLog); + } + + @Test + public void testV1SnapshotAccountingFailsClosedWithoutIo() { + Snapshot snapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifests\":[\"/manifest/v1-a.avro\",\"/manifest/v1-b.avro\"]}"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(snapshot))); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertFalse(estimate.isComplete()); + } + + @Test + public void testSnapshotWithoutSummaryRemainsCacheable() { + Snapshot snapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\"}"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(snapshot))); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + } + + @Test + public void testMaterializedV2SnapshotPayloadFailsClosed() throws Exception { + String snapshotJson = "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\"}"; + Snapshot unloaded = SnapshotParser.fromJson(snapshotJson); + MetaCacheSizeEstimate unloadedEstimate = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(unloaded))) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertTrue(unloadedEstimate.getIncompleteReason(), unloadedEstimate.isComplete()); + + for (String fieldName : new String[] { + "allManifests", "dataManifests", "deleteManifests", + "addedDataFiles", "removedDataFiles", + "addedDeleteFiles", "removedDeleteFiles"}) { + Snapshot loaded = SnapshotParser.fromJson(snapshotJson); + Field retainedField = loaded.getClass().getDeclaredField(fieldName); + retainedField.setAccessible(true); + retainedField.set(loaded, Collections.emptyList()); + + MetaCacheSizeEstimate loadedEstimate = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(loaded))) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertFalse(fieldName, loadedEstimate.isComplete()); + } + } + + @Test + public void testSnapshotKeyIdPayloadIsAccounted() { + String longKeyId = repeatedCharacter('k', 64 * 1024); + Snapshot shortSnapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\",\"key-id\":\"k\"}"); + Snapshot longSnapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\",\"key-id\":\"" + + longKeyId + "\"}"); + + long shortBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithSnapshots(shortSnapshot))); + long longBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithSnapshots(longSnapshot))); + + Assert.assertTrue(longBytes - shortBytes >= longKeyId.length() - 8L); + } + + @Test + public void testManifestEstimateAccountsForSkewedFilePaths() { + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/x.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(largePath) + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-skew.avro", ManifestContent.DATA); + + long smallBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue).getBytes(); + long largeBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue).getBytes(); + + Assert.assertTrue(largeBytes - smallBytes >= largePath.length() - "/data/x.parquet".length()); + } + + @Test + public void testManifestEstimateAccountsForSkewedBufferPayload() { + Metrics smallMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(32)), Collections.emptyMap()); + Metrics largeMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(64 * 1024)), Collections.emptyMap()); + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(smallMetrics) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(largeMetrics) + .build())); + + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/encrypted.avro", ManifestContent.DATA); + MetaCacheSizeEstimate smallEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue); + MetaCacheSizeEstimate largeEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue); + + Assert.assertTrue(smallEstimate.getIncompleteReason(), smallEstimate.isComplete()); + Assert.assertTrue(largeEstimate.getIncompleteReason(), largeEstimate.isComplete()); + Assert.assertEquals(1L, smallValue.getDataFileMetricEntryCount()); + Assert.assertEquals(1L, largeValue.getDataFileMetricEntryCount()); + Assert.assertTrue(largeEstimate.getBytes() - smallEstimate.getBytes() >= 64 * 1024 - 32); + } + + @Test + public void testManifestEstimateAccountsForDeleteFileAuxiliaryPayload() { + String largeReference = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + List largeOffsets = IntStream.range(0, 4096) + .mapToObj(index -> (long) index).collect(Collectors.toList()); + DeleteFile smallPositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile("/data/x.parquet") + .withSplitOffsets(Collections.singletonList(0L)) + .build(); + DeleteFile largePositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile(largeReference) + .withSplitOffsets(largeOffsets) + .build(); + int[] largeEqualityIds = IntStream.range(0, 4096).toArray(); + DeleteFile smallEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(1) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + DeleteFile largeEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(largeEqualityIds) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/delete.avro", ManifestContent.DELETES); + + long smallPositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallPositionDelete))).getBytes(); + long largePositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largePositionDelete))).getBytes(); + long smallEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallEqualityDelete))).getBytes(); + long largeEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largeEqualityDelete))).getBytes(); + + Assert.assertTrue(largePositionBytes > smallPositionBytes); + Assert.assertTrue(largeEqualityBytes > smallEqualityBytes); + } + + @Test + public void testV1SnapshotPublicationDoesNotPolluteManifestIo() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/a.avro\",\"/manifest/b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); + Table table = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), Optional.empty(), table); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate( + NameMapping.createForTest(1L, "db", "tbl"), table).get(); + + value.prepareForCachePublication(key); + + Assert.assertFalse(value.getSizeEstimate().isComplete()); + Table queryTable = value.getIcebergTable().get(); + Assert.assertSame(table.currentSnapshot(), queryTable.currentSnapshot()); + Mockito.verifyNoInteractions(fileIO); + queryTable.currentSnapshot().allManifests(fileIO); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/a.avro"); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/b.avro"); + } @Test public void testInvalidateTableKeepsManifestCache() { @@ -52,10 +1717,16 @@ public void testInvalidateTableKeepsManifestCache() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + + Table snapshotTable = tableWithMetadataLocation("/metadata/invalidate-v1.json"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(t1, snapshotTable).get(); + MetaCacheEntry snapshotEntry = cache.entry(catalogId, + IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, + new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); MetaCacheEntry viewEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_VIEW, NameMapping.class, org.apache.iceberg.view.View.class); @@ -77,6 +1748,7 @@ public void testInvalidateTableKeepsManifestCache() { Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); Assert.assertNull(viewEntry.getIfPresent(t1)); Assert.assertNotNull(viewEntry.getIfPresent(t2)); Assert.assertNotNull(manifestEntry.getIfPresent(m1)); @@ -98,10 +1770,8 @@ public void testInvalidateDbAndStats() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); MetaCacheEntry schemaEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class); @@ -228,6 +1898,577 @@ private Map manifestCacheEnabledProperties() { return properties; } + private long snapshotWeight(IcebergSnapshotEntryKey key, int partitionCount) { + IcebergPartitionInfo partitionInfo = Mockito.mock(IcebergPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Map> aliases = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToIcebergPartition()).thenReturn(partitions); + Mockito.when(partitionInfo.getNameToIcebergPartitionNames()).thenReturn(aliases); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(key.getSnapshotId(), key.getSchemaId())); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateSnapshotEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private long manifestWeight(IcebergManifestEntryKey key, int fileCount) { + ManifestCacheValue value = Mockito.mock(ManifestCacheValue.class); + List dataFiles = sizeOnlyList(fileCount); + Mockito.when(value.getDataFiles()).thenReturn(dataFiles); + Mockito.when(value.getDeleteFiles()).thenReturn(Collections.emptyList()); + Mockito.when(value.isAccountingComplete()).thenReturn(true); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private void assertLinearScale(long base, long oneThousand, long tenThousand, long oneHundredThousand) { + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + @SuppressWarnings("unchecked") + private List sizeOnlyList(int size) { + List list = Mockito.mock(List.class); + Mockito.when(list.size()).thenReturn(size); + return list; + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Table tableWithMetadataLocation(String metadataLocation) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation(metadataLocation).build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + + private IcebergTableCacheValue tableValueWithFields(int fieldCount) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + TableMetadata metadata = TableMetadata.newTableMetadata( + new Schema(fields), PartitionSpec.unpartitioned(), + "file:/warehouse/jol-table", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithNestedFields(int nestedFieldCount) { + List nestedFields = IntStream.range(0, nestedFieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 2, "nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema(Types.NestedField.optional( + 1, "payload", Types.StructType.of(nestedFields))); + return tableValueWithSchemaAndProperties(schema, Collections.emptyMap()); + } + + private Schema unicodeNestedSchema(int nestedFieldCount) { + List nestedFields = new ArrayList<>(); + for (int index = 0; index < nestedFieldCount; index++) { + nestedFields.add(Types.NestedField.optional( + index + 10, "Nested_" + DOTTED_CAPITAL_I + "_" + index, Types.StringType.get())); + } + Types.StructType element = Types.StructType.of(Types.NestedField.optional( + 3, "Leaf_" + DOTTED_CAPITAL_I, Types.StringType.get())); + return new Schema( + Types.NestedField.optional(1, "Payload_" + DOTTED_CAPITAL_I, Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "List_" + DOTTED_CAPITAL_I, + Types.ListType.ofOptional(4, element)), + Types.NestedField.optional(5, "Map_" + DOTTED_CAPITAL_I, Types.MapType.ofOptional( + 6, 7, Types.StringType.get(), Types.StringType.get()))); + } + + private IcebergTableCacheValue tableValueWithIdentityPartitionedFields(int fieldCount) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema(fields); + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + } + return tableValueWithSchemaAndSpec(schema, specBuilder.build()); + } + + private IcebergTableCacheValue tableValueWithNestedMixedCaseFields(int nestedFieldCount) { + // Uncached field ids, upper-case names and list/map synthetic fields exercise the boxed + // key, lower-case String and short-alias terms of the schema formula. + List nestedFields = IntStream.range(0, nestedFieldCount) + .mapToObj(index -> Types.NestedField.optional( + 1000 + index, "Nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema( + Types.NestedField.optional(1, "payload", Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "list", Types.ListType.ofOptional(3, + Types.StructType.of(Types.NestedField.optional( + 4, "leaf", Types.StringType.get())))), + Types.NestedField.optional(5, "map", Types.MapType.ofOptional( + 6, 7, Types.StringType.get(), Types.LongType.get())), + Types.NestedField.optional(8, "id", Types.LongType.get())); + return tableValueWithSchemaAndSpec( + schema, PartitionSpec.builderFor(schema).identity("id").build()); + } + + private IcebergTableCacheValue tableValueWithSchemaAndSpec(Schema schema, PartitionSpec spec) { + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, spec, "file:/warehouse/jol-table", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithProperties(int propertyCount) { + Map properties = IntStream.range(0, propertyCount).boxed() + .collect(Collectors.toMap(index -> "key_" + index, index -> "value_" + index)); + Schema schema = new Schema(Types.NestedField.required( + 1, "id", Types.IntegerType.get())); + return tableValueWithSchemaAndProperties(schema, properties); + } + + private IcebergTableCacheValue tableValueWithSchemaAndProperties( + Schema schema, Map properties) { + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/jol-table", properties); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithSnapshotHistory( + int snapshotCount, boolean includeSnapshotLog) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"jol-table\",") + .append("\"location\":\"file:/warehouse/jol-table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index); + if (index > 0) { + json.append(",\"parent-snapshot-id\":").append(1000L + index - 1L); + } + json.append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/jol/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + if (includeSnapshotLog) { + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "/metadata/jol-history-v1.json", json.toString()); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Exception { + return realPartitionInfo(partitionCount, 1); + } + + private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partitionColumnCount) + throws Exception { + return realPartitionInfo(partitionCount, partitionColumnCount, false); + } + + private IcebergPartitionInfo realPartitionInfo( + int partitionCount, int partitionColumnCount, boolean mergeAllIntoFirst) throws Exception { + Map partitionItems = new java.util.HashMap<>(); + Map partitions = new java.util.HashMap<>(); + List partitionColumns = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + partitionColumns.add(new org.apache.doris.catalog.Column( + "part" + column, org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + } + for (int index = 0; index < partitionCount; index++) { + String value = Integer.toString(index); + String name = "part=" + value; + partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( + IcebergUtils.getPartitionRange(value, "day", partitionColumns))); + // Loaded partitions own one String per value and transform. + List values = new ArrayList<>(); + List transforms = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + values.add(new String(value)); + transforms.add(new String("day")); + } + partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); + } + Map> aliases = Collections.emptyMap(); + if (mergeAllIntoFirst && partitionCount > 0) { + // mergeOverlapPartitions() shape: the surviving name owns a set of every enclosed name. + aliases = Collections.singletonMap("part=0", new java.util.HashSet<>(partitions.keySet())); + } + return new IcebergPartitionInfo(partitionItems, partitions, aliases); + } + + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { + Map columnSizes = metricLongMap(index, 0); + Map valueCounts = metricLongMap(index, 1); + Map nullCounts = metricLongMap(index, 2); + Map nanCounts = metricLongMap(index, 3); + Map lowerBounds = metricBufferMap(); + Map upperBounds = metricBufferMap(); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build() + .copy(); + } + + private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/" + repeatedCharacter('x', pathLength) + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .build(); + } + + private DeleteFile deleteFileWithMetrics(int index) { + Map columnSizes = metricLongMap(index, 0); + Map valueCounts = metricLongMap(index, 1); + Map nullCounts = metricLongMap(index, 2); + Map nanCounts = metricLongMap(index, 3); + Map lowerBounds = metricBufferMap(); + Map upperBounds = metricBufferMap(); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .withReferencedDataFile("/data/jol-" + index + ".parquet") + .withMetrics(metrics) + .build() + .copy(); + } + + private Map metricLongMap(int fileIndex, int mapIndex) { + Map values = new java.util.HashMap<>(); + for (int column = 0; column < 8; column++) { + values.put(Integer.valueOf(10_000 + column), + Long.valueOf(10_000L + fileIndex * 100L + mapIndex * 10L + column)); + } + return values; + } + + private Map metricBufferMap() { + Map values = new java.util.HashMap<>(); + for (int column = 0; column < 8; column++) { + values.put(Integer.valueOf(10_000 + column), ByteBuffer.allocate(32)); + } + return values; + } + + private List manifestFilesWithIntegerPartitions( + int fileCount, int partitionFieldCount) { + return manifestFilesWithPartitions( + fileCount, partitionFieldCount, Types.IntegerType.get(), + (fileIndex, fieldIndex) -> Integer.valueOf( + 10_000 + fileIndex * partitionFieldCount + fieldIndex)); + } + + private List manifestFilesWithPartitions( + int fileCount, int partitionFieldCount, Type.PrimitiveType partitionType, + BiFunction valueFactory) { + if (partitionFieldCount == 0) { + return IntStream.range(0, fileCount) + .mapToObj(index -> DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/partition-data/file-" + index + ".parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()) + .collect(Collectors.toList()); + } + List fields = IntStream.range(0, partitionFieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "partition_" + index, partitionType)) + .collect(Collectors.toList()); + Schema schema = new Schema(fields); + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + fields.forEach(field -> specBuilder.identity(field.name())); + PartitionSpec spec = specBuilder.build(); + DataFiles.Builder fileBuilder = DataFiles.builder(spec); + PartitionData partitionData = new PartitionData(spec.partitionType()); + List files = new ArrayList<>(fileCount); + for (int fileIndex = 0; fileIndex < fileCount; fileIndex++) { + for (int fieldIndex = 0; fieldIndex < partitionFieldCount; fieldIndex++) { + partitionData.set(fieldIndex, valueFactory.apply(fileIndex, fieldIndex)); + } + files.add(fileBuilder.withPartition(partitionData) + .withPath("/partition-data/file-" + fileIndex + ".parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()); + } + return files; + } + + private Table tableWithMetadata(TableMetadata metadata) { + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(metadata); + return new BaseTable(operations, "db.tbl"); + } + + private TableMetadata roundTripMetadata(TableMetadata metadata, String metadataLocation) { + return TableMetadataParser.fromJson(metadataLocation, TableMetadataParser.toJson(metadata)); + } + + private void materializeAllLazyState(IcebergTableCacheValue value) { + Table table = value.getRetainedIcebergTable(); + materializeAllLazyState(((HasTableOperations) table).operations().current()); + } + + private void materializeAllLazyState(TableMetadata metadata) { + for (Schema schema : metadata.schemas()) { + materializeSchemaAndStruct(schema); + } + for (PartitionSpec spec : metadata.specs()) { + spec.fields(); + spec.javaClasses(); + Types.StructType partitionType = spec.partitionType(); + spec.rawPartitionType(); + if (!spec.fields().isEmpty()) { + spec.getFieldsBySourceId(spec.fields().get(0).sourceId()); + } + materializeStructAndSecondarySchema(partitionType); + } + } + + private void materializeSchemaAndStruct(Schema schema) { + if (schema.columns().isEmpty()) { + return; + } + Types.NestedField first = schema.columns().get(0); + materializeSchemaIndexes(schema, first); + materializeStructAndSecondarySchema(schema.asStruct()); + } + + private void materializeStructAndSecondarySchema(Types.StructType struct) { + if (struct.fields().isEmpty()) { + return; + } + Types.NestedField first = struct.fields().get(0); + materializeStructIndexes(struct, first); + Schema secondary = struct.asSchema(); + materializeSchemaIndexes(secondary, first); + materializeStructIndexes(secondary.asStruct(), first); + } + + private void materializeSchemaIndexes(Schema schema, Types.NestedField first) { + schema.findField(first.name()); + schema.findField(first.fieldId()); + schema.caseInsensitiveFindField(first.name().toUpperCase(java.util.Locale.ROOT)); + schema.idToName(); + schema.identifierFieldIds(); + schema.accessorForField(first.fieldId()); + } + + private void materializeStructIndexes(Types.StructType struct, Types.NestedField first) { + struct.fields(); + struct.field(first.name()); + struct.caseInsensitiveField(first.name().toUpperCase(java.util.Locale.ROOT)); + struct.field(first.fieldId()); + } + + /** + * Delta between the first two columns and the whole schema: measures the schema graph. Both + * sides reach the same shared type singletons (StringType, ListType element names, ...) so + * only per-column growth is compared. + */ + private void assertSchemaLookupFormula(Schema schema, String fixture) { + Schema firstColumns = new Schema(schema.schemaId(), schema.columns().subList(0, 2)); + TableMetadata empty = TableMetadata.newTableMetadata( + firstColumns, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + TableMetadata populated = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + assertRetainedPayloadDelta(fixture, empty, populated, "jol-schema-lookup"); + } + + /** Delta between an unpartitioned spec and one identity field: measures the spec graph. */ + private void assertPartitionSpecFormula(Schema schema, String fixture) { + TableMetadata empty = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + TableMetadata populated = TableMetadata.newTableMetadata( + schema, PartitionSpec.builderFor(schema).identity(schema.columns().get(0).name()).build(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + assertRetainedPayloadDelta(fixture, empty, populated, "jol-partition-spec"); + } + + private void assertRetainedPayloadDelta( + String fixture, TableMetadata empty, TableMetadata populated, String locationPrefix) { + empty = roundTripMetadata(empty, + "/metadata/" + locationPrefix + "-none-" + fixture.hashCode() + ".json"); + populated = roundTripMetadata(populated, + "/metadata/" + locationPrefix + "-with-" + fixture.hashCode() + ".json"); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(empty)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populated)); + materializeAllLazyState(empty); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg " + fixture, emptyEstimate, populatedEstimate, empty, populated); + } + + private TableMetadata metadataWithMaterializedPayload(String payload, int bufferBytes) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.uuid()).thenReturn("stable-uuid"); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + payload, Mockito.mock(SnapshotRef.class))); + TableMetadata.MetadataLogEntry metadataLogEntry = + Mockito.mock(TableMetadata.MetadataLogEntry.class); + Mockito.when(metadataLogEntry.file()).thenReturn(payload); + Mockito.when(metadata.previousFiles()).thenReturn( + Collections.singletonList(metadataLogEntry)); + GenericBlobMetadata blob = new GenericBlobMetadata( + payload, 1L, 1L, Collections.singletonList(1), + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.singletonList( + new GenericStatisticsFile(1L, payload, 1L, 1L, + Collections.singletonList(blob)))); + org.apache.iceberg.PartitionStatisticsFile partitionStatistics = + Mockito.mock(org.apache.iceberg.PartitionStatisticsFile.class); + Mockito.when(partitionStatistics.path()).thenReturn(payload); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn( + Collections.singletonList(partitionStatistics)); + EncryptedKey encryptedKey = Mockito.mock(EncryptedKey.class); + Mockito.when(encryptedKey.keyId()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedById()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedKeyMetadata()).thenReturn( + ByteBuffer.allocateDirect(bufferBytes)); + Mockito.when(encryptedKey.properties()).thenReturn( + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.singletonList(encryptedKey)); + return metadata; + } + + private TableMetadata metadataWithSnapshots(Snapshot snapshot) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshots()).thenReturn(Collections.singletonList(snapshot)); + Mockito.when(metadata.currentSnapshot()).thenReturn(snapshot); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.refs()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/snapshot.json"); + return metadata; + } + + private TableMetadata metadataWithStatisticsFile(StatisticsFile statisticsFile) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshots()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.refs()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.statisticsFiles()).thenReturn( + Collections.singletonList(statisticsFile)); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/statistics.json"); + return metadata; + } + + private TableMetadata metadataWithSnapshotSequence(long lastSequenceNumber) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.singletonList( + Mockito.mock(org.apache.iceberg.HistoryEntry.class))); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + "branch-tip", Mockito.mock(SnapshotRef.class))); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.lastSequenceNumber()).thenReturn(lastSequenceNumber); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/sequence.json"); + return metadata; + } + + private Table tableWithNestedSchemaAndProperty(String nestedFieldName, String propertyValue) { + Schema schema = new Schema(Types.NestedField.optional(1, "payload", + Types.StructType.of(Types.NestedField.optional( + 2, nestedFieldName, Types.StringType.get())))); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.singletonMap("payload", propertyValue)); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/nested.json").build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + private IcebergManifestEntryKey mockManifestKey(String path) { return IcebergManifestEntryKey.of(new TestingManifestFile(path, ManifestContent.DATA)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index 720f66fd5f9e3f..de3ab8c9746397 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -92,9 +92,9 @@ public void setUp() throws IOException { Mockito.doReturn(db).when(catalog).getDbNullable(Mockito.any()); Mockito.doReturn(dorisTable).when(db).getTableNullable(Mockito.any()); - // mock IcebergUtils.getIcebergTable to return our test icebergTable + // Mock writable access used by branch and tag mutations. mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(Mockito.any())) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any())) .thenReturn(icebergTable); // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a4b882c4497132..cc019c73ed12a7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -138,7 +138,8 @@ public void testTopLevelVariantModifyOnlyUpdatesMetadataOnOrcTable() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("payload"), column, ColumnPosition.FIRST, 1L); } @@ -165,7 +166,8 @@ public void testTopLevelVariantModifyRejectsTypeConversions() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("variant_col"), new Column("variant_col", Type.STRING, true), null, 1L), @@ -294,7 +296,7 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), new Column("new_field", Type.LARGEINT, true), null, 1L), @@ -328,7 +330,7 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); @@ -363,7 +365,7 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); } @@ -391,7 +393,7 @@ public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComm try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -433,7 +435,7 @@ public void testFullStructModifyPreservesOmittedChildComments() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), new Column("payload", payloadType, true), null, 1L); @@ -463,7 +465,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -489,7 +491,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("id"), new Column("id", Type.BIGINT, true), null, 1L); @@ -515,7 +517,7 @@ public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L), @@ -537,7 +539,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L); @@ -568,7 +570,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, @@ -603,7 +605,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); @@ -633,7 +635,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, @@ -661,7 +663,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, @@ -692,7 +694,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); // Iceberg schema columns are represented as keys in Doris, so the legacy API must not // interpret isKey as an explicit KEY clause. @@ -723,7 +725,7 @@ public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, column, null, 1L); } @@ -757,7 +759,7 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); @@ -810,7 +812,7 @@ public void execute(Runnable task) { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(staleTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(staleTable); try { conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), @@ -861,7 +863,7 @@ public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); icebergTable.refresh(); @@ -911,7 +913,7 @@ public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, "a", "renamed", 1L); icebergTable.refresh(); @@ -940,7 +942,7 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), nestedAddDefaultColumn, null, 1L), @@ -971,7 +973,7 @@ public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: id"); @@ -1002,7 +1004,7 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), @@ -1035,7 +1037,7 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); @@ -1082,7 +1084,7 @@ public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, new Column("info", infoType, true), null, 1L), @@ -1158,7 +1160,7 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn( dorisTable, new Column("id", Type.STRING, true), null, 1L), @@ -1194,7 +1196,7 @@ public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); } @@ -1216,7 +1218,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), null, 1L); @@ -1239,7 +1241,7 @@ public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), @@ -1270,7 +1272,7 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), "struct comment", 1L); @@ -1296,7 +1298,7 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), @@ -1336,7 +1338,7 @@ public void testRejectsTopLevelRowLineageMutationsForV3Tables() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L), @@ -1392,8 +1394,10 @@ public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v3DorisTable)).thenReturn(v3IcebergTable); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v2DorisTable)).thenReturn(v2IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v3DorisTable)) + .thenReturn(v3IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v2DorisTable)) + .thenReturn(v2IcebergTable); ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java index 74c8c3f6954a97..c184debd3d70ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java @@ -17,16 +17,39 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.Map; import java.util.Set; public class IcebergPartitionInfoTest { + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + String largeValue = repeatedCharacter('x', 64 * 1024); + IcebergPartition small = new IcebergPartition("p=x", 0, 0, 0, 0, 1, 101, + Collections.singletonList("x"), Collections.singletonList("identity")); + IcebergPartition large = new IcebergPartition("p=" + largeValue, 0, 0, 0, 0, 1, 101, + Collections.singletonList(largeValue), Collections.singletonList("identity")); + + long expectedDelta = MetaCacheWeightUtils.estimatedStringBytes("p=" + largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("p=x") + + MetaCacheWeightUtils.estimatedStringBytes(largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("x"); + Assertions.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= expectedDelta); + IcebergPartitionInfo info = new IcebergPartitionInfo( + Collections.emptyMap(), Collections.singletonMap(large.getPartitionName(), large), + Collections.emptyMap()); + Assertions.assertEquals(large.getRetainedPayloadBytes(), info.getRetainedPayloadBytes()); + } + @Test public void testGetLatestSnapshotId() { IcebergPartition p1 = new IcebergPartition("p1", 0, 0, 0, 0, 1, 101, null, null); @@ -50,4 +73,10 @@ public void testGetLatestSnapshotId() { Assertions.assertEquals(102, snapshot2); Assertions.assertEquals(103, snapshot3); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 03d217c25d16e5..5e73286fb8260d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -17,11 +17,20 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; + import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.Optional; + public class IcebergSysExternalTableTest { @Test public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { @@ -45,4 +54,66 @@ public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { sourceTable, MetadataTableType.DATA_FILES.name()); Assertions.assertTrue(dataFiles.supportsSnapshotSelection()); } + + @Test + public void testMetadataSchemaReloadsAfterSourceEvolution() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table firstGeneration = Mockito.mock(Table.class); + Table evolvedGeneration = Mockito.mock(Table.class); + Mockito.when(firstGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()))); + Mockito.when(evolvedGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.optional(2, "evolved_partition", Types.StringType.get()))); + IcebergSysExternalTable sysTable = Mockito.spy(new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name())); + Mockito.doReturn(firstGeneration, evolvedGeneration).when(sysTable).getSysIcebergTable(); + + Assertions.assertEquals(1, sysTable.getFullSchema().size()); + Assertions.assertEquals(2, sysTable.getFullSchema().size()); + Mockito.verify(sysTable, Mockito.times(2)).getSysIcebergTable(); + } + + @Test + public void testSnapshotSelectableSchemaFollowsRelationSnapshot() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table frozenGeneration = Mockito.mock(Table.class); + Table latestGeneration = Mockito.mock(Table.class); + IcebergSnapshotCacheValue snapshotValue = Mockito.mock(IcebergSnapshotCacheValue.class); + Mockito.when(snapshotValue.getIcebergTable()).thenReturn(Optional.of(frozenGeneration)); + Optional relationSnapshot = Optional.of(new IcebergMvccSnapshot(snapshotValue)); + + try (MockedStatic mvccUtil = Mockito.mockStatic(MvccUtil.class); + MockedStatic icebergUtils = Mockito.mockStatic(IcebergUtils.class)) { + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(relationSnapshot); + icebergUtils.when(() -> IcebergUtils.getQueryScopedIcebergTable(sourceTable)) + .thenReturn(latestGeneration); + + // $partitions is snapshot-selectable: analysis must see the generation the scan uses. + IcebergSysExternalTable partitions = new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name()); + Assertions.assertSame(frozenGeneration, partitions.resolveBaseTable()); + + // $snapshots ignores a selected snapshot and keeps reading the latest generation. + IcebergSysExternalTable snapshots = new IcebergSysExternalTable( + sourceTable, MetadataTableType.SNAPSHOTS.name()); + Assertions.assertSame(latestGeneration, snapshots.resolveBaseTable()); + + // Without a bound relation snapshot the latest generation is used. + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(Optional.empty()); + Assertions.assertSame(latestGeneration, partitions.resolveBaseTable()); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index e2d923f3438863..bd966a75332568 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -34,6 +34,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; @@ -202,7 +203,7 @@ public void testPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); // Allow parsePartitionValueFromString to call the real implementation mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( @@ -318,7 +319,7 @@ public void testUnPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -429,7 +430,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -455,7 +456,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -500,7 +501,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -517,7 +518,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th checkPushDownByPartition(table, Expressions.equal("str1", "partition-b"), 1); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -590,7 +591,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(3); @@ -651,7 +652,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(formatVersion); @@ -706,6 +707,43 @@ public void testBeginDeleteUsesRetainedTargetTable() throws UserException { Mockito.verify(retainedTable).newTransaction(); } + @Test + public void testQueryScopedGenerationCommitsThroughWritableOperations() throws UserException { + // A weight-bounded snapshot cache hands query-scoped (read-only) tables to the sink; + // commits must still be re-based onto the live table operations. + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(liveTable); + tableValue.prepareForCachePublication(NameMapping.createForTest(dbName, tbWithoutPartition)); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson()); + Table queryScopedTable = cacheValue.getIcebergTable().get(); + Assert.assertFalse(IcebergSnapshotCacheValue.isFrozenGeneration(queryScopedTable)); + Assert.assertTrue(IcebergSnapshotCacheValue.isRetainedGeneration(queryScopedTable)); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("query-scoped-generation.parquet"); + commitData.setFileContent(TFileContent.DATA); + commitData.setRowCount(1); + commitData.setFileSize(1); + + try (MockedStatic mockedUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(Collections.singletonList(commitData)); + txn.beginInsert(dorisTable, queryScopedTable, Optional.empty()); + txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); + txn.commit(); + } + + Assert.assertNotNull(ops.getCatalog().loadTable( + TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); + } + @Test public void testRetainedGenerationCommitsThroughWritableOperations() throws UserException { Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); @@ -724,7 +762,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -736,6 +774,26 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); } + @Test + public void testWeightedTableSupportsSchemaAndPartitionSpecCommits() { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergTableCacheValue cacheValue = new IcebergTableCacheValue(liveTable); + cacheValue.prepareForCachePublication(NameMapping.createForTest(dbName, tbWithoutPartition)); + + Table writableTable = cacheValue.getWritableIcebergTable(liveTable); + writableTable.updateSchema() + .addColumn("new_col", Types.StringType.get()) + .commit(); + writableTable.updateSpec() + .addField("int1") + .commit(); + + Table refreshed = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + Assert.assertNotNull(refreshed.schema().findField("new_col")); + Assert.assertEquals(1, refreshed.spec().fields().size()); + Assert.assertEquals("int1", refreshed.spec().fields().get(0).name()); + } + @Test public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws UserException { Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); @@ -754,7 +812,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -783,7 +841,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -805,6 +863,30 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User Assert.assertEquals(2, refreshedTable.history().size()); } + @Test + public void testRetainedGenerationRefusesRetryAgainstRecreatedTable() { + HadoopCatalog icebergCatalog = (HadoopCatalog) ops.getCatalog(); + TableIdentifier identifier = TableIdentifier.of(dbName, tbWithoutPartition); + Table originalTable = icebergCatalog.loadTable(identifier); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), originalTable); + Table retainedTable = cacheValue.getIcebergTable().get(); + String retainedUuid = ((HasTableOperations) retainedTable).operations().current().uuid(); + + // Drop and recreate at the same location: schema, spec and sort-order ids restart, and + // the writer contract looks identical except for the table UUID. + icebergCatalog.dropTable(identifier, true); + Table recreatedTable = icebergCatalog.createTable(identifier, originalTable.schema()); + Assert.assertNotEquals(retainedUuid, + ((HasTableOperations) recreatedTable).operations().current().uuid()); + + Table writableTable = IcebergSnapshotCacheValue.createWritableTable(retainedTable, recreatedTable); + CommitFailedException failure = Assert.assertThrows(CommitFailedException.class, + () -> ((HasTableOperations) writableTable).operations().refresh()); + Assert.assertTrue(failure.getMessage(), failure.getMessage().contains("table UUID")); + } + @Test public void testStaticPartitionFilterRejectsUnknownKey() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 6407d540d1ef03..3decfbef1b0483 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -28,8 +28,13 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class AbstractExternalMetaCacheTest { @@ -97,6 +102,34 @@ public void testEntryFailsFastAfterCatalogRemoved() { } } + @Test + public void testCapturedCatalogGroupReturnsClosedEntryDuringConcurrentRemoval() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newSingleThreadExecutor(); + CountDownLatch groupCaptured = new CountDownLatch(1); + CountDownLatch releaseEntryLookup = new CountDownLatch(1); + LookupRaceExternalMetaCache cache = + new LookupRaceExternalMetaCache(refreshExecutor, groupCaptured, releaseEntryLookup); + try { + cache.initCatalog(1L, Maps.newHashMap()); + Future> lookup = workers.submit( + () -> cache.entry(1L, "value", String.class, Integer.class)); + Assert.assertTrue(groupCaptured.await(3L, TimeUnit.SECONDS)); + + cache.invalidateCatalog(1L); + releaseEntryLookup.countDown(); + + MetaCacheEntry capturedClosedEntry = lookup.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(Integer.valueOf(1), capturedClosedEntry.get("k")); + Assert.assertNull(capturedClosedEntry.peekIfPresent("k")); + } finally { + releaseEntryLookup.countDown(); + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testEntryLevelInvalidationUsesRegisteredMatcher() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -121,6 +154,127 @@ public void testEntryLevelInvalidationUsesRegisteredMatcher() { } } + @Test + public void testGlobalWeightAutomaticallyActivatesEntriesWithEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(600L))); + try { + cache.initCatalog(1L, Maps.newHashMap()); + MetaCacheEntry entry = cache.entry(1L, "value", String.class, Integer.class); + + Assert.assertTrue(entry.isWeightBounded()); + Assert.assertEquals(600L, entry.stats().getMaxWeight()); + entry.put("first", 60); + entry.put("second", 60); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(60), entry.getIfPresent("second")); + Assert.assertEquals(60L + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES, + entry.stats().getGlobalEstimatedWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitIgnoresEntryWeightWithoutEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); + Map properties = Maps.newHashMap(); + properties.put("meta.cache.test_engine.schema.max-weight", "1KB"); + + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertFalse(cache.entry(1L, "schema", SchemaCacheKey.class, SchemaCacheValue.class) + .isWeightBounded()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitIgnoresEntryWeightAboveCatalogWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(4L * 1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertEquals(1024L, cache.entry(1L, "value", String.class, Integer.class) + .stats().getMaxWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitClampsCatalogAcceptedOnLargerFe() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "4KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + Assert.assertThrows(IllegalArgumentException.class, + () -> cache.validateCatalogProperties(properties)); + + cache.initCatalog(1L, properties); + + MetaCacheEntryStats stats = cache.stats(1L).get("value"); + Assert.assertEquals(1024L, stats.getMaxWeight()); + Assert.assertEquals(1024L, stats.getCatalogMaxWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCatalogRemoveAndInitDoesNotDuplicateBudgetScope() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newFixedThreadPool(2); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(100L))); + CountDownLatch start = new CountDownLatch(1); + try { + Future first = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + Future second = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + start.countDown(); + first.get(10L, TimeUnit.SECONDS); + second.get(10L, TimeUnit.SECONDS); + cache.initCatalog(1L, Maps.newHashMap()); + Assert.assertTrue(cache.isCatalogInitialized(1L)); + } finally { + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + private static void repeatedlyRebuildCatalog(WeightedExternalMetaCache cache, CountDownLatch start) { + try { + Assert.assertTrue(start.await(3L, TimeUnit.SECONDS)); + for (int i = 0; i < 100; i++) { + cache.initCatalog(1L, Maps.newHashMap()); + cache.invalidateCatalog(1L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + private static final class TestExternalMetaCache extends AbstractExternalMetaCache { private TestExternalMetaCache(ExecutorService refreshExecutor) { super("test_engine", refreshExecutor); @@ -135,4 +289,44 @@ private TestExternalMetaCache(ExecutorService refreshExecutor) { MetaCacheEntryInvalidation.forNameMapping(SchemaCacheKey::getNameMapping))); } } + + private static final class WeightedExternalMetaCache extends AbstractExternalMetaCache { + private WeightedExternalMetaCache( + ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super("weighted_test", refreshExecutor, budgetManager); + registerEntry(MetaCacheEntryDef.of( + "value", + String.class, + Integer.class, + key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(value.longValue()))); + } + } + + private static final class LookupRaceExternalMetaCache extends AbstractExternalMetaCache { + private final CountDownLatch groupCaptured; + private final CountDownLatch releaseEntryLookup; + + private LookupRaceExternalMetaCache(ExecutorService refreshExecutor, + CountDownLatch groupCaptured, CountDownLatch releaseEntryLookup) { + super("lookup_race", refreshExecutor); + this.groupCaptured = groupCaptured; + this.releaseEntryLookup = releaseEntryLookup; + registerEntry(MetaCacheEntryDef.of( + "value", String.class, Integer.class, key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L))); + } + + @Override + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + groupCaptured.countDown(); + try { + Assert.assertTrue(releaseEntryLookup.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java index 05acbb539a26d6..6ea171abd6d717 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.DdlException; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -68,6 +69,7 @@ public void testFromPropertiesWithPropertySpecBuilder() { public void testFromPropertiesWithEngineEntryKeys() { Map properties = Maps.newHashMap(); properties.put("meta.cache.hive.schema.ttl-second", "0"); + properties.put("meta.cache.hive.schema.max-weight", "2KB"); CacheSpec defaultSpec = CacheSpec.fromProperties( Maps.newHashMap(), @@ -79,6 +81,8 @@ public void testFromPropertiesWithEngineEntryKeys() { Assert.assertTrue(spec.isEnable()); Assert.assertEquals(0, spec.getTtlSecond()); Assert.assertEquals(100, spec.getCapacity()); + Assert.assertTrue(spec.isWeightBounded()); + Assert.assertEquals(2048L, spec.getMaxWeight().getAsLong()); } @Test @@ -108,6 +112,7 @@ public void testOfSemantics() { Assert.assertTrue(enabled.isEnable()); Assert.assertEquals(60, enabled.getTtlSecond()); Assert.assertEquals(100, enabled.getCapacity()); + Assert.assertFalse(enabled.isWeightBounded()); CacheSpec zeroTtl = CacheSpec.of(true, 0, 100); Assert.assertTrue(zeroTtl.isEnable()); @@ -147,6 +152,10 @@ public void testIsCacheEnabled() { Assert.assertFalse(CacheSpec.isCacheEnabled(false, CacheSpec.CACHE_NO_TTL, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, 0, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, CacheSpec.CACHE_NO_TTL, 0)); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 0L, 1L).isCacheEnabled()); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 1L, 0L).isCacheEnabled()); } @Test @@ -166,4 +175,51 @@ public void testToExpireAfterAccess() { Assert.assertTrue(negativeOther.isPresent()); Assert.assertEquals(0, negativeOther.getAsLong()); } + + @Test + public void testParseWeight() { + Assert.assertEquals(1L, CacheSpec.parseWeight("1", "weight", false, 0L)); + Assert.assertEquals(1024L, CacheSpec.parseWeight("1KB", "weight", false, 0L)); + Assert.assertEquals(2L * 1024L * 1024L, + CacheSpec.parseWeight("2 mb", "weight", false, 0L)); + Assert.assertEquals(250L, CacheSpec.parseWeight("25%", "weight", true, 1000L)); + Assert.assertEquals(0L, CacheSpec.parseWeight("0", "weight", false, 0L)); + + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1.5GB", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("101%", "weight", true, 1000L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1PB000", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("999999999999999999PB", "weight", false, 0L)); + } + + @Test + public void testStrictEnginePropertyAllowlist() { + Map properties = Maps.newHashMap(); + properties.put("meta.cache.hive.partition_values.enable", "true"); + properties.put("meta.cache.hive.partition_values.ttl-second", "-1"); + properties.put("meta.cache.hive.partition_values.capacity", "10"); + properties.put("meta.cache.hive.partition_values.max-weight", "2MB"); + CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values")); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partiton_values.capacity"); + + properties.put("meta.cache.hive.partition_values.enabel", "true"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partition_values.enabel"); + + properties.put("meta.cache.hive.schema.max-weight", "1MB"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java new file mode 100644 index 00000000000000..d4215518c86356 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.junit.Assert; +import org.openjdk.jol.info.GraphLayout; +import org.openjdk.jol.info.GraphPathRecord; + +import java.lang.reflect.Field; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +/** JOL oracle used only by estimator calibration tests. */ +public final class EstimatorCalibrationAssertions { + private static final double MAX_CONSERVATIVE_FACTOR = 1.10D; + private static final boolean PRINT_RESULT = Boolean.getBoolean( + "metacache.estimator.calibration.print"); + // Integer.valueOf/Long.valueOf serve -128..127 from JVM-wide static caches. A populated + // fixture that reaches those shared instances (field ids, list indexes, small partition + // values) must not be charged for them as retained growth, so every graph is measured + // together with the same cache roots and the shared instances cancel out of the delta. + private static final Integer[] SHARED_INTEGER_CACHE = + IntStream.rangeClosed(-128, 127).boxed().toArray(Integer[]::new); + private static final Long[] SHARED_LONG_CACHE = + LongStream.rangeClosed(-128L, 127L).boxed().toArray(Long[]::new); + // Accessor objects reference java.lang.Class instances (String.class, StructLike.class, ...). + // JOL follows them into the JVM's per-class reflection and ClassValue caches, whose size + // depends on unrelated reflective use earlier in the same JVM (Mockito, layout fingerprints, + // JOL itself). Everything reached through a Class object is shared JVM state, not retained + // cache payload, and is excluded from every measurement. + private static final Field GRAPH_PATH_PARENT = graphPathParentField(); + + private static Field graphPathParentField() { + try { + Field field = GraphPathRecord.class.getDeclaredField("parent"); + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("JOL GraphPathRecord.parent is unavailable", e); + } + } + + static { + // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their + // offsets through the regular instrumentation path on JDK 17, so enable its Unsafe + // fallback for these test-only retained-graph measurements. Skip all attach attempts: + // Iceberg/Paimon calibration tests share their fork with Mockito's inline mock maker. + System.setProperty("jol.magicFieldOffset", "true"); + System.setProperty("jol.skipInstallAttach", "true"); + System.setProperty("jol.skipDynamicAttach", "true"); + System.setProperty("jol.skipHotspotSAAttach", "true"); + } + + private EstimatorCalibrationAssertions() { + } + + public static void assertConservativeDelta( + String fixture, long emptyEstimate, long populatedEstimate, + Object emptyGraph, Object populatedGraph) { + long actualDelta = graphSize(populatedGraph) - graphSize(emptyGraph); + long estimatedDelta = populatedEstimate - emptyEstimate; + if (PRINT_RESULT) { + System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", + fixture, estimatedDelta, actualDelta, + actualDelta == 0L ? Double.NaN : (double) estimatedDelta / actualDelta); + } + Assert.assertTrue(fixture + " must add retained heap", actualDelta > 0L); + Assert.assertTrue(fixture + " underestimates retained heap: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta >= actualDelta); + Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta <= Math.ceil(actualDelta * MAX_CONSERVATIVE_FACTOR)); + } + + /** Retained size of the graph excluding JVM-shared boxed-value caches and Class metadata. */ + public static long graphSize(Object graph) { + long sharedCacheBytes = GraphLayout.parseInstance( + SHARED_INTEGER_CACHE, SHARED_LONG_CACHE).totalSize(); + GraphLayout layout = GraphLayout.parseInstance( + graph, SHARED_INTEGER_CACHE, SHARED_LONG_CACHE); + long bytes = 0L; + for (long address : layout.addresses()) { + GraphPathRecord record = layout.record(address); + if (!reachedThroughClassObject(record)) { + bytes += record.size(); + } + } + return bytes - sharedCacheBytes; + } + + private static boolean reachedThroughClassObject(GraphPathRecord record) { + try { + for (GraphPathRecord current = record; current != null; + current = (GraphPathRecord) GRAPH_PATH_PARENT.get(current)) { + if (current.klass() == Class.class) { + return true; + } + } + return false; + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java new file mode 100644 index 00000000000000..d4b76b97c0131e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -0,0 +1,291 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExternalMetaCacheBudgetManagerTest { + + @Test + public void testGlobalCatalogAndEntryLimits() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget first = manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(60L)); + EntryBudget second = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(80L), OptionalLong.of(50L)); + + AdmissionReservation firstReservation = first.tryReserve(60L).get(); + Assert.assertFalse(second.tryReserve(30L).isPresent()); + AdmissionReservation secondReservation = second.tryReserve(20L).get(); + Assert.assertEquals(80L, manager.getGlobalUsedWeight()); + Assert.assertFalse(secondReservation.tryResize(30L)); + + firstReservation.release(); + Assert.assertTrue(secondReservation.tryResize(30L)); + Assert.assertEquals(30L, manager.getGlobalUsedWeight()); + + secondReservation.release(); + first.close(); + second.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testConcurrentReservationNeverExceedsGlobalLimit() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "iceberg", "manifest", OptionalLong.empty(), OptionalLong.empty()); + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + List reservations = Collections.synchronizedList(new ArrayList<>()); + try { + for (int i = 0; i < 200; i++) { + executor.submit(() -> { + await(start); + Optional reservation = budget.tryReserve(1L); + reservation.ifPresent(reservations::add); + }); + } + start.countDown(); + executor.shutdown(); + Assert.assertTrue(executor.awaitTermination(10L, TimeUnit.SECONDS)); + Assert.assertEquals(100, reservations.size()); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + } finally { + executor.shutdownNow(); + reservations.forEach(AdmissionReservation::release); + budget.close(); + } + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testRejectChildLargerThanParent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(90L))); + + Map properties = new HashMap<>(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "120"); + Assert.assertEquals(120L, manager.parseCatalogMaxWeight(properties).getAsLong()); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.validateCatalogMaxWeight(properties)); + } + + @Test + public void testRuntimeBudgetClampsReplayedCatalogToLocalGlobalLimit() { + ExternalMetaCacheBudgetManager observerManager = manager(100L); + + EntryBudget budget = observerManager.createEntryBudget( + 1L, "iceberg", "table", OptionalLong.of(400L), OptionalLong.of(300L)); + + Assert.assertEquals(100L, budget.getEffectiveMaxWeight()); + Assert.assertEquals(100L, budget.getCatalogMaxWeight()); + AdmissionReservation reservation = budget.tryReserve(100L).get(); + Assert.assertFalse(budget.tryReserve(1L).isPresent()); + reservation.release(); + budget.close(); + } + + @Test + public void testGlobalConfigSupportsPercentageAndDisabledZero() { + String original = Config.external_meta_cache_max_weight; + try { + Config.external_meta_cache_max_weight = "25%"; + ExternalMetaCacheBudgetManager percentageManager = ExternalMetaCacheBudgetManager.fromConfig(); + Assert.assertEquals(Runtime.getRuntime().maxMemory() / 4L, + percentageManager.getGlobalMaxWeight().getAsLong()); + + Config.external_meta_cache_max_weight = "0"; + Assert.assertFalse(ExternalMetaCacheBudgetManager.fromConfig().getGlobalMaxWeight().isPresent()); + + Config.external_meta_cache_max_weight = "0%"; + Assert.assertThrows(IllegalArgumentException.class, ExternalMetaCacheBudgetManager::fromConfig); + } finally { + Config.external_meta_cache_max_weight = original; + } + } + + @Test + public void testReservationReleaseIsIdempotent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = budget.tryReserve(40L).get(); + + reservation.release(); + reservation.release(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + budget.close(); + } + + @Test + public void testClosedBudgetRejectsStaleHandleAndReservationResize() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation zeroByteReservation = staleBudget.tryReserve(0L).get(); + + staleBudget.close(); + staleBudget.close(); + + Assert.assertFalse(staleBudget.tryReserve(1L).isPresent()); + Assert.assertFalse(zeroByteReservation.tryResize(1L)); + Assert.assertEquals(0L, staleBudget.getRejectedCount()); + Assert.assertEquals(0L, manager.getGlobalRejectedCount()); + zeroByteReservation.release(); + + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation replacementReservation = replacement.tryReserve(100L).get(); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + replacementReservation.release(); + replacement.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testCloseForceReleasesOutstandingAccounting() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation staleReservation = staleBudget.tryReserve(40L).get(); + + staleBudget.close(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + staleReservation.release(); + Assert.assertFalse(staleReservation.isActive()); + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + replacement.close(); + } + + @Test + public void testPeerReclaimCoalescesConcurrentMissesToLargestAdmission() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget owner = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.empty(), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 2L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = owner.tryReserve(100L).get(); + CountDownLatch firstReclaimStarted = new CountDownLatch(1); + CountDownLatch releaseFirstReclaim = new CountDownLatch(1); + CountDownLatch secondReclaimFinished = new CountDownLatch(1); + AtomicInteger invocation = new AtomicInteger(); + List targets = Collections.synchronizedList(new ArrayList<>()); + owner.setReclaimer(target -> { + targets.add(target); + if (invocation.getAndIncrement() == 0) { + firstReclaimStarted.countDown(); + await(releaseFirstReclaim); + } else { + secondReclaimFinished.countDown(); + } + return 0L; + }); + try { + requester.requestPeerReclaim(10L); + Assert.assertTrue(firstReclaimStarted.await(3L, TimeUnit.SECONDS)); + + requester.requestPeerReclaim(10L); + requester.requestPeerReclaim(20L); + requester.requestPeerReclaim(15L); + releaseFirstReclaim.countDown(); + + Assert.assertTrue(secondReclaimFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(2, targets.size()); + Assert.assertEquals(Long.valueOf(10L), targets.get(0)); + Assert.assertEquals(Long.valueOf(20L), targets.get(1)); + } finally { + releaseFirstReclaim.countDown(); + reservation.release(); + owner.close(); + requester.close(); + } + } + + @Test + public void testCatalogOnlyDeficitReclaimsSiblingWithoutTouchingOtherCatalog() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(200L); + EntryBudget sibling = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget otherCatalog = manager.createEntryBudget( + 2L, "paimon", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + AdmissionReservation siblingReservation = sibling.tryReserve(100L).get(); + AdmissionReservation otherReservation = otherCatalog.tryReserve(50L).get(); + CountDownLatch siblingReclaimed = new CountDownLatch(1); + AtomicInteger otherCatalogReclaims = new AtomicInteger(); + sibling.setReclaimer(target -> { + siblingReservation.release(); + siblingReclaimed.countDown(); + return 100L; + }); + otherCatalog.setReclaimer(target -> { + otherCatalogReclaims.incrementAndGet(); + return 0L; + }); + try { + requester.requestPeerReclaim(20L); + + Assert.assertTrue(siblingReclaimed.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, otherCatalogReclaims.get()); + Assert.assertEquals(0L, sibling.getUsedWeight()); + Assert.assertEquals(50L, manager.getGlobalUsedWeight()); + } finally { + siblingReservation.release(); + otherReservation.release(); + sibling.close(); + requester.close(); + otherCatalog.close(); + } + } + + private static ExternalMetaCacheBudgetManager manager(long maxWeight) { + return new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + } + + private static void await(CountDownLatch latch) { + try { + Assert.assertTrue(latch.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index ef1090dd5300c6..5607d604e1b01e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -20,21 +20,44 @@ import org.apache.doris.common.Config; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class MetaCacheEntryTest { + @Test + public void testCompactStringPayloadEstimate() { + // Three Latin-1 bytes and two UTF-16 characters both occupy one aligned slot; the exact + // slot size follows the JVM object alignment (8 by default, 16 with large heaps). + long latin1 = MetaCacheWeightUtils.estimatedStringPayloadBytes("abc"); + long utf16 = MetaCacheWeightUtils.estimatedStringPayloadBytes("中文"); + Assert.assertEquals(latin1, utf16); + Assert.assertTrue(latin1 >= 4L && latin1 <= 16L); + Assert.assertTrue(MetaCacheWeightUtils.estimatedStringPayloadBytes("abcdefghijklmnopq") + > latin1); + } + @Test public void testRefreshUsesConfiguredLoader() throws Exception { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; @@ -311,6 +334,976 @@ void beforeManualCachePutForTest(String key, Integer loaded) { } } + @Test + public void testExplicitPutWinsAgainstInFlightManualLoad() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePutStarted = new CountDownLatch(1); + CountDownLatch releaseBeforePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePutStarted.countDown(); + awaitLatch(releaseBeforePut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePutStarted.await(3L, TimeUnit.SECONDS)); + entry.put("k", 2); + releaseBeforePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + } finally { + releaseBeforePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedReplacementDoesNotQueueOldValuesOnRefreshExecutor() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "test", "value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "value", key -> new byte[1], CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), entryBudget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + entry.put("k", new byte[100]); + for (int i = 0; i < 100; i++) { + entry.put("k", new byte[100]); + } + + Assert.assertTrue("removal callbacks must not retain replaced values in the executor queue", + refreshExecutor.getQueue().isEmpty()); + Assert.assertEquals(accountedWeight(100L), entry.stats().getEstimatedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + } + } + + @Test + public void testWeightedFirstPublicationInvokesReplacementListener() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "publication", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger publications = new AtomicInteger(); + AtomicReference previous = new AtomicReference<>(); + byte[] value = new byte[10]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> value, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, loaded) -> MetaCacheSizeEstimate.complete(loaded.length), budget, + (key, oldValue, currentValue) -> { + publications.incrementAndGet(); + previous.set(oldValue); + Assert.assertSame(value, currentValue); + }); + try { + entry.put("k", value); + + Assert.assertEquals(1, publications.get()); + Assert.assertNull(previous.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCountEntryLoadAndRefreshInvokeReplacementListener() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loads = new AtomicInteger(); + AtomicInteger publications = new AtomicInteger(); + AtomicReference refreshPrevious = new AtomicReference<>(); + AtomicReference refreshCurrent = new AtomicReference<>(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> loads.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, null, null, + (key, previousValue, currentValue) -> { + publications.incrementAndGet(); + if (previousValue != null) { + refreshPrevious.set(previousValue); + refreshCurrent.set(currentValue); + } + }); + try { + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + Assert.assertEquals(1, publications.get()); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + Assert.assertEquals(2, loads.get()); + Assert.assertEquals(2, publications.get()); + Assert.assertEquals(Integer.valueOf(1), refreshPrevious.get()); + Assert.assertEquals(Integer.valueOf(2), refreshCurrent.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testQueuedWeightedRefreshDoesNotCaptureCurrentValue() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-capture", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-capture", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + + Runnable queuedRefresh = refreshExecutor.getQueue().peek(); + Assert.assertNotNull(queuedRefresh); + for (Field field : queuedRefresh.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("queued refresh must not directly retain the cached value", + currentValue, field.get(queuedRefresh)); + } + + entry.invalidateAll(); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, loaderCalls.get()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderWeightedRefresh() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-fence", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-fence", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertFalse(refreshExecutor.getQueue().isEmpty()); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + + Assert.assertEquals("the older refresh must be rejected before it calls the loader", + 0, loaderCalls.get()); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderCountRefresh() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch loaderEntered = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + CountDownLatch loaderFinished = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-refresh-fence", key -> { + loaderEntered.countDown(); + awaitLatch(releaseLoader); + loaderFinished.countDown(); + return "stale"; + }, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + String currentValue = new String("current"); + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertTrue(loaderEntered.await(3L, TimeUnit.SECONDS)); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseLoader.countDown(); + Assert.assertTrue(loaderFinished.await(3L, TimeUnit.SECONDS)); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertSame("the event fence must suppress refresh write-back", + currentValue, entry.peekIfPresent("k")); + } finally { + releaseLoader.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentWeightedRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-weighted-refresh", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-weighted-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte["a".equals(key) ? 2 : 3]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("a", new byte[1]); + entry.put("b", new byte[1]); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValueLength(entry, "a", 2); + awaitValueLength(entry, "b", 3); + Assert.assertEquals(accountedWeight(2L) + accountedWeight(3L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCountRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-count-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return key + "-refreshed"; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("a", "a-current"); + entry.put("b", "b-current"); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValue(entry, "a", "a-refreshed"); + awaitValue(entry, "b", "b-refreshed"); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testInvalidatingOneKeyDoesNotSuppressAnotherKeysConcurrentMissAdmission() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-miss", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-miss", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte[1]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + Future first = queryExecutor.submit(() -> entry.get("a")); + Future second = queryExecutor.submit(() -> entry.get("b")); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + entry.invalidateKey("a"); + releaseLoaders.countDown(); + first.get(3L, TimeUnit.SECONDS); + second.get(3L, TimeUnit.SECONDS); + + Assert.assertNull(entry.peekIfPresent("a")); + Assert.assertNotNull(entry.peekIfPresent("b")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRemovalCleanupDoesNotDeadlockWithInvalidateAll() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "deadlock", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch removalListenerEntered = new CountDownLatch(1); + CountDownLatch invalidateHasAdmissionLock = new CountDownLatch(1); + CountDownLatch releaseRemovalListener = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "deadlock", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + removalListenerEntered.countDown(); + awaitLatch(releaseRemovalListener); + } + } + + @Override + void beforeWeightedInvalidateAllForTest() { + invalidateHasAdmissionLock.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future eviction = queryExecutor.submit( + () -> loadingCache.policy().eviction().get().setMaximum(1L)); + Assert.assertTrue(removalListenerEntered.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertTrue(invalidateHasAdmissionLock.await(3L, TimeUnit.SECONDS)); + + releaseRemovalListener.countDown(); + eviction.get(3L, TimeUnit.SECONDS); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + releaseRemovalListener.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedRemovalDoesNotReleaseSameIdentityReinsert() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "same-identity-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "same-identity-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCountRemovalDoesNotDropSameIdentityRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry( + "count-same-identity-aba", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeNonWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("the replacement refresh owner must remain usable", 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentCountRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-expired-same-identity", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null); + try { + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new refresh owner", + 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentWeightedReservation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted-expired-same-identity", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted-expired-same-identity", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new reservation", + accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "soft-value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft-value", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] value = new byte[1]; + entry.put("k", value); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + Assert.assertTrue("weighted values must be held through Caffeine SoftReference", + valueReference instanceof SoftReference); + Map owners = (Map) readField(entry, "reservations"); + Object owner = owners.get("k"); + Assert.assertNotNull(owner); + for (Field field : owner.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("reservation ownership must not strongly retain V", value, + field.get(owner)); + } + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + + awaitGlobalWeight(manager, 0L); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testAutomaticEvictionTelemetryKeepsExactWeightAboveWeigherLimit() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long hugeEstimate = 3L << 30; // above Caffeine's int weigher limit + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(8L << 30)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "huge-eviction", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "huge-eviction", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 8L << 30), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(hugeEstimate), budget); + try { + entry.put("k", new byte[1]); + Assert.assertEquals(accountedWeight(hugeEstimate), manager.getGlobalUsedWeight()); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + // A soft-value collection is an automatic eviction reported through Caffeine, whose + // weigher saw at most Integer.MAX_VALUE; the statistics must report the reservation. + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + awaitGlobalWeight(manager, 0L); + + Assert.assertEquals(accountedWeight(hugeEstimate), entry.stats().getEvictionWeight()); + Assert.assertTrue(entry.stats().getEvictionWeight() > Integer.MAX_VALUE); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testStrongQueryReferenceSurvivesSoftValueCollectionChecks() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "query-reference", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "query-reference", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] queryReference = entry.get("k"); + WeakReference observed = new WeakReference<>(queryReference); + + for (int i = 0; i < 3; i++) { + System.gc(); + extractLoadingCache(entry).cleanUp(); + } + + Assert.assertSame(queryReference, observed.get()); + Assert.assertSame(queryReference, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCollectedCallbackCannotReleaseReplacementGeneration() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "collected-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch collectedBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseCollectedCallback = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch collectedCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "collected-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + collectedBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseCollectedCallback); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + collectedCleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference oldValueReference = extractValueReference(loadingCache); + armRemovalHook.set(true); + + Future collection = queryExecutor.submit(() -> { + oldValueReference.clear(); + oldValueReference.enqueue(); + loadingCache.cleanUp(); + }); + Assert.assertTrue(collectedBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + + byte[] replacement = new byte[1]; + armPutHook.set(true); + Future replacementPut = queryExecutor.submit(() -> entry.put("k", replacement)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseCollectedCallback.countDown(); + collection.get(3L, TimeUnit.SECONDS); + releaseReplacementPut.countDown(); + replacementPut.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(collectedCleanupFinished.await(3L, TimeUnit.SECONDS)); + + Assert.assertSame(replacement, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseCollectedCallback.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOwnershipRecordsHaveNoGenericValueReference() { + for (Class nested : MetaCacheEntry.class.getDeclaredClasses()) { + if (nested.getSimpleName().equals("ReservationRecord") + || nested.getSimpleName().equals("RefreshRecord")) { + Assert.assertFalse(nested.getSimpleName() + " must not retain V", + Arrays.stream(nested.getDeclaredFields()) + .anyMatch(field -> field.getType() == Object.class)); + } + } + } + + @Test + public void testRemovalCleanupRetriesAfterTransientFailure() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "removal-retry", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch firstAttempt = new CountDownLatch(1); + CountDownLatch cleanupFinished = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "removal-retry", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalCleanupLockForTest(String key) { + if (attempts.incrementAndGet() == 1) { + firstAttempt.countDown(); + throw new IllegalStateException("transient cleanup failure"); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + cleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + extractLoadingCache(entry).invalidate("k"); + + Assert.assertTrue(firstAttempt.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue(cleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue("cleanup should be retried", attempts.get() >= 2); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testBulkInvalidateDoesNotEnqueueOneCleanupPerEntry() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_000L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "bulk-invalidate", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger queuedCleanupCount = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "bulk-invalidate", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 2_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + queuedCleanupCount.incrementAndGet(); + } + }; + try { + for (int i = 0; i < 1_000; i++) { + entry.put("k-" + i, new byte[1]); + } + + entry.invalidateAll(); + + Assert.assertEquals(0, queuedCleanupCount.get()); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(() -> entry.invalidateKey("k")); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateAllLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testManualMissLoadAllowsNullWithoutCaching() { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; @@ -373,6 +1366,449 @@ public void testManualMissLoadDoesNotCacheWhenEntryDisabled() { } } + @Test + public void testClosedEntryCanNotBeRepopulated() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + AtomicInteger loadCounter = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", + key -> loadCounter.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, + false); + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + + entry.close(); + + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(Integer.valueOf(2), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testManualMissLoadDoesNotWriteBackAcrossClose() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePut = new CountDownLatch(1); + CountDownLatch releasePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePut.countDown(); + awaitLatch(releasePut); + } + }; + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePut.await(3L, TimeUnit.SECONDS)); + entry.close(); + releasePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releasePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCompareAndReplaceUsesIdentityAndPeekDoesNotPolluteStats() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", key -> "loaded", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false); + String current = new String("same"); + entry.put("k", current); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.NOT_CURRENT, + entry.tryReplace("k", new String("same"), "wrong")); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", current, "new")); + Assert.assertEquals("new", entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionAndReplacementAccounting() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + + entry.put("k", 40); + Assert.assertEquals(Integer.valueOf(40), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(40L), manager.getGlobalUsedWeight()); + + entry.put("k", 10); + Assert.assertEquals(Integer.valueOf(10), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(10L), manager.getGlobalUsedWeight()); + + entry.invalidateKey("k"); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCaffeineWeigherOnlyReadsPreparedReservationWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Integer first = Integer.valueOf(20); + entry.put("k", first); + Assert.assertEquals(1, estimateCalls.get()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", first, Integer.valueOf(30))); + Assert.assertEquals(2, estimateCalls.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIncompleteEstimateReturnsValueWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.incomplete("unclassified_field"), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNegativeEstimateFailsImmediately() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(-1L), + budget); + try { + Assert.assertThrows(IllegalArgumentException.class, () -> entry.put("k", 30)); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testZeroEstimateIsRejectedWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(0L), budget); + try { + entry.put("k", 1); + + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + Assert.assertEquals("invalid_estimate", entry.stats().getLastWeightRejectReason()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedEntryEvictsItsOwnColdestValueBeforeAdmission() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + entry.put("first", 30); + entry.put("second", 30); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(30), entry.getIfPresent("second")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getEvictionCount()); + Assert.assertEquals(accountedWeight(30L), entry.stats().getEvictionWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionCanReclaimMoreThanOneThousandSmallValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_500L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "many-small-values", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "many-small-values", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + for (int i = 0; i < 1_500; i++) { + entry.put("small-" + i, new byte[1]); + } + + byte[] large = new byte[600_000]; + entry.put("large", large); + + Assert.assertSame(large, entry.peekIfPresent("large")); + Assert.assertTrue(entry.stats().getEvictionCount() > 1_024L); + Assert.assertTrue(entry.stats().getEstimatedWeight() <= maxWeight); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOversizedValueIsRejectedWithoutEvictingUsefulValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_100L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_100L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + entry.put("first", 20); + entry.put("second", 20); + entry.put("oversized", 600); + + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("first")); + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("second")); + Assert.assertNull(entry.peekIfPresent("oversized")); + Assert.assertEquals(2L * accountedWeight(20L), manager.getGlobalUsedWeight()); + Assert.assertEquals(0L, entry.stats().getEvictionCount()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRejectedAtomicReplacementKeepsExpectedValueUntilConditionalInvalidation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(600L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + Integer current = Integer.valueOf(30); + entry.put("k", current); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REJECTED, + entry.tryReplace("k", current, Integer.valueOf(100))); + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(entry.invalidateKeyIfSame("k", current)); + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRejectedWeightedRefreshRetainsPreviousValue() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(600L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-reject", OptionalLong.empty(), OptionalLong.empty()); + byte[] current = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-reject", key -> new byte[100], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRefreshFailureRetainsPreviousValueAndExecutorThread() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + String current = new String("current"); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-failure", key -> { + throw new IllegalStateException("temporary metastore failure"); + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + AtomicBoolean executorStillAlive = new AtomicBoolean(); + refreshExecutor.submit(() -> executorStillAlive.set(true)).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(executorStillAlive.get()); + Assert.assertEquals(1L, entry.stats().getLoadFailureCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testPeerReclamationPreventsGlobalBudgetStarvation() throws Exception { + long valueWeight = accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(2L * valueWeight)); + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager.EntryBudget firstBudget = manager.createEntryBudget( + 1L, "test", "first", OptionalLong.empty(), OptionalLong.empty()); + ExternalMetaCacheBudgetManager.EntryBudget secondBudget = manager.createEntryBudget( + 2L, "test", "second", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry first = new MetaCacheEntry<>( + "first", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), firstBudget); + MetaCacheEntry second = new MetaCacheEntry<>( + "second", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), secondBudget); + try { + first.put("a", new byte[1]); + first.put("b", new byte[1]); + second.put("c", new byte[1]); + Assert.assertNull(second.peekIfPresent("c")); + + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() > valueWeight && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + second.put("c", new byte[1]); + + Assert.assertNotNull(second.peekIfPresent("c")); + Assert.assertTrue(manager.getGlobalUsedWeight() <= 2L * valueWeight); + } finally { + first.close(); + second.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDisabledWeightedEntrySkipsEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 30, CacheSpec.ofWeight(false, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(0, estimateCalls.get()); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + // Keep the loader blocking helper in one place so concurrent tests stay readable. private void awaitLatch(CountDownLatch latch) { try { @@ -383,12 +1819,87 @@ private void awaitLatch(CountDownLatch latch) { } } + private void awaitValueLength(MetaCacheEntry entry, String key, int expectedLength) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + byte[] value = entry.peekIfPresent(key); + if (value != null && value.length == expectedLength) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expectedLength, entry.peekIfPresent(key).length); + } + + private void awaitValue(MetaCacheEntry entry, String key, String expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + if (expected.equals(entry.peekIfPresent(key))) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expected, entry.peekIfPresent(key)); + } + + private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() != expected && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + Assert.assertEquals(expected, manager.getGlobalUsedWeight()); + } + + private Reference extractValueReference(LoadingCache loadingCache) throws Exception { + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + Assert.assertEquals(1, nodes.size()); + Object node = nodes.values().iterator().next(); + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + Object valueReference = valueReferenceMethod.invoke(node); + Assert.assertTrue(valueReference instanceof Reference); + return (Reference) valueReference; + } + + private Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } + @SuppressWarnings("unchecked") - private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { + private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { Field dataField = MetaCacheEntry.class.getDeclaredField("loadingData"); dataField.setAccessible(true); Object raw = dataField.get(entry); Assert.assertTrue(raw instanceof LoadingCache); - return (LoadingCache) raw; + return (LoadingCache) raw; + } + + private static long accountedWeight(long estimatedPayloadBytes) { + return estimatedPayloadBytes + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 228bb112ff0016..3d9152e5eb1900 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -21,12 +21,17 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; @@ -38,18 +43,33 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.privilege.PrivilegeChecker; +import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.PrimaryKeyFileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.StreamTableCommit; import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.DataTypeVisitor; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.FloatType; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import org.junit.Assert; import org.junit.Assume; import org.junit.Rule; @@ -58,22 +78,727 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; public class PaimonExternalMetaCacheTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testSnapshotWeightScalesLinearlyToOneHundredThousandPartitions() throws Exception { + FileStoreTable table = newPartitionedTable("linear_snapshot_estimate", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + long base = snapshotWeight(key, table, 0); + long oneThousand = snapshotWeight(key, table, 1_000); + long tenThousand = snapshotWeight(key, table, 10_000); + long oneHundredThousand = snapshotWeight(key, table, 100_000); + + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @Test + public void testSnapshotWeightAccountsForSkewedTableOptions() throws Exception { + FileStoreTable smallTable = newPartitionedTable( + "option_small", Collections.singletonMap("payload", "x")); + String largePayload = repeatedCharacter('x', 64 * 1024); + FileStoreTable largeTable = newPartitionedTable( + "option_large", Collections.singletonMap("payload", largePayload)); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largePayload) + - MetaCacheWeightUtils.estimatedStringBytes("x")); + } + + @Test + public void testSnapshotWeightAccountsForNestedSchemaPayload() throws Exception { + String largeFieldName = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTableWithNestedField("nested_small", "x"); + FileStoreTable largeTable = newPartitionedTableWithNestedField( + "nested_large", largeFieldName); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largeFieldName) + - MetaCacheWeightUtils.estimatedStringBytes("x")); + } + + @Test + public void testSnapshotWeightAccountsForTableComment() throws Exception { + String largeComment = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTable( + "comment_small", Collections.emptyMap(), "x"); + FileStoreTable largeTable = newPartitionedTable( + "comment_large", Collections.emptyMap(), largeComment); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + + long smallBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L), smallTable, 0); + long largeBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L), largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largeComment) + - MetaCacheWeightUtils.estimatedStringBytes("x")); + } + + @Test + public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable table = newStringPartitionedTable("jol_snapshot"); + FileStoreTable intTable = newPartitionedTable("jol_int_snapshot", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16, Type.STRING); + PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions( + table, 32, 16, Type.STRING); + PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions( + table, 1, 16, Type.STRING); + PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions( + table, 1, 4096, Type.STRING); + + long emptyEstimate = empty.prepareForCachePublication(key).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(key).getBytes(); + long shortTailEstimate = shortTail.prepareForCachePublication(key).getBytes(); + long longTailEstimate = longTail.prepareForCachePublication(key).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon snapshot partitions", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + + PaimonSnapshotEntryKey intKey = new PaimonSnapshotEntryKey( + mapping, 1L, intTable.schema().id(), 1L); + PaimonSnapshotCacheValue emptyInts = snapshotValueWithRealPartitions( + intTable, 0, 0, Type.INT); + PaimonSnapshotCacheValue populatedInts = snapshotValueWithRealPartitions( + intTable, 32, 0, Type.INT); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon int snapshot partitions", + emptyInts.prepareForCachePublication(intKey).getBytes(), + populatedInts.prepareForCachePublication(intKey).getBytes(), + emptyInts, populatedInts); + + // Every partition column beyond the first adds a literal to each ListPartitionItem key + // and an entry to the Partition spec; the estimate scales with the loaded width. + PaimonSnapshotCacheValue wideInts = snapshotValueWithRealPartitions( + intTable, 32, 0, Type.INT, 3); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon wide snapshot partitions", + populatedInts.prepareForCachePublication(intKey).getBytes(), + wideInts.prepareForCachePublication(intKey).getBytes(), + populatedInts, wideInts); + } + + @Test + public void testTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable emptyTable = newTableWithExtraFields("jol_empty_schema", 0); + FileStoreTable populatedTable = newTableWithExtraFields("jol_populated_schema", 32); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey emptyKey = new PaimonSnapshotEntryKey( + mapping, 1L, emptyTable.schema().id(), 1L); + PaimonSnapshotEntryKey populatedKey = new PaimonSnapshotEntryKey( + mapping, 1L, populatedTable.schema().id(), 1L); + PaimonSnapshotCacheValue empty = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, emptyTable.schema().id(), emptyTable)); + PaimonSnapshotCacheValue populated = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, populatedTable.schema().id(), populatedTable)); + + long emptyEstimate = empty.prepareForCachePublication(emptyKey).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + materializeStoreGraph(emptyTable); + materializeStoreGraph(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon table fields", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testStoreGraphFormulaAgainstJolOwnedGraph() throws Exception { + // AppendOnlyFileStore copies every field for its non-null row type; KeyValueFileStore + // copies the primary-key fields and shares the rest. Both derive RowTypes with lazy + // lookup maps and copy the table options; the estimate must cover them after + // newReadBuilder().newScan() runs on the admitted table. + assertTableDeltaAgainstJol("paimon append store fields", + newTableWithExtraFields("jol_store_append_narrow", 10, false, 0), + newTableWithExtraFields("jol_store_append_wide", 300, false, 0)); + assertTableDeltaAgainstJol("paimon primary-key store fields", + newTableWithExtraFields("jol_store_pk_narrow", 10, true, 0), + newTableWithExtraFields("jol_store_pk_wide", 300, true, 0)); + assertTableDeltaAgainstJol("paimon store options", + newTableWithExtraFields("jol_store_options_none", 10, false, 0), + newTableWithExtraFields("jol_store_options_many", 10, false, 100)); + } + + @Test + public void testNestedTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable smallTable = newTableWithNestedFields("jol_nested_small", 1); + FileStoreTable populatedTable = newTableWithNestedFields("jol_nested_large", 33); + assertTableDeltaAgainstJol("paimon nested fields", smallTable, populatedTable); + } + + @Test + public void testTableOptionFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable emptyTable = newTableWithOptions("jol_options_empty", 0); + FileStoreTable populatedTable = newTableWithOptions("jol_options_large", 32); + assertTableDeltaAgainstJol("paimon table options", emptyTable, populatedTable); + } + + @Test + public void testCompositeTypeFormulaAgainstJolOwnedGraph() { + assertTableDeltaAgainstJol("paimon array type", + newTableWithPayloadType("array", nestedArrayType(1)), + newTableWithPayloadType("array", nestedArrayType(100))); + assertTableDeltaAgainstJol("paimon map type", + newTableWithPayloadType("map", nestedMapType(1)), + newTableWithPayloadType("map", nestedMapType(100))); + assertTableDeltaAgainstJol("paimon multiset type", + newTableWithPayloadType("multiset", nestedMultisetType(1)), + newTableWithPayloadType("multiset", nestedMultisetType(100))); + assertTableDeltaAgainstJol("paimon row type", + newTableWithPayloadType("row", nestedRowType(1)), + newTableWithPayloadType("row", nestedRowType(100))); + assertTableDeltaAgainstJol("paimon vector type", + newTableWithPayloadType("vector", rowOfLeafTypes(1, VectorType.class)), + newTableWithPayloadType("vector", rowOfLeafTypes(100, VectorType.class))); + assertTableDeltaAgainstJol("paimon decimal type", + newTableWithPayloadType("decimal", rowOfLeafTypes(1, DecimalType.class)), + newTableWithPayloadType("decimal", rowOfLeafTypes(100, DecimalType.class))); + } + + @Test + public void testUnknownDataTypeFailsClosedWithoutFailingLoad() { + DataType unknownType = new DataType(true, DataTypeRoot.INTEGER) { + @Override + public int defaultSize() { + return Integer.BYTES; + } + + @Override + public DataType copy(boolean isNullable) { + return this; + } + + @Override + public String asSQLString() { + return "UNKNOWN"; + } + + @Override + public R accept(DataTypeVisitor visitor) { + return new IntType().accept(visitor); + } + }; + FileStoreTable table = newTableWithPayloadType("unknown-type", unknownType); + Assert.assertThrows(IllegalStateException.class, + () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + + Assert.assertFalse(estimate.isComplete()); + Assert.assertSame(table, value.getSnapshot().getTable()); + } + + @Test + public void testTableEntryWeightCoversBaseOnlyScanAndReleasesOnInvalidate() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.singletonMap( + "meta.cache.paimon.table.max-weight", "8MB")); + Assert.assertTrue(cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE).isWeightBounded()); + + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + FileStoreTable table = newTableWithExtraFields("table_entry_weight", 64, true, 8); + Object lazyStoreBefore = readField(table, table.getClass(), "lazyStore"); + PaimonTableCacheValue value = new PaimonTableCacheValue(table); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, value); + + Assert.assertSame(value, tables.peekIfPresent(mapping)); + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + long estimate = value.getSizeEstimate().getBytes(); + Assert.assertTrue(estimate > 0L); + MetaCacheEntryStats stats = cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE); + // The entry adds a fixed per-record overhead on top of the value estimate. + long reservedWeight = stats.getEstimatedWeight(); + Assert.assertTrue(reservedWeight >= estimate); + Assert.assertSame("table publication must not materialize FileStoreTable.store()", + lazyStoreBefore, readField(table, table.getClass(), "lazyStore")); + + // A base-only scan (fetchRowCount, table-only paths) opens the store and its RowType + // indexes on the admitted handle; the reserved weight already covers that graph. + long beforeScan = EstimatorCalibrationAssertions.graphSize(value); + materializeStoreGraph(table); + materializeRowTypeIndexes(table.schema()); + long afterScan = EstimatorCalibrationAssertions.graphSize(value); + Assert.assertTrue("scan must grow the retained graph", afterScan > beforeScan); + Assert.assertTrue("estimate " + estimate + " must cover the grown graph " + afterScan, + estimate >= afterScan); + Assert.assertEquals(reservedWeight, + cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE).getEstimatedWeight()); + + tables.invalidateKey(mapping); + Assert.assertNull(tables.peekIfPresent(mapping)); + Assert.assertEquals(0L, + cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE).getEstimatedWeight()); + + // Unsupported table implementations fail closed and stay outside the cache. + PaimonTableCacheValue unsupported = new PaimonTableCacheValue(Mockito.mock(Table.class)); + tables.put(mapping, unsupported); + Assert.assertNull(tables.peekIfPresent(mapping)); + Assert.assertFalse(unsupported.getSizeEstimate().isComplete()); + Assert.assertTrue(unsupported.getSizeEstimate().getIncompleteReason() + .startsWith("unsupported_paimon_table:")); + MetaCacheEntryStats rejected = cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE); + Assert.assertEquals(1L, rejected.getWeightAdmissionRejectedCount()); + Assert.assertEquals(0L, rejected.getEstimatedWeight()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testTableEntryFormulaAgainstJolOwnedGraph() throws Exception { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + FileStoreTable smallTable = newTableWithExtraFields("jol_table_entry_narrow", 10, true, 0); + FileStoreTable populatedTable = newTableWithExtraFields("jol_table_entry_wide", 300, true, 50); + PaimonTableCacheValue small = new PaimonTableCacheValue(smallTable); + PaimonTableCacheValue populated = new PaimonTableCacheValue(populatedTable); + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeRowTypeIndexes(smallTable.schema()); + materializeRowTypeIndexes(populatedTable.schema()); + materializeStoreGraph(smallTable); + materializeStoreGraph(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon table entry", smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.paimon.snapshot.max-weight", "8MB")); + Assert.assertTrue(cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + FileStoreTable table = newPartitionedTable("snapshot_estimate", Collections.emptyMap()); + Object lazyStoreBefore = readField(table, table.getClass(), "lazyStore"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertTrue(value.getSizeEstimate().getBytes() > 0L); + Assert.assertSame("cache admission must not materialize FileStoreTable.store()", + lazyStoreBefore, readField(table, table.getClass(), "lazyStore")); + + PaimonSnapshotCacheValue unsupportedValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, Mockito.mock(Table.class))); + unsupportedValue.prepareForCachePublication(new PaimonSnapshotEntryKey(mapping, 1L, 1L, 1L)); + Assert.assertFalse(unsupportedValue.getSizeEstimate().isComplete()); + Assert.assertTrue(unsupportedValue.getSizeEstimate().getIncompleteReason() + .startsWith("unsupported_paimon_table:")); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testTablePayloadAccountingWorkIsBounded() { + FileStoreTable table = Mockito.mock(FileStoreTable.class); + TableSchema schema = Mockito.mock(TableSchema.class); + @SuppressWarnings("unchecked") + Map oversizedOptions = Mockito.mock(Map.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(schema.fields()).thenReturn(Collections.emptyList()); + Mockito.when(schema.options()).thenReturn(oversizedOptions); + Mockito.when(oversizedOptions.size()).thenReturn(50_001); + + Assert.assertThrows(IllegalStateException.class, + () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + } + + @Test + public void testRowTypeLazyLookupReservationCoversPostAdmissionGrowth() throws Exception { + // Field ids above the Integer cache make every lazy map box its keys and values. + RowType small = wideRowType(1); + RowType populated = wideRowType(200); + FileStoreTable smallTable = newTableWithPayloadType("row-lazy-small", small); + FileStoreTable populatedTable = newTableWithPayloadType("row-lazy-large", populated); + for (String fieldName : ROW_TYPE_LAZY_FIELDS) { + Assert.assertNull(fieldName, readField(populated, fieldName)); + } + + // The oracle inside assertTableDeltaAgainstJol materializes the four maps after the + // estimate is taken; the estimate reserved at admission must already cover them. + assertTableDeltaAgainstJol("paimon row lazy lookup maps", smallTable, populatedTable); + for (String fieldName : ROW_TYPE_LAZY_FIELDS) { + Assert.assertNotNull(fieldName, readField(populated, fieldName)); + } + + // A RowType whose maps were materialized before admission is estimated identically. + RowType preloaded = wideRowType(200); + long unloadedEstimate = PaimonCacheSizeEstimator.retainedTablePayloadBytes( + newTableWithPayloadType("row-unloaded", wideRowType(200))); + materializeRowTypeIndexes(preloaded); + Assert.assertEquals(unloadedEstimate, PaimonCacheSizeEstimator.retainedTablePayloadBytes( + newTableWithPayloadType("row-loaded", preloaded))); + } + + @Test + public void testSnapshotKeySeparatesReloadedTableGenerations() { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, null); + PaimonSnapshotCacheValue fenceValue = new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue first = new PaimonTableCacheValue(null, fenceValue); + PaimonTableCacheValue reloaded = new PaimonTableCacheValue(null, fenceValue); + + PaimonSnapshotEntryKey firstKey = PaimonSnapshotEntryKey.of( + mapping, fence, first.getGeneration()); + PaimonSnapshotEntryKey reloadedKey = PaimonSnapshotEntryKey.of( + mapping, fence, reloaded.getGeneration()); + + Assert.assertNotEquals(firstKey, reloadedKey); + Assert.assertNotEquals(firstKey.getTableGeneration(), reloadedKey.getTableGeneration()); + } + + @Test + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(Mockito.mock(Table.class)); + PaimonTableCacheValue second = new PaimonTableCacheValue(Mockito.mock(Table.class)); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + PaimonSnapshotEntryKey oldSnapshotKey = new PaimonSnapshotEntryKey( + mapping, 1L, 2L, first.getGeneration()); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, first.getPaimonTable()))); + PaimonSchemaCacheKey oldSchemaKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 2L); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotHitRefreshesFenceWithoutReloadingProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "remote_db", "remote_tbl"); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + FileStoreTable pinnedTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + Mockito.when(table.copyWithLatestSchema()).thenReturn(table); + Mockito.when(table.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(table.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(table.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(pinnedTable); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, pinnedTable); + PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + mapping, fence, tableValue.getGeneration()); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class).put(mapping, tableValue); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class).put(key, snapshotValue); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + + Mockito.verify(table, Mockito.times(4)).copyWithLatestSchema(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testContextualSnapshotAndSchemaMissesRunAuthenticated() { + AtomicInteger authenticationDepth = new AtomicInteger(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + authenticationDepth.incrementAndGet(); + try { + return task.call(); + } finally { + authenticationDepth.decrementAndGet(); + } + } + }; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Mockito.doAnswer(invocation -> { + Assert.assertTrue("schema history must be read under authentication", + authenticationDepth.get() > 0); + Column partitionColumn = new Column("part", Type.INT); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenAnswer(invocation -> { + Assert.assertTrue("snapshot fence must be read under authentication", + authenticationDepth.get() > 0); + return latestSchemaTable; + }); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenAnswer(invocation -> { + Assert.assertTrue("snapshot pinning must run under authentication", + authenticationDepth.get() > 0); + return snapshotTable; + }); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenAnswer(invocation -> { + Assert.assertTrue("partition enumeration must run under authentication", + authenticationDepth.get() > 0); + return readBuilder; + }); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenAnswer(invocation -> { + Assert.assertTrue("partition manifest access must run under authentication", + authenticationDepth.get() > 0); + return Collections.emptyList(); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(baseTable); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + PaimonSnapshotCacheValue snapshot = cache.getSnapshotCache(dorisTable); + + Assert.assertEquals(7L, snapshot.getSnapshot().getSnapshotId()); + Assert.assertEquals(0, authenticationDepth.get()); + + PaimonTableCacheValue second = new PaimonTableCacheValue(baseTable); + tables.put(mapping, second); + PaimonSchemaCacheKey staleKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 99L); + cache.getPaimonSchemaCacheValue(mapping, 99L, first.getGeneration(), baseTable); + Assert.assertNull("a concurrent old-generation schema load must not repopulate the cache", + cache.entry(1L, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class).peekIfPresent(staleKey)); + Assert.assertEquals(0, authenticationDepth.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { + FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); + FileStoreTable privileged = PrivilegedFileStoreTable.wrap( + table, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "tbl")); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, table.schema().id(), privileged)); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + } + + @Test + public void testSnapshotEstimateDoesNotMaterializeNestedRowTypeIndexes() throws Exception { + RowType nested = DataTypes.ROW( + DataTypes.FIELD(10, "nested_id", DataTypes.INT()), + DataTypes.FIELD(11, "nested_name", DataTypes.STRING())); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "payload", nested)), + 11, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder("nested_row_estimate").toURI()), + schema, + CatalogEnvironment.empty()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, schema.id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, schema.id(), table)); + + Map stateBefore = new HashMap<>(); + for (String fieldName : java.util.Arrays.asList( + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex")) { + stateBefore.put(fieldName, readField(nested, fieldName)); + } + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + for (Map.Entry entry : stateBefore.entrySet()) { + Assert.assertSame(entry.getKey() + " must not be changed by cache admission", + entry.getValue(), readField(nested, entry.getKey())); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + Map properties = new HashMap<>(); + properties.put("meta.cache.paimon.table.enable", "false"); + properties.put("meta.cache.paimon.table.ttl-second", "17"); + properties.put("meta.cache.paimon.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + @Test public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -113,7 +838,7 @@ public void testFullLatestProjectionCapsManifestParallelismBeforePartitionLoad() .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -153,7 +878,7 @@ public void testLatestFenceDoesNotLoadSchemaOrPartitions() { PaimonPartitionInfoLoader partitionLoader = Mockito.mock(PaimonPartitionInfoLoader.class); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> { + (nameMapping, schemaId, tableGeneration, retainedTable) -> { throw new AssertionError("a version-only fence must not load schema metadata"); }); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); @@ -187,7 +912,7 @@ public void testFenceHydrationKeepsCapturedTableGeneration() throws Exception { .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable captured = Mockito.mock(FileStoreTable.class); @@ -224,7 +949,7 @@ public void testTagProjectionKeepsOnlyRepinnedSnapshotSelector() throws Exceptio table, Collections.singletonMap(CoreOptions.SCAN_TAG_NAME.key(), "stable")); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); PaimonSnapshotCacheValue value = loader.load( @@ -320,6 +1045,11 @@ public void testPartitionProjectionIgnoresReaderOnlyPhysicalOptions() throws Exc } private FileStoreTable newPartitionedTable(String name, Map options) throws Exception { + return newPartitionedTable(name, options, null); + } + + private FileStoreTable newPartitionedTable( + String name, Map options, String comment) throws Exception { TableSchema schema = new TableSchema( 0, java.util.Arrays.asList( @@ -329,6 +1059,24 @@ private FileStoreTable newPartitionedTable(String name, Map opti Collections.singletonList("part"), Collections.emptyList(), options, + comment); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newStringPartitionedTable(String name) throws Exception { + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "part", DataTypes.STRING())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), null); return new AppendOnlyFileStoreTable( LocalFileIO.create(), @@ -337,6 +1085,301 @@ private FileStoreTable newPartitionedTable(String name, Map opti CatalogEnvironment.empty()); } + private FileStoreTable newPartitionedTableWithNestedField( + String name, String nestedFieldName) throws Exception { + RowType nestedType = new RowType(Collections.singletonList( + new DataField(2, nestedFieldName, DataTypes.STRING()))); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithExtraFields(String name, int fieldCount) throws Exception { + return newTableWithExtraFields(name, fieldCount, false, 0); + } + + private FileStoreTable newTableWithExtraFields( + String name, int fieldCount, boolean primaryKey, int optionCount) throws Exception { + ArrayList fields = new ArrayList<>(); + fields.add(new DataField(0, "part", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new IntType())); + } + if (primaryKey) { + fields.add(new DataField(fieldCount + 1, "id", new IntType(false))); + } + Map options = new HashMap<>(); + for (int index = 0; index < optionCount; index++) { + options.put("option_" + index, "value_" + index); + } + TableSchema schema = new TableSchema( + 0, + fields, + fields.size(), + Collections.singletonList("part"), + primaryKey ? java.util.Arrays.asList("id", "part") : Collections.emptyList(), + options, + null); + Path location = new Path(temporaryFolder.newFolder(name).toURI()); + return primaryKey + ? new PrimaryKeyFileStoreTable(LocalFileIO.create(), location, schema, CatalogEnvironment.empty()) + : new AppendOnlyFileStoreTable(LocalFileIO.create(), location, schema, CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithNestedFields(String name, int nestedFieldCount) throws Exception { + ArrayList nestedFields = new ArrayList<>(); + for (int index = 0; index < nestedFieldCount; index++) { + nestedFields.add(new DataField(index + 2, "nested_" + index, new IntType())); + } + RowType nestedType = new RowType(nestedFields); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path(temporaryFolder.newFolder(name).toURI()), + schema, CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithOptions(String name, int optionCount) throws Exception { + Map options = new HashMap<>(); + for (int index = 0; index < optionCount; index++) { + options.put("key_" + index, "value_" + index); + } + return newPartitionedTable(name, options); + } + + private FileStoreTable newTableWithPayloadType(String name, DataType type) { + TableSchema schema = new TableSchema( + 0, Collections.singletonList(new DataField(0, "payload", type)), 1, + Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/paimon-composite-" + name), + schema, CatalogEnvironment.empty()); + } + + private DataType nestedArrayType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new ArrayType(type); + } + return type; + } + + private DataType nestedMapType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new MapType(new IntType(), type); + } + return type; + } + + private DataType nestedMultisetType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new MultisetType(type); + } + return type; + } + + private DataType nestedRowType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new RowType(Collections.singletonList( + new DataField(index + 1, "nested_" + index, type))); + } + return type; + } + + private void assertTableDeltaAgainstJol( + String fixture, FileStoreTable smallTable, FileStoreTable populatedTable) { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey populatedKey = new PaimonSnapshotEntryKey( + mapping, 1L, populatedTable.schema().id(), 1L); + PaimonSnapshotCacheValue small = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, smallTable.schema().id(), smallTable)); + PaimonSnapshotCacheValue populated = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, populatedTable.schema().id(), populatedTable)); + long smallEstimate = small.prepareForCachePublication(smallKey).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + // The estimate reserves the lookup maps every nested RowType can materialize after + // admission and the store graph scan planning creates, so the JOL oracle measures the + // fully grown graph. + materializeRowTypeIndexes(smallTable.schema()); + materializeRowTypeIndexes(populatedTable.schema()); + materializeStoreGraph(smallTable); + materializeStoreGraph(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + fixture, smallEstimate, populatedEstimate, small, populated); + } + + /** What scan planning materializes after admission: the store and its RowType indexes. */ + private void materializeStoreGraph(FileStoreTable table) { + table.newReadBuilder().newScan(); + try { + Object store = readField(table, table.getClass(), "lazyStore"); + for (Class owner = store.getClass(); owner != null && owner != Object.class; + owner = owner.getSuperclass()) { + for (Field field : owner.getDeclaredFields()) { + if (RowType.class.isAssignableFrom(field.getType())) { + field.setAccessible(true); + Object rowType = field.get(store); + if (rowType != null) { + materializeRowTypeIndexes((RowType) rowType); + } + } + } + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static final String[] ROW_TYPE_LAZY_FIELDS = { + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex"}; + + private void materializeRowTypeIndexes(TableSchema schema) { + for (DataField field : schema.fields()) { + materializeRowTypeIndexes(field.type()); + } + } + + private void materializeRowTypeIndexes(DataType type) { + if (type instanceof RowType) { + RowType rowType = (RowType) type; + if (!rowType.getFields().isEmpty()) { + DataField first = rowType.getFields().get(0); + rowType.getField(first.name()); + rowType.getFieldIndex(first.name()); + rowType.getField(first.id()); + rowType.getFieldIndexByFieldId(first.id()); + } + for (DataField field : rowType.getFields()) { + materializeRowTypeIndexes(field.type()); + } + } else if (type instanceof ArrayType) { + materializeRowTypeIndexes(((ArrayType) type).getElementType()); + } else if (type instanceof MultisetType) { + materializeRowTypeIndexes(((MultisetType) type).getElementType()); + } else if (type instanceof MapType) { + materializeRowTypeIndexes(((MapType) type).getKeyType()); + materializeRowTypeIndexes(((MapType) type).getValueType()); + } else if (type instanceof VectorType) { + materializeRowTypeIndexes(((VectorType) type).getElementType()); + } + } + + private RowType wideRowType(int fieldCount) { + ArrayList fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(1000 + index, "wide_" + index, new IntType())); + } + return new RowType(fields); + } + + private DataType rowOfLeafTypes(int fieldCount, Class leafType) { + ArrayList fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + DataType type = leafType == VectorType.class + ? new VectorType(4, new FloatType()) : new DecimalType(10, 2); + fields.add(new DataField(index + 1, "leaf_" + index, type)); + } + return new RowType(fields); + } + + private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, int partitionCount) { + PaimonPartitionInfo partitionInfo = Mockito.mock(PaimonPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToPartition()).thenReturn(partitions); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength, Type partitionType) + throws AnalysisException { + return snapshotValueWithRealPartitions(table, partitionCount, valueLength, partitionType, 1); + } + + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength, Type partitionType, + int partitionColumnCount) throws AnalysisException { + Map partitionItems = new HashMap<>(); + Map partitions = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = partitionType == Type.INT + ? Integer.toString(index) + : "p" + index + repeatedCharacter('x', valueLength); + String name = "part=" + value; + List values = new ArrayList<>(); + List types = new ArrayList<>(); + Map spec = new java.util.LinkedHashMap<>(); + for (int column = 0; column < partitionColumnCount; column++) { + // Each loaded column owns its own value String. + String columnValue = new String(value); + values.add(columnValue); + types.add(partitionType); + spec.put("part" + column, columnValue); + } + partitionItems.put(name, PaimonUtil.toListPartitionItem(values, types)); + partitions.put(name, new org.apache.paimon.partition.Partition( + spec, 100L, 1024L, 1L, 1L, 1, true)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(partitionItems, partitions); + return new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + private Object readField(RowType rowType, String fieldName) throws Exception { + return readField(rowType, RowType.class, fieldName); + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Object readField(Object target, Class owner, String fieldName) throws Exception { + Field field = owner.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } + @Test public void testInvalidateTablePrecise() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -350,15 +1393,24 @@ public void testInvalidateTablePrecise() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(t1, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(t2, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(t1, new PaimonTableCacheValue(null, fence)); + tableEntry.put(t2, new PaimonTableCacheValue(null, fence)); + + PaimonSnapshotEntryKey snapshotKey = new PaimonSnapshotEntryKey(t1, 1L, 2L, 1L); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshotEntry = cache.entry(catalogId, + PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, null))); cache.invalidateTable(catalogId, "db1", "tbl1"); Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); } finally { executor.shutdownNow(); } @@ -377,10 +1429,10 @@ public void testInvalidateDbAndStats() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(db1Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(db2Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(db1Table, new PaimonTableCacheValue(null, fence)); + tableEntry.put(db2Table, new PaimonTableCacheValue(null, fence)); org.apache.doris.datasource.metacache.MetaCacheEntry schemaEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java index a0f01f25cdfaeb..7f5483928c8a97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -107,7 +107,7 @@ public void testStatementContextDefersPhysicalManifestValidationUntilRelationOpt PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( Mockito.mock(PaimonPartitionInfoLoader.class), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "db", "table"); Mockito.doAnswer(ignored -> new PaimonMvccSnapshot( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 09bcee53985f5d..97597d774d85ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.catalog.VariantType; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TFieldPtr; @@ -36,6 +37,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Timestamp; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; @@ -87,6 +89,19 @@ private static PartitionEntry partitionEntry(BinaryRow partition, long sequence) return new PartitionEntry(partition, sequence, sequence, sequence, sequence, 1); } + @Test + public void testCompatibilityConstructorDerivesRetainedPartitionPayload() { + String largeValue = repeatedCharacter('x', 64 * 1024); + Partition partition = new Partition( + Collections.singletonMap("part", largeValue), + 1L, 1L, 1L, 1L, 1, false); + + PaimonPartitionInfo info = new PaimonPartitionInfo( + Collections.emptyMap(), Collections.singletonMap("part=" + largeValue, partition)); + + Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 2L); + } + @Test public void testSchemaForVarcharAndChar() { DataField c1 = new DataField(1, "c1", new VarCharType(32)); @@ -247,6 +262,7 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; + Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length()); Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) @@ -257,6 +273,22 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { "s1"), actualValues); } + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + List partitionColumns = Collections.singletonList(new Column("part", Type.STRING)); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + PaimonPartitionInfo small = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow("x"), 1L))); + String largeValue = repeatedCharacter('x', 64 * 1024); + PaimonPartitionInfo large = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow(largeValue), 1L))); + + Assert.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= MetaCacheWeightUtils.estimatedStringBytes(largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("x")); + } + @Test public void testGeneratePartitionInfoUsesPartitionColumnOrder() { List partitionColumns = Arrays.asList( @@ -532,4 +564,10 @@ public void testAuditLogHistorySchemaWithoutSequenceNumber() { Assert.assertEquals("id", fields.get(1).getFieldPtr().getName()); Assert.assertEquals("name", fields.get(2).getFieldPtr().getName()); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/pom.xml b/fe/pom.xml index 3783d2660e6b9b..f709024e526e32 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -272,6 +272,7 @@ under the License. 3.1.0 18.3.14-doris-SNAPSHOT 1.49 + 0.17 2.18.0 1.11.0 1.1.1 @@ -438,6 +439,12 @@ under the License. + + benchmark + + fe-benchmark + + @@ -1925,6 +1932,11 @@ under the License. mockito-inline ${mockito.version} + + org.openjdk.jol + jol-core + ${jol.version} + it.unimi.dsi fastutil-core diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 2e2a2ea8e9b5c9..0c2bebc52fed56 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -28,8 +28,24 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" String warehouse = "${default_fs}/warehouse" - // 1. test default catalog + // DDL validation must reject misspelled memory-governance options. sql """drop catalog if exists ${catalog_name};""" + test { + sql """ + create catalog ${catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'meta.cache.iceberg.snapshot.max-weigth' = '16MB' + ); + """ + exception "Unknown external meta cache" + } + + // 1. test a catalog-level memory bound without a global bound. The existing + // create/insert/select/refresh flow below is the weighted-cache happy path. sql """ create catalog ${catalog_name} properties ( 'type'='iceberg', @@ -37,6 +53,7 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', 'fs.defaultFS' = '${default_fs}', 'warehouse' = '${warehouse}', + 'meta.cache.max-weight' = '128MB', 'meta.cache.iceberg.manifest.enable' = 'true' ); """ @@ -63,6 +80,19 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern sql """refresh table test_iceberg_meta_cache_db.sales""" // select 3 rows sql """select * from test_iceberg_meta_cache_db.sales""" + // The weight-bounded entries expose their budget hierarchy in the statistics view. + def weightStats = sql """ + select entry_name, weight_bounded, max_weight, estimated_weight, catalog_max_weight, + weight_reject_count, last_weight_reject_reason + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalog_name}" and engine_name = "iceberg" and weight_bounded = true + order by entry_name; + """ + assertTrue(weightStats.size() > 0) + for (row in weightStats) { + assertTrue((row[2] as long) > 0L) + assertTrue((row[3] as long) >= 0L) + } sql """drop table test_iceberg_meta_cache_db.sales""" // 2. test catalog with meta.cache.iceberg.table.ttl-second diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy index 2a3176688f505f..ee15e4da2fcd8b 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy @@ -27,10 +27,12 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa String catalogWithCache = "test_paimon_table_cache_with_cache" String catalogNoCache = "test_paimon_table_cache_no_cache" + String catalogWeighted = "test_paimon_table_cache_weighted" String testDb = "paimon_cache_test_db" sql """drop catalog if exists ${catalogWithCache}""" sql """drop catalog if exists ${catalogNoCache}""" + sql """drop catalog if exists ${catalogWeighted}""" sql """ CREATE CATALOG ${catalogWithCache} PROPERTIES ( @@ -55,6 +57,19 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa ); """ + // A catalog-level memory bound puts the table handle and snapshot projections under weight. + sql """ + CREATE CATALOG ${catalogWeighted} PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true', + 'meta.cache.max-weight' = '128MB' + ); + """ + try { spark_paimon "CREATE DATABASE IF NOT EXISTS paimon.${testDb}" @@ -87,6 +102,48 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa def result3 = sql """select * from ${testDb}.test_insert order by id""" assertEquals(2, result3.size()) + // ==================== Test 1b: weight-bounded cache ==================== + logger.info("========== Test 1b: weight-bounded cache ==========") + sql """switch ${catalogWeighted}""" + def resultWeighted = sql """select * from ${testDb}.test_insert order by id""" + assertEquals(2, resultWeighted.size()) + // desc/table-only paths admit the base table handle; the scan above admits the snapshot. + sql """desc ${testDb}.test_insert""" + def weightStats = sql """ + select entry_name, weight_bounded, max_weight, estimated_weight, catalog_max_weight, + weight_reject_count, last_weight_reject_reason + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalogWeighted}" and engine_name = "paimon" and weight_bounded = true + order by entry_name; + """ + def weightedEntries = weightStats.collect { it[0] as String } + assertTrue(weightedEntries.contains("table")) + assertTrue(weightedEntries.contains("snapshot")) + for (row in weightStats) { + assertTrue((row[2] as long) > 0L) + assertEquals(0L, row[5] as long) + if (row[0] == "table" || row[0] == "snapshot") { + assertTrue((row[3] as long) > 0L) + } + } + // Refresh releases the table handle reservation with its projections; the next scan + // re-admits both without rejections. + sql """refresh table ${testDb}.test_insert""" + def resultWeightedRefreshed = sql """select * from ${testDb}.test_insert order by id""" + assertEquals(2, resultWeightedRefreshed.size()) + def weightStatsRefreshed = sql """ + select entry_name, estimated_weight, weight_reject_count + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalogWeighted}" and engine_name = "paimon" + and entry_name in ("table", "snapshot"); + """ + assertEquals(2, weightStatsRefreshed.size()) + for (row in weightStatsRefreshed) { + assertTrue((row[1] as long) > 0L) + assertEquals(0L, row[2] as long) + } + sql """switch ${catalogWithCache}""" + // ==================== Test 2: Schema Change (ADD COLUMN) ==================== logger.info("========== Test 2: Schema Change (ADD COLUMN) ==========") spark_paimon "DROP TABLE IF EXISTS paimon.${testDb}.test_add_column" @@ -124,5 +181,6 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa } sql """drop catalog if exists ${catalogWithCache}""" sql """drop catalog if exists ${catalogNoCache}""" + sql """drop catalog if exists ${catalogWeighted}""" } }