From b89e0445f9b93cce331fee633a15cd9ea9dd7675 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 13 Aug 2026 14:11:11 +0800 Subject: [PATCH 1/5] branch-4.1: add external metadata cache memory governance --- fe/fe-benchmark/pom.xml | 42 + .../run-external-meta-cache-size-benchmark.sh | 56 + .../doris/benchmark/BenchmarkHarness.java | 101 ++ .../HivePartitionValuesSizeBenchmark.java | 295 ++++ .../iceberg/IcebergCacheSizeBenchmark.java | 411 ++++++ .../MetaCacheSoftValueBenchmark.java | 175 +++ .../paimon/PaimonCacheSizeBenchmark.java | 272 ++++ .../java/org/apache/doris/common/Config.java | 5 + .../org/apache/doris/common/CacheFactory.java | 37 +- .../doris/datasource/ExternalCatalog.java | 30 +- .../datasource/ExternalMetaCacheMgr.java | 159 +- .../doris/DorisExternalMetaCache.java | 7 +- .../datasource/hive/HMSExternalCatalog.java | 6 - .../hive/HiveCacheSizeEstimator.java | 54 + .../hive/HiveExternalMetaCache.java | 287 +++- .../hudi/HudiExternalMetaCache.java | 7 +- .../iceberg/IcebergCacheSizeEstimator.java | 254 ++++ .../iceberg/IcebergExternalMetaCache.java | 221 ++- .../iceberg/IcebergExternalTable.java | 4 + .../iceberg/IcebergMetadataOps.java | 36 +- .../datasource/iceberg/IcebergPartition.java | 34 + .../iceberg/IcebergPartitionInfo.java | 43 +- .../iceberg/IcebergSnapshotCacheValue.java | 526 ++++++- .../iceberg/IcebergSnapshotEntryKey.java | 115 ++ .../iceberg/IcebergTableCacheValue.java | 85 +- .../iceberg/IcebergTransaction.java | 2 +- .../datasource/iceberg/IcebergUtils.java | 34 +- .../IcebergCherrypickSnapshotAction.java | 2 +- .../action/IcebergExpireSnapshotsAction.java | 2 +- .../action/IcebergFastForwardAction.java | 2 +- .../action/IcebergPublishChangesAction.java | 2 +- .../action/IcebergRewriteManifestsAction.java | 2 +- .../IcebergRollbackToSnapshotAction.java | 2 +- .../IcebergRollbackToTimestampAction.java | 2 +- .../IcebergSetCurrentSnapshotAction.java | 2 +- .../iceberg/cache/ManifestCacheValue.java | 174 ++- .../iceberg/source/IcebergScanNode.java | 5 +- .../MaxComputeExternalMetaCache.java | 8 +- .../metacache/AbstractExternalMetaCache.java | 111 +- .../doris/datasource/metacache/CacheSpec.java | 223 ++- .../metacache/CatalogEntryGroup.java | 8 + .../metacache/ExternalMetaCache.java | 4 + .../ExternalMetaCacheBudgetManager.java | 414 ++++++ .../datasource/metacache/MetaCacheEntry.java | 1060 ++++++++++++- .../metacache/MetaCacheEntryDef.java | 23 +- .../metacache/MetaCacheEntryStats.java | 72 +- .../metacache/MetaCacheSizeEstimate.java | 64 + .../metacache/MetaCacheSizeEstimator.java | 46 + .../metacache/MetaCacheWeightUtils.java | 70 + .../paimon/PaimonCacheSizeEstimator.java | 193 +++ .../paimon/PaimonExternalMetaCache.java | 44 +- .../paimon/PaimonPartitionInfo.java | 52 + .../paimon/PaimonSnapshotCacheValue.java | 23 + .../paimon/PaimonSnapshotEntryKey.java | 80 + .../paimon/PaimonTableCacheValue.java | 26 +- .../doris/datasource/paimon/PaimonUtil.java | 9 +- .../ExternalMetaCacheRouteResolverTest.java | 124 ++ .../hive/HiveMetaStoreCacheTest.java | 119 ++ .../iceberg/IcebergDDLAndDMLPlanTest.java | 2 + .../iceberg/IcebergExternalMetaCacheTest.java | 894 ++++++++++- .../IcebergExternalTableBranchAndTagTest.java | 4 +- .../IcebergMetadataOpsValidationTest.java | 74 +- .../iceberg/IcebergPartitionInfoTest.java | 23 + .../iceberg/IcebergTransactionTest.java | 42 +- .../AbstractExternalMetaCacheTest.java | 192 +++ .../datasource/metacache/CacheSpecTest.java | 56 + .../ExternalMetaCacheBudgetManagerTest.java | 191 +++ .../metacache/MetaCacheEntryTest.java | 1310 ++++++++++++++++- .../paimon/PaimonExternalMetaCacheTest.java | 327 +++- .../datasource/paimon/PaimonUtilTest.java | 36 + fe/pom.xml | 6 + 71 files changed, 9035 insertions(+), 388 deletions(-) create mode 100644 fe/fe-benchmark/pom.xml create mode 100755 fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java diff --git a/fe/fe-benchmark/pom.xml b/fe/fe-benchmark/pom.xml new file mode 100644 index 00000000000000..2de619284a9f69 --- /dev/null +++ b/fe/fe-benchmark/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + + org.apache.doris + fe + ${revision} + ../pom.xml + + + fe-benchmark + Doris FE Benchmarks + + + + ${project.groupId} + fe-core + ${project.version} + + + diff --git a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh new file mode 100755 index 00000000000000..12e28e0dd9a7a4 --- /dev/null +++ b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd) +CLASSPATH_FILE=$(mktemp) +trap 'rm -f "${CLASSPATH_FILE}"' EXIT + +( + cd "${FE_DIR}" + mvn -Pbenchmark -pl fe-benchmark -am compile -DskipTests -Dskip.clean=true + mvn -Pbenchmark -pl fe-benchmark -am dependency:build-classpath \ + -Dskip.clean=true \ + -DincludeScope=test \ + -Dmdep.outputFile="${CLASSPATH_FILE}" +) + +REACTOR_CLASSES=$(find "${FE_DIR}" -type d -path '*/target/classes' -printf '%p:') +DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") +BENCHMARK_FILTER=${BENCHMARK_FILTER:-'HivePartitionValuesSizeBenchmark|IcebergCacheSizeBenchmark|PaimonCacheSizeBenchmark|MetaCacheSoftValueBenchmark'} + +BENCHMARK_CLASSES=( + org.apache.doris.datasource.hive.HivePartitionValuesSizeBenchmark + org.apache.doris.datasource.iceberg.IcebergCacheSizeBenchmark + org.apache.doris.datasource.paimon.PaimonCacheSizeBenchmark + org.apache.doris.datasource.metacache.MetaCacheSoftValueBenchmark +) + +for BENCHMARK_CLASS in "${BENCHMARK_CLASSES[@]}"; do + if [[ "${BENCHMARK_CLASS##*.}" =~ ${BENCHMARK_FILTER} ]]; then + java \ + -Xms1g \ + -Xmx4g \ + -classpath "${REACTOR_CLASSES}${DEPENDENCY_CLASSES}" \ + "${BENCHMARK_CLASS}" \ + "$@" + fi +done diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java new file mode 100644 index 00000000000000..c1e9234c70fa67 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.benchmark; + +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** Small dependency-free harness for opt-in FE microbenchmarks. */ +public final class BenchmarkHarness { + private static final long WARMUP_MILLIS = Long.getLong("benchmark.warmup.millis", 500L); + private static final long MEASUREMENT_MILLIS = Long.getLong("benchmark.measurement.millis", 500L); + private static final int MEASUREMENT_ITERATIONS = Integer.getInteger("benchmark.iterations", 3); + private static final boolean PRINT_RESULT = Boolean.getBoolean("benchmark.print.result"); + private static volatile Object sink; + + private BenchmarkHarness() { + } + + @FunctionalInterface + public interface Operation { + Object run() throws Exception; + } + + public static void measure(String name, TimeUnit outputUnit, Operation operation) throws Exception { + runWindow(operation, WARMUP_MILLIS); + double totalNanosPerOperation = 0.0D; + long totalOperations = 0L; + for (int iteration = 0; iteration < MEASUREMENT_ITERATIONS; iteration++) { + Window result = runWindow(operation, MEASUREMENT_MILLIS); + totalNanosPerOperation += result.nanosPerOperation; + totalOperations += result.operations; + } + double averageNanos = totalNanosPerOperation / MEASUREMENT_ITERATIONS; + String result = PRINT_RESULT ? ", result=" + sink : ""; + System.out.printf(Locale.ROOT, "%-72s %12.3f %s/op (%d ops%s)%n", + name, convertFromNanos(averageNanos, outputUnit), unitName(outputUnit), totalOperations, result); + } + + private static Window runWindow(Operation operation, long minimumMillis) throws Exception { + long start = System.nanoTime(); + long deadline = start + TimeUnit.MILLISECONDS.toNanos(minimumMillis); + long operations = 0L; + do { + sink = operation.run(); + operations++; + } while (System.nanoTime() < deadline); + long elapsed = System.nanoTime() - start; + return new Window(operations, (double) elapsed / operations); + } + + private static double convertFromNanos(double nanos, TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return nanos; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return nanos / 1_000.0D; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return nanos / 1_000_000.0D; + } else if (outputUnit == TimeUnit.SECONDS) { + return nanos / 1_000_000_000.0D; + } + throw new IllegalArgumentException("unsupported benchmark time unit: " + outputUnit); + } + + private static String unitName(TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return "ns"; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return "us"; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return "ms"; + } else if (outputUnit == TimeUnit.SECONDS) { + return "s"; + } + return outputUnit.name().toLowerCase(Locale.ROOT); + } + + private static final class Window { + private final long operations; + private final double nanosPerOperation; + + private Window(long operations, double nanosPerOperation) { + this.operations = operations; + this.nanosPerOperation = nanosPerOperation; + } + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java new file mode 100644 index 00000000000000..7e3acd9e2f97ac --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; +import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; + +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.MoreExecutors; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** Measures count publication, weighted preparation, and prepared-value admission separately. */ +public class HivePartitionValuesSizeBenchmark { + private static final int TAIL_PAYLOAD_BYTES = 1024 * 1024; + private static final long MAX_WEIGHT_BYTES = 4L * 1024L * 1024L * 1024L; + + public int countPublicationBaseline(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + return state.partitionValues.getSortedPartitionRanges() + .map(ranges -> ranges.sortedPartitions.size() + ranges.defaultPartitions.size()) + .orElse(0); + } + + public int sealPublicationWithoutEstimate(UnsealedState state) { + state.partitionValues.sealForPublication(); + return state.partitionValues.getIdToPartitionItem().size(); + } + + public long weightedPublication(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + state.partitionValues.prepareForCachePublication(state.key); + return requireComplete(state.partitionValues); + } + + public long eventCopySealAndEstimate(PreparedState state) { + HivePartitionValues copy = state.partitionValues.mutableCopy(); + copy.rebuildSortedPartitionRangesForPublication(); + copy.prepareForCachePublication(state.key); + return requireComplete(copy); + } + + public long preparedSizeProvider(PreparedState state) { + return state.partitionValues.getSizeEstimate().getBytes(); + } + + public long estimateFormula(PreparedState state) { + state.partitionValues.prepareSizeEstimate(state.key); + return requireComplete(state.partitionValues); + } + + public void replacementAdmission(PreparedState state) { + MetaCacheEntry.ReplaceResult result = state.cacheEntry.tryReplace( + state.key, state.currentPartitionValues, state.nextPartitionValues); + if (result != MetaCacheEntry.ReplaceResult.REPLACED) { + throw new IllegalStateException("replacement failed: " + result); + } + state.currentPartitionValues = state.nextPartitionValues; + state.nextPartitionValues = state.nextPartitionValues == state.partitionValues + ? state.replacementPartitionValues : state.partitionValues; + } + + public HivePartitionValues countStrongCacheHit(PreparedState state) { + return state.countCacheEntry.getIfPresent(state.key); + } + + public HivePartitionValues weightedSoftCacheHit(PreparedState state) { + return state.cacheEntry.getIfPresent(state.key); + } + + public static void main(String[] args) throws Exception { + HivePartitionValuesSizeBenchmark benchmark = new HivePartitionValuesSizeBenchmark(); + for (int partitionCount : new int[] {1000, 10000, 100000}) { + for (String distribution : new String[] {"uniform", "tail_skew"}) { + String suffix = "[partitions=" + partitionCount + ",distribution=" + distribution + "]"; + BenchmarkHarness.measure("hive.countPublicationBaseline" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.countPublicationBaseline(state); + }); + BenchmarkHarness.measure("hive.sealPublicationWithoutEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.sealPublicationWithoutEstimate(state); + }); + BenchmarkHarness.measure("hive.weightedPublication" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.weightedPublication(state); + }); + + PreparedState prepared = new PreparedState(); + prepared.partitionCount = partitionCount; + prepared.distribution = distribution; + prepared.setup(); + try { + BenchmarkHarness.measure("hive.eventCopySealAndEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> benchmark.eventCopySealAndEstimate(prepared)); + BenchmarkHarness.measure("hive.preparedSizeProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedSizeProvider(prepared)); + BenchmarkHarness.measure("hive.estimateFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.estimateFormula(prepared)); + BenchmarkHarness.measure("hive.countStrongCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.countStrongCacheHit(prepared)); + BenchmarkHarness.measure("hive.weightedSoftCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.weightedSoftCacheHit(prepared)); + BenchmarkHarness.measure("hive.replacementAdmission" + suffix, + TimeUnit.NANOSECONDS, () -> { + benchmark.replacementAdmission(prepared); + return null; + }); + } finally { + prepared.tearDown(); + } + } + } + } + + private static UnsealedState unsealedState(int partitionCount, String distribution) throws Exception { + UnsealedState state = new UnsealedState(); + state.partitionCount = partitionCount; + state.distribution = distribution; + state.setupInvocation(); + return state; + } + + /** Fresh graph per invocation so all publication work stays inside the measured method. */ + public static class UnsealedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + + public void setupInvocation() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + } + } + + public static class PreparedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + private HivePartitionValues replacementPartitionValues; + private HivePartitionValues currentPartitionValues; + private HivePartitionValues nextPartitionValues; + private MetaCacheEntry cacheEntry; + private MetaCacheEntry countCacheEntry; + private ExecutorService cacheExecutor; + + public void setup() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + partitionValues.prepareForCachePublication(key); + requireComplete(partitionValues); + + // A distinct value root makes Caffeine perform a real replacement while sharing + // immutable payload objects to keep the fixture's resident heap bounded. + replacementPartitionValues = new HivePartitionValues( + partitionValues.getIdToPartitionItem(), + partitionValues.getPartitionNameToIdMap(), + partitionValues.getPartitionValuesMap()); + replacementPartitionValues.prepareForCachePublication(key); + requireComplete(replacementPartitionValues); + + cacheExecutor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "hive", "partition_values_benchmark", OptionalLong.empty(), OptionalLong.empty()); + cacheEntry = new MetaCacheEntry<>( + "partition_values_benchmark", + ignored -> partitionValues, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 1L, MAX_WEIGHT_BYTES), + cacheExecutor, + false, + false, + (entryKey, value) -> value.prepareForCachePublication(entryKey), + entryBudget); + cacheEntry.put(key, partitionValues); + countCacheEntry = new MetaCacheEntry<>( + "partition_values_count_benchmark", + ignored -> partitionValues, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L), + cacheExecutor, + false, + false); + countCacheEntry.put(key, partitionValues); + currentPartitionValues = partitionValues; + nextPartitionValues = replacementPartitionValues; + } + + public void tearDown() { + if (cacheEntry != null) { + cacheEntry.close(); + } + if (countCacheEntry != null) { + countCacheEntry.close(); + } + if (cacheExecutor != null) { + cacheExecutor.shutdownNow(); + } + } + } + + private static List benchmarkTypes() { + return Collections.singletonList(Type.STRING); + } + + private static PartitionValueCacheKey benchmarkKey(List types) { + return new PartitionValueCacheKey( + NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"), types); + } + + private static long requireComplete(HivePartitionValues value) { + MetaCacheSizeEstimate estimate = value.getSizeEstimate(); + if (!estimate.isComplete()) { + throw new IllegalStateException("benchmark graph is not fully measurable: " + + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private static HivePartitionValues createPartitionValues( + int count, String distribution, List types) throws Exception { + HashBiMap nameToId = HashBiMap.create(count); + Map idToItem = Maps.newHashMapWithExpectedSize(count); + Map> idToValues = Maps.newHashMapWithExpectedSize(count); + String tailPayload = "tail_skew".equals(distribution) ? repeat('x', TAIL_PAYLOAD_BYTES) : null; + long partitionNameCharacterCount = 0L; + + for (int i = 0; i < count; i++) { + long id = i; + String value = i == count - 1 && tailPayload != null ? tailPayload : "value-" + i; + String name = "p=" + value; + partitionNameCharacterCount += name.length(); + List rawValues = Collections.singletonList(new PartitionValue(value)); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(rawValues, types, true); + List keys = new ArrayList<>(1); + keys.add(partitionKey); + + nameToId.put(name, id); + idToItem.put(id, new ListPartitionItem(keys)); + idToValues.put(id, new ArrayList<>(Collections.singletonList(value))); + } + return new HivePartitionValues( + idToItem, nameToId, idToValues, partitionNameCharacterCount, types.size()); + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, value); + return new String(chars); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java new file mode 100644 index 00000000000000..2cca17a879b625 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java @@ -0,0 +1,411 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** Measures Iceberg table, long-history table, snapshot and manifest publication. */ +public class IcebergCacheSizeBenchmark { + public int tableValueConstruction(TablePublicationState state) { + return new IcebergTableCacheValue(state.table).getIcebergTable().schema().schemaId(); + } + + public long tablePublication(TablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(TablePublicationState state) { + return IcebergCacheSizeEstimator.retainedTablePayloadBytes(state.table); + } + + public long longHistoryTablePublication(LongHistoryTablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long snapshotPublication(SnapshotPublicationState state) { + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table); + value.prepareForCachePublication(state.snapshotKey); + return requireComplete(value.getSizeEstimate()); + } + + public int snapshotValueConstruction(SnapshotPublicationState state) { + return new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table) + .getPartitionInfo().getNameToIcebergPartition().size(); + } + + public long preparedSnapshotCacheHit(SnapshotPublicationState state) { + return state.preparedSnapshotValue.getIcebergTable().get() + .currentSnapshot().snapshotId(); + } + + public long manifestPublication(ManifestState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateManifestEntry(state.key, state.value)); + } + + public int manifestValueConstruction(ManifestState state) { + return ManifestCacheValue.forDataFiles(state.files).getDataFiles().size(); + } + + public int denseManifestReaderBaseline(DenseManifestState state) { + List collected = new ArrayList<>(); + for (DataFile file : state.files) { + collected.add(file.copy()); + } + return ImmutableList.copyOf(collected).size(); + } + + public int denseManifestValueConstruction(DenseManifestState state) { + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); + for (DataFile file : state.files) { + builder.addDataFile(file.copy()); + } + return builder.build().getDataFiles().size(); + } + + public long preparedWeightProvider(PreparedState state) { + return state.preparedTableValue.getSizeEstimate().getBytes(); + } + + public long tableFormula(PreparedState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateTableEntry( + state.mapping, state.preparedTableValue)); + } + + public int preparedTableCacheHit(PreparedState state) { + return state.preparedTableValue.getIcebergTable().schema().schemaId(); + } + + public static void main(String[] args) throws Exception { + IcebergCacheSizeBenchmark benchmark = new IcebergCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState tableState = new TablePublicationState(); + tableState.fieldCount = fieldCount; + tableState.setup(); + BenchmarkHarness.measure("iceberg.tableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(tableState)); + BenchmarkHarness.measure("iceberg.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(tableState)); + BenchmarkHarness.measure("iceberg.tablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(tableState)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.setup(); + BenchmarkHarness.measure("iceberg.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("iceberg.tableFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableFormula(prepared)); + BenchmarkHarness.measure("iceberg.preparedTableCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedTableCacheHit(prepared)); + } + for (int snapshotCount : new int[] {1000, 10000}) { + String suffix = "[snapshots=" + snapshotCount + "]"; + LongHistoryTablePublicationState state = new LongHistoryTablePublicationState(); + state.snapshotCount = snapshotCount; + state.setup(); + BenchmarkHarness.measure("iceberg.longHistoryTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.longHistoryTablePublication(state)); + } + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + SnapshotPublicationState state = new SnapshotPublicationState(); + state.fieldCount = fieldCount; + state.partitionCount = partitionCount; + state.setup(); + BenchmarkHarness.measure("iceberg.snapshotValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotValueConstruction(state)); + BenchmarkHarness.measure("iceberg.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(state)); + BenchmarkHarness.measure("iceberg.preparedSnapshotCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedSnapshotCacheHit(state)); + } + } + for (int fileCount : new int[] {100, 10000}) { + ManifestState state = new ManifestState(); + state.fileCount = fileCount; + state.setup(); + BenchmarkHarness.measure("iceberg.manifestPublication[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestPublication(state)); + BenchmarkHarness.measure("iceberg.manifestValueConstruction[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestValueConstruction(state)); + } + for (int metricColumns : new int[] {100, 1000}) { + for (int fileCount : new int[] {100, 10000}) { + DenseManifestState state = new DenseManifestState(); + state.metricColumns = metricColumns; + state.fileCount = fileCount; + state.setup(); + String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + "]"; + BenchmarkHarness.measure("iceberg.denseManifestReaderBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestReaderBaseline(state)); + BenchmarkHarness.measure("iceberg.denseManifestValueConstruction" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestValueConstruction(state)); + } + } + } + + public static class TablePublicationState { + public int fieldCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + } + } + + public static class SnapshotPublicationState { + public int fieldCount; + + public int partitionCount; + + private Table table; + private IcebergSnapshotEntryKey snapshotKey; + private IcebergPartitionInfo partitionInfo; + private IcebergSnapshotCacheValue preparedSnapshotValue; + + public void setup() { + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + partitionInfo = newPartitionInfo(partitionCount); + snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table) + .orElseThrow(() -> new IllegalStateException("benchmark table has no generation key")); + preparedSnapshotValue = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), table); + preparedSnapshotValue.prepareForCachePublication(snapshotKey); + requireComplete(preparedSnapshotValue.getSizeEstimate()); + } + } + + public static class LongHistoryTablePublicationState { + public int snapshotCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newLongHistoryTable(snapshotCount); + } + } + + public static class PreparedState { + public int fieldCount; + + private IcebergTableCacheValue preparedTableValue; + private NameMapping mapping; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + Table table = newTable(fieldCount); + preparedTableValue = new IcebergTableCacheValue(table); + preparedTableValue.prepareForCachePublication(mapping); + requireComplete(preparedTableValue.getSizeEstimate()); + } + } + + public static class ManifestState { + public int fileCount; + + private IcebergManifestEntryKey key; + private ManifestCacheValue value; + private List files; + + public void setup() { + key = new IcebergManifestEntryKey("/benchmark/manifest.avro", ManifestContent.DATA); + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/file-" + index + ".parquet") + .withFileSizeInBytes(1024L + index) + .withRecordCount(10L + index) + .build()); + } + value = ManifestCacheValue.forDataFiles(files); + } + } + + public static class DenseManifestState { + public int fileCount; + + public int metricColumns; + + private List files; + + public void setup() { + int poolSize = Math.min(fileCount, 256); + List filePool = new ArrayList<>(poolSize); + for (int fileIndex = 0; fileIndex < poolSize; fileIndex++) { + HashMap lowerBounds = new HashMap<>(metricColumns); + HashMap upperBounds = new HashMap<>(metricColumns); + for (int columnIndex = 0; columnIndex < metricColumns; columnIndex++) { + lowerBounds.put(columnIndex, ByteBuffer.allocate(16)); + upperBounds.put(columnIndex, ByteBuffer.allocate(32)); + } + Metrics metrics = new Metrics( + 10L, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + lowerBounds, + upperBounds); + filePool.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/dense-" + fileIndex + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build()); + } + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(filePool.get(index % poolSize)); + } + } + } + + private static Table newTable(int fieldCount) { + List fields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional(index + 1, "field_" + index, Types.StringType.get())); + } + Schema schema = new Schema(fields); + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:/benchmark/table", Collections.emptyMap()); + InMemoryFileIO fileIO = new InMemoryFileIO(); + StringBuilder snapshotJson = new StringBuilder("{\"snapshot-id\":7,\"timestamp-ms\":1,") + .append("\"summary\":{\"operation\":\"append\"},\"manifests\":["); + for (int index = 0; index < 10; index++) { + if (index > 0) { + snapshotJson.append(','); + } + String manifestPath = "/benchmark/manifest-" + index + ".avro"; + snapshotJson.append('"').append(manifestPath).append('"'); + fileIO.addFile(manifestPath, new byte[0]); + } + Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.append("],\"schema-id\":0}").toString()); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges() + .withMetadataLocation("file:/benchmark/table/metadata/v1.json").build(); + return new BaseTable(new StaticTableOperations(metadata, fileIO), "benchmark.table"); + } + + private static Table newLongHistoryTable(int snapshotCount) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"benchmark-table\",") + .append("\"location\":\"file:/benchmark/table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index) + .append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/benchmark/history/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "file:/benchmark/table/metadata/v1.json", json.toString()); + return new BaseTable( + new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); + } + + private static IcebergPartitionInfo newPartitionInfo(int partitionCount) { + HashMap partitions = new HashMap<>(partitionCount); + long retainedPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=value_" + index; + IcebergPartition partition = new IcebergPartition(name, 0, 10L + index, 1024L + index, + 1L, 1_700_000_000_000L + index, 7L, + Collections.singletonList("value_" + index), Collections.singletonList("identity")); + partitions.put(name, partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); + } + return new IcebergPartitionInfo( + Collections.emptyMap(), partitions, Collections.emptyMap(), retainedPayloadBytes); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java new file mode 100644 index 00000000000000..b488f428e8b9a9 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.util.concurrent.MoreExecutors; + +import java.lang.ref.Reference; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Measures reservation cleanup after Caffeine reports soft values as COLLECTED. */ +public final class MetaCacheSoftValueBenchmark { + private static final int SAMPLE_COUNT = 3; + private static final int VALUE_COUNT = 10_000; + private static final long MAX_WEIGHT_BYTES = 16L * 1024L * 1024L; + + private MetaCacheSoftValueBenchmark() { + } + + public static void main(String[] args) throws Exception { + long totalNanos = 0L; + for (int sample = 0; sample < SAMPLE_COUNT; sample++) { + CollectedState state = CollectedState.create(VALUE_COUNT); + try { + state.enqueueAll(); + long start = System.nanoTime(); + state.cleanUp(); + totalNanos += System.nanoTime() - start; + } finally { + state.close(); + } + } + double averageNanos = (double) totalNanos / SAMPLE_COUNT; + System.out.printf(Locale.ROOT, + "%-72s %12.3f us/batch (%.3f ns/value, %d samples)%n", + "metacache.collectedCleanup[values=" + VALUE_COUNT + "]", + averageNanos / TimeUnit.MICROSECONDS.toNanos(1L), + averageNanos / VALUE_COUNT, + SAMPLE_COUNT); + } + + private static final class CollectedState implements AutoCloseable { + private final ExecutorService executor; + private final ExternalMetaCacheBudgetManager budgetManager; + private final MetaCacheEntry entry; + private final LoadingCache loadingCache; + private final List> references; + + private CollectedState(ExecutorService executor, + ExternalMetaCacheBudgetManager budgetManager, + MetaCacheEntry entry, + LoadingCache loadingCache, + List> references) { + this.executor = executor; + this.budgetManager = budgetManager; + this.entry = entry; + this.loadingCache = loadingCache; + this.references = references; + } + + private static CollectedState create(int valueCount) throws Exception { + ExecutorService executor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget budget = budgetManager.createEntryBudget( + 1L, "benchmark", "soft_cleanup", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft_cleanup", + key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, valueCount, MAX_WEIGHT_BYTES), + executor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), + budget); + for (int index = 0; index < valueCount; index++) { + entry.put("key-" + index, new byte[1]); + } + + LoadingCache loadingCache = (LoadingCache) readField(entry, "loadingData"); + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + List> references = new ArrayList<>(nodes.size()); + for (Object node : nodes.values()) { + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + references.add((Reference) valueReferenceMethod.invoke(node)); + } + if (references.size() != valueCount) { + entry.close(); + executor.shutdownNow(); + throw new IllegalStateException( + "benchmark admission retained " + references.size() + " of " + valueCount + " values"); + } + return new CollectedState(executor, budgetManager, entry, loadingCache, references); + } + + private void enqueueAll() { + for (Reference reference : references) { + reference.clear(); + reference.enqueue(); + } + } + + private void cleanUp() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); + while (budgetManager.getGlobalUsedWeight() != 0L + && System.nanoTime() < deadline) { + loadingCache.cleanUp(); + LockSupport.parkNanos(TimeUnit.MICROSECONDS.toNanos(100L)); + } + if (budgetManager.getGlobalUsedWeight() != 0L) { + throw new IllegalStateException( + "COLLECTED cleanup retained " + budgetManager.getGlobalUsedWeight() + " bytes"); + } + } + + @Override + public void close() { + entry.close(); + executor.shutdownNow(); + } + } + + private static Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private static Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java new file mode 100644 index 00000000000000..f73eca412f3c44 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.AppendOnlyFileStoreTable; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Measures Paimon 1.4.2 nested-schema/non-empty snapshot publication and prepared weight lookup. */ +public class PaimonCacheSizeBenchmark { + public long snapshotPublication(PublicationState state) { + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue(state.partitionInfo, state.snapshot); + value.prepareForCachePublication(state.key); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(PublicationState state) { + return PaimonCacheSizeEstimator.retainedTablePayloadBytes(state.snapshot.getTable()); + } + + public long preparedWeightProvider(PreparedState state) { + return state.value.getSizeEstimate().getBytes(); + } + + public long snapshotFormula(PreparedState state) { + return requireComplete(PaimonCacheSizeEstimator.estimateSnapshotEntry(state.key, state.value)); + } + + public long partitionMapBaseline(PartitionPayloadState state) { + return buildPartitionInfo(state, false); + } + + public long partitionMapWithRetainedCounter(PartitionPayloadState state) { + return buildPartitionInfo(state, true); + } + + public static void main(String[] args) throws Exception { + PaimonCacheSizeBenchmark benchmark = new PaimonCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + PublicationState publication = new PublicationState(); + publication.fieldCount = fieldCount; + publication.partitionCount = partitionCount; + publication.setup(); + BenchmarkHarness.measure("paimon.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(publication)); + BenchmarkHarness.measure("paimon.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(publication)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.partitionCount = partitionCount; + prepared.setup(); + BenchmarkHarness.measure("paimon.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("paimon.snapshotFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotFormula(prepared)); + } + } + for (int partitionCount : new int[] {1000, 10000}) { + for (boolean tailSkew : new boolean[] {false, true}) { + PartitionPayloadState state = new PartitionPayloadState(); + state.partitionCount = partitionCount; + state.tailSkew = tailSkew; + state.setup(); + String suffix = "[partitions=" + partitionCount + + ",distribution=" + (tailSkew ? "tail-skew" : "uniform") + "]"; + BenchmarkHarness.measure("paimon.partitionMapBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapBaseline(state)); + BenchmarkHarness.measure("paimon.partitionMapWithRetainedCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapWithRetainedCounter(state)); + } + } + } + + public static class PublicationState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonPartitionInfo partitionInfo; + private PaimonSnapshot snapshot; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + partitionInfo = fixture.value.getPartitionInfo(); + snapshot = fixture.value.getSnapshot(); + } + } + + public static class PreparedState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonSnapshotCacheValue value; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + value = fixture.value; + value.prepareForCachePublication(fixture.key); + requireComplete(value.getSizeEstimate()); + } + } + + public static class PartitionPayloadState { + public int partitionCount; + + public boolean tailSkew; + + private List partitions; + + public void setup() { + partitions = new ArrayList<>(partitionCount); + String longTail = String.join("", Collections.nCopies(64 * 1024, "x")); + for (int index = 0; index < partitionCount; index++) { + String value = tailSkew && index % 997 == 0 ? longTail : String.valueOf(index); + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < 4; field++) { + typedSpec.put("partition_key_" + field, value + '_' + field); + } + String displayName = "partition_key_0=" + value; + partitions.add(new PartitionPayload( + displayName, new ArrayList<>(typedSpec.values()), index)); + } + } + } + + private static Fixture newFixture(int fieldCount, int partitionCount) throws Exception { + List fields = new ArrayList<>(fieldCount + 2); + fields.add(new DataField(0, "partition_key", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new VarCharType())); + } + List nestedFields = new ArrayList<>(); + for (int index = 0; index < 8; index++) { + nestedFields.add(DataTypes.FIELD(fieldCount + index + 1, + "nested_field_" + index, DataTypes.STRING())); + } + fields.add(new DataField(fieldCount + 9, "nested_payload", new RowType(nestedFields))); + TableSchema schema = new TableSchema( + 0L, fields, fieldCount + 9, Collections.singletonList("partition_key"), + Collections.emptyList(), Collections.emptyMap(), null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/doris-paimon-cache-size-benchmark"), + schema, CatalogEnvironment.empty()); + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 7L, schema.id(), 1L); + HashMap partitions = new HashMap<>(partitionCount); + long retainedPartitionPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=" + index; + String value = String.valueOf(index); + partitions.put(name, new Partition(Collections.singletonMap("partition_key", value), + 10L + index, 1024L + index, 1L, 1_700_000_000_000L + index, 1, true, + 1_700_000_000_000L, "benchmark", 1_700_000_000_000L + index, "benchmark", + Collections.singletonMap("source", "benchmark"))); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(name)); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes("partition_key")); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + new PaimonPartitionInfo(Collections.emptyMap(), partitions, retainedPartitionPayloadBytes), + new PaimonSnapshot(7L, schema.id(), table)); + return new Fixture(key, value); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private long buildPartitionInfo(PartitionPayloadState state, boolean countPayload) { + HashMap partitions = new HashMap<>(state.partitionCount); + long retainedPayloadBytes = 0L; + for (PartitionPayload payload : state.partitions) { + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < payload.values.size(); field++) { + String fieldName = "partition_key_" + field; + String fieldValue = payload.values.get(field); + typedSpec.put(fieldName, fieldValue); + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldName); + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldValue); + } + } + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, payload.displayName); + } + int index = payload.index; + partitions.put(payload.displayName, new Partition( + typedSpec, 10L + index, 1024L + index, 1L, + 1_700_000_000_000L + index, 1, false)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo( + Collections.emptyMap(), partitions, retainedPayloadBytes); + return MetaCacheWeightUtils.saturatedAdd( + partitionInfo.getNameToPartition().size(), partitionInfo.getRetainedPayloadBytes()); + } + + private static class PartitionPayload { + private final String displayName; + private final List values; + private final int index; + + private PartitionPayload( + String displayName, List values, int index) { + this.displayName = displayName; + this.values = values; + this.index = index; + } + } + + private static class Fixture { + private final PaimonSnapshotEntryKey key; + private final PaimonSnapshotCacheValue value; + + private Fixture(PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + this.key = key; + this.value = value; + } + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4e535950aae0e3..c3e95e2d4ff24a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2462,6 +2462,11 @@ public class Config extends ConfigBase { }) public static long external_cache_refresh_time_minutes = 10; // 10 mins + @ConfField(mutable = false, masterOnly = false, + description = {"FE-wide maximum weight for managed external metadata caches. Supports byte units " + + "or a percentage of the JVM max heap; 0 disables the global quota."}) + public static String external_meta_cache_max_weight = "0"; + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. @ConfField(mutable = true, masterOnly = false, description = {"Whether external meta cache uses manual miss load instead of Caffeine sync load."}) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java index 674bf0aa39cd5b..7ad55174e303e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import com.github.benmanes.caffeine.cache.Weigher; import org.jetbrains.annotations.NotNull; import java.time.Duration; @@ -49,7 +50,10 @@ public class CacheFactory { private OptionalLong expireAfterAccessSec; private OptionalLong refreshAfterWriteSec; private long maxSize; + private OptionalLong maxWeight; + private Weigher weigher; private boolean enableStats; + private boolean softValues; // Ticker is used to provide a time source for the cache. // Only used for test, to provide a fake time source. // If not provided, the system time is used. @@ -61,11 +65,34 @@ public CacheFactory( long maxSize, boolean enableStats, Ticker ticker) { + this(expireAfterAccessSec, refreshAfterWriteSec, maxSize, OptionalLong.empty(), null, enableStats, ticker); + } + + @SuppressWarnings("unchecked") + public CacheFactory( + OptionalLong expireAfterAccessSec, + OptionalLong refreshAfterWriteSec, + long maxSize, + OptionalLong maxWeight, + Weigher weigher, + boolean enableStats, + Ticker ticker) { this.expireAfterAccessSec = expireAfterAccessSec; this.refreshAfterWriteSec = refreshAfterWriteSec; this.maxSize = maxSize; + this.maxWeight = maxWeight; + this.weigher = (Weigher) weigher; this.enableStats = enableStats; this.ticker = ticker; + if (maxWeight.isPresent() && this.weigher == null) { + throw new IllegalArgumentException("maximumWeight requires a weigher"); + } + } + + /** Configure values as soft references so unused cache entries may be reclaimed under heap pressure. */ + public CacheFactory withSoftValues() { + softValues = true; + return this; } // Build a loading cache, without executor, it will use fork-join pool for refresh @@ -116,7 +143,11 @@ public AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cac @NotNull private Caffeine buildWithParams() { Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); + if (maxWeight.isPresent()) { + builder.maximumWeight(maxWeight.getAsLong()).weigher(weigher); + } else { + builder.maximumSize(maxSize); + } if (expireAfterAccessSec.isPresent()) { builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); @@ -129,6 +160,10 @@ private Caffeine buildWithParams() { builder.recordStats(); } + if (softValues) { + builder.softValues(); + } + if (ticker != null) { builder.ticker(ticker); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 481c1fcb3aca5f..f32c77a7015818 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase; import org.apache.doris.datasource.lance.LanceExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; @@ -446,6 +447,19 @@ protected void checkProperties(CatalogProperty property) throws DdlException { } } + try { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr extMetaCacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (extMetaCacheMgr == null) { + // This fallback is only for isolated construction tests before Env is initialized. + ExternalMetaCacheBudgetManager.fromConfig().validateCatalogMaxWeight(properties); + } else { + extMetaCacheMgr.validateCatalogCacheProperties(this, properties); + } + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + // check schema.cache.ttl-second parameter String schemaCacheTtlSecond = property.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL) @@ -1367,9 +1381,21 @@ public int hashCode() { public void notifyPropertiesUpdated(Map updatedProps) { CatalogIf.super.notifyPropertiesUpdated(updatedProps); String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); - if (java.util.Objects.nonNull(schemaCacheTtl)) { - ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + if (java.util.Objects.nonNull(schemaCacheTtl) + || updatedProps.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { extMetaCacheMgr.removeCatalog(id); + return; + } + for (String key : updatedProps.keySet()) { + if (key == null || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + if (separator > 0) { + extMetaCacheMgr.removeCatalogByEngine(id, remainder.substring(0, separator)); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 007e850e54e24e..a80d0c8b71485f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalMetaCache; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.ExternalMetaCacheRegistry; import org.apache.doris.datasource.metacache.ExternalMetaCacheRouteResolver; import org.apache.doris.datasource.metacache.LegacyMetaCacheFactory; @@ -38,6 +39,7 @@ import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Striped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,8 +49,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.Lock; import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -95,6 +100,10 @@ public class ExternalMetaCacheMgr { private final ExternalMetaCacheRegistry cacheRegistry; private final ExternalMetaCacheRouteResolver routeResolver; private final LegacyMetaCacheFactory legacyMetaCacheFactory; + private final ExternalMetaCacheBudgetManager budgetManager; + // Catalog property publication and cache-group replacement share this striped lifecycle fence. + // Initialized lookups retain the lock-free fast path above it. + private final Striped catalogLifecycleLocks = Striped.lock(64); // all catalogs could share the same fsCache. private FileSystemCache fsCache; @@ -102,6 +111,7 @@ public class ExternalMetaCacheMgr { private ExternalRowCountCache rowCountCache; public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { + budgetManager = ExternalMetaCacheBudgetManager.fromConfig(); rowCountRefreshExecutor = newThreadPool(isCheckpointCatalog, Config.max_external_cache_loader_thread_pool_size, Config.max_external_cache_loader_thread_pool_size * 1000, @@ -191,28 +201,114 @@ public DorisExternalMetaCache doris(long catalogId) { } public void prepareCatalog(long catalogId) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalog"); - return; + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalog"); + return; + } + validateCatalogCachePropertiesForRuntime(catalogProperties); + routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); + } finally { + lifecycleLock.unlock(); } - routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); } public void prepareCatalogByEngine(long catalogId, String engine) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { return; } - prepareCatalogByEngine(catalogId, engine, catalogProperties); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } public void prepareCatalogByEngine(long catalogId, String engine, Map catalogProperties) { + ExternalMetaCache targetCache = this.engine(engine); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } + } + + private void prepareCatalogByEngineLocked( + long catalogId, ExternalMetaCache targetCache, Map catalogProperties) { Map safeCatalogProperties = catalogProperties == null ? Maps.newHashMap() : Maps.newHashMap(catalogProperties); - routeSpecifiedEngine(engine, cache -> cache.initCatalog(catalogId, safeCatalogProperties)); + validateCatalogCachePropertiesForRuntime(safeCatalogProperties); + targetCache.initCatalog(catalogId, safeCatalogProperties); + } + + public void validateCatalogCacheProperties(Map catalogProperties) { + budgetManager.validateCatalogMaxWeight(catalogProperties); + validateCatalogCachePropertyNamespaces(catalogProperties); + cacheRegistry.allCaches().forEach(cache -> cache.validateCatalogProperties(catalogProperties)); + } + + private void validateCatalogCachePropertiesForRuntime(Map catalogProperties) { + budgetManager.parseCatalogMaxWeight(catalogProperties); + validateCatalogCachePropertyNamespaces(catalogProperties); + } + + private void validateCatalogCachePropertyNamespaces(Map catalogProperties) { + String globalKey = ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY; + String prefix = "meta.cache."; + for (String key : catalogProperties.keySet()) { + if (key == null || globalKey.equals(key) || !key.startsWith(prefix)) { + continue; + } + String remainder = key.substring(prefix.length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String configuredEngine = remainder.substring(0, separator); + ExternalMetaCache resolved = cacheRegistry.resolve(configuredEngine); + if (!resolved.engine().equals(configuredEngine)) { + throw new IllegalArgumentException("External meta cache properties must use canonical engine '" + + resolved.engine() + "' instead of alias '" + configuredEngine + "': " + key); + } + } + } + + /** Strict DDL validation also rejects a valid engine namespace not routed by the catalog type. */ + public void validateCatalogCacheProperties(CatalogIf catalog, Map catalogProperties) { + validateCatalogCacheProperties(catalogProperties); + Set routedEngines = routeResolver.resolveCatalogCaches(catalog.getId(), catalog).stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + for (String key : catalogProperties.keySet()) { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + String configuredEngine = separator < 0 ? remainder : remainder.substring(0, separator); + if (!routedEngines.contains(configuredEngine)) { + throw new IllegalArgumentException("External meta cache engine '" + configuredEngine + + "' is not supported by catalog type " + catalog.getClass().getSimpleName() + ": " + key); + } + } } public void invalidateCatalog(long catalogId) { @@ -228,15 +324,27 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { } public void removeCatalog(long catalogId) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "removeCatalog", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalog", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void removeCatalogByEngine(long catalogId, String engine) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "removeCatalogByEngine", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "removeCatalogByEngine", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void invalidateDb(long catalogId, String dbName) { @@ -302,13 +410,13 @@ private void initEngineCaches() { } private void registerBuiltinEngineCaches() { - cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor)); - cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor)); - cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor)); + cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor, budgetManager)); + cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor, budgetManager)); } private void routeCatalogEngines(long catalogId, Consumer action) { @@ -428,8 +536,9 @@ void replaceEngineCachesForTest(List caches) { * loading/invalidation. No engine-specific metadata (partitions/files/snapshots) is cached. */ private static class DefaultExternalMetaCache extends AbstractExternalMetaCache { - DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor) { - super(engine, refreshExecutor); + DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(engine, refreshExecutor, budgetManager); registerEntry(MetaCacheEntryDef.of( ENTRY_SCHEMA, SchemaCacheKey.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java index d14ba5645bf269..e7487c065b5ce3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -72,7 +73,11 @@ public class DorisExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public DorisExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public DorisExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); backendsEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_BACKENDS, String.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e2d73fd7a16edf..883a38780149d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -28,10 +28,8 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.hudi.HudiExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.iceberg.IcebergUtils; -import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; import org.apache.doris.fs.FileSystemProvider; @@ -218,10 +216,6 @@ public void notifyPropertiesUpdated(Map updatedProps) { if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); } - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java new file mode 100644 index 00000000000000..c26a021890e880 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +/** Constant-time retained-weight formula for Hive partition-value cache entries. */ +final class HiveCacheSizeEstimator { + // Calibrated against complete 4.1 object graphs. The per-character reserve covers the + // partition name plus derived value/literal strings and therefore remains skew-sensitive. + private static final long ENTRY_BASE_BYTES = 2L * 1024L; + private static final long PARTITION_BASE_BYTES = 4L * 1024L; + private static final long PARTITION_COLUMN_BYTES = 384L; + private static final long PARTITION_NAME_CHARACTER_BYTES = 8L; + + private HiveCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimatePartitionValuesEntry( + PartitionValueCacheKey key, HivePartitionValues value) { + long partitionCount = value.getIdToPartitionItem() == null + ? 0L : value.getIdToPartitionItem().size(); + long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( + PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionColumnCount(), PARTITION_COLUMN_BYTES)); + long bytes = MetaCacheWeightUtils.saturatedAdd( + ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionNameCharacterCount(), PARTITION_NAME_CHARACTER_BYTES)); + return MetaCacheSizeEstimate.complete(bytes); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 73986138c51cb0..527497237b1bf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -42,8 +42,12 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.FileSystemDirectoryLister; @@ -59,10 +63,12 @@ import com.google.common.base.Strings; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import lombok.Data; +import lombok.Getter; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Partition; @@ -108,6 +114,7 @@ */ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private static final Logger LOG = LogManager.getLogger(HiveExternalMetaCache.class); + private static final int PARTITION_EVENT_REPLACE_MAX_RETRIES = 8; public static final String ENGINE = "hive"; public static final String ENTRY_SCHEMA = "schema"; @@ -127,7 +134,12 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private final PartitionCacheCoordinator partitionCacheCoordinator = new PartitionCacheCoordinator(); public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, fileListingExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); this.fileListingExecutor = fileListingExecutor; schemaEntry = registerEntry(MetaCacheEntryDef.of( @@ -144,7 +156,8 @@ public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fi CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, - Config.max_hive_partition_table_cache_num))); + Config.max_hive_partition_table_cache_num)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); partitionEntry = registerEntry(MetaCacheEntryDef.of( ENTRY_PARTITION, PartitionCacheKey.class, @@ -292,9 +305,12 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); + long partitionNameCharacterCount = 0L; String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); for (String partitionName : partitionNames) { + partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( + partitionNameCharacterCount, partitionName.length()); long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); idToPartitionItem.put(partitionId, listPartitionItem); @@ -302,7 +318,15 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap); + HivePartitionValues partitionValues = + new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + partitionNameCharacterCount, key.types == null ? 0 : key.types.size()); + preparePartitionValuesForPublication(partitionValues); + return partitionValues; + } + + private void preparePartitionValuesForPublication(HivePartitionValues partitionValues) { + partitionValues.rebuildSortedPartitionRangesForPublication(); } private ListPartitionItem toListPartitionItem(String partitionName, List types, String catalogName) { @@ -635,7 +659,7 @@ private void invalidatePartitionCache(NameMapping nameMapping, String partitionN List values = HiveUtil.toPartitionValues(partitionName); PartitionCacheKey partKey = new PartitionCacheKey(nameMapping, values); - HivePartition partition = partitionEntry.getIfPresent(partKey); + HivePartition partition = partitionEntry.peekIfPresent(partKey); if (partition == null) { // Partition metadata cache miss: the exact FileCacheKey cannot be rebuilt here because it // needs the partition path and input format carried by HivePartition. Invalidate this @@ -715,41 +739,62 @@ private void addPartitionsCache(NameMapping nameMapping, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, partitionColumnTypes); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; - } - - HivePartitionValues copy = partitionValues.copy(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItem = new HashMap<>(); - HMSExternalCatalog catalog = hmsCatalog(catalogId); String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); - for (String partitionName : partitionNames) { - if (partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", - partitionName, localTblName); - continue; + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; } - long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); - ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); - idToPartitionItemBefore.put(partitionId, listPartitionItem); - idToPartitionItem.put(partitionId, listPartitionItem); - partitionNameToIdMapBefore.put(partitionName, partitionId); - } - Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); - Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - partitionValuesMapBefore.putAll(partitionValuesMap); - copy.rebuildSortedPartitionRanges(); - - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); + HivePartitionValues copy = current.mutableCopy(); + Map allItems = copy.getIdToPartitionItem(); + Map allNames = copy.getPartitionNameToIdMap(); + Map addedItems = new HashMap<>(); + for (String partitionName : partitionNames) { + if (allNames.containsKey(partitionName)) { + LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", + partitionName, localTblName); + continue; + } + long partitionId = Util.genIdByName( + catalog.getName(), localDbName, localTblName, partitionName); + ListPartitionItem item = toListPartitionItem(partitionName, key.types, catalog.getName()); + allItems.put(partitionId, item); + addedItems.put(partitionId, item); + allNames.put(partitionName, partitionId); + copy.addPartitionNameCharacters(partitionName.length()); + } + if (addedItems.isEmpty()) { + // Even a replay/no-op event must fence a refresh that started before the event. + // Otherwise that refresh could replace this already-correct graph with stale HMS data. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } + continue; + } + copy.getPartitionValuesMap().putAll( + ListPartitionPrunerV2.getPartitionValuesMap(addedItems)); + preparePartitionValuesForPublication(copy); + + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after add event was rejected: {}", key); + return; + } } + // Repeated conflicts mean we cannot prove the cached graph contains this event. Force + // the next reader to rebuild it from HMS rather than retaining a possibly stale value. + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated add-event conflicts: {}", key); } private void dropPartitionsCache(ExternalTable dorisTable, @@ -765,41 +810,63 @@ private void dropPartitionsCache(ExternalTable dorisTable, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, null); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; + if (invalidPartitionCache) { + for (String partitionName : partitionNames) { + invalidatePartitionCache(nameMapping, partitionName); + } } - HivePartitionValues copy = partitionValues.copy(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map> partitionValuesMap = copy.getPartitionValuesMap(); - - for (String partitionName : partitionNames) { - if (!partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", - partitionName, nameMapping.getFullLocalName()); + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; + } + HivePartitionValues copy = current.mutableCopy(); + Map allNames = copy.getPartitionNameToIdMap(); + Map allItems = copy.getIdToPartitionItem(); + Map> allValues = copy.getPartitionValuesMap(); + boolean changed = false; + for (String partitionName : partitionNames) { + Long partitionId = allNames.remove(partitionName); + if (partitionId == null) { + LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", + partitionName, nameMapping.getFullLocalName()); + continue; + } + allItems.remove(partitionId); + allValues.remove(partitionId); + copy.removePartitionNameCharacters(partitionName.length()); + changed = true; + } + if (!changed) { + // See the add-event no-op path: event ordering still has to win over an older refresh. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } continue; } - Long partitionId = partitionNameToIdMapBefore.remove(partitionName); - idToPartitionItemBefore.remove(partitionId); - partitionValuesMap.remove(partitionId); - - if (invalidPartitionCache) { - invalidatePartitionCache(nameMapping, partitionName); + preparePartitionValuesForPublication(copy); + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after drop event was rejected: {}", key); + return; } } - - copy.rebuildSortedPartitionRanges(); - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); - } + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated drop-event conflicts: {}", key); } } @VisibleForTesting public void putPartitionValuesCacheForTest(PartitionValueCacheKey key, HivePartitionValues values) { + preparePartitionValuesForPublication(values); partitionValuesEntry.get(key.getNameMapping().getCtlId()).put(key, values); } @@ -842,15 +909,15 @@ public List getFilesByTransaction(List partitions /** * The key of hive partition values cache. */ - @Data + @Getter public static class PartitionValueCacheKey { - private NameMapping nameMapping; + private final NameMapping nameMapping; // Not part of cache identity. - private List types; + private final List types; public PartitionValueCacheKey(NameMapping nameMapping, List types) { this.nameMapping = nameMapping; - this.types = types; + this.types = types == null ? null : ImmutableList.copyOf(types); } @Override @@ -1037,7 +1104,7 @@ public static class HiveFileStatus { AcidInfo acidInfo; } - @Data + @Getter public static class HivePartitionValues { private BiMap partitionNameToIdMap; private Map idToPartitionItem; @@ -1045,6 +1112,12 @@ public static class HivePartitionValues { // Sorted partition ranges for binary search filtering. private SortedPartitionRanges sortedPartitionRanges; + // Prepared once after construction/update; the cache weigher only reads this value. + private transient volatile MetaCacheSizeEstimate sizeEstimate; + // Maintained while the metadata is already being loaded or updated. Admission only reads it. + private long partitionNameCharacterCount; + private int partitionColumnCount; + private transient boolean sortedPartitionRangesPrepared; public HivePartitionValues() { } @@ -1052,22 +1125,98 @@ public HivePartitionValues() { public HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap) { + this(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + countPartitionNameCharacters(partitionNameToIdMap), + inferPartitionColumnCount(partitionValuesMap)); + } + + HivePartitionValues(Map idToPartitionItem, + BiMap partitionNameToIdMap, + Map> partitionValuesMap, + long partitionNameCharacterCount, + int partitionColumnCount) { this.idToPartitionItem = idToPartitionItem; this.partitionNameToIdMap = partitionNameToIdMap; this.partitionValuesMap = partitionValuesMap; - this.sortedPartitionRanges = buildSortedPartitionRanges(); + this.partitionNameCharacterCount = partitionNameCharacterCount; + this.partitionColumnCount = partitionColumnCount; } - public HivePartitionValues copy() { + HivePartitionValues mutableCopy() { HivePartitionValues copy = new HivePartitionValues(); - copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); - copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); - copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); + copy.partitionNameToIdMap = partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap); + copy.idToPartitionItem = idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem); + copy.partitionValuesMap = partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap); + copy.partitionNameCharacterCount = partitionNameCharacterCount; + copy.partitionColumnCount = partitionColumnCount; return copy; } - public void rebuildSortedPartitionRanges() { - this.sortedPartitionRanges = buildSortedPartitionRanges(); + /** Compatibility hook for tests and benchmarks; publication uses copy-on-write updates. */ + void sealForPublication() { + if (!sortedPartitionRangesPrepared) { + rebuildSortedPartitionRangesForPublication(); + } + } + + void rebuildSortedPartitionRangesForPublication() { + sortedPartitionRanges = buildSortedPartitionRanges(); + sortedPartitionRangesPrepared = true; + } + + MetaCacheSizeEstimate prepareForCachePublication(PartitionValueCacheKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely( + "hive_partition_values_preparation_failed", () -> { + prepareSizeEstimate(key); + return getSizeEstimate(); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + MetaCacheSizeEstimate result = sizeEstimate; + return result == null ? MetaCacheSizeEstimate.incomplete("estimate_not_prepared") : result; + } + + void prepareSizeEstimate(PartitionValueCacheKey key) { + sizeEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, this); + } + + long getPartitionNameCharacterCount() { + return partitionNameCharacterCount; + } + + int getPartitionColumnCount() { + return partitionColumnCount; + } + + private void addPartitionNameCharacters(int characters) { + partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( + partitionNameCharacterCount, characters); + } + + private void removePartitionNameCharacters(int characters) { + partitionNameCharacterCount = Math.max(0L, partitionNameCharacterCount - characters); + } + + private static long countPartitionNameCharacters(BiMap names) { + long characters = 0L; + if (names != null) { + for (String name : names.keySet()) { + characters = MetaCacheWeightUtils.saturatedAdd(characters, name.length()); + } + } + return characters; + } + + private static int inferPartitionColumnCount(Map> values) { + if (values == null || values.isEmpty()) { + return 0; + } + List first = values.values().iterator().next(); + return first == null ? 0 : first.size(); } public java.util.Optional> getSortedPartitionRanges() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 74d2aa99900340..a83777b1e53e68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -83,7 +84,11 @@ public class HudiExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public HudiExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_PARTITION, HudiPartitionCacheKey.class, TablePartitionValues.class, this::loadPartitionValuesCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiPartitionCacheKey::getNameMapping))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..e60f78eef90182 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** Constant-time retained-weight formulas for Iceberg cache entries. */ +final class IcebergCacheSizeEstimator { + private static final long KEY_BASE_BYTES = 128L; + private static final long TABLE_BASE_BYTES = 16L * 1024L; + private static final long SCHEMA_VERSION_BYTES = 512L; + private static final long SCHEMA_FIELD_BYTES = 512L; + private static final long NESTED_SCHEMA_FIELD_BYTES = 512L; + private static final long PARTITION_SPEC_BYTES = 256L; + private static final long PARTITION_SPEC_FIELD_BYTES = 384L; + private static final long SORT_ORDER_BYTES = 256L; + private static final long SORT_FIELD_BYTES = 256L; + private static final long TABLE_PROPERTY_BYTES = 256L; + private static final long CURRENT_SNAPSHOT_BYTES = 512L; + private static final long PARTITION_BYTES = 512L; + private static final long PARTITION_ALIAS_BYTES = 256L; + private static final long NAME_MAPPING_ENTRY_BYTES = 256L; + private static final long MANIFEST_ENTRY_BASE_BYTES = 256L; + private static final long DATA_FILE_BYTES = 16L * 1024L; + private static final long DELETE_FILE_BYTES = 18L * 1024L; + private static final long FILE_METRIC_ENTRY_BYTES = 160L; + + private IcebergCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + Table table = value.getRetainedIcebergTable(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + long bytes = KEY_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation())); + + IcebergPartitionInfo partitionInfo = value.getPartitionInfo(); + bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); + bytes = addCount(bytes, value.getNameMapping().map(java.util.Map::size).orElse(0), + NAME_MAPPING_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedNameMappingPayloadBytes()); + + if (value.getRetainedIcebergTable().isPresent()) { + Table table = value.getRetainedIcebergTable().get(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + } + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateManifestEntry( + IcebergManifestEntryKey key, ManifestCacheValue value) { + long bytes = MetaCacheWeightUtils.saturatedAdd( + MANIFEST_ENTRY_BASE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); + bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_BYTES); + bytes = addCount(bytes, value.getDeleteFiles().size(), DELETE_FILE_BYTES); + bytes = addCount(bytes, value.getDataFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + private static MetaCacheSizeEstimate checkSupportedTable(Table table) { + if (table == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); + } + if (!(table instanceof HasTableOperations)) { + return MetaCacheSizeEstimate.incomplete( + "unsupported_iceberg_table:" + table.getClass().getName()); + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata"); + } + if (metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location"); + } + return MetaCacheSizeEstimate.complete(1L); + } + + /** Reads only metadata collection sizes and a constant number of strings; no FileIO is used. */ + private static long estimateTable(Table table) { + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + long bytes = MetaCacheWeightUtils.saturatedAdd( + TABLE_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.location())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.metadataFileLocation())); + + bytes = addCount(bytes, metadata.properties().size(), TABLE_PROPERTY_BYTES); + if (metadata.currentSnapshot() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_BYTES); + } + return bytes; + } + + /** Captures exact historical cardinalities and skew-sensitive payload once before admission. */ + static long retainedTablePayloadBytes(Table table) { + if (!(table instanceof HasTableOperations)) { + return 0L; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return 0L; + } + + long bytes = 0L; + for (Schema schema : metadata.schemas()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SCHEMA_VERSION_BYTES); + for (Types.NestedField field : schema.columns()) { + bytes = addFieldPayload(bytes, field, false); + } + } + for (PartitionSpec spec : metadata.specs()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_BYTES); + for (org.apache.iceberg.PartitionField field : spec.fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_FIELD_BYTES); + bytes = addString(bytes, field.name()); + } + } + for (SortOrder sortOrder : metadata.sortOrders()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_BYTES); + bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_BYTES); + } + for (Map.Entry property : metadata.properties().entrySet()) { + bytes = addString(bytes, property.getKey()); + bytes = addString(bytes, property.getValue()); + } + bytes = addString(bytes, metadata.uuid()); + return bytes; + } + + private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.doc()); + bytes = addDefaultPayload(bytes, field.initialDefault()); + bytes = addDefaultPayload(bytes, field.writeDefault()); + return addTypePayload(bytes, field.type()); + } + + private static long addTypePayload(long bytes, Type type) { + if (type.isStructType()) { + for (Types.NestedField field : type.asStructType().fields()) { + bytes = addFieldPayload(bytes, field, true); + } + } else if (type.isListType()) { + bytes = addTypePayload(bytes, type.asListType().elementType()); + } else if (type.isMapType()) { + bytes = addTypePayload(bytes, type.asMapType().keyType()); + bytes = addTypePayload(bytes, type.asMapType().valueType()); + } + return bytes; + } + + private static long addDefaultPayload(long bytes, Object value) { + if (value instanceof CharSequence) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + return MetaCacheWeightUtils.saturatedAdd(bytes, ((ByteBuffer) value).capacity()); + } else if (value instanceof byte[]) { + return MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addStringMap(long bytes, Map values, long entryBytes) { + if (values == null) { + return bytes; + } + bytes = addCount(bytes, values.size(), entryBytes); + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 8407a29d8908a9..e6402dd234839c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; @@ -29,9 +28,12 @@ import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -47,6 +49,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; @@ -55,8 +59,8 @@ * *

Registered entries: *

    - *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping, each - * memoizing its latest snapshot runtime projection
    • + *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping
    • + *
  • {@code snapshot}: immutable snapshot projections keyed by a stable metadata generation
    • *
  • {@code view}: loaded Iceberg {@link View} instances
    • *
  • {@code manifest}: parsed manifest payload ({@link ManifestCacheValue}) keyed by * manifest path and content type
    • @@ -69,7 +73,7 @@ *

    Invalidation behavior: *

      *
    • catalog invalidation clears all entries and drops Iceberg {@link ManifestFiles} IO cache
      • - *
    • db/table invalidation clears table/view/schema entries, while keeping manifest entries
      • + *
    • db/table invalidation clears table/snapshot/view/schema entries, while keeping manifest entries
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ @@ -78,26 +82,40 @@ public class IcebergExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "iceberg"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_VIEW = "view"; public static final String ENTRY_MANIFEST = "manifest"; public static final String ENTRY_SCHEMA = "schema"; private static final long DEFAULT_MANIFEST_CACHE_CAPACITY = 100_000L; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle viewEntry; private final EntryHandle manifestEntry; private final EntryHandle schemaEntry; public IcebergExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withSizeEstimator(this::prepareTableForCachePublication)); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); viewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_VIEW, NameMapping.class, View.class, this::loadView, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); manifestEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_MANIFEST, IcebergManifestEntryKey.class, ManifestCacheValue.class, - CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY))); + CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimator.estimateSafely( + "iceberg_manifest_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateManifestEntry(key, value)))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSchemaCacheKey::getNameMapping))); @@ -108,13 +126,95 @@ public Table getIcebergTable(ExternalTable dorisTable) { return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getIcebergTable(); } + public Table getWritableIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + IcebergTableCacheValue tableValue = + tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + + " when loading a writable Iceberg table"); + } + IcebergMetadataOps ops = resolveMetadataOps(catalog); + Table liveTable = executeAuthenticated(catalog, () -> ops.loadTable( + nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + try { + return tableValue.getWritableIcebergTable(liveTable); + } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { + MetaCacheEntry entry = + tableEntry.get(nameMapping.getCtlId()); + entry.invalidateKeyIfSame(nameMapping, tableValue); + IcebergTableCacheValue refreshedValue = entry.get(nameMapping); + Table refreshedLiveTable = executeAuthenticated(catalog, () -> ops.loadTable( + nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + return refreshedValue.getWritableIcebergTable(refreshedLiveTable); + } + } + + Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + MetaCacheEntry entry = + tableEntry.get(nameMapping.getCtlId()); + IcebergTableCacheValue tableValue = + entry.get(nameMapping); + try { + return createQueryTable(nameMapping, tableValue); + } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { + entry.invalidateKeyIfSame(nameMapping, tableValue); + return createQueryTable(nameMapping, entry.get(nameMapping)); + } + } + + private Table createQueryTable( + NameMapping nameMapping, IcebergTableCacheValue tableValue) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || snapshotEntry.get(nameMapping.getCtlId()).isWeightBounded(); + if (!isolateForQueries) { + return tableValue.getIcebergTable(); + } + Table queryTable = tableValue.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(queryTable); + return queryTable; + } + public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + IcebergTableCacheValue tableValue = + tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + Table retainedTable = tableValue.getRetainedIcebergTable(); + java.util.Optional optionalKey = + IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!optionalKey.isPresent()) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared(); + return executeAuthenticated(nameMapping.getCtlId(), + () -> loadSnapshotProjection( + dorisTable, + isolateForQueries ? tableValue.newQueryScopedTable() + : tableValue.getIcebergTable(), + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)); + } + IcebergSnapshotEntryKey key = optionalKey.get(); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || entry.isWeightBounded(); + return entry.get(key, ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + Table projectionTable = isolateForQueries + ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); + IcebergSnapshotCacheValue value = loadSnapshotProjection( + dorisTable, projectionTable, + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + if (entry.isWeightBounded()) { + value.prepareForCachePublication(key); + } + return value; + })); } public List getSnapshotList(ExternalTable dorisTable) { - Table icebergTable = getIcebergTable(dorisTable); + Table icebergTable = getQueryScopedIcebergTable(dorisTable); List snapshots = com.google.common.collect.Lists.newArrayList(); com.google.common.collect.Iterables.addAll(snapshots, icebergTable.snapshots()); return snapshots; @@ -139,11 +239,12 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, MetaCacheEntry manifestEntry = this.manifestEntry.get(nameMapping.getCtlId()); IcebergManifestEntryKey key = IcebergManifestEntryKey.of(manifest); - boolean hit = manifestEntry.getIfPresent(key) != null; + boolean hit = manifestEntry.peekIfPresent(key) != null; if (cacheHitRecorder != null) { cacheHitRecorder.accept(hit); } - return manifestEntry.get(key, ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); + return manifestEntry.get(key, + ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); } @Override @@ -159,25 +260,33 @@ public void invalidateCatalogEntries(long catalogId) { } private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { throw new RuntimeException(String.format("Cannot find catalog %d when loading table %s/%s.", nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); } IcebergMetadataOps ops = resolveMetadataOps(catalog); - try { - Table table = ((ExternalCatalog) catalog).getExecutionAuthenticator() - .execute(() -> ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - ExternalTable dorisTable = findExternalTable(nameMapping, ENGINE); - return new IcebergTableCacheValue(table, () -> loadSnapshotProjection(dorisTable, table)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } + return executeAuthenticated(catalog, () -> { + Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + IcebergTableCacheValue value = new IcebergTableCacheValue( + table, ((ExternalCatalog) catalog).getExecutionAuthenticator()); + MetaCacheEntry currentEntry = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + if (currentEntry != null && currentEntry.isWeightBounded()) { + prepareTableForCachePublication(nameMapping, value); + } + return value; + }); + } + + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + return value.prepareForCachePublication(nameMapping); } private View loadView(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (!(catalog instanceof IcebergExternalCatalog)) { return null; } @@ -199,10 +308,9 @@ private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFil } try { if (content == ManifestContent.DELETES) { - return ManifestCacheValue.forDeleteFiles( - loadDeleteFiles(manifest, icebergTable)); + return loadDeleteFiles(manifest, icebergTable); } - return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, icebergTable)); + return loadDataFiles(manifest, icebergTable); } catch (IOException e) { throw new CacheException("Failed to read manifest %s", e, manifest.path()); } @@ -216,27 +324,32 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } - private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table icebergTable) { + private IcebergSnapshotCacheValue loadSnapshotProjection( + ExternalTable dorisTable, Table projectionTable, Table retainedTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); } try { - // Freeze before deriving snapshot, partitions, and aliases; BaseTable accessors share - // refreshable operations and otherwise could mix two concurrent metadata generations. - Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; - IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(retainedTable); + IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(projectionTable); IcebergPartitionInfo icebergPartitionInfo; if (!table.isValidRelatedTable()) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { - icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, retainedTable, + icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue( - icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(retainedTable), - retainedTable); + Optional>> nameMapping = + IcebergUtils.getNameMapping(projectionTable); + return isolateForQueries + ? new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable, retainedCurrentSnapshotJson) + : new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } @@ -251,32 +364,56 @@ private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) { throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); } + private T executeAuthenticated(long catalogId, Callable task) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + catalogId + " when loading Iceberg metadata."); + } + return executeAuthenticated(catalog, task); + } + + private T executeAuthenticated(CatalogIf catalog, Callable task) { + if (!(catalog instanceof ExternalCatalog)) { + throw new RuntimeException("Iceberg metadata cache requires an external catalog"); + } + try { + return ((ExternalCatalog) catalog).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.iceberg.table.enable", "meta.cache.iceberg.snapshot.enable"); + compatibility.put("meta.cache.iceberg.table.ttl-second", "meta.cache.iceberg.snapshot.ttl-second"); + compatibility.put("meta.cache.iceberg.table.capacity", "meta.cache.iceberg.snapshot.capacity"); + return compatibility; } - private List loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) throws IOException { - List dataFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); try (ManifestReader reader = ManifestFiles.read(manifest, table.io())) { for (org.apache.iceberg.DataFile dataFile : reader) { - dataFiles.add(dataFile.copy()); + builder.addDataFile(dataFile.copy()); } } - return dataFiles; + return builder.build(); } - private List loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) throws IOException { - List deleteFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(); try (ManifestReader reader = ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { for (org.apache.iceberg.DeleteFile deleteFile : reader) { - deleteFiles.add(deleteFile.copy()); + builder.addDeleteFile(deleteFile.copy()); } } - return deleteFiles; + return builder.build(); } private void dropManifestFileIoCacheForCatalog(long catalogId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 5d59440a5a62b1..9c9ee6d53b6416 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -148,6 +148,10 @@ public Table getIcebergTable() { return IcebergUtils.getIcebergTable(this); } + public Table getWritableIcebergTable() { + return IcebergUtils.getWritableIcebergTable(this); + } + @Override public String getComment() { return properties().getOrDefault(TABLE_COMMENT_PROP, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index bcbe03de6e6d01..d996f80754a37f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -483,7 +483,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); BranchOptions branchOptions = branchInfo.getBranchOptions(); Long snapshotId = branchOptions.getSnapshotId() @@ -571,7 +571,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); TagOptions tagOptions = tagInfo.getTagOptions(); Long snapshotId = tagOptions.getSnapshotId() .orElse( @@ -623,7 +623,7 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(tagName); if (snapshotRef != null || !ifExists) { @@ -644,7 +644,7 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(branchName); if (snapshotRef != null || !ifExists) { @@ -747,7 +747,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -778,7 +778,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); if (!parentPath.getType().isStructType()) { @@ -808,7 +808,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); for (Column column : columns) { validateAddColumnMetadata(column, true); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); @@ -831,7 +831,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -851,7 +851,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -868,7 +868,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, oldName, "rename"); validateRowLineageColumnMutation(icebergTable, newName, "rename to"); Schema schema = icebergTable.schema(); @@ -893,7 +893,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), @@ -955,7 +955,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() .caseInsensitiveField(columnPath.getTopLevelName()); @@ -1024,7 +1024,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCollectionPseudoFieldComment( @@ -1075,7 +1075,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); if (!columnPath.isNested()) { validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); } @@ -1642,7 +1642,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String columnName : newOrder) { @@ -1709,7 +1709,7 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); String transformName = clause.getTransformName(); @@ -1738,7 +1738,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); if (clause.getPartitionFieldName() != null) { @@ -1765,7 +1765,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); // remove old partition field diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java index cccc6244a0d0cc..96ed0a0dcc1d8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import java.util.List; public class IcebergPartition { @@ -29,10 +31,19 @@ public class IcebergPartition { private final long lastUpdateTime; private final long lastSnapshotId; private final List transforms; + private final long retainedPayloadBytes; public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, long lastUpdateTime, long lastSnapshotId, List partitionValues, List transforms) { + this(partitionName, specId, recordCount, fileSizeInBytes, fileCount, lastUpdateTime, + lastSnapshotId, partitionValues, transforms, + estimateRetainedPayloadBytes(partitionName, partitionValues, transforms)); + } + + public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, + long lastUpdateTime, long lastSnapshotId, List partitionValues, + List transforms, long retainedPayloadBytes) { this.partitionName = partitionName; this.specId = specId; this.recordCount = recordCount; @@ -42,6 +53,7 @@ public IcebergPartition(String partitionName, int specId, long recordCount, long this.lastSnapshotId = lastSnapshotId; this.partitionValues = partitionValues; this.transforms = transforms; + this.retainedPayloadBytes = retainedPayloadBytes; } public String getPartitionName() { @@ -79,4 +91,26 @@ public List getPartitionValues() { public List getTransforms() { return transforms; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long estimateRetainedPayloadBytes( + String partitionName, List partitionValues, List transforms) { + long bytes = MetaCacheWeightUtils.estimatedStringBytes(partitionName); + bytes = addStrings(bytes, partitionValues); + return addStrings(bytes, transforms); + } + + private static long addStrings(long bytes, List values) { + if (values == null) { + return bytes; + } + for (String value : values) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(value)); + } + return bytes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index de36f0855ddd14..5c43cc56f8bbf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; -import com.google.common.collect.Maps; - +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -28,21 +28,32 @@ public class IcebergPartitionInfo { private final Map nameToPartitionItem; private final Map nameToIcebergPartition; private final Map> nameToIcebergPartitionNames; + private final long retainedPayloadBytes; private static final IcebergPartitionInfo EMPTY = new IcebergPartitionInfo(); private IcebergPartitionInfo() { - this.nameToPartitionItem = Maps.newHashMap(); - this.nameToIcebergPartition = Maps.newHashMap(); - this.nameToIcebergPartitionNames = Maps.newHashMap(); + this.nameToPartitionItem = Collections.emptyMap(); + this.nameToIcebergPartition = Collections.emptyMap(); + this.nameToIcebergPartitionNames = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public IcebergPartitionInfo(Map nameToPartitionItem, Map nameToIcebergPartition, Map> nameToIcebergPartitionNames) { + this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, + retainedPayloadBytes(nameToIcebergPartition)); + } + + public IcebergPartitionInfo(Map nameToPartitionItem, + Map nameToIcebergPartition, + Map> nameToIcebergPartitionNames, + long retainedPayloadBytes) { this.nameToPartitionItem = nameToPartitionItem; this.nameToIcebergPartition = nameToIcebergPartition; this.nameToIcebergPartitionNames = nameToIcebergPartitionNames; + this.retainedPayloadBytes = retainedPayloadBytes; } static IcebergPartitionInfo empty() { @@ -57,6 +68,28 @@ public Map getNameToIcebergPartition() { return nameToIcebergPartition; } + Map> getNameToIcebergPartitionNames() { + return nameToIcebergPartitionNames; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (IcebergPartition partition : partitions.values()) { + if (partition != null) { + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, partition.getRetainedPayloadBytes()); + } + } + return bytes; + } + public long getLatestSnapshotId(String partitionName) { Set icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName); if (icebergPartitionNames == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 30cf64fcfc6bfa..406d4ee11ab003 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,17 +17,33 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -40,36 +56,65 @@ public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; private final Optional>> nameMapping; - private final Optional icebergTable; + private Optional
    icebergTable; + private final long retainedNameMappingPayloadBytes; + private String retainedCurrentSnapshotJson; + private boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { - this(partitionInfo, snapshot, Optional.empty(), Optional.empty()); + this(partitionInfo, snapshot, Optional.empty(), Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping) { - this(partitionInfo, snapshot, nameMapping, Optional.empty()); + this(partitionInfo, snapshot, nameMapping, Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping, Table icebergTable) { - this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable)); + this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable), null, false); + } + + IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping, Table retainedTable, + String retainedCurrentSnapshotJson) { + this(partitionInfo, snapshot, nameMapping, Optional.of(retainedTable), + retainedCurrentSnapshotJson, true); } private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, - Optional>> nameMapping, Optional
    icebergTable) { + Optional>> nameMapping, Optional
    icebergTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; // A cached BaseTable shares live TableOperations; retain a metadata-only generation so a // later commit through that same Table cannot move an already bound statement forward. this.icebergTable = icebergTable.map(IcebergSnapshotCacheValue::retainTableGeneration); - this.nameMapping = nameMapping.map(mapping -> { + this.retainedCurrentSnapshotJson = retainedCurrentSnapshotJson; + if (isolateForQueries) { + this.icebergTable = this.icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + this.queryIsolationPrepared = true; + } + if (nameMapping.isPresent()) { Map> copy = new HashMap<>(); - // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. - mapping.forEach((id, names) -> copy.put(id, - Collections.unmodifiableList(new ArrayList<>(names)))); - return Collections.unmodifiableMap(copy); - }); + long payloadBytes = 0L; + for (Map.Entry> entry : nameMapping.get().entrySet()) { + List names = ImmutableList.copyOf(entry.getValue()); + copy.put(entry.getKey(), names); + for (String name : names) { + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(name)); + } + } + this.nameMapping = Optional.of(Collections.unmodifiableMap(copy)); + this.retainedNameMappingPayloadBytes = payloadBytes; + } else { + this.nameMapping = Optional.empty(); + this.retainedNameMappingPayloadBytes = 0L; + } } public IcebergPartitionInfo getPartitionInfo() { @@ -85,26 +130,153 @@ public Optional>> getNameMapping() { } public Optional
    getIcebergTable() { + return queryIsolationPrepared + ? icebergTable.map(table -> createQueryScopedTable( + table, retainedCurrentSnapshotJson)) + : icebergTable; + } + + MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { + if (sizeEstimate == null) { + if (retainedCurrentSnapshotJson == null) { + retainedCurrentSnapshotJson = icebergTable + .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); + } + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this)); + if (sizeEstimate.isComplete()) { + icebergTable = icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + long getRetainedNameMappingPayloadBytes() { + return retainedNameMappingPayloadBytes; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return retainedSnapshotJsonBytes(retainedCurrentSnapshotJson); + } + + Optional
    getRetainedIcebergTable() { return icebergTable; } static Table retainTableGeneration(Table table) { + return retainTableGeneration(table, null); + } + + static Table retainTableGeneration(Table table, ExecutionAuthenticator authenticator) { if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { return table; } TableOperations operations = ((HasTableOperations) table).operations(); // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. - TableOperations frozenOperations = new FrozenTableOperations(operations, operations.current()); + TableOperations frozenOperations = new FrozenTableOperations( + operations, operations.current(), authenticator); return tableWithOperations(table, frozenOperations); } + static Table retainNonGrowingGeneration(Table table) { + if (!isFrozenGeneration(table)) { + return table; + } + TableOperations retainedOperations = ((HasTableOperations) table).operations(); + TableMetadata source = retainedOperations.current(); + if (source.schemas().isEmpty() || source.specs().isEmpty() + || source.sortOrders().isEmpty()) { + return table; + } + TableMetadata.Builder builder = TableMetadata.buildFromEmpty(source.formatVersion()); + if (source.uuid() != null) { + builder.assignUUID(source.uuid()); + } + for (Schema schema : source.schemas()) { + builder.addSchema(schema); + } + builder.setCurrentSchema(source.currentSchemaId()); + for (PartitionSpec spec : source.specs()) { + builder.addPartitionSpec(spec); + } + builder.setDefaultPartitionSpec(source.defaultSpecId()); + for (SortOrder sortOrder : source.sortOrders()) { + builder.addSortOrder(sortOrder); + } + builder.setDefaultSortOrder(source.defaultSortOrderId()); + builder.setLocation(source.location()); + builder.setProperties(source.properties()); + if (source.currentSnapshot() != null) { + builder.setBranchSnapshot( + new NonGrowingSnapshot(source.currentSnapshot()), SnapshotRef.MAIN_BRANCH); + } + TableMetadata retainedMetadata = builder.discardChanges() + .withMetadataLocation(source.metadataFileLocation()).build(); + return tableWithOperations(table, new FrozenTableOperations( + retainedOperations, retainedMetadata)); + } + + static String retainCurrentSnapshotJson(Table table) { + if (!(table instanceof HasTableOperations)) { + return null; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + Snapshot snapshot = metadata == null ? null : metadata.currentSnapshot(); + return snapshot == null ? null : SnapshotParser.toJson(snapshot, false); + } + + static long retainedSnapshotJsonBytes(String snapshotJson) { + return MetaCacheWeightUtils.estimatedStringBytes(snapshotJson); + } + + static Table createQueryScopedTable(Table retainedTable, String currentSnapshotJson) { + if (!isFrozenGeneration(retainedTable)) { + return retainedTable; + } + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + if (retainedTable instanceof BaseTable) { + return new QueryScopedTable(retainedOperations, retainedTable.name(), + ((BaseTable) retainedTable).reporter(), currentSnapshotJson); + } + return new QueryScopedTable(retainedOperations, retainedTable.name(), null, + currentSnapshotJson); + } + + static void loadQueryMetadataForStatement(Table table) { + if (table instanceof QueryScopedTable) { + ((QueryScopedTable) table).queryMetadata(); + } + } + static boolean isFrozenGeneration(Table table) { return table instanceof HasTableOperations && ((HasTableOperations) table).operations() instanceof FrozenTableOperations; } - static Table createWritableTable(Table retainedTable, Table liveTable) { + static TableOperations unwrapRetainedTableOperations(TableOperations operations) { + TableOperations current = Objects.requireNonNull(operations, "operations can not be null"); + while (current instanceof RetainedTableOperations) { + current = ((RetainedTableOperations) current).delegate; + } + return current; + } + + static Table createWritableTable( + Table retainedTable, Table liveTable, boolean reloadRetainedMetadata) { if (!isFrozenGeneration(retainedTable)) { return retainedTable; } @@ -113,12 +285,46 @@ static Table createWritableTable(Table retainedTable, Table liveTable) { throw new IllegalArgumentException( "Iceberg commit table must provide writable table operations"); } - TableMetadata retainedMetadata = ((HasTableOperations) retainedTable).operations().current(); - TableOperations liveOperations = ((HasTableOperations) liveTable).operations(); + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + TableMetadata retainedMetadata = reloadRetainedMetadata + ? loadQueryMetadata(retainedOperations) : retainedOperations.current(); + TableOperations liveOperations = unwrapRetainedTableOperations( + ((HasTableOperations) liveTable).operations()); return tableWithOperations(retainedTable, new WritableTableOperations(liveOperations, retainedMetadata)); } + static Table createWritableTable(Table retainedTable, Table liveTable) { + return createWritableTable(retainedTable, liveTable, + isNonGrowingGeneration(retainedTable)); + } + + private static boolean isNonGrowingGeneration(Table table) { + return isFrozenGeneration(table) + && ((FrozenTableOperations) ((HasTableOperations) table).operations()).nonGrowing; + } + + private static TableMetadata loadQueryMetadata(TableOperations retainedOperations) { + TableMetadata retainedMetadata = retainedOperations.current(); + TableOperations serviceOperations = unwrapRetainedTableOperations(retainedOperations); + String metadataLocation = retainedMetadata.metadataFileLocation(); + if (metadataLocation != null && !metadataLocation.isEmpty() && serviceOperations.io() != null) { + ExecutionAuthenticator authenticator = retainedOperations instanceof FrozenTableOperations + ? ((FrozenTableOperations) retainedOperations).authenticator : null; + try { + return authenticator == null + ? TableMetadataParser.read(serviceOperations.io(), metadataLocation) + : authenticator.execute( + () -> TableMetadataParser.read(serviceOperations.io(), metadataLocation)); + } catch (Exception e) { + throw new StaleMetadataException( + "Iceberg metadata generation is no longer readable: " + metadataLocation, e); + } + } + throw new IllegalStateException( + "Iceberg query-local metadata requires a stable metadata location and FileIO"); + } + private static Table tableWithOperations(Table table, TableOperations operations) { if (table instanceof BaseTable) { return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); @@ -166,15 +372,75 @@ public LocationProvider locationProvider() { } } - private static class FrozenTableOperations extends RetainedTableOperations { - private FrozenTableOperations(TableOperations delegate, TableMetadata metadata) { - super(delegate, metadata); + private static class FrozenTableOperations implements TableOperations { + private final TableMetadata metadata; + private final FileIO fileIO; + private final EncryptionManager encryptionManager; + private final LocationProvider locationProvider; + private final ExecutionAuthenticator authenticator; + private final boolean nonGrowing; + + private FrozenTableOperations(TableOperations source, TableMetadata metadata) { + this(source, metadata, source instanceof FrozenTableOperations + ? ((FrozenTableOperations) source).authenticator : null, true); + } + + private FrozenTableOperations(TableOperations source, TableMetadata metadata, + ExecutionAuthenticator authenticator) { + this(source, metadata, authenticator, false); + } + + private FrozenTableOperations(TableOperations source, TableMetadata metadata, + ExecutionAuthenticator authenticator, boolean nonGrowing) { + this.metadata = metadata; + this.fileIO = source.io(); + this.encryptionManager = source.encryption(); + this.locationProvider = source.locationProvider(); + this.authenticator = authenticator; + this.nonGrowing = nonGrowing; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; } @Override public void commit(TableMetadata base, TableMetadata newMetadata) { throw new UnsupportedOperationException("Frozen Iceberg table generation is read-only"); } + + @Override + public FileIO io() { + return fileIO; + } + + @Override + public EncryptionManager encryption() { + return encryptionManager; + } + + @Override + public String metadataFileLocation(String fileName) { + String metadataLocation = metadata.metadataFileLocation(); + if (metadataLocation == null) { + throw new UnsupportedOperationException( + "Frozen Iceberg table has no metadata directory"); + } + int separator = metadataLocation.lastIndexOf('/'); + return separator < 0 ? fileName + : metadataLocation.substring(0, separator + 1) + fileName; + } + + @Override + public LocationProvider locationProvider() { + return locationProvider; + } } private static class WritableTableOperations extends RetainedTableOperations { @@ -208,8 +474,9 @@ public TableMetadata refresh() { @Override public void commit(TableMetadata base, TableMetadata newMetadata) { - delegate.commit(base, newMetadata); - currentMetadata = newMetadata; + TableMetadata delegateBase = prepareDelegateCommit(delegate, base, currentMetadata); + delegate.commit(delegateBase, newMetadata); + currentMetadata = delegate.current(); } private boolean isWriterCompatible(TableMetadata refreshedMetadata) { @@ -221,4 +488,221 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { && Objects.equals(retainedMetadata.properties(), refreshedMetadata.properties()); } } + + /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ + private static final class QueryScopedTable extends BaseTable { + private final TableOperations retainedOperations; + private final Snapshot currentSnapshot; + private TableMetadata queryMetadata; + + private QueryScopedTable(TableOperations retainedOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + super(retainedOperations, name, reporter == null + ? org.apache.iceberg.metrics.LoggingMetricsReporter.instance() : reporter); + this.retainedOperations = retainedOperations; + this.currentSnapshot = currentSnapshotJson == null + ? null : SnapshotParser.fromJson(currentSnapshotJson); + } + + @Override + public Snapshot currentSnapshot() { + return currentSnapshot; + } + + @Override + public Snapshot snapshot(long snapshotId) { + if (currentSnapshot != null && currentSnapshot.snapshotId() == snapshotId) { + return currentSnapshot; + } + return queryMetadata().snapshot(snapshotId); + } + + @Override + public Iterable snapshots() { + return queryMetadata().snapshots(); + } + + @Override + public List history() { + return queryMetadata().snapshotLog(); + } + + @Override + public Map refs() { + return queryMetadata().refs(); + } + + @Override + public List statisticsFiles() { + return queryMetadata().statisticsFiles(); + } + + @Override + public List partitionStatisticsFiles() { + return queryMetadata().partitionStatisticsFiles(); + } + + private synchronized TableMetadata queryMetadata() { + if (queryMetadata == null) { + queryMetadata = loadQueryMetadata(retainedOperations); + } + return queryMetadata; + } + } + + static final class StaleMetadataException extends RuntimeException { + private StaleMetadataException(String message, Throwable cause) { + super(message, cause); + } + } + + /** Scalar-only snapshot retained by the cache-owned metadata generation. */ + private static final class NonGrowingSnapshot implements Snapshot { + private final long sequenceNumber; + private final long snapshotId; + private final Long parentId; + private final long timestampMillis; + private final Integer schemaId; + private final Long firstRowId; + private final Long addedRows; + + private NonGrowingSnapshot(Snapshot snapshot) { + this.sequenceNumber = snapshot.sequenceNumber(); + this.snapshotId = snapshot.snapshotId(); + this.parentId = snapshot.parentId(); + this.timestampMillis = snapshot.timestampMillis(); + this.schemaId = snapshot.schemaId(); + this.firstRowId = snapshot.firstRowId(); + this.addedRows = snapshot.addedRows(); + } + + @Override + public long sequenceNumber() { + return sequenceNumber; + } + + @Override + public long snapshotId() { + return snapshotId; + } + + @Override + public Long parentId() { + return parentId; + } + + @Override + public long timestampMillis() { + return timestampMillis; + } + + @Override + public List allManifests(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public List dataManifests(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public List deleteManifests(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public String operation() { + return null; + } + + @Override + public Map summary() { + return Collections.emptyMap(); + } + + @Override + public Iterable addedDataFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public Iterable removedDataFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public Iterable addedDeleteFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public Iterable removedDeleteFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public String manifestListLocation() { + return null; + } + + @Override + public Integer schemaId() { + return schemaId; + } + + @Override + public Long firstRowId() { + return firstRowId; + } + + @Override + public Long addedRows() { + return addedRows; + } + + @Override + public String keyId() { + return null; + } + + private UnsupportedOperationException queryScopedSnapshotRequired() { + return new UnsupportedOperationException( + "Cache-owned Iceberg snapshots cannot materialize manifests or files"); + } + } + + private static TableMetadata prepareDelegateCommit(TableOperations delegate, + TableMetadata base, TableMetadata wrapperCurrent) { + if (base != wrapperCurrent) { + throw new CommitFailedException("Cannot commit from a stale Iceberg table view"); + } + TableMetadata delegateCurrent = delegate.current(); + if (!isSameGeneration(base, delegateCurrent)) { + throw new CommitFailedException("Cannot commit from a stale Iceberg metadata generation"); + } + return delegateCurrent; + } + + private static boolean isSameGeneration(TableMetadata retained, TableMetadata live) { + if (retained == live) { + return true; + } + if (retained == null || live == null) { + return false; + } + if (!Objects.equals(retained.uuid(), live.uuid())) { + return false; + } + if (retained.metadataFileLocation() != null || live.metadataFileLocation() != null) { + return Objects.equals(retained.metadataFileLocation(), live.metadataFileLocation()); + } + return retained.lastUpdatedMillis() == live.lastUpdatedMillis() + && retained.lastSequenceNumber() == live.lastSequenceNumber() + && retained.currentSchemaId() == live.currentSchemaId() + && retained.defaultSpecId() == live.defaultSpecId() + && retained.defaultSortOrderId() == live.defaultSortOrderId() + && Objects.equals(retained.location(), live.location()) + && Objects.equals(retained.properties(), live.properties()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java new file mode 100644 index 00000000000000..782a51d5f135db --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; + +import java.util.Objects; +import java.util.Optional; + +/** Stable identity for an Iceberg snapshot projection built from one frozen metadata generation. */ +public final class IcebergSnapshotEntryKey { + private final NameMapping nameMapping; + private final String tableUuid; + private final String metadataFileLocation; + private final long snapshotId; + private final int schemaId; + private final int defaultSpecId; + + private IcebergSnapshotEntryKey(NameMapping nameMapping, String tableUuid, String metadataFileLocation, + long snapshotId, int schemaId, int defaultSpecId) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.tableUuid = Objects.requireNonNull(tableUuid, "tableUuid can not be null"); + this.metadataFileLocation = Objects.requireNonNull( + metadataFileLocation, "metadataFileLocation can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.defaultSpecId = defaultSpecId; + } + + /** + * Build a key from the same retained table generation that will be used by the value loader. + * Tables without a stable metadata location intentionally bypass the snapshot cache. + */ + public static Optional tryCreate(NameMapping nameMapping, Table retainedTable) { + if (!(retainedTable instanceof HasTableOperations)) { + return Optional.empty(); + } + TableMetadata metadata = ((HasTableOperations) retainedTable).operations().current(); + if (metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + || metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return Optional.empty(); + } + Snapshot snapshot = metadata.currentSnapshot(); + long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); + return Optional.of(new IcebergSnapshotEntryKey(nameMapping, metadata.uuid(), metadata.metadataFileLocation(), + snapshotId, metadata.currentSchemaId(), metadata.defaultSpecId())); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public String getMetadataFileLocation() { + return metadataFileLocation; + } + + public String getTableUuid() { + return tableUuid; + } + + public long getSnapshotId() { + return snapshotId; + } + + public int getSchemaId() { + return schemaId; + } + + public int getDefaultSpecId() { + return defaultSpecId; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof IcebergSnapshotEntryKey)) { + return false; + } + IcebergSnapshotEntryKey that = (IcebergSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && defaultSpecId == that.defaultSpecId + && nameMapping.equals(that.nameMapping) + && tableUuid.equals(that.tableUuid) + && metadataFileLocation.equals(that.metadataFileLocation); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, tableUuid, metadataFileLocation, snapshotId, schemaId, defaultSpecId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index cdef77346ade27..6403e6bfdbafa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -17,25 +17,90 @@ package org.apache.doris.datasource.iceberg; -import com.google.common.base.Suppliers; -import org.apache.iceberg.Table; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; -import java.util.function.Supplier; +import org.apache.iceberg.Table; public class IcebergTableCacheValue { - private final Table icebergTable; - private final Supplier latestSnapshotCacheValue; + private Table icebergTable; + private String retainedCurrentSnapshotJson; + private volatile boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; - public IcebergTableCacheValue(Table icebergTable, Supplier latestSnapshotCacheValue) { - this.icebergTable = icebergTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + public IcebergTableCacheValue(Table icebergTable) { + this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); + } + + IcebergTableCacheValue(Table icebergTable, ExecutionAuthenticator authenticator) { + this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration( + icebergTable, authenticator); } public Table getIcebergTable() { + return queryIsolationPrepared + ? IcebergSnapshotCacheValue.createQueryScopedTable( + icebergTable, retainedCurrentSnapshotJson) + : icebergTable; + } + + public Table getWritableIcebergTable(Table liveTable) { + return IcebergSnapshotCacheValue.createWritableTable( + icebergTable, liveTable, queryIsolationPrepared); + } + + MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + if (sizeEstimate == null) { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + retainedTablePayloadBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateTableEntry(key, this)); + if (sizeEstimate.isComplete()) { + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + Table getRetainedIcebergTable() { return icebergTable; } - public IcebergSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + synchronized Table newQueryScopedTable() { + if (!queryIsolationPrepared) { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + return IcebergSnapshotCacheValue.createQueryScopedTable( + icebergTable, retainedCurrentSnapshotJson); + } + + String getRetainedCurrentSnapshotJson() { + return retainedCurrentSnapshotJson; + } + + boolean isQueryIsolationPrepared() { + return queryIsolationPrepared; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return IcebergSnapshotCacheValue.retainedSnapshotJsonBytes( + retainedCurrentSnapshotJson); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 71935cbd88157b..138e4fc4b1289c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -299,7 +299,7 @@ private Table createTransactionTable(ExternalTable dorisTable, Table retainedTab // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, // while writer-contract changes still invalidate files produced for the retained metadata. return IcebergSnapshotCacheValue.createWritableTable( - retainedTable, IcebergUtils.getIcebergTable(dorisTable)); + retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable)); } /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 0b375c70d6791e..1c370222d71a83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -56,6 +56,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.metastore.HMSBaseProperties; @@ -1057,6 +1058,10 @@ public static Table getIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTable(dorisTable); } + public static Table getWritableIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); + } + private static IcebergExternalMetaCache icebergExternalMetaCache(ExternalCatalog catalog) { Preconditions.checkNotNull(catalog, "catalog can not be null"); return Env.getCurrentEnv().getExtMetaCacheMgr().iceberg(catalog.getId()); @@ -1751,10 +1756,13 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T } Map nameToPartition = Maps.newHashMap(); Map nameToPartitionItem = Maps.newHashMap(); + long retainedPayloadBytes = 0L; List partitionColumns = IcebergUtils.getSchemaCacheValue(dorisTable, schemaId).getPartitionColumns(); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); @@ -1762,7 +1770,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); - return new IcebergPartitionInfo(nameToPartitionItem, nameToPartition, partitionNameMap); + return new IcebergPartitionInfo( + nameToPartitionItem, nameToPartition, partitionNameMap, retainedPayloadBytes); } private static List loadIcebergPartition(Table table, long snapshotId) { @@ -1802,6 +1811,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike StringBuilder sb = new StringBuilder(); List partitionValues = Lists.newArrayList(); List transforms = Lists.newArrayList(); + long retainedPayloadBytes = 0L; for (int i = 0; i < partitionSpec.fields().size(); ++i) { PartitionField partitionField = partitionSpec.fields().get(i); Class fieldClass = partitionSpec.javaClasses()[i]; @@ -1817,12 +1827,19 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike sb.append(fieldValue); sb.append("/"); partitionValues.add(fieldValue); - transforms.add(partitionField.transform().toString()); + String transform = partitionField.transform().toString(); + transforms.add(transform); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(fieldValue)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(transform)); } if (sb.length() > 0) { sb.delete(sb.length() - 1, sb.length()); } String partitionName = sb.toString(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(partitionName)); long recordCount = row.get(2, Long.class); long fileCount = row.get(3, Integer.class); long fileSizeInBytes = row.get(4, Long.class); @@ -1841,7 +1858,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike lastUpdateSnapShotId = UNKNOWN_SNAPSHOT_ID; } return new IcebergPartition(partitionName, specId, recordCount, fileSizeInBytes, fileCount, - lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms); + lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms, retainedPayloadBytes); } @VisibleForTesting @@ -2000,7 +2017,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( Optional scanParams) { if (tableSnapshot.isPresent() || IcebergUtils.isIcebergBranchOrTag(scanParams)) { // If a snapshot is specified, use the specified snapshot and the corresponding schema (not latest). - Table icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(getIcebergTable(dorisTable)); + IcebergExternalMetaCache metaCache = icebergExternalMetaCache(dorisTable); + Table icebergTable = metaCache.getQueryScopedIcebergTable(dorisTable); IcebergTableQueryInfo info; try { info = getQuerySpecSnapshot(icebergTable, tableSnapshot, scanParams); @@ -2010,8 +2028,7 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), - getNameMapping(icebergTable), - icebergTable); + getNameMapping(icebergTable), icebergTable); } return getLatestSnapshotCacheValue(dorisTable); } @@ -2027,11 +2044,12 @@ public static List getIcebergSchema(ExternalTable dorisTable, Optional getIcebergPartitionColumns(Optional snapshot, ExternalTable dorisTable) { IcebergSnapshotCacheValue snapshotValue = getSnapshotCacheValue(snapshot, dorisTable); - if (snapshotValue.getIcebergTable().isPresent()) { + Optional
    snapshotTable = snapshotValue.getIcebergTable(); + if (snapshotTable.isPresent()) { // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep // the same schema and snapshot IDs while changing spec(), so derive both from T0. return buildTableSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), - snapshotValue.getIcebergTable().get()).getPartitionColumns(); + snapshotTable.get()).getPartitionColumns(); } return getSchemaCacheValue(dorisTable, snapshotValue).getPartitionColumns(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java index e303f0e9111486..94924514a58a48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long sourceSnapshotId = namedArguments.getLong(SNAPSHOT_ID); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java index 0937af8ba4cac4..82a93022354067 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java @@ -149,7 +149,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); // Parse parameters String olderThan = namedArguments.getString(OLDER_THAN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java index a5560db65520e4..cd746a7dbe6959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String sourceBranch = namedArguments.getString(BRANCH); String desBranch = namedArguments.getString(TO); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java index e1bf8cbdad4472..bf3f116d1cba81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java @@ -66,7 +66,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String targetWapId = namedArguments.getString(WAP_ID); // Find the target WAP snapshot diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java index 430e9fe9d5e22d..dce45c2729693b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { try { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot current = icebergTable.currentSnapshot(); if (current == null) { // No current snapshot means the table is empty, no manifests to rewrite diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java index 8d6b3842a9dc80..a5609f83439d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long targetSnapshotId = namedArguments.getLong(SNAPSHOT_ID); Snapshot targetSnapshot = icebergTable.snapshot(targetSnapshotId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java index 6957c563512657..de7e2a680791c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java @@ -96,7 +96,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String timestampStr = namedArguments.getString(TIMESTAMP); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java index 44df40f8f492b9..5b2c5bd220eb7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java @@ -87,7 +87,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot previousSnapshot = icebergTable.currentSnapshot(); Long previousSnapshotId = previousSnapshot != null ? previousSnapshot.snapshotId() : null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index e98ca6b2fb2808..47f8c8cde3e7df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -17,30 +17,62 @@ package org.apache.doris.datasource.iceberg.cache; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.StructLike; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Cached manifest payload containing parsed files. */ public class ManifestCacheValue { + private static final long AUXILIARY_LIST_ENTRY_BYTES = 32L; + private final List dataFiles; private final List deleteFiles; + private final long dataFileMetricEntryCount; + private final long deleteFileMetricEntryCount; + private final long retainedPayloadBytes; - private ManifestCacheValue(List dataFiles, List deleteFiles) { - this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; - this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + private ManifestCacheValue(List dataFiles, List deleteFiles, + long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes) { + this.dataFiles = ImmutableList.copyOf(dataFiles); + this.deleteFiles = ImmutableList.copyOf(deleteFiles); + this.dataFileMetricEntryCount = dataFileMetricEntryCount; + this.deleteFileMetricEntryCount = deleteFileMetricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; } public static ManifestCacheValue forDataFiles(List dataFiles) { - return new ManifestCacheValue(dataFiles, Collections.emptyList()); + Builder builder = dataFilesBuilder(); + if (dataFiles != null) { + dataFiles.forEach(builder::addDataFile); + } + return builder.build(); } public static ManifestCacheValue forDeleteFiles(List deleteFiles) { - return new ManifestCacheValue(Collections.emptyList(), deleteFiles); + Builder builder = deleteFilesBuilder(); + if (deleteFiles != null) { + deleteFiles.forEach(builder::addDeleteFile); + } + return builder.build(); + } + + public static Builder dataFilesBuilder() { + return new Builder(true); + } + + public static Builder deleteFilesBuilder() { + return new Builder(false); } public List getDataFiles() { @@ -50,4 +82,134 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + public long getDataFileMetricEntryCount() { + return dataFileMetricEntryCount; + } + + public long getDeleteFileMetricEntryCount() { + return deleteFileMetricEntryCount; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + /** Accumulates retained-size counters in the manifest reader's existing file loop. */ + public static final class Builder { + private final boolean dataContent; + private final List dataFiles = new ArrayList<>(); + private final List deleteFiles = new ArrayList<>(); + private long metricEntryCount; + private long retainedPayloadBytes; + + private Builder(boolean dataContent) { + this.dataContent = dataContent; + } + + public void addDataFile(DataFile file) { + if (!dataContent) { + throw new IllegalStateException("delete manifest builder cannot accept a data file"); + } + dataFiles.add(file); + account(file); + } + + public void addDeleteFile(DeleteFile file) { + if (dataContent) { + throw new IllegalStateException("data manifest builder cannot accept a delete file"); + } + deleteFiles.add(file); + account(file); + } + + public ManifestCacheValue build() { + return new ManifestCacheValue(dataFiles, deleteFiles, + dataContent ? metricEntryCount : 0L, + dataContent ? 0L : metricEntryCount, + retainedPayloadBytes); + } + + private void account(ContentFile file) { + metricEntryCount = MetaCacheWeightUtils.saturatedAdd( + metricEntryCount, metricEntryCount(file)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, retainedPayloadBytes(file)); + } + } + + private static long metricEntryCount(ContentFile file) { + long count = mapSize(file.columnSizes()); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.valueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nullValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nanValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.lowerBounds())); + return MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.upperBounds())); + } + + private static long retainedPayloadBytes(ContentFile file) { + long bytes = MetaCacheWeightUtils.estimatedCharSequenceBytes(file.path()); + bytes = addBuffer(bytes, file.keyMetadata()); + bytes = addBuffers(bytes, file.lowerBounds()); + bytes = addBuffers(bytes, file.upperBounds()); + bytes = addListEntries(bytes, file.splitOffsets()); + bytes = addListEntries(bytes, file.equalityFieldIds()); + if (file instanceof DeleteFile) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes( + ((DeleteFile) file).referencedDataFile())); + } + return addPartitionPayload(bytes, file.partition()); + } + + private static long addBuffers(long bytes, Map buffers) { + if (buffers == null) { + return bytes; + } + for (ByteBuffer buffer : buffers.values()) { + bytes = addBuffer(bytes, buffer); + } + return bytes; + } + + private static long addBuffer(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + + private static long addListEntries(long bytes, List values) { + if (values == null) { + return bytes; + } + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + values.size(), AUXILIARY_LIST_ENTRY_BYTES)); + } + + private static long addPartitionPayload(long bytes, StructLike partition) { + if (partition == null) { + return bytes; + } + try { + for (int index = 0; index < partition.size(); index++) { + Object value = partition.get(index, Object.class); + if (value instanceof CharSequence) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + bytes = addBuffer(bytes, (ByteBuffer) value); + } else if (value instanceof byte[]) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + } + } + } catch (RuntimeException ignored) { + // A third-party StructLike may reject Object.class. The fixed per-file allowance + // remains conservative, and cache accounting must never fail manifest loading. + } + return bytes; + } + + private static int mapSize(Map map) { + return map == null ? 0 : map.size(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 957ab6ed55e193..570652f245ff71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -781,8 +781,9 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); - if (cacheValue.getIcebergTable().isPresent()) { - Table frozenBaseTable = cacheValue.getIcebergTable().get(); + Optional
    frozenTable = cacheValue.getIcebergTable(); + if (frozenTable.isPresent()) { + Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); if (systemTable.supportsSnapshotSelection()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java index 46e58f1e380081..da2980f7185ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -52,7 +53,12 @@ public class MaxComputeExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public MaxComputeExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public MaxComputeExternalMetaCache(ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionValuesEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_PARTITION_VALUES, NameMapping.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index a3a44151e45e2f..4cf326b9110b8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; import java.util.function.Function; import java.util.function.Predicate; @@ -62,12 +63,19 @@ protected static CacheSpec defaultSchemaCacheSpec() { private final String engine; private final ExecutorService refreshExecutor; + private final ExternalMetaCacheBudgetManager budgetManager; private final Map catalogEntries = Maps.newConcurrentMap(); private final Map> metaCacheEntryDefs = Maps.newConcurrentMap(); protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor) { + this(engine, refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.empty())); + } + + protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { this.engine = engine; this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); } @Override @@ -81,10 +89,43 @@ public Collection aliases() { } @Override - public void initCatalog(long catalogId, Map catalogProperties) { + public void validateCatalogProperties(Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, id -> buildCatalogEntryGroup(safeCatalogProperties)); + validateMappedCatalogProperties(safeCatalogProperties, true); + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + synchronized (this) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + validateMappedCatalogProperties(safeCatalogProperties, false); + catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); + } + } + + private void validateMappedCatalogProperties( + Map catalogProperties, boolean validateAgainstLocalGlobalLimit) { + CacheSpec.validateEngineProperties(catalogProperties, engine, metaCacheEntryDefs); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + metaCacheEntryDefs.values().stream() + .filter(entryDef -> entryDef.getSizeEstimator() != null) + .map(entryDef -> CacheSpec.fromProperties( + catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec())) + .forEach(cacheSpec -> { + if (validateAgainstLocalGlobalLimit) { + budgetManager.validateHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } else { + budgetManager.validateCatalogEntryHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } + }); } @Override @@ -114,6 +155,7 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class MetaCacheEntryDef def = requireMetaCacheEntryDef(entryName); ensureTypeCompatible(def, keyType, valueType); + beforeCatalogEntryLookupForTest(catalogId, entryName); MetaCacheEntry cacheEntry = group.get(entryName); if (cacheEntry == null) { throw new IllegalStateException(String.format( @@ -124,10 +166,10 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class } @Override - public void invalidateCatalog(long catalogId) { + public synchronized void invalidateCatalog(long catalogId) { CatalogEntryGroup removed = catalogEntries.remove(catalogId); if (removed != null) { - removed.invalidateAll(); + removed.close(); } } @@ -162,8 +204,8 @@ public Map stats(long catalogId) { } @Override - public void close() { - catalogEntries.values().forEach(CatalogEntryGroup::invalidateAll); + public synchronized void close() { + catalogEntries.values().forEach(CatalogEntryGroup::close); catalogEntries.clear(); } @@ -191,6 +233,10 @@ protected final MetaCacheEntry entry(long catalogId, MetaCacheEntry return entry(catalogId, entryDef.getName(), entryDef.getKeyType(), entryDef.getValueType()); } + // Let tests pause after capturing a group and before looking up its entry. + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + } + protected final String metaCacheTtlKey(String entryName) { return "meta.cache." + engine + "." + entryName + ".ttl-second"; } @@ -283,23 +329,52 @@ private void invalidateEntryIfMatched(CatalogEntryGroup group, MetaCacheE } } - private CatalogEntryGroup buildCatalogEntryGroup(Map catalogProperties) { + private CatalogEntryGroup buildCatalogEntryGroup(long catalogId, Map catalogProperties) { CatalogEntryGroup group = new CatalogEntryGroup(); - metaCacheEntryDefs.values() - .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(entryDef, catalogProperties))); - return group; + try { + metaCacheEntryDefs.values().forEach(entryDef -> group.put( + entryDef.getName(), newMetaCacheEntry(catalogId, entryDef, catalogProperties))); + return group; + } catch (RuntimeException | Error e) { + group.close(); + throw e; + } } @SuppressWarnings("unchecked") private MetaCacheEntry newMetaCacheEntry( - MetaCacheEntryDef rawEntryDef, Map catalogProperties) { + long catalogId, MetaCacheEntryDef rawEntryDef, Map catalogProperties) { MetaCacheEntryDef entryDef = (MetaCacheEntryDef) rawEntryDef; CacheSpec cacheSpec = CacheSpec.fromProperties( catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); - return new MetaCacheEntry<>(entryDef.getName(), - wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), - cacheSpec, - refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly()); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + if (cacheSpec.isWeightBounded() && entryDef.getSizeEstimator() == null) { + throw new IllegalArgumentException(String.format( + "Entry '%s' for engine '%s' configures max-weight but has no estimator.", + entryDef.getName(), engine)); + } + boolean enableWeight = entryDef.getSizeEstimator() != null + && (cacheSpec.isWeightBounded() + || catalogMaxWeight.isPresent() + || budgetManager.getGlobalMaxWeight().isPresent()); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = null; + if (enableWeight) { + entryBudget = budgetManager.createEntryBudget( + catalogId, engine, entryDef.getName(), catalogMaxWeight, cacheSpec.getMaxWeight()); + cacheSpec = cacheSpec.withMaxWeight(entryBudget.getEffectiveMaxWeight()); + } + try { + return new MetaCacheEntry<>(entryDef.getName(), + wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), + cacheSpec, + refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), + entryDef.getSizeEstimator(), entryBudget); + } catch (RuntimeException | Error e) { + if (entryBudget != null) { + entryBudget.close(); + } + throw e; + } } private Function wrapSchemaValidator(Function loader, Class valueType) { @@ -327,8 +402,12 @@ public MetaCacheEntry get(long catalogId) { return entry(catalogId, entryDef); } + @SuppressWarnings("unchecked") public MetaCacheEntry getIfInitialized(long catalogId) { - return isCatalogInitialized(catalogId) ? get(catalogId) : null; + // Read the group once. A concurrent invalidation may close that captured entry, which + // is safe; looking the group up a second time could instead throw after the first check. + CatalogEntryGroup group = catalogEntries.get(catalogId); + return group == null ? null : (MetaCacheEntry) group.get(entryDef.getName()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 0bb640ad0d753c..1cea5d20a3a591 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -21,10 +21,16 @@ import org.apache.commons.lang3.math.NumberUtils; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -43,19 +49,36 @@ public final class CacheSpec { private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_VOLUME_PATTERN = Pattern.compile("^([0-9]+)\\s*(B|KB|MB|GB|TB|PB)?$", + Pattern.CASE_INSENSITIVE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + if (maxWeight < 0) { + throw new IllegalArgumentException("maxWeight can not be negative: " + maxWeight); + } + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); + } + + public CacheSpec withMaxWeight(long effectiveMaxWeight) { + return ofWeight(enable, ttlSecond, capacity, effectiveMaxWeight); } public static PropertySpec.Builder propertySpecBuilder() { @@ -77,7 +100,8 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getWeightProperty(properties, propertySpec.getMaxWeightKey()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** @@ -95,6 +119,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT) .build(); } @@ -151,6 +176,68 @@ public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capaci return enable && ttlSecond != 0 && capacity != 0; } + /** + * Parse an exact byte value with an optional binary unit. Percentages are accepted only + * when {@code allowPercent} is true and are resolved against {@code maxHeapBytes}. + */ + public static long parseWeight(String value, String key, boolean allowPercent, long maxHeapBytes) { + String normalized = Objects.requireNonNull(value, "value").trim(); + if (normalized.isEmpty()) { + throw invalidWeight(key, value); + } + if (normalized.endsWith("%")) { + if (!allowPercent || maxHeapBytes <= 0) { + throw invalidWeight(key, value); + } + String percentageText = normalized.substring(0, normalized.length() - 1).trim(); + try { + BigDecimal percentage = new BigDecimal(percentageText); + if (percentage.signum() < 0 || percentage.compareTo(BigDecimal.valueOf(100L)) > 0) { + throw invalidWeight(key, value); + } + BigInteger bytes = BigDecimal.valueOf(maxHeapBytes) + .multiply(percentage) + .divide(BigDecimal.valueOf(100L)) + .toBigInteger(); + return checkedLong(bytes, key, value); + } catch (NumberFormatException e) { + throw invalidWeight(key, value); + } + } + + Matcher matcher = DATA_VOLUME_PATTERN.matcher(normalized); + if (!matcher.matches()) { + throw invalidWeight(key, value); + } + BigInteger amount = new BigInteger(matcher.group(1)); + String rawUnit = matcher.group(2); + String unit = rawUnit == null ? "B" : rawUnit.toUpperCase(Locale.ROOT); + int power; + switch (unit) { + case "B": + power = 0; + break; + case "KB": + power = 1; + break; + case "MB": + power = 2; + break; + case "GB": + power = 3; + break; + case "TB": + power = 4; + break; + case "PB": + power = 5; + break; + default: + throw invalidWeight(key, value); + } + return checkedLong(amount.multiply(BigInteger.valueOf(1024L).pow(power)), key, value); + } + /** * Build standard external meta cache key prefix for one engine. * Example: {@code meta.cache.iceberg.} @@ -166,6 +253,85 @@ public static boolean isMetaCacheKeyForEngine(String key, String engine) { return key != null && engine != null && key.startsWith(metaCacheKeyPrefix(engine)); } + /** + * Strictly validate one engine namespace so misspelled entries/options cannot be silently ignored. + * The catalog-wide {@code meta.cache.max-weight} key is validated by the budget manager. + */ + static void validateEngineProperties(Map properties, String engine, + Map> entryDefs) { + Set weightedEntries = new java.util.HashSet<>(); + for (MetaCacheEntryDef entryDef : entryDefs.values()) { + if (entryDef.getSizeEstimator() != null) { + weightedEntries.add(entryDef.getName()); + } + } + validateEngineProperties(properties, engine, entryDefs.keySet(), weightedEntries); + } + + public static void validateEngineProperties(Map properties, String engine, + Set entryNames, Set weightedEntryNames) { + if (properties == null || properties.isEmpty()) { + return; + } + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + String remainder = key.substring(enginePrefix.length()); + int optionSeparator = remainder.lastIndexOf('.'); + if (optionSeparator <= 0 || optionSeparator == remainder.length() - 1) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String entryName = remainder.substring(0, optionSeparator); + String option = remainder.substring(optionSeparator + 1); + if (!entryNames.contains(entryName)) { + throw new IllegalArgumentException("Unknown external meta cache entry property: " + key); + } + String value = property.getValue(); + switch (option) { + case "enable": + requireStrictBoolean(key, value); + break; + case "ttl-second": + requireLongAtLeast(key, value, CACHE_NO_TTL); + break; + case "capacity": + requireLongAtLeast(key, value, 0L); + break; + case "max-weight": + if (!weightedEntryNames.contains(entryName)) { + throw new IllegalArgumentException( + "External meta cache entry does not support max-weight: " + key); + } + parseWeight(value, key, false, 0L); + break; + default: + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + } + } + + private static void requireStrictBoolean(String key, String value) { + if (value == null || (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value))) { + throw new IllegalArgumentException("Invalid boolean cache property '" + key + "': " + value); + } + } + + private static void requireLongAtLeast(String key, String value, long minimum) { + final long parsed; + try { + parsed = Long.parseLong(value); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid integer cache property '" + key + "': " + value, e); + } + if (parsed < minimum) { + throw new IllegalArgumentException("Cache property '" + key + "' must be >= " + minimum + + ", but was " + value); + } + } + /** * Convert ttlSecond to OptionalLong for CacheFactory. * ttlSecond=-1 means no expiration; ttlSecond=0 disables cache. @@ -193,6 +359,27 @@ private static long getLongProperty(Map properties, String key, return NumberUtils.toLong(value, defaultValue); } + private static OptionalLong getWeightProperty(Map properties, String key) { + if (key == null) { + return OptionalLong.empty(); + } + String value = properties.get(key); + return value == null + ? OptionalLong.empty() + : OptionalLong.of(parseWeight(value, key, false, 0L)); + } + + private static long checkedLong(BigInteger value, String key, String rawValue) { + if (value.signum() < 0 || value.compareTo(LONG_MAX) > 0) { + throw invalidWeight(key, rawValue); + } + return value.longValue(); + } + + private static IllegalArgumentException invalidWeight(String key, String value) { + return new IllegalArgumentException("Invalid cache weight for '" + key + "': " + value); + } + public boolean isEnable() { return enable; } @@ -205,6 +392,19 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return isCacheEnabled(enable, ttlSecond, capacity) + && (!maxWeight.isPresent() || maxWeight.getAsLong() != 0L); + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -212,15 +412,17 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, String maxWeightKey) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; } public String getEnableKey() { @@ -247,6 +449,10 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -254,6 +460,7 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -273,6 +480,11 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key) { + this.maxWeightKey = Objects.requireNonNull(key, "key"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -280,7 +492,8 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java index c195087f415bfc..09376a1a129051 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java @@ -46,4 +46,12 @@ public Map stats() { public void invalidateAll() { entries.values().forEach(MetaCacheEntry::invalidateAll); } + + public void close() { + entries.values().forEach(MetaCacheEntry::close); + // Keep the closed entries reachable from this retired group. A query may have captured the + // group immediately before its catalog is removed; returning a closed entry lets that query + // serve an uncached load instead of spuriously observing an uninitialized entry. The group + // is already absent from the owner map and is reclaimed with the last concurrent reader. + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 1a067726ec9136..8b874fac8f659e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -41,6 +41,10 @@ public interface ExternalMetaCache { */ Collection aliases(); + /** Validate cache properties in this engine's canonical namespace. */ + default void validateCatalogProperties(Map catalogProperties) { + } + /** * Initialize all registered entries for one catalog under current engine. * Entry instances are created eagerly at this stage. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java new file mode 100644 index 00000000000000..a41e53bfb35b95 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -0,0 +1,414 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicLong; + +/** + * FE-wide admission accounting for managed external metadata caches. + * + *

    All changes are serialized by one short critical section. Cache loads and + * estimators run outside it, so the lock only protects a few arithmetic and map + * operations while making global/catalog/entry reservation atomic. + */ +public final class ExternalMetaCacheBudgetManager { + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; + + private final Object lock = new Object(); + private final OptionalLong globalMaxWeight; + private final Map catalogBuckets = new HashMap<>(); + private final Map entryBuckets = new HashMap<>(); + private long globalUsedWeight; + private final AtomicLong globalRejectedCount = new AtomicLong(); + + public ExternalMetaCacheBudgetManager(OptionalLong globalMaxWeight) { + this.globalMaxWeight = Objects.requireNonNull(globalMaxWeight, "globalMaxWeight"); + if (globalMaxWeight.isPresent() && globalMaxWeight.getAsLong() <= 0) { + throw new IllegalArgumentException("global max weight must be positive when enabled"); + } + } + + public static ExternalMetaCacheBudgetManager fromConfig() { + String configured = Config.external_meta_cache_max_weight; + long parsed = CacheSpec.parseWeight( + configured, + "external_meta_cache_max_weight", + true, + Runtime.getRuntime().maxMemory()); + if (configured.trim().endsWith("%") && parsed == 0L) { + throw new IllegalArgumentException( + "external_meta_cache_max_weight percentage must be greater than 0%"); + } + return new ExternalMetaCacheBudgetManager(parsed == 0L ? OptionalLong.empty() : OptionalLong.of(parsed)); + } + + public OptionalLong parseCatalogMaxWeight(Map catalogProperties) { + String configured = catalogProperties.get(CATALOG_MAX_WEIGHT_PROPERTY); + if (configured == null) { + return OptionalLong.empty(); + } + long parsed = CacheSpec.parseWeight(configured, CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } + + /** Validate a catalog limit at DDL time against this FE's configured global bound. */ + public OptionalLong validateCatalogMaxWeight(Map catalogProperties) { + OptionalLong catalogMaxWeight = parseCatalogMaxWeight(catalogProperties); + validateHierarchy(catalogMaxWeight, OptionalLong.empty()); + return catalogMaxWeight; + } + + /** + * Create the budget handle used by one physical per-catalog cache entry. + */ + public EntryBudget createEntryBudget(long catalogId, String engine, String entryName, + OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + Objects.requireNonNull(engine, "engine"); + Objects.requireNonNull(entryName, "entryName"); + Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight"); + Objects.requireNonNull(entryMaxWeight, "entryMaxWeight"); + validateCatalogEntryHierarchy(catalogMaxWeight, entryMaxWeight); + + OptionalLong effectiveMax = minimumPresent(globalMaxWeight, catalogMaxWeight, entryMaxWeight); + if (!effectiveMax.isPresent()) { + throw new IllegalArgumentException("entry budget requires at least one configured weight bound"); + } + + EntryScope scope = new EntryScope(catalogId, engine, entryName); + synchronized (lock) { + Bucket catalogBucket = catalogBuckets.get(catalogId); + long catalogLimit = minimumLimit(globalMaxWeight, catalogMaxWeight); + if (catalogBucket == null) { + catalogBucket = new Bucket(catalogLimit); + catalogBuckets.put(catalogId, catalogBucket); + } else if (catalogBucket.maxWeight != catalogLimit) { + throw new IllegalStateException("Conflicting catalog cache max weight for catalog " + catalogId); + } + + if (entryBuckets.containsKey(scope)) { + throw new IllegalStateException("Duplicated external meta cache budget: " + scope); + } + Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + entryBuckets.put(scope, entryBucket); + return new EntryBudget(this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); + } + } + + public OptionalLong getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalUsedWeight() { + synchronized (lock) { + return globalUsedWeight; + } + } + + public long getGlobalRejectedCount() { + return globalRejectedCount.get(); + } + + public void validateHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + if (globalMaxWeight.isPresent() && catalogMaxWeight.isPresent() + && catalogMaxWeight.getAsLong() > globalMaxWeight.getAsLong()) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " can not exceed FE global max weight"); + } + OptionalLong parent = catalogMaxWeight.isPresent() ? catalogMaxWeight : globalMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + /** + * Validate persisted catalog-to-entry hierarchy without comparing it with this FE's local + * global bound. Catalog properties are validated on the master, while the global percentage + * is resolved independently from each FE's heap. Runtime admission therefore clamps to the + * local global limit instead of rejecting a catalog accepted on a larger master. + */ + public void validateCatalogEntryHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + OptionalLong parent = catalogMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + private Optional tryReserve(EntryBudget entryBudget, long bytes) { + checkWeight(bytes); + synchronized (lock) { + if (entryBudget.closed) { + return Optional.empty(); + } + if (!fits(limitOf(globalMaxWeight), globalUsedWeight, bytes) + || !fits(entryBudget.catalogBucket.maxWeight, entryBudget.catalogBucket.usedWeight, bytes) + || !fits(entryBudget.entryBucket.maxWeight, entryBudget.entryBucket.usedWeight, bytes)) { + entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return Optional.empty(); + } + addUsed(entryBudget, bytes); + return Optional.of(new AdmissionReservation(this, entryBudget, bytes)); + } + } + + private boolean resize(AdmissionReservation reservation, long newBytes) { + checkWeight(newBytes); + synchronized (lock) { + if (!reservation.active || reservation.entryBudget.closed) { + return false; + } + long delta = newBytes - reservation.bytes; + if (delta > 0 && (!fits(limitOf(globalMaxWeight), globalUsedWeight, delta) + || !fits(reservation.entryBudget.catalogBucket.maxWeight, + reservation.entryBudget.catalogBucket.usedWeight, delta) + || !fits(reservation.entryBudget.entryBucket.maxWeight, + reservation.entryBudget.entryBucket.usedWeight, delta))) { + reservation.entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return false; + } + if (delta >= 0) { + addUsed(reservation.entryBudget, delta); + } else { + subtractUsed(reservation.entryBudget, -delta); + } + reservation.bytes = newBytes; + return true; + } + } + + private void release(AdmissionReservation reservation) { + synchronized (lock) { + if (!reservation.active) { + return; + } + subtractUsed(reservation.entryBudget, reservation.bytes); + reservation.bytes = 0L; + reservation.active = false; + } + } + + private void close(EntryBudget entryBudget) { + synchronized (lock) { + if (entryBudget.closed) { + return; + } + if (entryBudget.entryBucket.usedWeight != 0L) { + throw new IllegalStateException("entry budget closed with active reservations: " + entryBudget.scope); + } + entryBudget.closed = true; + entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); + Bucket catalogBucket = entryBudget.catalogBucket; + boolean catalogStillReferenced = entryBuckets.keySet().stream() + .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); + if (!catalogStillReferenced && catalogBucket.usedWeight == 0L) { + catalogBuckets.remove(entryBudget.scope.catalogId, catalogBucket); + } + } + } + + private void addUsed(EntryBudget entryBudget, long bytes) { + globalUsedWeight += bytes; + entryBudget.catalogBucket.usedWeight += bytes; + entryBudget.entryBucket.usedWeight += bytes; + } + + private void subtractUsed(EntryBudget entryBudget, long bytes) { + if (bytes > globalUsedWeight + || bytes > entryBudget.catalogBucket.usedWeight + || bytes > entryBudget.entryBucket.usedWeight) { + throw new IllegalStateException("external meta cache budget accounting underflow"); + } + globalUsedWeight -= bytes; + entryBudget.catalogBucket.usedWeight -= bytes; + entryBudget.entryBucket.usedWeight -= bytes; + } + + private static boolean fits(long maxWeight, long usedWeight, long delta) { + return delta >= 0 && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; + } + + private static long limitOf(OptionalLong configured) { + return configured.isPresent() ? configured.getAsLong() : Long.MAX_VALUE; + } + + private static long minimumLimit(OptionalLong first, OptionalLong second) { + return Math.min(limitOf(first), limitOf(second)); + } + + private static OptionalLong minimumPresent(OptionalLong first, OptionalLong second, OptionalLong third) { + if (!first.isPresent() && !second.isPresent() && !third.isPresent()) { + return OptionalLong.empty(); + } + long minimum = Math.min(limitOf(first), Math.min(limitOf(second), limitOf(third))); + return OptionalLong.of(minimum); + } + + private static void checkWeight(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache reservation can not be negative: " + bytes); + } + } + + private static final class Bucket { + private final long maxWeight; + private long usedWeight; + + private Bucket(long maxWeight) { + this.maxWeight = maxWeight; + } + } + + private static final class EntryScope { + private final long catalogId; + private final String engine; + private final String entryName; + + private EntryScope(long catalogId, String engine, String entryName) { + this.catalogId = catalogId; + this.engine = engine; + this.entryName = entryName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryScope)) { + return false; + } + EntryScope that = (EntryScope) other; + return catalogId == that.catalogId && engine.equals(that.engine) && entryName.equals(that.entryName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, engine, entryName); + } + + @Override + public String toString() { + return catalogId + "/" + engine + "/" + entryName; + } + } + + public static final class EntryBudget { + private final ExternalMetaCacheBudgetManager manager; + private final EntryScope scope; + private final Bucket catalogBucket; + private final Bucket entryBucket; + private final long effectiveMaxWeight; + private final AtomicLong rejectedCount = new AtomicLong(); + // Guarded by manager.lock. A closed handle must never re-enter accounting. + private boolean closed; + + private EntryBudget(ExternalMetaCacheBudgetManager manager, EntryScope scope, + Bucket catalogBucket, Bucket entryBucket, long effectiveMaxWeight) { + this.manager = manager; + this.scope = scope; + this.catalogBucket = catalogBucket; + this.entryBucket = entryBucket; + this.effectiveMaxWeight = effectiveMaxWeight; + } + + public Optional tryReserve(long bytes) { + return manager.tryReserve(this, bytes); + } + + public long getEffectiveMaxWeight() { + return effectiveMaxWeight; + } + + public long getUsedWeight() { + synchronized (manager.lock) { + return entryBucket.usedWeight; + } + } + + public long getCatalogUsedWeight() { + synchronized (manager.lock) { + return catalogBucket.usedWeight; + } + } + + public long getCatalogMaxWeight() { + return catalogBucket.maxWeight == Long.MAX_VALUE ? -1L : catalogBucket.maxWeight; + } + + public long getRejectedCount() { + return rejectedCount.get(); + } + + public long getGlobalUsedWeight() { + return manager.getGlobalUsedWeight(); + } + + public long getGlobalMaxWeight() { + return manager.globalMaxWeight.isPresent() ? manager.globalMaxWeight.getAsLong() : -1L; + } + + public void close() { + manager.close(this); + } + } + + public static final class AdmissionReservation { + private final ExternalMetaCacheBudgetManager manager; + private final EntryBudget entryBudget; + private long bytes; + private boolean active = true; + + private AdmissionReservation(ExternalMetaCacheBudgetManager manager, EntryBudget entryBudget, long bytes) { + this.manager = manager; + this.entryBudget = entryBudget; + this.bytes = bytes; + } + + public boolean tryResize(long newBytes) { + return manager.resize(this, newBytes); + } + + public void release() { + manager.release(this); + } + + public long getBytes() { + synchronized (manager.lock) { + return bytes; + } + } + + public boolean isActive() { + synchronized (manager.lock) { + return active; + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 30668163539d3b..01998d16f2d3ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -19,14 +19,29 @@ import org.apache.doris.common.CacheFactory; import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.Weigher; import com.github.benmanes.caffeine.cache.stats.CacheStats; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -40,8 +55,22 @@ * key/predicate/full invalidation, and lightweight runtime stats. */ public class MetaCacheEntry { + private static final Logger LOG = LogManager.getLogger(MetaCacheEntry.class); // Use striped locks to deduplicate slow external loads without managing per-key lock lifecycle. private static final int LOAD_LOCK_STRIPES = 128; + private static final int LOCAL_EVICTION_BATCH_SIZE = 16; + private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; + // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced + // generation map per physical entry after callbacks return; cleanup tasks never capture values. + private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-removal-cleanup"); + thread.setDaemon(true); + return thread; + }); + // Conservative retained cost outside the estimator-owned key/value graph: Caffeine's data + // node and policy links plus the reservation ConcurrentHashMap node, record and token. This + // deliberately overestimates common compressed-oops layouts; calibrate downward only with JOL. + static final long FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES = 512L; private final String name; @Nullable @@ -49,15 +78,39 @@ public class MetaCacheEntry { private final CacheSpec cacheSpec; private final boolean effectiveEnabled; private final boolean autoRefresh; + private final ExecutorService refreshExecutor; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final EntryBudget entryBudget; + private final boolean weightBounded; + // Estimator-backed entries use the same generation-fenced refresh protocol even before a + // max-weight is configured. This keeps event ordering stable when weight governance is toggled. + private final boolean generationFencedRefresh; // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. private final LoadingCache loadingData; // Use the plain cache view for manual miss load so slow I/O does not happen in Caffeine's sync load path. private final Cache data; // Protect one key stripe at a time to deduplicate concurrent miss loads with bounded lock count. private final Object[] loadLocks = new Object[LOAD_LOCK_STRIPES]; + // Serialize weighted cache mutation with reservation ownership changes. + private final Object admissionLock = new Object(); + // Ownership records deliberately contain no V reference. A weighted cache's Caffeine soft + // reference must be the only cache-owned path to its value, while generation fencing keeps + // delayed removal callbacks from releasing a replacement reservation. + private final Map reservations = new ConcurrentHashMap<>(); + private final Map refreshRecords = new ConcurrentHashMap<>(); + private final Map pendingRemovalGenerations = new ConcurrentHashMap<>(); + private final AtomicBoolean removalCleanupScheduled = new AtomicBoolean(false); + private final Map refreshesInFlight = new ConcurrentHashMap<>(); + // A state exists only while a miss/refresh for the key is in flight. Mutations advance that + // state's epoch, fencing stale publication without retaining every key ever observed. + private final Map keyMutationStates = new ConcurrentHashMap<>(); private final AtomicLong invalidateCount = new AtomicLong(0); - // Bump generation before invalidation so in-flight manual loads do not repopulate stale values. - private final AtomicLong invalidateGeneration = new AtomicLong(0); + // Full invalidation is the only cross-key fence. Ordinary mutations use the per-key state. + private final AtomicLong fullInvalidationGeneration = new AtomicLong(0); + // Primitive owner id lets queued refresh work fence a reservation without retaining its value. + private final AtomicLong reservationGeneration = new AtomicLong(0); // Track load statistics outside Caffeine because manual miss loads bypass the built-in load counters. private final AtomicLong loadSuccessCount = new AtomicLong(0); private final AtomicLong loadFailureCount = new AtomicLong(0); @@ -65,6 +118,11 @@ public class MetaCacheEntry { private final AtomicLong lastLoadSuccessTimeMs = new AtomicLong(-1L); private final AtomicLong lastLoadFailureTimeMs = new AtomicLong(-1L); private final AtomicReference lastError = new AtomicReference<>(""); + private final AtomicLong weightAdmissionRejectedCount = new AtomicLong(0L); + private final AtomicLong localEvictionCount = new AtomicLong(0L); + private final AtomicLong localEvictionWeight = new AtomicLong(0L); + private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final AtomicBoolean closed = new AtomicBoolean(false); public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { this(name, loader, cacheSpec, refreshExecutor, true, false); @@ -77,6 +135,12 @@ public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, E public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, null, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -91,23 +155,43 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); this.autoRefresh = autoRefresh; - Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); + this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.sizeEstimator = sizeEstimator; + this.entryBudget = entryBudget; + this.weightBounded = this.cacheSpec.isWeightBounded(); + this.generationFencedRefresh = autoRefresh && sizeEstimator != null; + if (weightBounded && (sizeEstimator == null || entryBudget == null)) { + throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); + } + this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); OptionalLong refreshAfterWriteSec = - effectiveEnabled && autoRefresh + effectiveEnabled && autoRefresh && !weightBounded && !generationFencedRefresh ? OptionalLong.of(Config.external_cache_refresh_time_minutes * 60) : OptionalLong.empty(); long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; + Weigher cacheWeigher = weightBounded ? this::weigh : null; CacheFactory cacheFactory = new CacheFactory( expireAfterAccessSec, refreshAfterWriteSec, maxSize, + weightBounded ? OptionalLong.of(effectiveEnabled ? this.cacheSpec.getMaxWeight().getAsLong() : 0L) + : OptionalLong.empty(), + cacheWeigher, true, null); - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + if (weightBounded) { + cacheFactory.withSoftValues(); + } + if (weightBounded || generationFencedRefresh) { + // Direct notification avoids queuing REPLACED values. The listener itself is lock-free + // and delegates only current-owner cleanup, so it is safe under Caffeine's eviction lock. + this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener( + this::loadFromDefaultLoader, this::onRemoval); + } else { + this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + } this.data = loadingData; // Initialize striped locks eagerly to keep the hot path allocation-free. for (int i = 0; i < loadLocks.length; i++) { @@ -120,6 +204,9 @@ public String name() { } public V get(K key) { + if (closed.get()) { + return loadAndTrack(key, this::applyDefaultLoader); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key); } @@ -128,6 +215,9 @@ public V get(K key) { public V get(K key, Function missLoader) { Function loadFunction = Objects.requireNonNull(missLoader, "missLoader can not be null"); + if (closed.get()) { + return loadAndTrack(key, loadFunction); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key, typedKey -> loadAndTrack(typedKey, loadFunction)); } @@ -135,42 +225,232 @@ public V get(K key, Function missLoader) { } public V getIfPresent(K key) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return null; } - return data.getIfPresent(key); + V value = data.getIfPresent(key); + if (value != null) { + maybeRefreshManagedValue(key, value); + } + return value; + } + + /** Return the current value without recording a user-visible cache request. */ + public V peekIfPresent(K key) { + if (!effectiveEnabled || closed.get()) { + return null; + } + return data.asMap().get(key); + } + + /** + * Fence loads and refreshes that started before an event, but only while the expected value is + * still current. Estimator-backed entries retain the known-good value and advance the key's + * mutation epoch. Other count-based entries must invalidate because their legacy Caffeine-managed + * refresh path does not participate in that generation protocol. + */ + public boolean fenceInFlightLoadIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + synchronized (admissionLock) { + if (!effectiveEnabled || closed.get() || data.asMap().get(key) != expectedCurrent) { + return false; + } + advanceKeyMutation(key); + if (!weightBounded && !generationFencedRefresh) { + if (!data.asMap().remove(key, expectedCurrent)) { + return false; + } + invalidateCount.incrementAndGet(); + } + return true; + } } public void put(K key, V value) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return; } - data.put(key, value); + if (weightBounded) { + admitWeightedValue(key, value, null, false, null, -1L, true); + } else { + synchronized (admissionLock) { + if (!closed.get()) { + advanceKeyMutation(key); + putNonWeightedValue(key, value); + } + } + } } - public void invalidateKey(K key) { - invalidateGeneration.incrementAndGet(); - if (data.asMap().remove(key) != null) { + /** Result of an atomic compare-and-replace operation. */ + public enum ReplaceResult { + REPLACED, + NOT_CURRENT, + REJECTED, + DISABLED + } + + /** + * Replace one cached value only when it is still the expected identity. + * + *

    Weighted entries perform the identity check, budget resize and Caffeine write under the + * same admission lock. Callers can therefore distinguish a concurrent update from admission + * rejection and avoid retaining a value they already know is stale. + */ + public ReplaceResult tryReplace(K key, V expectedCurrent, V newValue) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + Objects.requireNonNull(newValue, "newValue can not be null"); + if (!effectiveEnabled || closed.get()) { + return ReplaceResult.DISABLED; + } + if (weightBounded) { + return toReplaceResult(admitWeightedValue( + key, newValue, expectedCurrent, true, null, -1L, true)); + } + synchronized (admissionLock) { + AtomicReference result = new AtomicReference<>(ReplaceResult.NOT_CURRENT); + AtomicReference published = new AtomicReference<>(); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (closed.get()) { + result.set(ReplaceResult.DISABLED); + return current; + } + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + published.set(publishRefreshRecord(key)); + result.set(ReplaceResult.REPLACED); + return newValue; + }); + RefreshRecord record = published.get(); + if (record != null && refreshRecords.get(key) == record + && data.asMap().get(key) == newValue) { + record.published = true; + } + return result.get(); + } + } + + /** Remove a key only if it still maps to the expected value identity. */ + public boolean invalidateKeyIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + if (!weightBounded) { + synchronized (admissionLock) { + AtomicBoolean removed = new AtomicBoolean(false); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + invalidateCount.incrementAndGet(); + refreshRecords.remove(key); + removed.set(true); + return null; + }); + return removed.get(); + } + } + synchronized (admissionLock) { + V current = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (current != expectedCurrent || record == null || !record.published) { + return false; + } + advanceKeyMutation(key); + if (!data.asMap().remove(key, current)) { + return false; + } + releaseReservation(key, record.generation); invalidateCount.incrementAndGet(); + return true; + } + } + + public void invalidateKey(K key) { + synchronized (admissionLock) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + if (removed != null) { + refreshRecords.remove(key); + invalidateCount.incrementAndGet(); + } + } } } public void invalidateIf(Predicate predicate) { - invalidateGeneration.incrementAndGet(); - data.asMap().keySet().removeIf(key -> { - if (predicate.test(key)) { - invalidateCount.incrementAndGet(); - return true; + synchronized (admissionLock) { + Set candidates = new HashSet<>(data.asMap().keySet()); + candidates.addAll(keyMutationStates.keySet()); + if (weightBounded) { + candidates.addAll(reservations.keySet()); + } else if (generationFencedRefresh) { + candidates.addAll(refreshRecords.keySet()); } - return false; - }); + for (K key : candidates) { + if (predicate.test(key)) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + refreshRecords.remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + } + } + } + } } public void invalidateAll() { - invalidateGeneration.incrementAndGet(); - long size = data.estimatedSize(); - data.invalidateAll(); - invalidateCount.addAndGet(size); + synchronized (admissionLock) { + fullInvalidationGeneration.incrementAndGet(); + if (weightBounded) { + long size = data.estimatedSize(); + beforeWeightedInvalidateAllForTest(); + data.invalidateAll(); + reservations.values().forEach(record -> record.reservation.release()); + reservations.clear(); + pendingRemovalGenerations.clear(); + invalidateCount.addAndGet(size); + } else { + long size = data.estimatedSize(); + data.invalidateAll(); + refreshRecords.clear(); + pendingRemovalGenerations.clear(); + invalidateCount.addAndGet(size); + } + } + } + + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + invalidateAll(); + if (entryBudget != null) { + entryBudget.close(); + } } public void forEach(BiConsumer consumer) { @@ -198,62 +478,581 @@ public MetaCacheEntryStats stats() { failureCount, totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - cacheStats.evictionCount(), + saturatedAdd(cacheStats.evictionCount(), localEvictionCount.get()), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), - lastError.get()); + lastError.get(), + weightBounded, + weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, + weightBounded ? entryBudget.getUsedWeight() : -1L, + weightBounded ? saturatedAdd(cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + weightBounded ? weightAdmissionRejectedCount.get() : -1L, + weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, + weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, + weightBounded ? entryBudget.getGlobalMaxWeight() : -1L, + weightBounded ? entryBudget.getGlobalUsedWeight() : -1L, + weightBounded ? lastWeightRejectReason.get() : ""); + } + + public boolean isWeightBounded() { + return weightBounded; + } + + private AdmissionResult admitWeightedValue( + K key, V value, @Nullable V expectedCurrent, boolean requireExpected, + @Nullable KeyMutationToken expectedMutation, long expectedReservationGeneration, + boolean advanceMutationOnAdmission) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + MetaCacheSizeEstimate estimate; + try { + estimate = Objects.requireNonNull(sizeEstimator.estimate(key, value), "size estimate"); + } catch (IllegalArgumentException e) { + throw e; + } catch (RuntimeException e) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + if (!estimate.isComplete()) { + rejectWeight(estimate.getIncompleteReason()); + return AdmissionResult.REJECTED; + } + + long estimatedPayloadBytes = estimate.getBytes(); + // A retained non-null key/value plus Caffeine node can never consume zero bytes. Treat a + // complete zero as an estimator contract violation so an omitted formula cannot bypass + // every quota and admit an unbounded number of zero-weight entries. + if (estimatedPayloadBytes == 0L) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + long newWeight = MetaCacheWeightUtils.saturatedAdd( + estimatedPayloadBytes, FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES); + synchronized (admissionLock) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + if (expectedMutation != null && !isKeyMutationCurrent(key, expectedMutation)) { + return AdmissionResult.NOT_CURRENT; + } + V oldValue = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (expectedReservationGeneration >= 0L + && (record == null || record.generation != expectedReservationGeneration)) { + return AdmissionResult.NOT_CURRENT; + } + if (requireExpected && oldValue != expectedCurrent) { + return AdmissionResult.NOT_CURRENT; + } + if (oldValue == null && record != null) { + reservations.remove(key, record); + record.reservation.release(); + record = null; + } + if (oldValue != null && (record == null || !record.published)) { + rejectWeight("missing_reservation"); + return AdmissionResult.REJECTED; + } + + if (record == null) { + Optional reservation = reserveWithLocalEviction(key, newWeight); + if (!reservation.isPresent()) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, reservation.get(), nextReservationGeneration()); + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + if (reservations.get(key) == newRecord && data.asMap().get(key) == value) { + newRecord.published = true; + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + reservations.remove(key, newRecord); + newRecord.reservation.release(); + throw e; + } + } + + ReservationRecord previousRecord = record; + long reservedWeight = Math.max(previousRecord.weight, newWeight); + if (!resizeWithLocalEviction(key, previousRecord.reservation, reservedWeight)) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, previousRecord.reservation, nextReservationGeneration()); + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + boolean retained = reservations.get(key) == newRecord && data.asMap().get(key) == value; + if (retained) { + newRecord.published = true; + } + if (retained && reservedWeight != newWeight && !newRecord.reservation.tryResize(newWeight)) { + throw new IllegalStateException("failed to release cache replacement reservation delta"); + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + if (reservations.replace(key, newRecord, previousRecord)) { + if (data.asMap().get(key) == null) { + reservations.remove(key, previousRecord); + previousRecord.reservation.release(); + } else if (!previousRecord.reservation.tryResize(previousRecord.weight)) { + throw new IllegalStateException("failed to roll back cache replacement reservation", e); + } + } + throw e; + } + } + } + + private Optional reserveWithLocalEviction(K incomingKey, long bytes) { + if (bytes > entryBudget.getEffectiveMaxWeight()) { + return Optional.empty(); + } + Optional reservation = entryBudget.tryReserve(bytes); + while (!reservation.isPresent()) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + break; + } + reservation = entryBudget.tryReserve(bytes); + } + return reservation; + } + + private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation reservation, long newBytes) { + if (newBytes > entryBudget.getEffectiveMaxWeight()) { + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + while (true) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + } + } + + private int evictLocalColdest(K incomingKey, int limit) { + if (!data.policy().eviction().isPresent()) { + return 0; + } + Map coldest = data.policy().eviction().get().coldest(limit); + int evicted = 0; + for (Map.Entry candidate : coldest.entrySet()) { + if (Objects.equals(candidate.getKey(), incomingKey)) { + continue; + } + V current = data.asMap().get(candidate.getKey()); + ReservationRecord record = reservations.get(candidate.getKey()); + long evictedWeight = record != null && record.published && current != null ? record.weight : 0L; + if (current == candidate.getValue() && data.asMap().remove(candidate.getKey(), current)) { + if (record != null) { + releaseReservation(candidate.getKey(), record.generation); + } + localEvictionCount.incrementAndGet(); + localEvictionWeight.accumulateAndGet(evictedWeight, MetaCacheWeightUtils::saturatedAdd); + evicted++; + } + } + return evicted; + } + + private int weigh(K key, V value) { + ReservationRecord record = reservations.get(key); + // Every supported write path installs the reservation record before calling data.put. + // Missing ownership is an invariant violation, so fail closed without invoking an O(n) + // estimator from Caffeine's hot weigher callback. + long weight = record == null ? Integer.MAX_VALUE : record.weight; + return weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight; + } + + private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { + if (key == null) { + return; + } + if (!weightBounded && !generationFencedRefresh) { + return; + } + if (closed.get()) { + return; + } + // Replacement transfers the existing reservation to the newly published generation. A + // soft-value collection instead reports a null value with COLLECTED and must release it. + if (cause == RemovalCause.REPLACED) { + return; + } + if (Thread.holdsLock(admissionLock)) { + // Other removals have already removed the Caffeine mapping and can release their owner + // inline. A stale callback cannot release a replacement while its mapping is visible. + if (data.asMap().get(key) == null) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null) { + releaseReservation(key, record.generation); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null) { + releaseRefreshRecord(key, record.generation); + } + } + } + return; + } + beforeRemovalOwnerSnapshotForTest(key); + long ownerGeneration = currentOwnerGeneration(key); + if (ownerGeneration >= 0L) { + beforeRemovalReleaseForTest(key); + if (closed.get()) { + return; + } + pendingRemovalGenerations.merge(key, ownerGeneration, Math::max); + if (closed.get()) { + pendingRemovalGenerations.remove(key, ownerGeneration); + return; + } + scheduleRemovalCleanup(); + } + } + + private void scheduleRemovalCleanup() { + if (closed.get()) { + return; + } + if (removalCleanupScheduled.compareAndSet(false, true)) { + try { + REMOVAL_CLEANUP_EXECUTOR.execute(this::drainRemovalCleanups); + } catch (RejectedExecutionException e) { + removalCleanupScheduled.set(false); + LOG.warn("Failed to schedule removal cleanup for external metadata cache entry {}", name, e); + } + } + } + + private void drainRemovalCleanups() { + try { + int processed = 0; + for (Map.Entry cleanup : pendingRemovalGenerations.entrySet()) { + if (processed++ >= REMOVAL_CLEANUP_BATCH_SIZE) { + break; + } + K key = cleanup.getKey(); + long generation = cleanup.getValue(); + // Claim before cleanup. If the same generation is reported again while cleanup + // runs, its notification creates a new pending item instead of being lost when + // this worker finishes. + if (!pendingRemovalGenerations.remove(key, generation)) { + continue; + } + try { + cleanupRemovedReservation(key, generation); + } catch (RuntimeException e) { + // Restore the generation for retry. The finally block requeues one bounded + // drain instead of permanently wedging this entry's scheduled flag. + pendingRemovalGenerations.merge(key, generation, Math::max); + LOG.warn("Failed to clean a removal reservation for external metadata cache entry {}", + name, e); + } + } + } finally { + removalCleanupScheduled.set(false); + if (!closed.get() && !pendingRemovalGenerations.isEmpty()) { + // One bounded task per turn prevents a hot entry from monopolizing the process-wide + // cleanup executor; a later task is queued behind already scheduled catalogs. + scheduleRemovalCleanup(); + } + } + } + + private void cleanupRemovedReservation(K key, long expectedReservationGeneration) { + beforeRemovalCleanupLockForTest(key); + synchronized (admissionLock) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null + && reservations.remove(key, record)) { + record.reservation.release(); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null) { + refreshRecords.remove(key, record); + } + } + } + afterRemovalCleanupForTest(key); + } + + private void releaseReservation(K key, long expectedGeneration) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedGeneration && reservations.remove(key, record)) { + record.reservation.release(); + } + } + + private void releaseRefreshRecord(K key, long expectedGeneration) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedGeneration) { + refreshRecords.remove(key, record); + } + } + + private long currentOwnerGeneration(K key) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + return record == null ? -1L : record.generation; + } + RefreshRecord record = refreshRecords.get(key); + return record == null ? -1L : record.generation; + } + + private void putNonWeightedValue(K key, V value) { + RefreshRecord previous = refreshRecords.get(key); + RefreshRecord next = publishRefreshRecord(key); + try { + beforeNonWeightedCachePutForTest(key, value); + data.put(key, value); + if (next != null && refreshRecords.get(key) == next && data.asMap().get(key) == value) { + next.published = true; + } + } catch (RuntimeException | Error e) { + if (next != null) { + if (previous == null) { + refreshRecords.remove(key, next); + } else { + refreshRecords.replace(key, next, previous); + } + } + throw e; + } + } + + @Nullable + private RefreshRecord publishRefreshRecord(K key) { + if (!generationFencedRefresh) { + return null; + } + RefreshRecord record = new RefreshRecord(nextReservationGeneration()); + refreshRecords.put(key, record); + return record; + } + + private long nextReservationGeneration() { + return reservationGeneration.incrementAndGet(); + } + + private void rejectWeight(String reason) { + weightAdmissionRejectedCount.incrementAndGet(); + lastWeightRejectReason.set(reason == null || reason.isEmpty() ? "unknown" : reason); + } + + private void maybeRefreshManagedValue(K key, V currentValue) { + if (closed.get() || !generationFencedRefresh || loader == null) { + return; + } + if (!weightBounded) { + maybeRefreshNonWeightedValue(key, currentValue); + return; + } + ReservationRecord record = reservations.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void maybeRefreshNonWeightedValue(K key, V currentValue) { + RefreshRecord record = refreshRecords.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void submitNonWeightedRefresh( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed == null) { + return; + } + synchronized (admissionLock) { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + advanceKeyMutation(key); + putNonWeightedValue(key, refreshed); + } + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isRefreshRecordCurrent( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + RefreshRecord record = refreshRecords.get(key); + return record != null && record.generation == expectedRefreshGeneration + && record.published && data.asMap().get(key) != null; + } + + private void submitWeightedRefresh( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isReservationCurrent(key, expectedReservationGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { + AdmissionResult result = admitWeightedValue( + key, refreshed, null, false, expectedMutation, + expectedReservationGeneration, true); + if (result == AdmissionResult.REJECTED) { + invalidateKeyIfReservationGeneration(key, expectedReservationGeneration); + } + } + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isReservationCurrent( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + ReservationRecord record = reservations.get(key); + return record != null && record.generation == expectedReservationGeneration + && record.published && data.asMap().get(key) != null; + } + + private void invalidateKeyIfReservationGeneration(K key, long expectedReservationGeneration) { + synchronized (admissionLock) { + ReservationRecord record = reservations.get(key); + if (record == null || record.generation != expectedReservationGeneration) { + return; + } + V current = data.asMap().get(key); + if (current != null && data.asMap().remove(key, current)) { + advanceKeyMutation(key); + releaseReservation(key, expectedReservationGeneration); + invalidateCount.incrementAndGet(); + } + } } // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { - return Config.enable_external_meta_cache_manual_miss_load; + return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; } // Execute slow miss loads outside Caffeine's sync load path and suppress stale write-back after invalidation. private V getWithManualLoad(K key, Function loadFunction) { - if (!effectiveEnabled) { - // Bypass cache entirely when the entry is disabled so manual miss load does not relax disable semantics. + if (!effectiveEnabled || closed.get()) { + // Disabled and closed entries may still serve the caller, but can not retain the loaded value. return loadAndTrack(key, loadFunction); } V value = data.getIfPresent(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } synchronized (loadLock(key)) { + if (!effectiveEnabled || closed.get()) { + return loadAndTrack(key, loadFunction); + } value = data.asMap().get(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } - long generation = invalidateGeneration.get(); - V loaded = loadAndTrack(key, loadFunction); - if (generation != invalidateGeneration.get()) { - return loaded; - } + KeyMutationToken mutation = beginKeyMutation(key); + try { + V loaded = loadAndTrack(key, loadFunction); + if (!isKeyMutationCurrent(key, mutation)) { + return loaded; + } - // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. - if (loaded == null) { - return null; - } + // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. + if (loaded == null) { + return null; + } - // Leave a narrow hook for tests to pause exactly before the cache put race window. - beforeManualCachePutForTest(key, loaded); - data.put(key, loaded); - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, loaded); + // Leave a narrow hook for tests to pause exactly before the cache put race window. + beforeManualCachePutForTest(key, loaded); + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + if (weightBounded) { + admitWeightedValue(key, loaded, null, false, mutation, -1L, false); + } else { + synchronized (admissionLock) { + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + beforeNonWeightedManualCachePutForTest(key, loaded); + putNonWeightedValue(key, loaded); + } + } + return loaded; + } finally { + endKeyMutation(key, mutation); } - return loaded; } } - // Remove only the value loaded by the current request and keep newer replacements intact. - private void removeLoadedValue(K key, V loaded) { - data.asMap().computeIfPresent(key, (ignored, currentValue) -> currentValue == loaded ? null : currentValue); - } - // Map keys to a fixed lock stripe set to bound memory usage while keeping same-key deduplication. private Object loadLock(K key) { int hash = key == null ? 0 : key.hashCode(); @@ -264,6 +1063,67 @@ private Object loadLock(K key) { void beforeManualCachePutForTest(K key, V loaded) { } + // Let tests pause after the final generation check while holding the admission lock. + void beforeNonWeightedManualCachePutForTest(K key, V loaded) { + } + + // Called inside Caffeine's direct removal callback; tests use it to force the eviction-lock race. + void beforeRemovalReleaseForTest(K key) { + } + + // Let tests pause a callback after Caffeine removal but before reservation-owner lookup. + void beforeRemovalOwnerSnapshotForTest(K key) { + } + + // Called after reservation ownership is published and before Caffeine receives the value. + void beforeWeightedCachePutForTest(K key, V value) { + } + + // Called after refresh ownership is published and before Caffeine receives a count-bounded value. + void beforeNonWeightedCachePutForTest(K key, V value) { + } + + // Called after one asynchronous generation-conditional removal cleanup has finished. + void afterRemovalCleanupForTest(K key) { + } + + // Called immediately before the asynchronous drain tries to acquire admissionLock. + void beforeRemovalCleanupLockForTest(K key) { + } + + // Let tests establish admissionLock -> Caffeine eviction-lock ordering deterministically. + void beforeWeightedInvalidateAllForTest() { + } + + // Invoke the direct-listener branch while holding the mutation lock. + void notifyRemovalUnderAdmissionLockForTest(K key, V value, RemovalCause cause) { + synchronized (admissionLock) { + onRemoval(key, value, cause); + } + } + + // Enqueue the same generation-only refresh task without waiting for the production interval. + void triggerRefreshForTest(K key) { + V current = data.asMap().get(key); + if (current == null || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.published) { + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } else if (generationFencedRefresh) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.published) { + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } + refreshesInFlight.remove(key); + } + private V loadFromDefaultLoader(K key) { return loadAndTrack(key, this::applyDefaultLoader); } @@ -294,4 +1154,104 @@ private V loadAndTrack(K key, Function loadFunction) { throw e; } } + + private KeyMutationToken beginKeyMutation(K key) { + synchronized (admissionLock) { + KeyMutationState state = keyMutationStates.computeIfAbsent(key, ignored -> new KeyMutationState()); + state.inFlight++; + return new KeyMutationToken(state, state.generation, fullInvalidationGeneration.get()); + } + } + + private void advanceKeyMutation(K key) { + // Callers already serialize cache mutation with admissionLock. Keeping the helper lock-free + // prevents accidental deadlock if it is used by a listener reached under that same lock. + KeyMutationState state = keyMutationStates.get(key); + if (state != null) { + state.generation++; + } + } + + private boolean isKeyMutationCurrent(K key, KeyMutationToken token) { + return token.fullInvalidationGeneration == fullInvalidationGeneration.get() + && token.state == keyMutationStates.get(key) + && token.generation == token.state.generation; + } + + private void endKeyMutation(K key, KeyMutationToken token) { + synchronized (admissionLock) { + if (--token.state.inFlight == 0) { + keyMutationStates.remove(key, token.state); + } + } + } + + private static final class ReservationRecord { + private final long weight; + private final long writeNanos; + private final AdmissionReservation reservation; + private final long generation; + private volatile boolean published; + + private ReservationRecord(long weight, AdmissionReservation reservation, long generation) { + this.weight = weight; + this.reservation = reservation; + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class RefreshRecord { + private final long writeNanos; + private final long generation; + private volatile boolean published; + + private RefreshRecord(long generation) { + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class KeyMutationState { + private volatile long generation; + private int inFlight; + } + + private static final class KeyMutationToken { + private final KeyMutationState state; + private final long generation; + private final long fullInvalidationGeneration; + + private KeyMutationToken( + KeyMutationState state, long generation, long fullInvalidationGeneration) { + this.state = state; + this.generation = generation; + this.fullInvalidationGeneration = fullInvalidationGeneration; + } + } + + private static ReplaceResult toReplaceResult(AdmissionResult result) { + switch (result) { + case ADMITTED: + return ReplaceResult.REPLACED; + case NOT_CURRENT: + return ReplaceResult.NOT_CURRENT; + case REJECTED: + return ReplaceResult.REJECTED; + case DISABLED: + default: + return ReplaceResult.DISABLED; + } + } + + private static long saturatedAdd(long left, long right) { + return left > Long.MAX_VALUE - right ? Long.MAX_VALUE : left + right; + } + + private enum AdmissionResult { + ADMITTED, + NOT_CURRENT, + REJECTED, + DISABLED + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 1f48057a44fc40..5963f3d8aa7b86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -101,10 +101,12 @@ public final class MetaCacheEntryDef { private final boolean autoRefresh; private final boolean contextualOnly; private final MetaCacheEntryInvalidation invalidation; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -123,6 +125,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.autoRefresh = autoRefresh; this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); + this.sizeEstimator = sizeEstimator; } /** @@ -142,7 +145,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation); + invalidation, null); } /** @@ -164,7 +167,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation); + invalidation, null); } /** @@ -179,7 +182,14 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation); + invalidation, null); + } + + /** Return a definition with a publication-time size estimator. */ + public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, + Objects.requireNonNull(estimator, "estimator")); } /** @@ -232,4 +242,9 @@ public boolean isContextualOnly() { public MetaCacheEntryInvalidation getInvalidation() { return invalidation; } + + @Nullable + public MetaCacheSizeEstimator getSizeEstimator() { + return sizeEstimator; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 495fd011083bb0..433cc027515873 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -51,6 +51,16 @@ public final class MetaCacheEntryStats { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final boolean weightBounded; + private final long maxWeight; + private final long estimatedWeight; + private final long evictionWeight; + private final long weightAdmissionRejectedCount; + private final long catalogMaxWeight; + private final long catalogEstimatedWeight; + private final long globalMaxWeight; + private final long globalEstimatedWeight; + private final String lastWeightRejectReason; /** * Build an immutable stats snapshot. @@ -74,7 +84,17 @@ public MetaCacheEntryStats( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + boolean weightBounded, + long maxWeight, + long estimatedWeight, + long evictionWeight, + long weightAdmissionRejectedCount, + long catalogMaxWeight, + long catalogEstimatedWeight, + long globalMaxWeight, + long globalEstimatedWeight, + String lastWeightRejectReason) { this.configEnabled = configEnabled; this.effectiveEnabled = effectiveEnabled; this.autoRefresh = autoRefresh; @@ -94,6 +114,16 @@ public MetaCacheEntryStats( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = Objects.requireNonNull(lastError, "lastError"); + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.evictionWeight = evictionWeight; + this.weightAdmissionRejectedCount = weightAdmissionRejectedCount; + this.catalogMaxWeight = catalogMaxWeight; + this.catalogEstimatedWeight = catalogEstimatedWeight; + this.globalMaxWeight = globalMaxWeight; + this.globalEstimatedWeight = globalEstimatedWeight; + this.lastWeightRejectReason = Objects.requireNonNull(lastWeightRejectReason, "lastWeightRejectReason"); } public boolean isConfigEnabled() { @@ -186,4 +216,44 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public boolean isWeightBounded() { + return weightBounded; + } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getEvictionWeight() { + return evictionWeight; + } + + public long getWeightAdmissionRejectedCount() { + return weightAdmissionRejectedCount; + } + + public long getCatalogMaxWeight() { + return catalogMaxWeight; + } + + public long getCatalogEstimatedWeight() { + return catalogEstimatedWeight; + } + + public long getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalEstimatedWeight() { + return globalEstimatedWeight; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java new file mode 100644 index 00000000000000..d44702ce559212 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; + +/** + * Immutable result value returned by {@link MetaCacheSizeEstimator}; this class is not an + * estimator implementation. An incomplete result carries no usable byte count and must fail + * cache admission closed. + */ +public final class MetaCacheSizeEstimate { + private final long bytes; + private final boolean complete; + private final String incompleteReason; + + private MetaCacheSizeEstimate(long bytes, boolean complete, String incompleteReason) { + this.bytes = bytes; + this.complete = complete; + this.incompleteReason = Objects.requireNonNull(incompleteReason, "incompleteReason"); + } + + public static MetaCacheSizeEstimate complete(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache size estimate can not be negative: " + bytes); + } + return new MetaCacheSizeEstimate(bytes, true, ""); + } + + public static MetaCacheSizeEstimate incomplete(String reason) { + String safeReason = Objects.requireNonNull(reason, "reason").trim(); + if (safeReason.isEmpty()) { + throw new IllegalArgumentException("incomplete cache size estimate requires a reason"); + } + return new MetaCacheSizeEstimate(0L, false, safeReason); + } + + public long getBytes() { + return bytes; + } + + public boolean isComplete() { + return complete; + } + + public String getIncompleteReason() { + return incompleteReason; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..f78fd5bc734b39 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Supplies the admission weight of one key/value pair. + * + *

    The callback runs after load and before admission. Implementations must use already available + * shape counters and constant-time collection sizes; they must not walk object graphs, perform IO, + * materialize lazy SDK state, or copy payloads. Caffeine's weigher reads only the admitted + * reservation record, so cache hits and eviction remain O(1). + */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + MetaCacheSizeEstimate estimate(K key, V value); + + /** Convert preparation failures into fail-closed incomplete estimates. */ + static MetaCacheSizeEstimate estimateSafely( + String failureReason, Supplier estimation) { + Objects.requireNonNull(failureReason, "failureReason"); + Objects.requireNonNull(estimation, "estimation"); + try { + return Objects.requireNonNull(estimation.get(), "size estimate"); + } catch (RuntimeException e) { + return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java new file mode 100644 index 00000000000000..7869ac5993f48e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.NameMapping; + +/** Constant-time helpers for conservative external metadata cache weights. */ +public final class MetaCacheWeightUtils { + private static final long STRING_BASE_BYTES = 40L; + private static final long STRING_BYTES_PER_CHARACTER = 2L; + private static final long NAME_MAPPING_BASE_BYTES = 64L; + + private MetaCacheWeightUtils() { + } + + /** + * Estimate a String without inspecting its contents. Two bytes per character deliberately + * avoids depending on CompactStrings or VM-private layout details. + */ + public static long estimatedStringBytes(String value) { + return estimatedCharSequenceBytes(value); + } + + /** Estimate retained character data without materializing a String copy. */ + public static long estimatedCharSequenceBytes(CharSequence value) { + return value == null ? 0L : saturatedAdd( + STRING_BASE_BYTES, saturatedMultiply(value.length(), STRING_BYTES_PER_CHARACTER)); + } + + /** Estimate the fixed set of names retained by a cache key. */ + public static long estimatedNameMappingBytes(NameMapping nameMapping) { + if (nameMapping == null) { + return 0L; + } + long bytes = NAME_MAPPING_BASE_BYTES; + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); + return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); + } + + public static long saturatedAdd(long left, long right) { + if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left < 0L || right < 0L || (left != 0L && right > Long.MAX_VALUE / left)) { + return Long.MAX_VALUE; + } + return left * right; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java new file mode 100644 index 00000000000000..b5a6178538a9c4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.privilege.PrivilegedFileStoreTable; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; + +import java.util.List; +import java.util.Map; + +/** Constant-time retained-weight formula for Paimon snapshot projections. */ +final class PaimonCacheSizeEstimator { + private static final long KEY_BASE_BYTES = 128L; + private static final long SNAPSHOT_BASE_BYTES = 4L * 1024L; + private static final long TABLE_BASE_BYTES = 16L * 1024L; + private static final long TABLE_FIELD_BYTES = 3584L; + private static final long TABLE_OPTION_BYTES = 256L; + private static final long TABLE_KEY_BYTES = 128L; + private static final long NESTED_FIELD_BYTES = 512L; + private static final long PARTITION_BYTES = 1280L; + private static final long PARTITION_ITEM_BYTES = 1024L; + private static final long WRAPPER_BYTES = 512L; + + private PaimonCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + if (!isSupportedTable(table)) { + return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:" + + (table == null ? "null" : table.getClass().getName())); + } + + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getPartitionInfo().getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); + } + + private static boolean isSupportedTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return isSupportedTable(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return isSupportedTable(fallback.wrapped()) && isSupportedTable(fallback.other()); + } + if (!(table instanceof FileStoreTable)) { + return false; + } + String className = table.getClass().getName(); + return "org.apache.paimon.table.AppendOnlyFileStoreTable".equals(className) + || "org.apache.paimon.table.PrimaryKeyFileStoreTable".equals(className); + } + + /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ + private static long estimateTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, + estimateTable(((PrivilegedFileStoreTable) table).wrapped())); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + long bytes = MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, estimateTable(fallback.wrapped())); + return MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(fallback.other())); + } + + FileStoreTable fileStoreTable = (FileStoreTable) table; + TableSchema schema = fileStoreTable.schema(); + long bytes = TABLE_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(fileStoreTable.location().toString())); + bytes = addCount(bytes, schema.fields().size(), TABLE_FIELD_BYTES); + bytes = addCount(bytes, schema.options().size(), TABLE_OPTION_BYTES); + bytes = addCount(bytes, schema.partitionKeys().size(), TABLE_KEY_BYTES); + bytes = addCount(bytes, schema.primaryKeys().size(), TABLE_KEY_BYTES); + return addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); + } + + /** + * Captures skew-sensitive schema text once when the snapshot cache value is constructed. + * All collections are already materialized in TableSchema; this never opens the table store. + */ + static long retainedTablePayloadBytes(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return retainedTablePayloadBytes(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return MetaCacheWeightUtils.saturatedAdd( + retainedTablePayloadBytes(fallback.wrapped()), + retainedTablePayloadBytes(fallback.other())); + } + if (!(table instanceof FileStoreTable)) { + return 0L; + } + + TableSchema schema = ((FileStoreTable) table).schema(); + if (schema == null) { + return 0L; + } + long bytes = addString(0L, schema.comment()); + for (DataField field : schema.fields()) { + bytes = addFieldPayload(bytes, field, false); + } + for (Map.Entry option : schema.options().entrySet()) { + bytes = addString(bytes, option.getKey()); + bytes = addString(bytes, option.getValue()); + } + bytes = addStrings(bytes, schema.partitionKeys()); + bytes = addStrings(bytes, schema.primaryKeys()); + return addStrings(bytes, schema.bucketKeys()); + } + + private static long addStrings(long bytes, List values) { + for (String value : values) { + bytes = addString(bytes, value); + } + return bytes; + } + + private static long addFieldPayload(long bytes, DataField field, boolean nested) { + if (nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + } + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.description()); + bytes = addString(bytes, field.defaultValue()); + return addTypePayload(bytes, field.type()); + } + + private static long addTypePayload(long bytes, DataType type) { + if (type instanceof RowType) { + for (DataField field : ((RowType) type).getFields()) { + bytes = addFieldPayload(bytes, field, true); + } + } else if (type instanceof ArrayType) { + bytes = addTypePayload(bytes, ((ArrayType) type).getElementType()); + } else if (type instanceof MapType) { + bytes = addTypePayload(bytes, ((MapType) type).getKeyType()); + bytes = addTypePayload(bytes, ((MapType) type).getValueType()); + } else if (type instanceof MultisetType) { + bytes = addTypePayload(bytes, ((MultisetType) type).getElementType()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index cde2bbb31efd18..ee5fdeebb2f793 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -23,6 +23,8 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; @@ -40,36 +42,48 @@ *

    Registered entries: *

      *
    • {@code table}: loaded Paimon table handle per table mapping
      • + *
    • {@code snapshot}: immutable partition projection keyed by a captured snapshot/schema fence
      • *
    • {@code schema}: schema cache keyed by table identity + schema id
      • *
    * - *

    Latest snapshot metadata is modeled as a runtime projection memoized inside the table cache - * value instead of as an independent cache entry. + *

    The latest main-branch snapshot is captured once as a fence and loaded through an independent + * contextual entry. Branch/tag/options projections remain statement-local and are not aliased to + * this main-snapshot key. * *

    Invalidation behavior: *

      - *
    • db/table invalidation clears table and schema entries by matching local names
      • + *
    • db/table invalidation clears table, snapshot and schema entries by matching local names
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ public class PaimonExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "paimon"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_SCHEMA = "schema"; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle schemaEntry; private final PaimonTableLoader tableLoader; private final PaimonLatestSnapshotProjectionLoader latestSnapshotProjectionLoader; public PaimonExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableLoader = new PaimonTableLoader(); latestSnapshotProjectionLoader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, PaimonSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSchemaCacheKey::getNameMapping))); @@ -86,7 +100,13 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + PaimonSnapshot fence = tableValue.getLatestSnapshotFence().getSnapshot(); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + nameMapping, fence, tableValue.getGeneration()); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + return entry.get(key, ignored -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); } public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { @@ -95,8 +115,7 @@ public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - Table table = tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getPaimonTable(); - return latestSnapshotProjectionLoader.loadFence(nameMapping, table); + return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotFence(); } public PaimonSnapshotCacheValue loadSnapshotAtFence( @@ -119,8 +138,8 @@ public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { Table paimonTable = tableLoader.load(nameMapping); - return new PaimonTableCacheValue(paimonTable, - () -> latestSnapshotProjectionLoader.load(nameMapping, paimonTable)); + PaimonSnapshotCacheValue fence = latestSnapshotProjectionLoader.loadFence(nameMapping, paimonTable); + return new PaimonTableCacheValue(paimonTable, fence); } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -133,6 +152,11 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.paimon.table.enable", "meta.cache.paimon.snapshot.enable"); + compatibility.put("meta.cache.paimon.table.ttl-second", "meta.cache.paimon.snapshot.ttl-second"); + compatibility.put("meta.cache.paimon.table.capacity", "meta.cache.paimon.snapshot.capacity"); + return compatibility; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 207810b66f5a68..004adc197061b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.paimon.partition.Partition; @@ -49,18 +50,26 @@ public enum PruningStatus { private final PruningStatus pruningStatus; private final Map nameToPartitionItem; private final Map nameToPartition; + private final long retainedPayloadBytes; private PaimonPartitionInfo(PruningStatus pruningStatus) { this.pruningStatus = pruningStatus; this.nameToPartitionItem = Collections.emptyMap(); this.nameToPartition = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public PaimonPartitionInfo(Map nameToPartitionItem, Map nameToPartition) { + this(nameToPartitionItem, nameToPartition, retainedPayloadBytes(nameToPartition)); + } + + public PaimonPartitionInfo(Map nameToPartitionItem, + Map nameToPartition, long retainedPayloadBytes) { this.pruningStatus = PruningStatus.PRUNABLE; this.nameToPartitionItem = nameToPartitionItem; this.nameToPartition = nameToPartition; + this.retainedPayloadBytes = retainedPayloadBytes; } public Map getNameToPartitionItem() { @@ -74,4 +83,47 @@ public Map getNameToPartition() { public PruningStatus getPruningStatus() { return pruningStatus; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + static long addRetainedStringPayload(long bytes, String value) { + return addString(bytes, value); + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (Map.Entry entry : partitions.entrySet()) { + bytes = addString(bytes, entry.getKey()); + Partition partition = entry.getValue(); + if (partition == null) { + continue; + } + bytes = addStrings(bytes, partition.spec()); + bytes = addString(bytes, partition.createdBy()); + bytes = addString(bytes, partition.updatedBy()); + bytes = addStrings(bytes, partition.options()); + } + return bytes; + } + + private static long addStrings(long bytes, Map values) { + if (values == null) { + return bytes; + } + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index 37be7c6a5f3585..630db395afbfa3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -17,11 +17,16 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; private final boolean schemaFromSnapshotTable; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { this(partitionInfo, snapshot, false); @@ -45,4 +50,22 @@ public PaimonSnapshot getSnapshot() { public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + MetaCacheSizeEstimate prepareForCachePublication(PaimonSnapshotEntryKey key) { + if (sizeEstimate == null) { + retainedTablePayloadBytes = PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_snapshot_preparation_failed", + () -> PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this)); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java new file mode 100644 index 00000000000000..d820d3c15e992b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; + +import java.util.Objects; + +/** Stable identity for a Paimon projection hydrated from one captured snapshot/schema fence. */ +public final class PaimonSnapshotEntryKey { + private final NameMapping nameMapping; + private final long snapshotId; + private final long schemaId; + private final long tableGeneration; + + public PaimonSnapshotEntryKey( + NameMapping nameMapping, long snapshotId, long schemaId, long tableGeneration) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.tableGeneration = tableGeneration; + } + + public static PaimonSnapshotEntryKey of( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return new PaimonSnapshotEntryKey( + nameMapping, fence.getSnapshotId(), fence.getSchemaId(), tableGeneration); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public long getSnapshotId() { + return snapshotId; + } + + public long getSchemaId() { + return schemaId; + } + + public long getTableGeneration() { + return tableGeneration; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PaimonSnapshotEntryKey)) { + return false; + } + PaimonSnapshotEntryKey that = (PaimonSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && tableGeneration == that.tableGeneration + && nameMapping.equals(that.nameMapping); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, snapshotId, schemaId, tableGeneration); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index 7539f28d770bf6..e5fba223ea92b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -17,28 +17,38 @@ package org.apache.doris.datasource.paimon; -import com.google.common.base.Suppliers; import org.apache.paimon.table.Table; -import java.util.function.Supplier; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; /** - * Cache value for Paimon table metadata and its latest runtime snapshot projection. + * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry so this + * value cannot grow after admission through a memoized supplier. */ public class PaimonTableCacheValue { + private static final AtomicLong NEXT_GENERATION = new AtomicLong(); + private final Table paimonTable; - private final Supplier latestSnapshotCacheValue; + private final PaimonSnapshotCacheValue latestSnapshotFence; + private final long generation; - public PaimonTableCacheValue(Table paimonTable, Supplier latestSnapshotCacheValue) { + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue latestSnapshotFence) { this.paimonTable = paimonTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + this.latestSnapshotFence = Objects.requireNonNull( + latestSnapshotFence, "latestSnapshotFence can not be null"); + this.generation = NEXT_GENERATION.incrementAndGet(); } public Table getPaimonTable() { return paimonTable; } - public PaimonSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + public long getGeneration() { + return generation; + } + + public PaimonSnapshotCacheValue getLatestSnapshotFence() { + return latestSnapshotFence; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index eaa59b28c2f4c1..a1eaf3100cdc28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -176,6 +176,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List candidates = Lists.newArrayListWithExpectedSize(partitionEntries.size()); Map> displayNameToTypedSpec = Maps.newHashMap(); + long retainedPayloadBytes = 0L; for (PartitionEntry partitionEntry : partitionEntries) { Map typedSpec = getPartitionInfoMap( @@ -193,6 +194,10 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List previousTypedSpec = displayNameToTypedSpec.putIfAbsent( displayName, orderedTypedSpec); if (previousTypedSpec != null) { @@ -247,7 +254,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List properties = new HashMap<>(); + + properties.put("meta.cache.hvie.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + properties.clear(); + + properties.put("meta.cache.hms.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + properties.clear(); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + } + + @Test + public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = Collections.singletonMap( + "meta.cache.hive.partition_values.capacity", "10"); + PaimonExternalCatalog catalog = new PaimonExternalCatalog( + 1L, "paimon", null, Collections.emptyMap(), ""); + + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); + } + @Test public void testRouteByCatalogType() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); @@ -110,6 +149,60 @@ public void testPrepareCatalogByEngineSkipsMissingCatalog() throws Exception { Assert.assertEquals(0, hive.initCatalogCalls); } + @Test + public void testPreparedEngineUsesLockFreeFastPath() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> true); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + long catalogId = 12L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + + Assert.assertEquals(1, hive.initCatalogCalls); + } + + @Test + public void testCatalogRemovalFencesInFlightFirstInitialization() throws Exception { + CountDownLatch initializationEntered = new CountDownLatch(1); + CountDownLatch releaseInitialization = new CountDownLatch(1); + BlockingRecordingExternalMetaCache hive = new BlockingRecordingExternalMetaCache( + "hive", initializationEntered, releaseInitialization); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + ExecutorService workers = Executors.newFixedThreadPool(2); + long catalogId = 13L; + Map oldProperties = Collections.singletonMap("generation", "old"); + Map newProperties = Collections.singletonMap("generation", "new"); + try { + Future initialization = workers.submit( + () -> metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", oldProperties)); + Assert.assertTrue(initializationEntered.await(3L, TimeUnit.SECONDS)); + + CountDownLatch removalStarted = new CountDownLatch(1); + Future removal = workers.submit(() -> { + removalStarted.countDown(); + metaCacheMgr.removeCatalogByEngine(catalogId, "hive"); + }); + Assert.assertTrue(removalStarted.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("removal must wait for the property snapshot publication", removal.isDone()); + + releaseInitialization.countDown(); + initialization.get(3L, TimeUnit.SECONDS); + removal.get(3L, TimeUnit.SECONDS); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", newProperties); + Assert.assertEquals("new", hive.lastCatalogProperties.get("generation")); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + } finally { + releaseInitialization.countDown(); + workers.shutdownNow(); + } + } + @Test public void testGetSchemaCacheValueReturnsEmptyWhenCatalogMissing() throws Exception { MissingCatalogSchemaExternalMetaCache schemaCache = new MissingCatalogSchemaExternalMetaCache("default"); @@ -373,4 +466,35 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class throw new IllegalStateException("catalog " + catalogId + " is not initialized"); } } + + private static final class BlockingRecordingExternalMetaCache extends RecordingExternalMetaCache { + private final CountDownLatch initializationEntered; + private final CountDownLatch releaseInitialization; + private final AtomicBoolean blockNextInitialization = new AtomicBoolean(true); + private Map lastCatalogProperties = Collections.emptyMap(); + + private BlockingRecordingExternalMetaCache(String engine, + CountDownLatch initializationEntered, CountDownLatch releaseInitialization) { + super(engine, Collections.emptyList(), catalog -> true); + this.initializationEntered = initializationEntered; + this.releaseInitialization = releaseInitialization; + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (blockNextInitialization.compareAndSet(true, false)) { + initializationEntered.countDown(); + try { + if (!releaseInitialization.await(3L, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to publish catalog initialization"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + lastCatalogProperties = new HashMap<>(catalogProperties); + super.initCatalog(catalogId, catalogProperties); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 151d46252d9084..cafeaf915ddaff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -17,23 +17,48 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import com.google.common.collect.HashBiMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; public class HiveMetaStoreCacheTest { + @Test + public void testPartitionValueWeightScalesLinearlyToOneHundredThousandPartitions() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.singletonList(Type.STRING)); + long base = partitionValueWeight(key, 0); + long oneThousand = partitionValueWeight(key, 1_000); + long tenThousand = partitionValueWeight(key, 10_000); + long oneHundredThousand = partitionValueWeight(key, 100_000); + + long oneThousandPayload = oneThousand - base; + Assertions.assertTrue(oneThousandPayload > 0L); + Assertions.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assertions.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + @Test public void testInvalidateTableCache() { ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( @@ -144,6 +169,82 @@ public void testInvalidatePartitionCacheClearsStaleFileCacheOnPartitionMiss() { } } + @Test + public void testPartitionValuesEstimateIsPreparedAgainAfterCopy() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.emptyList()); + PartitionKey partitionKey = new PartitionKey(); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + partitionItem.setDefaultPartition(true); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("p", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, Collections.emptyList()); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + Assertions.assertTrue(values.getSizeEstimate().isComplete()); + Assertions.assertTrue(values.getSizeEstimate().getBytes() > 0L); + + HiveExternalMetaCache.HivePartitionValues copy = values.mutableCopy(); + Assertions.assertFalse(copy.getSizeEstimate().isComplete()); + copy.sealForPublication(); + copy.prepareSizeEstimate(new HiveExternalMetaCache.PartitionValueCacheKey( + key.getNameMapping(), null)); + Assertions.assertTrue(copy.getSizeEstimate().isComplete()); + Assertions.assertTrue(copy.getSizeEstimate().getBytes() > 0L); + ListPartitionItem publishedItem = (ListPartitionItem) values.getIdToPartitionItem().get(1L); + Assertions.assertSame(partitionItem, publishedItem, + "cache publication must not rewrite common catalog partition objects"); + Assertions.assertSame(partitionKey, publishedItem.getItems().get(0)); + } + + @Test + public void testPartitionValuesEstimateSupportsRealLiteralGraph() throws Exception { + List types = java.util.Arrays.asList(Type.STRING, Type.INT, Type.DATEV2, Type.DECIMALV2); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + java.util.Arrays.asList( + new PartitionValue("tail-value"), + new PartitionValue("42"), + new PartitionValue("2026-08-12"), + new PartitionValue("123456789.0123")), + types, true); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("s=tail-value/i=42/d=2026-08-12/n=123456789.0123", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, java.util.Arrays.asList( + "tail-value", "42", "2026-08-12", "123456789.0123")); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + + Assertions.assertTrue(values.getSizeEstimate().isComplete(), + values.getSizeEstimate().getIncompleteReason()); + long estimatedBytes = values.getSizeEstimate().getBytes(); + PartitionKey publishedKey = ((ListPartitionItem) values.getIdToPartitionItem().get(1L)).getItems().get(0); + StringLiteral publishedString = (StringLiteral) publishedKey.getKeys().get(0); + // Exercise normal read-only lazy paths after publication. Their bounded memoized state is + // covered by estimator headroom without changing or cloning common expression classes. + publishedString.getExprName(); + values.getSortedPartitionRanges().orElseThrow(AssertionError::new).sortedPartitions + .forEach(partition -> partition.range.toString()); + values.prepareSizeEstimate(key); + Assertions.assertEquals(estimatedBytes, values.getSizeEstimate().getBytes()); + Assertions.assertSame(partitionKey, publishedKey, + "cache publication must not rewrite common catalog partition objects"); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, @@ -178,4 +279,22 @@ private long entrySize(MetaCacheEntry entry) { entry.forEach((k, v) -> count.incrementAndGet()); return count.get(); } + + private long partitionValueWeight( + HiveExternalMetaCache.PartitionValueCacheKey key, int partitionCount) { + Map items = sizeOnlyMap(partitionCount); + HiveExternalMetaCache.HivePartitionValues values = + new HiveExternalMetaCache.HivePartitionValues( + items, null, null, partitionCount * 16L, 1); + MetaCacheSizeEstimate estimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, values); + Assertions.assertTrue(estimate.isComplete(), estimate.getIncompleteReason()); + return estimate.getBytes(); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index af3be4475f71ac..9992238170c2c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -225,6 +225,8 @@ protected void runBeforeAll() throws Exception { } return invocation.callRealMethod(); }); + icebergUtilsMock.when(() -> IcebergUtils.getWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class))).thenReturn(mockedIcebergTable); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 00dde9f4cc0a74..bc8a7cc7a74ff0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -17,28 +17,768 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.types.Types; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testSnapshotAndManifestWeightsScaleLinearlyToOneHundredThousandItems() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, tableWithMetadataLocation("/metadata/linear-v1.json")).get(); + long snapshotBase = snapshotWeight(snapshotKey, 0); + long snapshotOneThousand = snapshotWeight(snapshotKey, 1_000); + assertLinearScale(snapshotBase, snapshotOneThousand, + snapshotWeight(snapshotKey, 10_000), snapshotWeight(snapshotKey, 100_000)); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey( + "/manifest/linear.avro", ManifestContent.DATA); + long manifestBase = manifestWeight(manifestKey, 0); + long manifestOneThousand = manifestWeight(manifestKey, 1_000); + assertLinearScale(manifestBase, manifestOneThousand, + manifestWeight(manifestKey, 10_000), manifestWeight(manifestKey, 100_000)); + } + + @Test + public void testWeightedEntriesAreRegistered() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.max-weight", "4MB"); + properties.put("meta.cache.iceberg.snapshot.max-weight", "8MB"); + properties.put("meta.cache.iceberg.manifest.enable", "true"); + properties.put("meta.cache.iceberg.manifest.max-weight", "16MB"); + cache.initCatalog(1L, properties); + + Map stats = cache.stats(1L); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_TABLE).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.enable", "false"); + properties.put("meta.cache.iceberg.table.ttl-second", "17"); + properties.put("meta.cache.iceberg.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(IcebergExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeyIncludesMetadataGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table first = tableWithMetadataLocation("/metadata/v1.json"); + Table second = tableWithMetadataLocation("/metadata/v2.json"); + Table recreated = tableWithMetadataLocation("/metadata/v1.json"); + + IcebergSnapshotEntryKey firstKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey sameKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey secondKey = IcebergSnapshotEntryKey.tryCreate(mapping, second).get(); + IcebergSnapshotEntryKey recreatedKey = IcebergSnapshotEntryKey.tryCreate(mapping, recreated).get(); + + Assert.assertEquals(firstKey, sameKey); + Assert.assertNotEquals(firstKey, secondKey); + Assert.assertNotEquals("drop/recreate may reuse HadoopCatalog's v1 path", firstKey, recreatedKey); + Assert.assertEquals("/metadata/v1.json", firstKey.getMetadataFileLocation()); + Assert.assertNotEquals(firstKey.getTableUuid(), recreatedKey.getTableUuid()); + Assert.assertFalse(IcebergSnapshotEntryKey.tryCreate(mapping, + newInterfaceProxy(Table.class)).isPresent()); + } + + @Test + public void testTableSnapshotAndManifestEstimatesArePrecomputed() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table table = tableWithMetadataLocation("/metadata/v1.json"); + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(table); + tableValue.prepareForCachePublication(mapping); + Assert.assertTrue(tableValue.getSizeEstimate().isComplete()); + Assert.assertTrue(tableValue.getSizeEstimate().getBytes() > 0L); + + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), table); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Assert.assertTrue(snapshotValue.getSizeEstimate().getBytes() > 0L); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey("/manifest/a.avro", ManifestContent.DATA); + ManifestCacheValue manifestValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/a.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + MetaCacheSizeEstimate manifestEstimate = + IcebergCacheSizeEstimator.estimateManifestEntry(manifestKey, manifestValue); + Assert.assertTrue(manifestEstimate.getIncompleteReason(), manifestEstimate.isComplete()); + Assert.assertTrue(manifestEstimate.getBytes() > 0L); + + Table unsupportedTable = newInterfaceProxy(Table.class); + MetaCacheSizeEstimate unsupported = IcebergCacheSizeEstimator.estimateTableEntry( + mapping, new IcebergTableCacheValue(unsupportedTable)); + Assert.assertFalse(unsupported.isComplete()); + Assert.assertTrue(unsupported.getIncompleteReason().startsWith("unsupported_iceberg_table:")); + } + + @Test + public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { + String largePayload = repeatedCharacter('x', 64 * 1024); + Table smallTable = tableWithNestedSchemaAndProperty("x", "x"); + Table largeTable = tableWithNestedSchemaAndProperty(largePayload, largePayload); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(smallTable); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue(largeTable); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + long expectedPayloadDelta = (largePayload.length() - 1L) * 4L; + Assert.assertTrue(largeValue.getSizeEstimate().getBytes() + - smallValue.getSizeEstimate().getBytes() >= expectedPayloadDelta); + } + + @Test + public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { + List fields = IntStream.range(0, 100) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema largeSchema = new Schema(0, fields); + Schema smallSchema = new Schema(1, fields.get(0)); + TableMetadata schemaHistory = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + schemaHistory = TableMetadata.buildFrom(schemaHistory) + .addSchema(smallSchema) + .setCurrentSchema(smallSchema.schemaId()) + .discardChanges() + .build(); + TableMetadata smallSchemaOnly = TableMetadata.newTableMetadata( + smallSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(largeSchema).withSpecId(0); + SortOrder.Builder sortBuilder = SortOrder.builderFor(largeSchema).withOrderId(1); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + sortBuilder.asc(field.name()); + } + TableMetadata fieldHistory = TableMetadata.newTableMetadata( + largeSchema, specBuilder.build(), sortBuilder.build(), + "file:/warehouse/field-history", Collections.emptyMap()); + fieldHistory = TableMetadata.buildFrom(fieldHistory) + .setDefaultPartitionSpec( + PartitionSpec.builderFor(largeSchema).withSpecId(1).build()) + .setDefaultSortOrder(SortOrder.unsorted()) + .discardChanges() + .build(); + TableMetadata emptyFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + + long schemaDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(schemaHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(smallSchemaOnly)); + long specAndSortDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(fieldHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + + Assert.assertTrue(schemaDelta >= 99L * 512L); + Assert.assertTrue(specAndSortDelta >= 100L * (384L + 256L)); + } + + @Test + public void testTablePayloadExcludesQueryLocalHistoricalMetadata() { + String largePayload = repeatedCharacter('x', 64 * 1024); + long smallBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload("x", 32))); + long largeBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload(largePayload, 64 * 1024))); + + Assert.assertEquals(smallBytes, largeBytes); + } + + @Test + public void testTableEstimateExcludesQueryLocalBranchHistory() { + TableMetadata oneCommit = metadataWithSnapshotSequence(1L); + TableMetadata tenThousandCommits = metadataWithSnapshotSequence(10_000L); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(tableWithMetadata(oneCommit)); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue( + tableWithMetadata(tenThousandCommits)); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + Assert.assertTrue(smallValue.getSizeEstimate().getIncompleteReason(), + smallValue.getSizeEstimate().isComplete()); + Assert.assertTrue(largeValue.getSizeEstimate().getIncompleteReason(), + largeValue.getSizeEstimate().isComplete()); + Assert.assertEquals(smallValue.getSizeEstimate().getBytes(), + largeValue.getSizeEstimate().getBytes()); + Mockito.verify(oneCommit, Mockito.never()).snapshots(); + Mockito.verify(tenThousandCommits, Mockito.never()).snapshots(); + Mockito.verify(oneCommit, Mockito.never()).lastSequenceNumber(); + Mockito.verify(tenThousandCommits, Mockito.never()).lastSequenceNumber(); + Mockito.verify(oneCommit, Mockito.never()).snapshotLog(); + Mockito.verify(tenThousandCommits, Mockito.never()).snapshotLog(); + Mockito.verify(oneCommit, Mockito.never()).refs(); + Mockito.verify(tenThousandCommits, Mockito.never()).refs(); + } + + @Test + public void testWeightedTablePreparationRunsInsideCatalogAuthenticator() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + AtomicBoolean authenticated = new AtomicBoolean(); + AtomicBoolean firstPreparation = new AtomicBoolean(true); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Table table = tableWithMetadataLocation("/metadata/authenticated-v1.json"); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenReturn(table); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + Assert.assertTrue(authenticated.compareAndSet(false, true)); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + if (firstPreparation.compareAndSet(true, false)) { + Assert.assertTrue("publication preparation must retain Kerberos scope", + authenticated.get()); + } + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = new NameMapping( + 1L, "db", "tbl", "remote_db", "remote_tbl"); + + IcebergTableCacheValue value = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + IcebergTableCacheValue cached = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertSame(value, cached); + Mockito.verify(metadataOps, Mockito.times(1)).loadTable("remote_db", "remote_tbl"); + Assert.assertFalse(authenticated.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testExactMetadataReadsRunInsideCatalogAuthenticator() throws Exception { + String tableLocation = temporaryFolder.newFolder("authenticated-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + AtomicBoolean authenticated = new AtomicBoolean(); + AtomicInteger metadataReads = new AtomicInteger(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + Assert.assertTrue(authenticated.compareAndSet(false, true)); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }; + FileIO trackingFileIO = Mockito.mock(FileIO.class); + Mockito.when(trackingFileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + Assert.assertTrue("metadata FileIO must retain catalog authentication", authenticated.get()); + metadataReads.incrementAndGet(); + return liveTable.io().newInputFile((String) invocation.getArgument(0)); + }); + TableMetadata metadata = ((HasTableOperations) liveTable).operations().current(); + Table trackedTable = new BaseTable( + new StaticTableOperations(metadata, trackingFileIO), liveTable.name()); + IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable, authenticator); + countValue.getWritableIcebergTable(liveTable); + Assert.assertEquals("count-based writes must not add metadata FileIO", 0, metadataReads.get()); + IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable, authenticator); + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Table statementTable = value.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(statementTable); + IcebergSnapshotCacheValue statementValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), statementTable); + Assert.assertSame(statementTable, statementValue.getIcebergTable().get()); + com.google.common.collect.Lists.newArrayList( + statementValue.getIcebergTable().get().snapshots()); + Assert.assertEquals("statement handoff must reuse parsed metadata", 1, metadataReads.get()); + value.getWritableIcebergTable(liveTable); + + Assert.assertEquals(2, metadataReads.get()); + Assert.assertFalse(authenticated.get()); + } + + @Test + public void testCountModeTimeTravelDoesNotEnableQueryIsolation() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/count-v1.json")); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, value); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertSame(value.getRetainedIcebergTable(), queryTable); + Assert.assertFalse(value.isQueryIsolationPrepared()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws Exception { + String firstLocation = temporaryFolder.newFolder("bundle-first").toURI().toString(); + String secondLocation = temporaryFolder.newFolder("bundle-second").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table firstTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), firstLocation); + firstTable.newAppend().appendFile(DataFiles.builder(firstTable.spec()) + .withPath(firstLocation + "/data/a.parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()).commit(); + Table secondTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), secondLocation); + secondTable.newAppend().appendFile(DataFiles.builder(secondTable.spec()) + .withPath(secondLocation + "/data/b.parquet") + .withFileSizeInBytes(20L).withRecordCount(2L).build()).commit(); + long firstSnapshotId = firstTable.currentSnapshot().snapshotId(); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + MetaCacheEntry entry = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + entry.put(mapping, new IcebergTableCacheValue(firstTable)); + ExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(dorisTable); + entry.put(mapping, new IcebergTableCacheValue(secondTable)); + + Assert.assertEquals(firstSnapshotId, + queryTable.currentSnapshot().snapshotId()); + Assert.assertEquals(firstTable.location(), + queryTable.location()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testMissingPinnedMetadataRefreshesBeforeStatementFence() throws Exception { + String staleLocation = temporaryFolder.newFolder("stale-metadata").toURI().toString(); + String freshLocation = temporaryFolder.newFolder("fresh-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table staleTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), staleLocation); + Table freshTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), freshLocation); + String staleMetadataLocation = ((HasTableOperations) staleTable) + .operations().current().metadataFileLocation(); + IcebergTableCacheValue staleValue = new IcebergTableCacheValue(staleTable); + staleValue.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + staleTable.io().deleteFile(staleMetadataLocation); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + Mockito.when(metadataOps.loadTable("db", "tbl")).thenReturn(freshTable); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, staleValue); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertEquals(freshTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.times(1)).loadTable("db", "tbl"); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testWeightedTablePublicationRetainsNonGrowingGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot currentSnapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/current-a.avro\"," + + "\"/manifest/current-b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .setBranchSnapshot(currentSnapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); + Table liveTable = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); + Assert.assertThrows(UnsupportedOperationException.class, + () -> retained.snapshot(7L).dataManifests(retained.io())); + Table firstUse = value.getIcebergTable(); + Table secondUse = value.getIcebergTable(); + Assert.assertNotSame(retained, firstUse); + Assert.assertNotSame(firstUse, secondUse); + Assert.assertNotSame(retained.currentSnapshot(), firstUse.currentSnapshot()); + Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); + Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); + Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(firstUse)); + + IcebergSnapshotEntryKey snapshotKey = + IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), + Optional.empty(), retained, value.getRetainedCurrentSnapshotJson()); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Table snapshotQuery = snapshotValue.getIcebergTable().get(); + Assert.assertEquals(2, + snapshotQuery.currentSnapshot().dataManifests(snapshotQuery.io()).size()); + } + + @Test + public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Exception { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + String tableLocation = temporaryFolder.newFolder("v2-table").toURI().toString(); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + liveTable.newAppend().appendFile( + DataFiles.builder(liveTable.spec()) + .withPath(tableLocation + "/data/a.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()).commit(); + Assert.assertNotNull(liveTable.currentSnapshot().manifestListLocation()); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Assert.assertThrows(UnsupportedOperationException.class, + () -> retained.currentSnapshot().dataManifests(retained.io())); + Table firstQuery = value.getIcebergTable(); + Table secondQuery = value.getIcebergTable(); + List firstManifests = + firstQuery.currentSnapshot().dataManifests(firstQuery.io()); + List secondManifests = + secondQuery.currentSnapshot().dataManifests(secondQuery.io()); + Assert.assertEquals(1, firstManifests.size()); + Assert.assertEquals(1, secondManifests.size()); + Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); + Assert.assertNotSame(firstManifests, secondManifests); + Assert.assertThrows(UnsupportedOperationException.class, + () -> retained.currentSnapshot().dataManifests(retained.io())); + } + + @Test + public void testTablePublicationDoesNotReadHistoricalManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot historical = SnapshotParser.fromJson("{\"snapshot-id\":6,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/history.avro\",\"schema-id\":0}"); + Snapshot current = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"},\"manifests\":[],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .addSnapshot(historical) + .setBranchSnapshot(current, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v2.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + IcebergTableCacheValue value = new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl")); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Mockito.verify(fileIO, Mockito.never()).newInputFile("/manifest-list/history.avro"); + } + + @Test + public void testManifestEstimateScalesWithFileCount() { + ManifestCacheValue oneFile = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/one.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + ManifestCacheValue twoFiles = ManifestCacheValue.forDataFiles(java.util.Arrays.asList( + oneFile.getDataFiles().get(0), oneFile.getDataFiles().get(0))); + IcebergManifestEntryKey key = new IcebergManifestEntryKey("/manifest/data.avro", ManifestContent.DATA); + + long oneFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, oneFile).getBytes(); + long twoFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, twoFiles).getBytes(); + + Assert.assertTrue(twoFileBytes > oneFileBytes); + } + + @Test + public void testManifestEstimateAccountsForSkewedFilePaths() { + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/x.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(largePath) + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-skew.avro", ManifestContent.DATA); + + long smallBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue).getBytes(); + long largeBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue).getBytes(); + + Assert.assertTrue(largeBytes - smallBytes >= (largePath.length() - "/data/x.parquet".length()) * 2L); + } + + @Test + public void testManifestEstimateAccountsForSkewedBufferPayload() { + Metrics smallMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(32)), Collections.emptyMap()); + Metrics largeMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(64 * 1024)), Collections.emptyMap()); + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(smallMetrics) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(largeMetrics) + .build())); + + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/encrypted.avro", ManifestContent.DATA); + MetaCacheSizeEstimate smallEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue); + MetaCacheSizeEstimate largeEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue); + + Assert.assertTrue(smallEstimate.getIncompleteReason(), smallEstimate.isComplete()); + Assert.assertTrue(largeEstimate.getIncompleteReason(), largeEstimate.isComplete()); + Assert.assertEquals(1L, smallValue.getDataFileMetricEntryCount()); + Assert.assertEquals(1L, largeValue.getDataFileMetricEntryCount()); + Assert.assertTrue(largeEstimate.getBytes() - smallEstimate.getBytes() >= 64 * 1024 - 32); + } + + @Test + public void testManifestEstimateAccountsForDeleteFileAuxiliaryPayload() { + String largeReference = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + List largeOffsets = IntStream.range(0, 4096) + .mapToObj(index -> (long) index).collect(Collectors.toList()); + DeleteFile smallPositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile("/data/x.parquet") + .withSplitOffsets(Collections.singletonList(0L)) + .build(); + DeleteFile largePositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile(largeReference) + .withSplitOffsets(largeOffsets) + .build(); + int[] largeEqualityIds = IntStream.range(0, 4096).toArray(); + DeleteFile smallEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(1) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + DeleteFile largeEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(largeEqualityIds) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/delete.avro", ManifestContent.DELETES); + + long smallPositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallPositionDelete))).getBytes(); + long largePositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largePositionDelete))).getBytes(); + long smallEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallEqualityDelete))).getBytes(); + long largeEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largeEqualityDelete))).getBytes(); + + Assert.assertTrue(largePositionBytes > smallPositionBytes); + Assert.assertTrue(largeEqualityBytes > smallEqualityBytes); + } + + @Test + public void testSnapshotPublicationDoesNotMaterializeManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/a.avro\",\"/manifest/b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Table table = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), Optional.empty(), table); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate( + NameMapping.createForTest(1L, "db", "tbl"), table).get(); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table queryTable = value.getIcebergTable().get(); + Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(queryTable)); + Mockito.verifyNoInteractions(fileIO); + } @Test public void testInvalidateTableKeepsManifestCache() { @@ -52,10 +792,16 @@ public void testInvalidateTableKeepsManifestCache() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + + Table snapshotTable = tableWithMetadataLocation("/metadata/invalidate-v1.json"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(t1, snapshotTable).get(); + MetaCacheEntry snapshotEntry = cache.entry(catalogId, + IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, + new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); MetaCacheEntry viewEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_VIEW, NameMapping.class, org.apache.iceberg.view.View.class); @@ -77,6 +823,7 @@ public void testInvalidateTableKeepsManifestCache() { Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); Assert.assertNull(viewEntry.getIfPresent(t1)); Assert.assertNotNull(viewEntry.getIfPresent(t2)); Assert.assertNotNull(manifestEntry.getIfPresent(m1)); @@ -98,10 +845,8 @@ public void testInvalidateDbAndStats() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); MetaCacheEntry schemaEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class); @@ -228,6 +973,139 @@ private Map manifestCacheEnabledProperties() { return properties; } + private long snapshotWeight(IcebergSnapshotEntryKey key, int partitionCount) { + IcebergPartitionInfo partitionInfo = Mockito.mock(IcebergPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Map> aliases = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToIcebergPartition()).thenReturn(partitions); + Mockito.when(partitionInfo.getNameToIcebergPartitionNames()).thenReturn(aliases); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(key.getSnapshotId(), key.getSchemaId())); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateSnapshotEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private long manifestWeight(IcebergManifestEntryKey key, int fileCount) { + ManifestCacheValue value = Mockito.mock(ManifestCacheValue.class); + List dataFiles = sizeOnlyList(fileCount); + Mockito.when(value.getDataFiles()).thenReturn(dataFiles); + Mockito.when(value.getDeleteFiles()).thenReturn(Collections.emptyList()); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private void assertLinearScale(long base, long oneThousand, long tenThousand, long oneHundredThousand) { + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + @SuppressWarnings("unchecked") + private List sizeOnlyList(int size) { + List list = Mockito.mock(List.class); + Mockito.when(list.size()).thenReturn(size); + return list; + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Table tableWithMetadataLocation(String metadataLocation) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation(metadataLocation).build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + + private Table tableWithMetadata(TableMetadata metadata) { + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(metadata); + return new BaseTable(operations, "db.tbl"); + } + + private TableMetadata metadataWithMaterializedPayload(String payload, int bufferBytes) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.uuid()).thenReturn("stable-uuid"); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + payload, Mockito.mock(SnapshotRef.class))); + TableMetadata.MetadataLogEntry metadataLogEntry = + Mockito.mock(TableMetadata.MetadataLogEntry.class); + Mockito.when(metadataLogEntry.file()).thenReturn(payload); + Mockito.when(metadata.previousFiles()).thenReturn( + Collections.singletonList(metadataLogEntry)); + GenericBlobMetadata blob = new GenericBlobMetadata( + payload, 1L, 1L, Collections.singletonList(1), + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.singletonList( + new GenericStatisticsFile(1L, payload, 1L, 1L, + Collections.singletonList(blob)))); + org.apache.iceberg.PartitionStatisticsFile partitionStatistics = + Mockito.mock(org.apache.iceberg.PartitionStatisticsFile.class); + Mockito.when(partitionStatistics.path()).thenReturn(payload); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn( + Collections.singletonList(partitionStatistics)); + EncryptedKey encryptedKey = Mockito.mock(EncryptedKey.class); + Mockito.when(encryptedKey.keyId()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedById()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedKeyMetadata()).thenReturn( + ByteBuffer.allocateDirect(bufferBytes)); + Mockito.when(encryptedKey.properties()).thenReturn( + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.singletonList(encryptedKey)); + return metadata; + } + + private TableMetadata metadataWithSnapshotSequence(long lastSequenceNumber) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.singletonList( + Mockito.mock(org.apache.iceberg.HistoryEntry.class))); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + "branch-tip", Mockito.mock(SnapshotRef.class))); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.lastSequenceNumber()).thenReturn(lastSequenceNumber); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/sequence.json"); + return metadata; + } + + private Table tableWithNestedSchemaAndProperty(String nestedFieldName, String propertyValue) { + Schema schema = new Schema(Types.NestedField.optional(1, "payload", + Types.StructType.of(Types.NestedField.optional( + 2, nestedFieldName, Types.StringType.get())))); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.singletonMap("payload", propertyValue)); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/nested.json").build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + private IcebergManifestEntryKey mockManifestKey(String path) { return IcebergManifestEntryKey.of(new TestingManifestFile(path, ManifestContent.DATA)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index 720f66fd5f9e3f..de3ab8c9746397 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -92,9 +92,9 @@ public void setUp() throws IOException { Mockito.doReturn(db).when(catalog).getDbNullable(Mockito.any()); Mockito.doReturn(dorisTable).when(db).getTableNullable(Mockito.any()); - // mock IcebergUtils.getIcebergTable to return our test icebergTable + // Mock writable access used by branch and tag mutations. mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(Mockito.any())) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any())) .thenReturn(icebergTable); // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a4b882c4497132..cc019c73ed12a7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -138,7 +138,8 @@ public void testTopLevelVariantModifyOnlyUpdatesMetadataOnOrcTable() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("payload"), column, ColumnPosition.FIRST, 1L); } @@ -165,7 +166,8 @@ public void testTopLevelVariantModifyRejectsTypeConversions() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("variant_col"), new Column("variant_col", Type.STRING, true), null, 1L), @@ -294,7 +296,7 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), new Column("new_field", Type.LARGEINT, true), null, 1L), @@ -328,7 +330,7 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); @@ -363,7 +365,7 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); } @@ -391,7 +393,7 @@ public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComm try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -433,7 +435,7 @@ public void testFullStructModifyPreservesOmittedChildComments() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), new Column("payload", payloadType, true), null, 1L); @@ -463,7 +465,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -489,7 +491,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("id"), new Column("id", Type.BIGINT, true), null, 1L); @@ -515,7 +517,7 @@ public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L), @@ -537,7 +539,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L); @@ -568,7 +570,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, @@ -603,7 +605,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); @@ -633,7 +635,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, @@ -661,7 +663,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, @@ -692,7 +694,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); // Iceberg schema columns are represented as keys in Doris, so the legacy API must not // interpret isKey as an explicit KEY clause. @@ -723,7 +725,7 @@ public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, column, null, 1L); } @@ -757,7 +759,7 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); @@ -810,7 +812,7 @@ public void execute(Runnable task) { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(staleTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(staleTable); try { conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), @@ -861,7 +863,7 @@ public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); icebergTable.refresh(); @@ -911,7 +913,7 @@ public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, "a", "renamed", 1L); icebergTable.refresh(); @@ -940,7 +942,7 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), nestedAddDefaultColumn, null, 1L), @@ -971,7 +973,7 @@ public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: id"); @@ -1002,7 +1004,7 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), @@ -1035,7 +1037,7 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); @@ -1082,7 +1084,7 @@ public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, new Column("info", infoType, true), null, 1L), @@ -1158,7 +1160,7 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn( dorisTable, new Column("id", Type.STRING, true), null, 1L), @@ -1194,7 +1196,7 @@ public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); } @@ -1216,7 +1218,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), null, 1L); @@ -1239,7 +1241,7 @@ public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), @@ -1270,7 +1272,7 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), "struct comment", 1L); @@ -1296,7 +1298,7 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), @@ -1336,7 +1338,7 @@ public void testRejectsTopLevelRowLineageMutationsForV3Tables() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L), @@ -1392,8 +1394,10 @@ public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v3DorisTable)).thenReturn(v3IcebergTable); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v2DorisTable)).thenReturn(v2IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v3DorisTable)) + .thenReturn(v3IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v2DorisTable)) + .thenReturn(v2IcebergTable); ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java index 74c8c3f6954a97..84745f815f828c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java @@ -22,11 +22,28 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.Map; import java.util.Set; public class IcebergPartitionInfoTest { + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + String largeValue = repeatedCharacter('x', 64 * 1024); + IcebergPartition small = new IcebergPartition("p=x", 0, 0, 0, 0, 1, 101, + Collections.singletonList("x"), Collections.singletonList("identity")); + IcebergPartition large = new IcebergPartition("p=" + largeValue, 0, 0, 0, 0, 1, 101, + Collections.singletonList(largeValue), Collections.singletonList("identity")); + + Assertions.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= (largeValue.length() - 1L) * 4L); + IcebergPartitionInfo info = new IcebergPartitionInfo( + Collections.emptyMap(), Collections.singletonMap(large.getPartitionName(), large), + Collections.emptyMap()); + Assertions.assertEquals(large.getRetainedPayloadBytes(), info.getRetainedPayloadBytes()); + } + @Test public void testGetLatestSnapshotId() { IcebergPartition p1 = new IcebergPartition("p1", 0, 0, 0, 0, 1, 101, null, null); @@ -50,4 +67,10 @@ public void testGetLatestSnapshotId() { Assertions.assertEquals(102, snapshot2); Assertions.assertEquals(103, snapshot3); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index e2d923f3438863..2df4ad7229dbe7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -202,7 +202,7 @@ public void testPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); // Allow parsePartitionValueFromString to call the real implementation mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( @@ -318,7 +318,7 @@ public void testUnPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -429,7 +429,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -455,7 +455,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -500,7 +500,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -517,7 +517,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th checkPushDownByPartition(table, Expressions.equal("str1", "partition-b"), 1); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -590,7 +590,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(3); @@ -651,7 +651,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(formatVersion); @@ -724,7 +724,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -736,6 +736,26 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); } + @Test + public void testWeightedTableSupportsSchemaAndPartitionSpecCommits() { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergTableCacheValue cacheValue = new IcebergTableCacheValue(liveTable); + cacheValue.prepareForCachePublication(NameMapping.createForTest(dbName, tbWithoutPartition)); + + Table writableTable = cacheValue.getWritableIcebergTable(liveTable); + writableTable.updateSchema() + .addColumn("new_col", Types.StringType.get()) + .commit(); + writableTable.updateSpec() + .addField("int1") + .commit(); + + Table refreshed = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + Assert.assertNotNull(refreshed.schema().findField("new_col")); + Assert.assertEquals(1, refreshed.spec().fields().size()); + Assert.assertEquals("int1", refreshed.spec().fields().get(0).name()); + } + @Test public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws UserException { Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); @@ -754,7 +774,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -783,7 +803,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 6407d540d1ef03..704c82df183e3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -28,8 +28,13 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class AbstractExternalMetaCacheTest { @@ -97,6 +102,34 @@ public void testEntryFailsFastAfterCatalogRemoved() { } } + @Test + public void testCapturedCatalogGroupReturnsClosedEntryDuringConcurrentRemoval() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newSingleThreadExecutor(); + CountDownLatch groupCaptured = new CountDownLatch(1); + CountDownLatch releaseEntryLookup = new CountDownLatch(1); + LookupRaceExternalMetaCache cache = + new LookupRaceExternalMetaCache(refreshExecutor, groupCaptured, releaseEntryLookup); + try { + cache.initCatalog(1L, Maps.newHashMap()); + Future> lookup = workers.submit( + () -> cache.entry(1L, "value", String.class, Integer.class)); + Assert.assertTrue(groupCaptured.await(3L, TimeUnit.SECONDS)); + + cache.invalidateCatalog(1L); + releaseEntryLookup.countDown(); + + MetaCacheEntry capturedClosedEntry = lookup.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(Integer.valueOf(1), capturedClosedEntry.get("k")); + Assert.assertNull(capturedClosedEntry.peekIfPresent("k")); + } finally { + releaseEntryLookup.countDown(); + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testEntryLevelInvalidationUsesRegisteredMatcher() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -121,6 +154,125 @@ public void testEntryLevelInvalidationUsesRegisteredMatcher() { } } + @Test + public void testGlobalWeightAutomaticallyActivatesEntriesWithEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(600L))); + try { + cache.initCatalog(1L, Maps.newHashMap()); + MetaCacheEntry entry = cache.entry(1L, "value", String.class, Integer.class); + + Assert.assertTrue(entry.isWeightBounded()); + Assert.assertEquals(600L, entry.stats().getMaxWeight()); + entry.put("first", 60); + entry.put("second", 60); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(60), entry.getIfPresent("second")); + Assert.assertEquals(60L + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES, + entry.stats().getGlobalEstimatedWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testEntryWeightWithoutEstimatorFailsCatalogInitialization() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); + Map properties = Maps.newHashMap(); + properties.put("meta.cache.test_engine.schema.max-weight", "1KB"); + + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); + Assert.assertTrue(exception.getMessage().contains("does not support max-weight")); + Assert.assertFalse(cache.isCatalogInitialized(1L)); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCommonValidationRejectsEntryWeightAboveCatalogWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(4L * 1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); + Assert.assertTrue(exception.getMessage().contains("entry max weight")); + Assert.assertFalse(cache.isCatalogInitialized(1L)); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitClampsCatalogAcceptedOnLargerFe() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "4KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + Assert.assertThrows(IllegalArgumentException.class, + () -> cache.validateCatalogProperties(properties)); + + cache.initCatalog(1L, properties); + + MetaCacheEntryStats stats = cache.stats(1L).get("value"); + Assert.assertEquals(1024L, stats.getMaxWeight()); + Assert.assertEquals(1024L, stats.getCatalogMaxWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCatalogRemoveAndInitDoesNotDuplicateBudgetScope() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newFixedThreadPool(2); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(100L))); + CountDownLatch start = new CountDownLatch(1); + try { + Future first = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + Future second = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + start.countDown(); + first.get(10L, TimeUnit.SECONDS); + second.get(10L, TimeUnit.SECONDS); + cache.initCatalog(1L, Maps.newHashMap()); + Assert.assertTrue(cache.isCatalogInitialized(1L)); + } finally { + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + private static void repeatedlyRebuildCatalog(WeightedExternalMetaCache cache, CountDownLatch start) { + try { + Assert.assertTrue(start.await(3L, TimeUnit.SECONDS)); + for (int i = 0; i < 100; i++) { + cache.initCatalog(1L, Maps.newHashMap()); + cache.invalidateCatalog(1L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + private static final class TestExternalMetaCache extends AbstractExternalMetaCache { private TestExternalMetaCache(ExecutorService refreshExecutor) { super("test_engine", refreshExecutor); @@ -135,4 +287,44 @@ private TestExternalMetaCache(ExecutorService refreshExecutor) { MetaCacheEntryInvalidation.forNameMapping(SchemaCacheKey::getNameMapping))); } } + + private static final class WeightedExternalMetaCache extends AbstractExternalMetaCache { + private WeightedExternalMetaCache( + ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super("weighted_test", refreshExecutor, budgetManager); + registerEntry(MetaCacheEntryDef.of( + "value", + String.class, + Integer.class, + key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(value.longValue()))); + } + } + + private static final class LookupRaceExternalMetaCache extends AbstractExternalMetaCache { + private final CountDownLatch groupCaptured; + private final CountDownLatch releaseEntryLookup; + + private LookupRaceExternalMetaCache(ExecutorService refreshExecutor, + CountDownLatch groupCaptured, CountDownLatch releaseEntryLookup) { + super("lookup_race", refreshExecutor); + this.groupCaptured = groupCaptured; + this.releaseEntryLookup = releaseEntryLookup; + registerEntry(MetaCacheEntryDef.of( + "value", String.class, Integer.class, key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L))); + } + + @Override + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + groupCaptured.countDown(); + try { + Assert.assertTrue(releaseEntryLookup.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java index 05acbb539a26d6..6ea171abd6d717 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.DdlException; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -68,6 +69,7 @@ public void testFromPropertiesWithPropertySpecBuilder() { public void testFromPropertiesWithEngineEntryKeys() { Map properties = Maps.newHashMap(); properties.put("meta.cache.hive.schema.ttl-second", "0"); + properties.put("meta.cache.hive.schema.max-weight", "2KB"); CacheSpec defaultSpec = CacheSpec.fromProperties( Maps.newHashMap(), @@ -79,6 +81,8 @@ public void testFromPropertiesWithEngineEntryKeys() { Assert.assertTrue(spec.isEnable()); Assert.assertEquals(0, spec.getTtlSecond()); Assert.assertEquals(100, spec.getCapacity()); + Assert.assertTrue(spec.isWeightBounded()); + Assert.assertEquals(2048L, spec.getMaxWeight().getAsLong()); } @Test @@ -108,6 +112,7 @@ public void testOfSemantics() { Assert.assertTrue(enabled.isEnable()); Assert.assertEquals(60, enabled.getTtlSecond()); Assert.assertEquals(100, enabled.getCapacity()); + Assert.assertFalse(enabled.isWeightBounded()); CacheSpec zeroTtl = CacheSpec.of(true, 0, 100); Assert.assertTrue(zeroTtl.isEnable()); @@ -147,6 +152,10 @@ public void testIsCacheEnabled() { Assert.assertFalse(CacheSpec.isCacheEnabled(false, CacheSpec.CACHE_NO_TTL, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, 0, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, CacheSpec.CACHE_NO_TTL, 0)); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 0L, 1L).isCacheEnabled()); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 1L, 0L).isCacheEnabled()); } @Test @@ -166,4 +175,51 @@ public void testToExpireAfterAccess() { Assert.assertTrue(negativeOther.isPresent()); Assert.assertEquals(0, negativeOther.getAsLong()); } + + @Test + public void testParseWeight() { + Assert.assertEquals(1L, CacheSpec.parseWeight("1", "weight", false, 0L)); + Assert.assertEquals(1024L, CacheSpec.parseWeight("1KB", "weight", false, 0L)); + Assert.assertEquals(2L * 1024L * 1024L, + CacheSpec.parseWeight("2 mb", "weight", false, 0L)); + Assert.assertEquals(250L, CacheSpec.parseWeight("25%", "weight", true, 1000L)); + Assert.assertEquals(0L, CacheSpec.parseWeight("0", "weight", false, 0L)); + + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1.5GB", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("101%", "weight", true, 1000L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1PB000", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("999999999999999999PB", "weight", false, 0L)); + } + + @Test + public void testStrictEnginePropertyAllowlist() { + Map properties = Maps.newHashMap(); + properties.put("meta.cache.hive.partition_values.enable", "true"); + properties.put("meta.cache.hive.partition_values.ttl-second", "-1"); + properties.put("meta.cache.hive.partition_values.capacity", "10"); + properties.put("meta.cache.hive.partition_values.max-weight", "2MB"); + CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values")); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partiton_values.capacity"); + + properties.put("meta.cache.hive.partition_values.enabel", "true"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partition_values.enabel"); + + properties.put("meta.cache.hive.schema.max-weight", "1MB"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java new file mode 100644 index 00000000000000..4d35dd15fd63b4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class ExternalMetaCacheBudgetManagerTest { + + @Test + public void testGlobalCatalogAndEntryLimits() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget first = manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(60L)); + EntryBudget second = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(80L), OptionalLong.of(50L)); + + AdmissionReservation firstReservation = first.tryReserve(60L).get(); + Assert.assertFalse(second.tryReserve(30L).isPresent()); + AdmissionReservation secondReservation = second.tryReserve(20L).get(); + Assert.assertEquals(80L, manager.getGlobalUsedWeight()); + Assert.assertFalse(secondReservation.tryResize(30L)); + + firstReservation.release(); + Assert.assertTrue(secondReservation.tryResize(30L)); + Assert.assertEquals(30L, manager.getGlobalUsedWeight()); + + secondReservation.release(); + first.close(); + second.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testConcurrentReservationNeverExceedsGlobalLimit() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "iceberg", "manifest", OptionalLong.empty(), OptionalLong.empty()); + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + List reservations = Collections.synchronizedList(new ArrayList<>()); + try { + for (int i = 0; i < 200; i++) { + executor.submit(() -> { + await(start); + Optional reservation = budget.tryReserve(1L); + reservation.ifPresent(reservations::add); + }); + } + start.countDown(); + executor.shutdown(); + Assert.assertTrue(executor.awaitTermination(10L, TimeUnit.SECONDS)); + Assert.assertEquals(100, reservations.size()); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + } finally { + executor.shutdownNow(); + reservations.forEach(AdmissionReservation::release); + budget.close(); + } + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testRejectChildLargerThanParent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(90L))); + + Map properties = new HashMap<>(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "120"); + Assert.assertEquals(120L, manager.parseCatalogMaxWeight(properties).getAsLong()); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.validateCatalogMaxWeight(properties)); + } + + @Test + public void testRuntimeBudgetClampsReplayedCatalogToLocalGlobalLimit() { + ExternalMetaCacheBudgetManager observerManager = manager(100L); + + EntryBudget budget = observerManager.createEntryBudget( + 1L, "iceberg", "table", OptionalLong.of(400L), OptionalLong.of(300L)); + + Assert.assertEquals(100L, budget.getEffectiveMaxWeight()); + Assert.assertEquals(100L, budget.getCatalogMaxWeight()); + AdmissionReservation reservation = budget.tryReserve(100L).get(); + Assert.assertFalse(budget.tryReserve(1L).isPresent()); + reservation.release(); + budget.close(); + } + + @Test + public void testGlobalConfigSupportsPercentageAndDisabledZero() { + String original = Config.external_meta_cache_max_weight; + try { + Config.external_meta_cache_max_weight = "25%"; + ExternalMetaCacheBudgetManager percentageManager = ExternalMetaCacheBudgetManager.fromConfig(); + Assert.assertEquals(Runtime.getRuntime().maxMemory() / 4L, + percentageManager.getGlobalMaxWeight().getAsLong()); + + Config.external_meta_cache_max_weight = "0"; + Assert.assertFalse(ExternalMetaCacheBudgetManager.fromConfig().getGlobalMaxWeight().isPresent()); + + Config.external_meta_cache_max_weight = "0%"; + Assert.assertThrows(IllegalArgumentException.class, ExternalMetaCacheBudgetManager::fromConfig); + } finally { + Config.external_meta_cache_max_weight = original; + } + } + + @Test + public void testReservationReleaseIsIdempotent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = budget.tryReserve(40L).get(); + + reservation.release(); + reservation.release(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + budget.close(); + } + + @Test + public void testClosedBudgetRejectsStaleHandleAndReservationResize() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation zeroByteReservation = staleBudget.tryReserve(0L).get(); + + staleBudget.close(); + staleBudget.close(); + + Assert.assertFalse(staleBudget.tryReserve(1L).isPresent()); + Assert.assertFalse(zeroByteReservation.tryResize(1L)); + Assert.assertEquals(0L, staleBudget.getRejectedCount()); + Assert.assertEquals(0L, manager.getGlobalRejectedCount()); + zeroByteReservation.release(); + + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation replacementReservation = replacement.tryReserve(100L).get(); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + replacementReservation.release(); + replacement.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + private static ExternalMetaCacheBudgetManager manager(long maxWeight) { + return new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + } + + private static void await(CountDownLatch latch) { + try { + Assert.assertTrue(latch.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index ef1090dd5300c6..fa013d58f4a653 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -20,17 +20,27 @@ import org.apache.doris.common.Config; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; public class MetaCacheEntryTest { @@ -311,6 +321,876 @@ void beforeManualCachePutForTest(String key, Integer loaded) { } } + @Test + public void testExplicitPutWinsAgainstInFlightManualLoad() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePutStarted = new CountDownLatch(1); + CountDownLatch releaseBeforePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePutStarted.countDown(); + awaitLatch(releaseBeforePut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePutStarted.await(3L, TimeUnit.SECONDS)); + entry.put("k", 2); + releaseBeforePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + } finally { + releaseBeforePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedReplacementDoesNotQueueOldValuesOnRefreshExecutor() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "test", "value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "value", key -> new byte[1], CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), entryBudget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + entry.put("k", new byte[100]); + for (int i = 0; i < 100; i++) { + entry.put("k", new byte[100]); + } + + Assert.assertTrue("removal callbacks must not retain replaced values in the executor queue", + refreshExecutor.getQueue().isEmpty()); + Assert.assertEquals(accountedWeight(100L), entry.stats().getEstimatedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + } + } + + @Test + public void testQueuedWeightedRefreshDoesNotCaptureCurrentValue() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-capture", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-capture", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + + Runnable queuedRefresh = refreshExecutor.getQueue().peek(); + Assert.assertNotNull(queuedRefresh); + for (Field field : queuedRefresh.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("queued refresh must not directly retain the cached value", + currentValue, field.get(queuedRefresh)); + } + + entry.invalidateAll(); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, loaderCalls.get()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderWeightedRefresh() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-fence", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-fence", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertFalse(refreshExecutor.getQueue().isEmpty()); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + + Assert.assertEquals("the older refresh must be rejected before it calls the loader", + 0, loaderCalls.get()); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderCountRefresh() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch loaderEntered = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + CountDownLatch loaderFinished = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-refresh-fence", key -> { + loaderEntered.countDown(); + awaitLatch(releaseLoader); + loaderFinished.countDown(); + return "stale"; + }, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + String currentValue = new String("current"); + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertTrue(loaderEntered.await(3L, TimeUnit.SECONDS)); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseLoader.countDown(); + Assert.assertTrue(loaderFinished.await(3L, TimeUnit.SECONDS)); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertSame("the event fence must suppress refresh write-back", + currentValue, entry.peekIfPresent("k")); + } finally { + releaseLoader.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentWeightedRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-weighted-refresh", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-weighted-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte["a".equals(key) ? 2 : 3]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("a", new byte[1]); + entry.put("b", new byte[1]); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValueLength(entry, "a", 2); + awaitValueLength(entry, "b", 3); + Assert.assertEquals(accountedWeight(2L) + accountedWeight(3L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCountRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-count-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return key + "-refreshed"; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("a", "a-current"); + entry.put("b", "b-current"); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValue(entry, "a", "a-refreshed"); + awaitValue(entry, "b", "b-refreshed"); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testInvalidatingOneKeyDoesNotSuppressAnotherKeysConcurrentMissAdmission() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-miss", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-miss", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte[1]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + Future first = queryExecutor.submit(() -> entry.get("a")); + Future second = queryExecutor.submit(() -> entry.get("b")); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + entry.invalidateKey("a"); + releaseLoaders.countDown(); + first.get(3L, TimeUnit.SECONDS); + second.get(3L, TimeUnit.SECONDS); + + Assert.assertNull(entry.peekIfPresent("a")); + Assert.assertNotNull(entry.peekIfPresent("b")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRemovalCleanupDoesNotDeadlockWithInvalidateAll() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "deadlock", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch removalListenerEntered = new CountDownLatch(1); + CountDownLatch invalidateHasAdmissionLock = new CountDownLatch(1); + CountDownLatch releaseRemovalListener = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "deadlock", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + removalListenerEntered.countDown(); + awaitLatch(releaseRemovalListener); + } + } + + @Override + void beforeWeightedInvalidateAllForTest() { + invalidateHasAdmissionLock.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future eviction = queryExecutor.submit( + () -> loadingCache.policy().eviction().get().setMaximum(1L)); + Assert.assertTrue(removalListenerEntered.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertTrue(invalidateHasAdmissionLock.await(3L, TimeUnit.SECONDS)); + + releaseRemovalListener.countDown(); + eviction.get(3L, TimeUnit.SECONDS); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + releaseRemovalListener.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedRemovalDoesNotReleaseSameIdentityReinsert() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "same-identity-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "same-identity-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCountRemovalDoesNotDropSameIdentityRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry( + "count-same-identity-aba", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeNonWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("the replacement refresh owner must remain usable", 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentCountRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-expired-same-identity", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null); + try { + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new refresh owner", + 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentWeightedReservation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted-expired-same-identity", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted-expired-same-identity", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new reservation", + accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "soft-value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft-value", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] value = new byte[1]; + entry.put("k", value); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + Assert.assertTrue("weighted values must be held through Caffeine SoftReference", + valueReference instanceof SoftReference); + Map owners = (Map) readField(entry, "reservations"); + Object owner = owners.get("k"); + Assert.assertNotNull(owner); + for (Field field : owner.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("reservation ownership must not strongly retain V", value, + field.get(owner)); + } + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + + awaitGlobalWeight(manager, 0L); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testStrongQueryReferenceSurvivesSoftValueCollectionChecks() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "query-reference", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "query-reference", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] queryReference = entry.get("k"); + WeakReference observed = new WeakReference<>(queryReference); + + for (int i = 0; i < 3; i++) { + System.gc(); + extractLoadingCache(entry).cleanUp(); + } + + Assert.assertSame(queryReference, observed.get()); + Assert.assertSame(queryReference, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCollectedCallbackCannotReleaseReplacementGeneration() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "collected-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch collectedBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseCollectedCallback = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch collectedCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "collected-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + collectedBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseCollectedCallback); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + collectedCleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference oldValueReference = extractValueReference(loadingCache); + armRemovalHook.set(true); + + Future collection = queryExecutor.submit(() -> { + oldValueReference.clear(); + oldValueReference.enqueue(); + loadingCache.cleanUp(); + }); + Assert.assertTrue(collectedBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + + byte[] replacement = new byte[1]; + armPutHook.set(true); + Future replacementPut = queryExecutor.submit(() -> entry.put("k", replacement)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseCollectedCallback.countDown(); + collection.get(3L, TimeUnit.SECONDS); + releaseReplacementPut.countDown(); + replacementPut.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(collectedCleanupFinished.await(3L, TimeUnit.SECONDS)); + + Assert.assertSame(replacement, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseCollectedCallback.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOwnershipRecordsHaveNoGenericValueReference() { + for (Class nested : MetaCacheEntry.class.getDeclaredClasses()) { + if (nested.getSimpleName().equals("ReservationRecord") + || nested.getSimpleName().equals("RefreshRecord")) { + Assert.assertFalse(nested.getSimpleName() + " must not retain V", + Arrays.stream(nested.getDeclaredFields()) + .anyMatch(field -> field.getType() == Object.class)); + } + } + } + + @Test + public void testRemovalCleanupRetriesAfterTransientFailure() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "removal-retry", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch firstAttempt = new CountDownLatch(1); + CountDownLatch cleanupFinished = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "removal-retry", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalCleanupLockForTest(String key) { + if (attempts.incrementAndGet() == 1) { + firstAttempt.countDown(); + throw new IllegalStateException("transient cleanup failure"); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + cleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + extractLoadingCache(entry).invalidate("k"); + + Assert.assertTrue(firstAttempt.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue(cleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue("cleanup should be retried", attempts.get() >= 2); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testBulkInvalidateDoesNotEnqueueOneCleanupPerEntry() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_000L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "bulk-invalidate", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger queuedCleanupCount = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "bulk-invalidate", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 2_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + queuedCleanupCount.incrementAndGet(); + } + }; + try { + for (int i = 0; i < 1_000; i++) { + entry.put("k-" + i, new byte[1]); + } + + entry.invalidateAll(); + + Assert.assertEquals(0, queuedCleanupCount.get()); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(() -> entry.invalidateKey("k")); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateAllLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testManualMissLoadAllowsNullWithoutCaching() { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; @@ -373,6 +1253,357 @@ public void testManualMissLoadDoesNotCacheWhenEntryDisabled() { } } + @Test + public void testClosedEntryCanNotBeRepopulated() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + AtomicInteger loadCounter = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", + key -> loadCounter.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, + false); + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + + entry.close(); + + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(Integer.valueOf(2), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testManualMissLoadDoesNotWriteBackAcrossClose() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePut = new CountDownLatch(1); + CountDownLatch releasePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePut.countDown(); + awaitLatch(releasePut); + } + }; + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePut.await(3L, TimeUnit.SECONDS)); + entry.close(); + releasePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releasePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCompareAndReplaceUsesIdentityAndPeekDoesNotPolluteStats() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", key -> "loaded", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false); + String current = new String("same"); + entry.put("k", current); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.NOT_CURRENT, + entry.tryReplace("k", new String("same"), "wrong")); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", current, "new")); + Assert.assertEquals("new", entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionAndReplacementAccounting() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + + entry.put("k", 40); + Assert.assertEquals(Integer.valueOf(40), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(40L), manager.getGlobalUsedWeight()); + + entry.put("k", 10); + Assert.assertEquals(Integer.valueOf(10), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(10L), manager.getGlobalUsedWeight()); + + entry.invalidateKey("k"); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCaffeineWeigherOnlyReadsPreparedReservationWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Integer first = Integer.valueOf(20); + entry.put("k", first); + Assert.assertEquals(1, estimateCalls.get()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", first, Integer.valueOf(30))); + Assert.assertEquals(2, estimateCalls.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIncompleteEstimateReturnsValueWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.incomplete("unclassified_field"), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNegativeEstimateFailsImmediately() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(-1L), + budget); + try { + Assert.assertThrows(IllegalArgumentException.class, () -> entry.put("k", 30)); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testZeroEstimateIsRejectedWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(0L), budget); + try { + entry.put("k", 1); + + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + Assert.assertEquals("invalid_estimate", entry.stats().getLastWeightRejectReason()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedEntryEvictsItsOwnColdestValueBeforeAdmission() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + entry.put("first", 30); + entry.put("second", 30); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(30), entry.getIfPresent("second")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getEvictionCount()); + Assert.assertEquals(accountedWeight(30L), entry.stats().getEvictionWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionCanReclaimMoreThanOneThousandSmallValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_500L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "many-small-values", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "many-small-values", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + for (int i = 0; i < 1_500; i++) { + entry.put("small-" + i, new byte[1]); + } + + byte[] large = new byte[600_000]; + entry.put("large", large); + + Assert.assertSame(large, entry.peekIfPresent("large")); + Assert.assertTrue(entry.stats().getEvictionCount() > 1_024L); + Assert.assertTrue(entry.stats().getEstimatedWeight() <= maxWeight); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOversizedValueIsRejectedWithoutEvictingUsefulValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_100L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_100L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + entry.put("first", 20); + entry.put("second", 20); + entry.put("oversized", 600); + + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("first")); + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("second")); + Assert.assertNull(entry.peekIfPresent("oversized")); + Assert.assertEquals(2L * accountedWeight(20L), manager.getGlobalUsedWeight()); + Assert.assertEquals(0L, entry.stats().getEvictionCount()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRejectedAtomicReplacementKeepsExpectedValueUntilConditionalInvalidation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(600L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + Integer current = Integer.valueOf(30); + entry.put("k", current); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REJECTED, + entry.tryReplace("k", current, Integer.valueOf(100))); + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(entry.invalidateKeyIfSame("k", current)); + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDisabledWeightedEntrySkipsEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 30, CacheSpec.ofWeight(false, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(0, estimateCalls.get()); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + // Keep the loader blocking helper in one place so concurrent tests stay readable. private void awaitLatch(CountDownLatch latch) { try { @@ -383,12 +1614,87 @@ private void awaitLatch(CountDownLatch latch) { } } + private void awaitValueLength(MetaCacheEntry entry, String key, int expectedLength) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + byte[] value = entry.peekIfPresent(key); + if (value != null && value.length == expectedLength) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expectedLength, entry.peekIfPresent(key).length); + } + + private void awaitValue(MetaCacheEntry entry, String key, String expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + if (expected.equals(entry.peekIfPresent(key))) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expected, entry.peekIfPresent(key)); + } + + private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() != expected && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + Assert.assertEquals(expected, manager.getGlobalUsedWeight()); + } + + private Reference extractValueReference(LoadingCache loadingCache) throws Exception { + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + Assert.assertEquals(1, nodes.size()); + Object node = nodes.values().iterator().next(); + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + Object valueReference = valueReferenceMethod.invoke(node); + Assert.assertTrue(valueReference instanceof Reference); + return (Reference) valueReference; + } + + private Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } + @SuppressWarnings("unchecked") - private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { + private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { Field dataField = MetaCacheEntry.class.getDeclaredField("loadingData"); dataField.setAccessible(true); Object raw = dataField.get(entry); Assert.assertTrue(raw instanceof LoadingCache); - return (LoadingCache) raw; + return (LoadingCache) raw; + } + + private static long accountedWeight(long estimatedPayloadBytes) { + return estimatedPayloadBytes + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 228bb112ff0016..066905dbfaa6ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -24,9 +24,11 @@ import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; @@ -38,6 +40,8 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.privilege.PrivilegeChecker; +import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -50,6 +54,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; import org.junit.Assert; import org.junit.Assume; import org.junit.Rule; @@ -58,6 +63,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -69,6 +75,240 @@ public class PaimonExternalMetaCacheTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testSnapshotWeightScalesLinearlyToOneHundredThousandPartitions() throws Exception { + FileStoreTable table = newPartitionedTable("linear_snapshot_estimate", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + long base = snapshotWeight(key, table, 0); + long oneThousand = snapshotWeight(key, table, 1_000); + long tenThousand = snapshotWeight(key, table, 10_000); + long oneHundredThousand = snapshotWeight(key, table, 100_000); + + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @Test + public void testSnapshotWeightAccountsForSkewedTableOptions() throws Exception { + FileStoreTable smallTable = newPartitionedTable( + "option_small", Collections.singletonMap("payload", "x")); + String largePayload = repeatedCharacter('x', 64 * 1024); + FileStoreTable largeTable = newPartitionedTable( + "option_large", Collections.singletonMap("payload", largePayload)); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largePayload.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightAccountsForNestedSchemaPayload() throws Exception { + String largeFieldName = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTableWithNestedField("nested_small", "x"); + FileStoreTable largeTable = newPartitionedTableWithNestedField( + "nested_large", largeFieldName); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largeFieldName.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightAccountsForTableComment() throws Exception { + String largeComment = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTable( + "comment_small", Collections.emptyMap(), "x"); + FileStoreTable largeTable = newPartitionedTable( + "comment_large", Collections.emptyMap(), largeComment); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + + long smallBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L), smallTable, 0); + long largeBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L), largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.paimon.snapshot.max-weight", "8MB")); + Assert.assertTrue(cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + FileStoreTable table = newPartitionedTable("snapshot_estimate", Collections.emptyMap()); + Object lazyStoreBefore = readField(table, table.getClass(), "lazyStore"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertTrue(value.getSizeEstimate().getBytes() > 0L); + Assert.assertSame("cache admission must not materialize FileStoreTable.store()", + lazyStoreBefore, readField(table, table.getClass(), "lazyStore")); + + PaimonSnapshotCacheValue unsupportedValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, Mockito.mock(Table.class))); + unsupportedValue.prepareForCachePublication(new PaimonSnapshotEntryKey(mapping, 1L, 1L, 1L)); + Assert.assertFalse(unsupportedValue.getSizeEstimate().isComplete()); + Assert.assertTrue(unsupportedValue.getSizeEstimate().getIncompleteReason() + .startsWith("unsupported_paimon_table:")); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeySeparatesReloadedTableGenerations() { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, null); + PaimonSnapshotCacheValue fenceValue = new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue first = new PaimonTableCacheValue(null, fenceValue); + PaimonTableCacheValue reloaded = new PaimonTableCacheValue(null, fenceValue); + + PaimonSnapshotEntryKey firstKey = PaimonSnapshotEntryKey.of( + mapping, fence, first.getGeneration()); + PaimonSnapshotEntryKey reloadedKey = PaimonSnapshotEntryKey.of( + mapping, fence, reloaded.getGeneration()); + + Assert.assertNotEquals(firstKey, reloadedKey); + Assert.assertNotEquals(firstKey.getTableGeneration(), reloadedKey.getTableGeneration()); + } + + @Test + public void testSnapshotHitReusesFenceCapturedByTableGeneration() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "remote_db", "remote_tbl"); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, table); + PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table, snapshotValue); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + mapping, fence, tableValue.getGeneration()); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class).put(mapping, tableValue); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class).put(key, snapshotValue); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); + Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); + + Mockito.verifyNoInteractions(table); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { + FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); + FileStoreTable privileged = PrivilegedFileStoreTable.wrap( + table, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "tbl")); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, table.schema().id(), privileged)); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + } + + @Test + public void testSnapshotEstimateDoesNotMaterializeNestedRowTypeIndexes() throws Exception { + RowType nested = DataTypes.ROW( + DataTypes.FIELD(10, "nested_id", DataTypes.INT()), + DataTypes.FIELD(11, "nested_name", DataTypes.STRING())); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "payload", nested)), + 11, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder("nested_row_estimate").toURI()), + schema, + CatalogEnvironment.empty()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, schema.id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, schema.id(), table)); + + Map stateBefore = new HashMap<>(); + for (String fieldName : java.util.Arrays.asList( + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex")) { + stateBefore.put(fieldName, readField(nested, fieldName)); + } + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + for (Map.Entry entry : stateBefore.entrySet()) { + Assert.assertSame(entry.getKey() + " must not be changed by cache admission", + entry.getValue(), readField(nested, entry.getKey())); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + Map properties = new HashMap<>(); + properties.put("meta.cache.paimon.table.enable", "false"); + properties.put("meta.cache.paimon.table.ttl-second", "17"); + properties.put("meta.cache.paimon.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + @Test public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( @@ -320,6 +560,11 @@ public void testPartitionProjectionIgnoresReaderOnlyPhysicalOptions() throws Exc } private FileStoreTable newPartitionedTable(String name, Map options) throws Exception { + return newPartitionedTable(name, options, null); + } + + private FileStoreTable newPartitionedTable( + String name, Map options, String comment) throws Exception { TableSchema schema = new TableSchema( 0, java.util.Arrays.asList( @@ -329,6 +574,27 @@ private FileStoreTable newPartitionedTable(String name, Map opti Collections.singletonList("part"), Collections.emptyList(), options, + comment); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newPartitionedTableWithNestedField( + String name, String nestedFieldName) throws Exception { + RowType nestedType = new RowType(Collections.singletonList( + new DataField(2, nestedFieldName, DataTypes.STRING()))); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), null); return new AppendOnlyFileStoreTable( LocalFileIO.create(), @@ -337,6 +603,42 @@ private FileStoreTable newPartitionedTable(String name, Map opti CatalogEnvironment.empty()); } + private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, int partitionCount) { + PaimonPartitionInfo partitionInfo = Mockito.mock(PaimonPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToPartition()).thenReturn(partitions); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + private Object readField(RowType rowType, String fieldName) throws Exception { + return readField(rowType, RowType.class, fieldName); + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Object readField(Object target, Class owner, String fieldName) throws Exception { + Field field = owner.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } + @Test public void testInvalidateTablePrecise() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -350,15 +652,24 @@ public void testInvalidateTablePrecise() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(t1, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(t2, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(t1, new PaimonTableCacheValue(null, fence)); + tableEntry.put(t2, new PaimonTableCacheValue(null, fence)); + + PaimonSnapshotEntryKey snapshotKey = new PaimonSnapshotEntryKey(t1, 1L, 2L, 1L); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshotEntry = cache.entry(catalogId, + PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, null))); cache.invalidateTable(catalogId, "db1", "tbl1"); Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); } finally { executor.shutdownNow(); } @@ -377,10 +688,10 @@ public void testInvalidateDbAndStats() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(db1Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(db2Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(db1Table, new PaimonTableCacheValue(null, fence)); + tableEntry.put(db2Table, new PaimonTableCacheValue(null, fence)); org.apache.doris.datasource.metacache.MetaCacheEntry schemaEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 09bcee53985f5d..17fa76bb5593bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -36,6 +36,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Timestamp; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; @@ -87,6 +88,19 @@ private static PartitionEntry partitionEntry(BinaryRow partition, long sequence) return new PartitionEntry(partition, sequence, sequence, sequence, sequence, 1); } + @Test + public void testCompatibilityConstructorDerivesRetainedPartitionPayload() { + String largeValue = repeatedCharacter('x', 64 * 1024); + Partition partition = new Partition( + Collections.singletonMap("part", largeValue), + 1L, 1L, 1L, 1L, 1, false); + + PaimonPartitionInfo info = new PaimonPartitionInfo( + Collections.emptyMap(), Collections.singletonMap("part=" + largeValue, partition)); + + Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 4L); + } + @Test public void testSchemaForVarcharAndChar() { DataField c1 = new DataField(1, "c1", new VarCharType(32)); @@ -247,6 +261,7 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; + Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length() * 2L); Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) @@ -257,6 +272,21 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { "s1"), actualValues); } + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + List partitionColumns = Collections.singletonList(new Column("part", Type.STRING)); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + PaimonPartitionInfo small = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow("x"), 1L))); + String largeValue = repeatedCharacter('x', 64 * 1024); + PaimonPartitionInfo large = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow(largeValue), 1L))); + + Assert.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= (largeValue.length() - 1L) * 4L); + } + @Test public void testGeneratePartitionInfoUsesPartitionColumnOrder() { List partitionColumns = Arrays.asList( @@ -532,4 +562,10 @@ public void testAuditLogHistorySchemaWithoutSequenceNumber() { Assert.assertEquals("id", fields.get(1).getFieldPtr().getName()); Assert.assertEquals("name", fields.get(2).getFieldPtr().getName()); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/pom.xml b/fe/pom.xml index 3783d2660e6b9b..6821b6c3fce2ab 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -438,6 +438,12 @@ under the License. + + benchmark + + fe-benchmark + + From 41e63e0efbf7cf666379078e77d7db52e4cf1699 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 17 Aug 2026 14:01:43 +0800 Subject: [PATCH 2/5] [fix](fe) Harden external metadata cache memory governance Address review findings around upgrade compatibility, generation retirement, live Iceberg writes, Paimon snapshot freshness, authenticated metadata access, refresh failure handling, and fair catalog/global budget reclaim. Make optional size accounting fail closed without failing queries, add JOL calibration and long-tail coverage, and keep retained-size traversal off default count-based cache paths. Tests: 170 targeted FE tests; external metadata cache size benchmark. --- .../run-external-meta-cache-size-benchmark.sh | 5 +- fe/fe-core/pom.xml | 5 + .../apache/doris/datasource/CatalogMgr.java | 10 +- .../datasource/ExternalMetaCacheMgr.java | 36 ++- .../hive/HiveExternalMetaCache.java | 6 +- .../iceberg/IcebergCacheSizeEstimator.java | 51 ++- .../iceberg/IcebergExternalMetaCache.java | 141 ++++++--- .../iceberg/IcebergSchemaCacheKey.java | 16 +- .../iceberg/IcebergSnapshotCacheValue.java | 282 ++++------------- .../iceberg/IcebergSnapshotEntryKey.java | 8 + .../iceberg/IcebergSysExternalTable.java | 49 +-- .../iceberg/IcebergTableCacheValue.java | 66 +++- .../datasource/iceberg/IcebergUtils.java | 27 +- .../iceberg/cache/ManifestCacheValue.java | 49 ++- .../metacache/AbstractExternalMetaCache.java | 40 ++- .../doris/datasource/metacache/CacheSpec.java | 30 +- .../metacache/CatalogEntryGroup.java | 13 +- .../ExternalMetaCacheBudgetManager.java | 181 ++++++++++- .../datasource/metacache/MetaCacheEntry.java | 121 ++++++-- .../metacache/MetaCacheEntryDef.java | 27 +- .../MetaCacheEntryReplacementListener.java | 26 ++ .../metacache/MetaCacheSizeEstimator.java | 9 +- .../PaimonLatestSnapshotProjectionLoader.java | 24 +- .../metacache/paimon/PaimonTableLoader.java | 11 + .../paimon/PaimonExternalMetaCache.java | 92 +++++- .../paimon/PaimonExternalTable.java | 8 + .../paimon/PaimonSchemaCacheKey.java | 14 +- .../paimon/PaimonSnapshotCacheValue.java | 20 +- .../paimon/PaimonTableCacheValue.java | 15 +- .../doris/datasource/paimon/PaimonUtils.java | 5 + .../ExternalMetaCacheRouteResolverTest.java | 46 ++- .../hive/HiveMetaStoreCacheTest.java | 42 +++ .../iceberg/IcebergExternalMetaCacheTest.java | 293 +++++++++++++++--- .../iceberg/IcebergSysExternalTableTest.java | 28 ++ .../AbstractExternalMetaCacheTest.java | 22 +- .../EstimatorCalibrationAssertions.java | 67 ++++ .../ExternalMetaCacheBudgetManagerTest.java | 100 ++++++ .../metacache/MetaCacheEntryTest.java | 160 ++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 244 ++++++++++++++- .../paimon/PaimonExternalTableTest.java | 2 +- fe/pom.xml | 6 + .../test_iceberg_table_meta_cache.groovy | 19 +- 42 files changed, 1929 insertions(+), 487 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java diff --git a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh index 12e28e0dd9a7a4..7ce2e0be8e3d4f 100755 --- a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh +++ b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh @@ -33,7 +33,10 @@ trap 'rm -f "${CLASSPATH_FILE}"' EXIT -Dmdep.outputFile="${CLASSPATH_FILE}" ) -REACTOR_CLASSES=$(find "${FE_DIR}" -type d -path '*/target/classes' -printf '%p:') +REACTOR_CLASSES= +while IFS= read -r -d '' CLASSES_DIR; do + REACTOR_CLASSES+="${CLASSES_DIR}:" +done < <(find "${FE_DIR}" -type d -path '*/target/classes' -print0) DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") BENCHMARK_FILTER=${BENCHMARK_FILTER:-'HivePartitionValuesSizeBenchmark|IcebergCacheSizeBenchmark|PaimonCacheSizeBenchmark|MetaCacheSoftValueBenchmark'} diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index a880c5ffad4db4..93ceeb49b58144 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -851,6 +851,11 @@ under the License. mockito-inline test + + org.openjdk.jol + jol-core + test + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index b44cb735f2ac61..3e8a8f813571ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -648,7 +648,15 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope // Only legacy validators publish a tentative candidate. Detached validators // leave the live CatalogProperty untouched while concurrent initialization runs. if (oldProperties != null && tentativelyMutated) { - ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null + ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + } else { + cacheMgr.rollbackCatalogProperties( + (ExternalCatalog) catalog, oldProperties); + } } if (validationException instanceof DdlException) { throw (DdlException) validationException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index a80d0c8b71485f..e58a984eacc462 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -209,8 +209,9 @@ public void prepareCatalog(long catalogId) { logMissingCatalogSkip(catalogId, "prepareCatalog"); return; } - validateCatalogCachePropertiesForRuntime(catalogProperties); - routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); + Map runtimeProperties = sanitizeCatalogCachePropertiesForRuntime( + catalogId, catalogProperties); + routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, runtimeProperties)); } finally { lifecycleLock.unlock(); } @@ -254,7 +255,7 @@ private void prepareCatalogByEngineLocked( Map safeCatalogProperties = catalogProperties == null ? Maps.newHashMap() : Maps.newHashMap(catalogProperties); - validateCatalogCachePropertiesForRuntime(safeCatalogProperties); + safeCatalogProperties = sanitizeCatalogCachePropertiesForRuntime(catalogId, safeCatalogProperties); targetCache.initCatalog(catalogId, safeCatalogProperties); } @@ -264,9 +265,17 @@ public void validateCatalogCacheProperties(Map catalogProperties cacheRegistry.allCaches().forEach(cache -> cache.validateCatalogProperties(catalogProperties)); } - private void validateCatalogCachePropertiesForRuntime(Map catalogProperties) { - budgetManager.parseCatalogMaxWeight(catalogProperties); - validateCatalogCachePropertyNamespaces(catalogProperties); + private Map sanitizeCatalogCachePropertiesForRuntime( + long catalogId, Map catalogProperties) { + Map sanitized = Maps.newHashMap(catalogProperties); + try { + budgetManager.parseCatalogMaxWeight(sanitized); + } catch (IllegalArgumentException e) { + sanitized.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' for catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, catalogId, e.getMessage()); + } + return sanitized; } private void validateCatalogCachePropertyNamespaces(Map catalogProperties) { @@ -335,6 +344,21 @@ public void removeCatalog(long catalogId) { } } + /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ + public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { + long catalogId = catalog.getId(); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + catalog.rollBackCatalogProps(oldProperties); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + } + public void removeCatalogByEngine(long catalogId, String engine) { Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); lifecycleLock.lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 527497237b1bf2..dfa5b00289feb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -756,8 +756,10 @@ private void addPartitionsCache(NameMapping nameMapping, Map addedItems = new HashMap<>(); for (String partitionName : partitionNames) { if (allNames.containsKey(partitionName)) { - LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", - partitionName, localTblName); + if (attempt == 0) { + LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", + partitionName, localTblName); + } continue; } long partitionId = Util.genIdByName( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index e60f78eef90182..8d483ecfb5153f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -24,17 +24,21 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.encryption.EncryptedKey; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import java.nio.ByteBuffer; import java.util.Map; -/** Constant-time retained-weight formulas for Iceberg cache entries. */ +/** Publication-time retained-weight formulas for Iceberg cache entries. */ final class IcebergCacheSizeEstimator { private static final long KEY_BASE_BYTES = 128L; private static final long TABLE_BASE_BYTES = 16L * 1024L; @@ -47,6 +51,13 @@ final class IcebergCacheSizeEstimator { private static final long SORT_FIELD_BYTES = 256L; private static final long TABLE_PROPERTY_BYTES = 256L; private static final long CURRENT_SNAPSHOT_BYTES = 512L; + private static final long HISTORICAL_SNAPSHOT_BYTES = 1024L; + private static final long SNAPSHOT_LOG_ENTRY_BYTES = 64L; + private static final long METADATA_LOG_ENTRY_BYTES = 128L; + private static final long SNAPSHOT_REF_BYTES = 128L; + private static final long STATISTICS_FILE_BYTES = 512L; + private static final long PARTITION_STATISTICS_FILE_BYTES = 256L; + private static final long ENCRYPTED_KEY_BYTES = 256L; private static final long PARTITION_BYTES = 512L; private static final long PARTITION_ALIAS_BYTES = 256L; private static final long NAME_MAPPING_ENTRY_BYTES = 256L; @@ -110,6 +121,9 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( static MetaCacheSizeEstimate estimateManifestEntry( IcebergManifestEntryKey key, ManifestCacheValue value) { + if (!value.isAccountingComplete()) { + return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); + } long bytes = MetaCacheWeightUtils.saturatedAdd( MANIFEST_ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); @@ -189,10 +203,45 @@ static long retainedTablePayloadBytes(Table table) { bytes = addString(bytes, property.getKey()); bytes = addString(bytes, property.getValue()); } + for (Snapshot snapshot : metadata.snapshots()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HISTORICAL_SNAPSHOT_BYTES); + bytes = addString(bytes, snapshot.operation()); + bytes = addString(bytes, snapshot.manifestListLocation()); + bytes = addStringMap(bytes, snapshot.summary(), TABLE_PROPERTY_BYTES); + } + bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); + bytes = addString(bytes, previousFile.file()); + } + bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); + for (String refName : metadata.refs().keySet()) { + bytes = addString(bytes, refName); + } + for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + bytes = addCount(bytes, statisticsFile.blobMetadata().size(), TABLE_PROPERTY_BYTES); + } + for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + } + for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); + bytes = addString(bytes, encryptedKey.keyId()); + bytes = addString(bytes, encryptedKey.encryptedById()); + bytes = addBufferPayload(bytes, encryptedKey.encryptedKeyMetadata()); + bytes = addStringMap(bytes, encryptedKey.properties(), TABLE_PROPERTY_BYTES); + } bytes = addString(bytes, metadata.uuid()); return bytes; } + private static long addBufferPayload(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index e6402dd234839c..e8b24299d53df2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -53,6 +53,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import javax.annotation.Nullable; /** * Iceberg engine implementation of {@link AbstractExternalMetaCache}. @@ -103,7 +104,8 @@ public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCac tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) - .withSizeEstimator(this::prepareTableForCachePublication)); + .withSizeEstimator(this::prepareTableForCachePublication) + .withReplacementListener(this::retireTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) @@ -128,27 +130,16 @@ public Table getIcebergTable(ExternalTable dorisTable) { public Table getWritableIcebergTable(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - IcebergTableCacheValue tableValue = - tableEntry.get(nameMapping.getCtlId()).get(nameMapping); CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + " when loading a writable Iceberg table"); } IcebergMetadataOps ops = resolveMetadataOps(catalog); - Table liveTable = executeAuthenticated(catalog, () -> ops.loadTable( + // DDL/actions must start from the live catalog generation. DML that was planned against a + // retained read generation wraps this live table separately in IcebergTransaction. + return executeAuthenticated(catalog, () -> ops.loadTable( nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - try { - return tableValue.getWritableIcebergTable(liveTable); - } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { - MetaCacheEntry entry = - tableEntry.get(nameMapping.getCtlId()); - entry.invalidateKeyIfSame(nameMapping, tableValue); - IcebergTableCacheValue refreshedValue = entry.get(nameMapping); - Table refreshedLiveTable = executeAuthenticated(catalog, () -> ops.loadTable( - nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - return refreshedValue.getWritableIcebergTable(refreshedLiveTable); - } } Table getQueryScopedIcebergTable(ExternalTable dorisTable) { @@ -157,12 +148,7 @@ Table getQueryScopedIcebergTable(ExternalTable dorisTable) { tableEntry.get(nameMapping.getCtlId()); IcebergTableCacheValue tableValue = entry.get(nameMapping); - try { - return createQueryTable(nameMapping, tableValue); - } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { - entry.invalidateKeyIfSame(nameMapping, tableValue); - return createQueryTable(nameMapping, entry.get(nameMapping)); - } + return createQueryTable(nameMapping, tableValue); } private Table createQueryTable( @@ -199,18 +185,26 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { snapshotEntry.get(nameMapping.getCtlId()); boolean isolateForQueries = tableValue.isQueryIsolationPrepared() || entry.isWeightBounded(); - return entry.get(key, ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { - Table projectionTable = isolateForQueries - ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); - IcebergSnapshotCacheValue value = loadSnapshotProjection( - dorisTable, projectionTable, - tableValue.getRetainedIcebergTable(), - tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); - if (entry.isWeightBounded()) { - value.prepareForCachePublication(key); - } - return value; - })); + IcebergSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + Table projectionTable = isolateForQueries + ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); + IcebergSnapshotCacheValue value = loadSnapshotProjection( + dorisTable, projectionTable, + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + if (entry.isWeightBounded()) { + value.prepareForCachePublication(key); + } + return value; + })); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && !tableValue.isSamePhysicalGeneration(currentTable)) { + // A query may have captured the previous table immediately before refresh publication. + // It can use that immutable value, but must not republish an unreachable old projection. + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public List getSnapshotList(ExternalTable dorisTable) { @@ -226,8 +220,31 @@ public View getIcebergView(ExternalTable dorisTable) { } public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { - IcebergSchemaCacheKey key = new IcebergSchemaCacheKey(nameMapping, schemaId); - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()).get(key); + IcebergTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable()); + } + + IcebergSchemaCacheValue getIcebergSchemaCacheValue( + NameMapping nameMapping, long schemaId, Table retainedTable) { + Optional generation = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!generation.isPresent()) { + return (IcebergSchemaCacheValue) loadSchemaCacheValue( + new IcebergSchemaCacheKey(nameMapping, schemaId), retainedTable); + } + IcebergSchemaCacheKey key = new IcebergSchemaCacheKey( + nameMapping, generation.get().getTableUuid(), schemaId); + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry + .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null) { + Optional currentGeneration = IcebergSnapshotEntryKey.tryCreate( + nameMapping, currentTable.getRetainedIcebergTable()); + if (!currentGeneration.isPresent() + || !currentGeneration.get().getTableUuid().equals(generation.get().getTableUuid())) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } + } return (IcebergSchemaCacheValue) schemaCacheValue; } @@ -244,7 +261,8 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, cacheHitRecorder.accept(hit); } return manifestEntry.get(key, - ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); + ignored -> loadManifestCacheValue( + manifest, icebergTable, key.getContent(), manifestEntry.isWeightBounded())); } @Override @@ -269,8 +287,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { IcebergMetadataOps ops = resolveMetadataOps(catalog); return executeAuthenticated(catalog, () -> { Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); - IcebergTableCacheValue value = new IcebergTableCacheValue( - table, ((ExternalCatalog) catalog).getExecutionAuthenticator()); + IcebergTableCacheValue value = new IcebergTableCacheValue(table); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); if (currentEntry != null && currentEntry.isWeightBounded()) { @@ -300,7 +317,7 @@ private View loadView(NameMapping nameMapping) { } private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFile manifest, Table icebergTable, - ManifestContent content) { + ManifestContent content, boolean accountRetainedSize) { if (manifest == null || icebergTable == null) { String manifestPath = manifest == null ? "null" : manifest.path(); throw new CacheException("Manifest cache loader context is missing for %s", @@ -308,9 +325,9 @@ private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFil } try { if (content == ManifestContent.DELETES) { - return loadDeleteFiles(manifest, icebergTable); + return loadDeleteFiles(manifest, icebergTable, accountRetainedSize); } - return loadDataFiles(manifest, icebergTable); + return loadDataFiles(manifest, icebergTable, accountRetainedSize); } catch (IOException e) { throw new CacheException("Failed to read manifest %s", e, manifest.path()); } @@ -324,6 +341,38 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } + private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + dorisTable.setUpdateTime(System.currentTimeMillis()); + boolean isView = dorisTable instanceof IcebergExternalTable + && ((IcebergExternalTable) dorisTable).isView(); + return IcebergUtils.loadSchemaCacheValue( + dorisTable, key.getSchemaId(), isView, retainedTable).orElseThrow(() -> + new CacheException("failed to load iceberg schema cache value for: %s.%s.%s, schemaId: %s", + null, key.getNameMapping().getCtlId(), key.getNameMapping().getLocalDbName(), + key.getNameMapping().getLocalTblName(), key.getSchemaId())); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable IcebergTableCacheValue previousValue, IcebergTableCacheValue currentValue) { + if (previousValue != null && previousValue.isSamePhysicalGeneration(currentValue)) { + return; + } + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.belongsTo(currentValue)); + } + Optional currentUuid = currentValue.getTableUuid(); + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.getTableUuid().equals(currentUuid)); + } + } + private IcebergSnapshotCacheValue loadSnapshotProjection( ExternalTable dorisTable, Table projectionTable, Table retainedTable, String retainedCurrentSnapshotJson, boolean isolateForQueries) { @@ -393,9 +442,10 @@ protected Map catalogPropertyCompatibilityMap() { return compatibility; } - private ManifestCacheValue loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDataFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.read(manifest, table.io())) { for (org.apache.iceberg.DataFile dataFile : reader) { builder.addDataFile(dataFile.copy()); @@ -404,9 +454,10 @@ private ManifestCacheValue loadDataFiles(org.apache.iceberg.ManifestFile manifes return builder.build(); } - private ManifestCacheValue loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDeleteFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(); + ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { for (org.apache.iceberg.DeleteFile deleteFile : reader) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java index 7c2d09511a2c93..9916d0afbb17e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java @@ -22,14 +22,26 @@ import com.google.common.base.Objects; +import java.util.Optional; + public class IcebergSchemaCacheKey extends SchemaCacheKey { + private final String tableUuid; private final long schemaId; public IcebergSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, "", schemaId); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId) { super(nameMapping); + this.tableUuid = java.util.Objects.requireNonNull(tableUuid, "tableUuid can not be null"); this.schemaId = schemaId; } + public Optional getTableUuid() { + return tableUuid.isEmpty() ? Optional.empty() : Optional.of(tableUuid); + } + public long getSchemaId() { return schemaId; } @@ -46,11 +58,11 @@ public boolean equals(Object o) { return false; } IcebergSchemaCacheKey that = (IcebergSchemaCacheKey) o; - return schemaId == that.schemaId; + return schemaId == that.schemaId && tableUuid.equals(that.tableUuid); } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableUuid, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 406d4ee11ab003..7cfc5bcc2bcd31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,27 +17,19 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import com.google.common.collect.ImmutableList; import org.apache.iceberg.BaseTable; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.HistoryEntry; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotParser; import org.apache.iceberg.SnapshotRef; -import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.CommitFailedException; @@ -138,14 +130,16 @@ public Optional
    getIcebergTable() { MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { if (sizeEstimate == null) { - if (retainedCurrentSnapshotJson == null) { - retainedCurrentSnapshotJson = icebergTable - .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); - } - retainedTablePayloadBytes = icebergTable - .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", - () -> IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this)); + () -> { + if (retainedCurrentSnapshotJson == null) { + retainedCurrentSnapshotJson = icebergTable + .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); + } + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); + return IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); if (sizeEstimate.isComplete()) { icebergTable = icebergTable.map( IcebergSnapshotCacheValue::retainNonGrowingGeneration); @@ -177,10 +171,6 @@ Optional
    getRetainedIcebergTable() { } static Table retainTableGeneration(Table table) { - return retainTableGeneration(table, null); - } - - static Table retainTableGeneration(Table table, ExecutionAuthenticator authenticator) { if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { return table; } @@ -188,46 +178,21 @@ static Table retainTableGeneration(Table table, ExecutionAuthenticator authentic // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. TableOperations frozenOperations = new FrozenTableOperations( - operations, operations.current(), authenticator); + operations, operations.current(), false); return tableWithOperations(table, frozenOperations); } static Table retainNonGrowingGeneration(Table table) { - if (!isFrozenGeneration(table)) { + if (!isFrozenGeneration(table) || isNonGrowingGeneration(table)) { return table; } TableOperations retainedOperations = ((HasTableOperations) table).operations(); - TableMetadata source = retainedOperations.current(); - if (source.schemas().isEmpty() || source.specs().isEmpty() - || source.sortOrders().isEmpty()) { - return table; - } - TableMetadata.Builder builder = TableMetadata.buildFromEmpty(source.formatVersion()); - if (source.uuid() != null) { - builder.assignUUID(source.uuid()); - } - for (Schema schema : source.schemas()) { - builder.addSchema(schema); - } - builder.setCurrentSchema(source.currentSchemaId()); - for (PartitionSpec spec : source.specs()) { - builder.addPartitionSpec(spec); - } - builder.setDefaultPartitionSpec(source.defaultSpecId()); - for (SortOrder sortOrder : source.sortOrders()) { - builder.addSortOrder(sortOrder); - } - builder.setDefaultSortOrder(source.defaultSortOrderId()); - builder.setLocation(source.location()); - builder.setProperties(source.properties()); - if (source.currentSnapshot() != null) { - builder.setBranchSnapshot( - new NonGrowingSnapshot(source.currentSnapshot()), SnapshotRef.MAIN_BRANCH); - } - TableMetadata retainedMetadata = builder.discardChanges() - .withMetadataLocation(source.metadataFileLocation()).build(); + // Do not rebuild parsed metadata with Iceberg's write-side Builder. Builder validation and + // ID reuse rules are intentionally stricter than metadata parsing and can reject legal + // upgraded tables or renumber sparse/equivalent schema histories. The frozen metadata is + // never exposed after query isolation; each caller receives exact query-local operations. return tableWithOperations(table, new FrozenTableOperations( - retainedOperations, retainedMetadata)); + retainedOperations, retainedOperations.current(), true)); } static String retainCurrentSnapshotJson(Table table) { @@ -275,8 +240,7 @@ static TableOperations unwrapRetainedTableOperations(TableOperations operations) return current; } - static Table createWritableTable( - Table retainedTable, Table liveTable, boolean reloadRetainedMetadata) { + static Table createWritableTable(Table retainedTable, Table liveTable) { if (!isFrozenGeneration(retainedTable)) { return retainedTable; } @@ -286,45 +250,18 @@ static Table createWritableTable( "Iceberg commit table must provide writable table operations"); } TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); - TableMetadata retainedMetadata = reloadRetainedMetadata - ? loadQueryMetadata(retainedOperations) : retainedOperations.current(); + TableMetadata retainedMetadata = retainedOperations.current(); TableOperations liveOperations = unwrapRetainedTableOperations( ((HasTableOperations) liveTable).operations()); return tableWithOperations(retainedTable, new WritableTableOperations(liveOperations, retainedMetadata)); } - static Table createWritableTable(Table retainedTable, Table liveTable) { - return createWritableTable(retainedTable, liveTable, - isNonGrowingGeneration(retainedTable)); - } - - private static boolean isNonGrowingGeneration(Table table) { + static boolean isNonGrowingGeneration(Table table) { return isFrozenGeneration(table) && ((FrozenTableOperations) ((HasTableOperations) table).operations()).nonGrowing; } - private static TableMetadata loadQueryMetadata(TableOperations retainedOperations) { - TableMetadata retainedMetadata = retainedOperations.current(); - TableOperations serviceOperations = unwrapRetainedTableOperations(retainedOperations); - String metadataLocation = retainedMetadata.metadataFileLocation(); - if (metadataLocation != null && !metadataLocation.isEmpty() && serviceOperations.io() != null) { - ExecutionAuthenticator authenticator = retainedOperations instanceof FrozenTableOperations - ? ((FrozenTableOperations) retainedOperations).authenticator : null; - try { - return authenticator == null - ? TableMetadataParser.read(serviceOperations.io(), metadataLocation) - : authenticator.execute( - () -> TableMetadataParser.read(serviceOperations.io(), metadataLocation)); - } catch (Exception e) { - throw new StaleMetadataException( - "Iceberg metadata generation is no longer readable: " + metadataLocation, e); - } - } - throw new IllegalStateException( - "Iceberg query-local metadata requires a stable metadata location and FileIO"); - } - private static Table tableWithOperations(Table table, TableOperations operations) { if (table instanceof BaseTable) { return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); @@ -377,26 +314,14 @@ private static class FrozenTableOperations implements TableOperations { private final FileIO fileIO; private final EncryptionManager encryptionManager; private final LocationProvider locationProvider; - private final ExecutionAuthenticator authenticator; private final boolean nonGrowing; - private FrozenTableOperations(TableOperations source, TableMetadata metadata) { - this(source, metadata, source instanceof FrozenTableOperations - ? ((FrozenTableOperations) source).authenticator : null, true); - } - - private FrozenTableOperations(TableOperations source, TableMetadata metadata, - ExecutionAuthenticator authenticator) { - this(source, metadata, authenticator, false); - } - private FrozenTableOperations(TableOperations source, TableMetadata metadata, - ExecutionAuthenticator authenticator, boolean nonGrowing) { + boolean nonGrowing) { this.metadata = metadata; this.fileIO = source.io(); this.encryptionManager = source.encryption(); this.locationProvider = source.locationProvider(); - this.authenticator = authenticator; this.nonGrowing = nonGrowing; } @@ -489,19 +414,39 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { } } + /** Query-local operations expose exact retained metadata without shared lazy table state. */ + private static final class QueryScopedTableOperations extends RetainedTableOperations { + private QueryScopedTableOperations(TableOperations retainedOperations) { + super(retainedOperations, retainedOperations.current()); + } + + @Override + public void commit(TableMetadata base, TableMetadata metadata) { + throw new UnsupportedOperationException("Query-scoped Iceberg table is read-only"); + } + } + /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ private static final class QueryScopedTable extends BaseTable { - private final TableOperations retainedOperations; + private final QueryScopedTableOperations queryOperations; private final Snapshot currentSnapshot; - private TableMetadata queryMetadata; + private final Map querySnapshots = new HashMap<>(); private QueryScopedTable(TableOperations retainedOperations, String name, org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { - super(retainedOperations, name, reporter == null + this(new QueryScopedTableOperations(retainedOperations), name, reporter, currentSnapshotJson); + } + + private QueryScopedTable(QueryScopedTableOperations queryOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + super(queryOperations, name, reporter == null ? org.apache.iceberg.metrics.LoggingMetricsReporter.instance() : reporter); - this.retainedOperations = retainedOperations; + this.queryOperations = queryOperations; this.currentSnapshot = currentSnapshotJson == null ? null : SnapshotParser.fromJson(currentSnapshotJson); + if (currentSnapshot != null) { + querySnapshots.put(currentSnapshot.snapshotId(), currentSnapshot); + } } @Override @@ -514,12 +459,16 @@ public Snapshot snapshot(long snapshotId) { if (currentSnapshot != null && currentSnapshot.snapshotId() == snapshotId) { return currentSnapshot; } - return queryMetadata().snapshot(snapshotId); + return copyForQuery(queryMetadata().snapshot(snapshotId)); } @Override public Iterable snapshots() { - return queryMetadata().snapshots(); + ImmutableList.Builder snapshots = ImmutableList.builder(); + for (Snapshot snapshot : queryMetadata().snapshots()) { + snapshots.add(copyForQuery(snapshot)); + } + return snapshots.build(); } @Override @@ -543,132 +492,15 @@ public List partitionStatisticsFiles } private synchronized TableMetadata queryMetadata() { - if (queryMetadata == null) { - queryMetadata = loadQueryMetadata(retainedOperations); - } - return queryMetadata; - } - } - - static final class StaleMetadataException extends RuntimeException { - private StaleMetadataException(String message, Throwable cause) { - super(message, cause); - } - } - - /** Scalar-only snapshot retained by the cache-owned metadata generation. */ - private static final class NonGrowingSnapshot implements Snapshot { - private final long sequenceNumber; - private final long snapshotId; - private final Long parentId; - private final long timestampMillis; - private final Integer schemaId; - private final Long firstRowId; - private final Long addedRows; - - private NonGrowingSnapshot(Snapshot snapshot) { - this.sequenceNumber = snapshot.sequenceNumber(); - this.snapshotId = snapshot.snapshotId(); - this.parentId = snapshot.parentId(); - this.timestampMillis = snapshot.timestampMillis(); - this.schemaId = snapshot.schemaId(); - this.firstRowId = snapshot.firstRowId(); - this.addedRows = snapshot.addedRows(); - } - - @Override - public long sequenceNumber() { - return sequenceNumber; - } - - @Override - public long snapshotId() { - return snapshotId; - } - - @Override - public Long parentId() { - return parentId; - } - - @Override - public long timestampMillis() { - return timestampMillis; - } - - @Override - public List allManifests(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public List dataManifests(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public List deleteManifests(FileIO fileIO) { - throw queryScopedSnapshotRequired(); + return queryOperations.current(); } - @Override - public String operation() { - return null; - } - - @Override - public Map summary() { - return Collections.emptyMap(); - } - - @Override - public Iterable addedDataFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public Iterable removedDataFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public Iterable addedDeleteFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public Iterable removedDeleteFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public String manifestListLocation() { - return null; - } - - @Override - public Integer schemaId() { - return schemaId; - } - - @Override - public Long firstRowId() { - return firstRowId; - } - - @Override - public Long addedRows() { - return addedRows; - } - - @Override - public String keyId() { - return null; - } - - private UnsupportedOperationException queryScopedSnapshotRequired() { - return new UnsupportedOperationException( - "Cache-owned Iceberg snapshots cannot materialize manifests or files"); + private synchronized Snapshot copyForQuery(Snapshot snapshot) { + if (snapshot == null) { + return null; + } + return querySnapshots.computeIfAbsent(snapshot.snapshotId(), ignored -> + SnapshotParser.fromJson(SnapshotParser.toJson(snapshot, false))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java index 782a51d5f135db..13db520e2d755d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java @@ -91,6 +91,14 @@ public int getDefaultSpecId() { return defaultSpecId; } + boolean belongsTo(IcebergTableCacheValue tableValue) { + Optional generation = tryCreate( + nameMapping, tableValue.getRetainedIcebergTable()); + return generation.isPresent() + && tableUuid.equals(generation.get().tableUuid) + && metadataFileLocation.equals(generation.get().metadataFileLocation); + } + @Override public boolean equals(Object object) { if (this == object) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index 28e45b47acd250..5bb537340ba6e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -44,9 +44,6 @@ public class IcebergSysExternalTable extends ExternalTable { private final IcebergExternalTable sourceTable; private final String sysTableType; - private volatile Table sysIcebergTable; - private volatile List fullSchema; - private volatile SchemaCacheValue schemaCacheValue; public IcebergSysExternalTable(IcebergExternalTable sourceTable, String sysTableType) { super(generateSysTableId(sourceTable.getId(), sysTableType), @@ -100,24 +97,19 @@ public boolean supportsSnapshotSelection() { } public Table getSysIcebergTable() { - if (sysIcebergTable == null) { - synchronized (this) { - if (sysIcebergTable == null) { - Table baseTable = sourceTable.getIcebergTable(); - MetadataTableType tableType = MetadataTableType.from(sysTableType); - if (tableType == null) { - throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); - } - sysIcebergTable = MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); - } - } + Table baseTable = IcebergUtils.getQueryScopedIcebergTable(sourceTable); + MetadataTableType tableType = MetadataTableType.from(sysTableType); + if (tableType == null) { + throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); } - return sysIcebergTable; + // Metadata tables capture their base operations. Keep them statement-local so exact + // previousFiles/history state and stale-generation retry never leak into this table object. + return MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); } @Override public List getFullSchema() { - return getOrCreateSchemaCacheValue().getSchema(); + return loadSchemaCacheValue().getSchema(); } @Override @@ -156,12 +148,12 @@ public long fetchRowCount() { @Override public Optional initSchema(SchemaCacheKey key) { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override public Optional getSchemaCacheValue() { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override @@ -178,19 +170,12 @@ private static long generateSysTableId(long sourceTableId, String sysTableType) return sourceTableId ^ (sysTableType.hashCode() * 31L); } - private SchemaCacheValue getOrCreateSchemaCacheValue() { - if (schemaCacheValue == null) { - synchronized (this) { - if (schemaCacheValue == null) { - if (fullSchema == null) { - fullSchema = IcebergUtils.parseSchema(getSysIcebergTable().schema(), - getCatalog().getEnableMappingVarbinary(), - getCatalog().getEnableMappingTimestampTz()); - } - schemaCacheValue = new SchemaCacheValue(fullSchema); - } - } - } - return schemaCacheValue; + private SchemaCacheValue loadSchemaCacheValue() { + // Metadata-table schemas may change after source schema or partition-spec evolution. + // Resolve the schema from the same latest-generation path instead of permanently pairing + // this long-lived system-table object with its first observed generation. + return new SchemaCacheValue(IcebergUtils.parseSchema(getSysIcebergTable().schema(), + getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 6403e6bfdbafa8..a4d4f600f66d6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -17,15 +17,19 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; + +import java.util.Objects; +import java.util.Optional; public class IcebergTableCacheValue { - private Table icebergTable; + private volatile Table icebergTable; private String retainedCurrentSnapshotJson; private volatile boolean queryIsolationPrepared; private long retainedTablePayloadBytes; @@ -35,30 +39,29 @@ public IcebergTableCacheValue(Table icebergTable) { this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); } - IcebergTableCacheValue(Table icebergTable, ExecutionAuthenticator authenticator) { - this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration( - icebergTable, authenticator); - } - public Table getIcebergTable() { - return queryIsolationPrepared + Table retainedTable = icebergTable; + return queryIsolationPrepared || IcebergSnapshotCacheValue.isNonGrowingGeneration(retainedTable) ? IcebergSnapshotCacheValue.createQueryScopedTable( - icebergTable, retainedCurrentSnapshotJson) - : icebergTable; + retainedTable, retainedCurrentSnapshotJson) + : retainedTable; } public Table getWritableIcebergTable(Table liveTable) { - return IcebergSnapshotCacheValue.createWritableTable( - icebergTable, liveTable, queryIsolationPrepared); + Table retainedTable = icebergTable; + return IcebergSnapshotCacheValue.createWritableTable(retainedTable, liveTable); } - MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { if (sizeEstimate == null) { - retainedCurrentSnapshotJson = - IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); - retainedTablePayloadBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", - () -> IcebergCacheSizeEstimator.estimateTableEntry(key, this)); + () -> { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + retainedTablePayloadBytes = + IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + return IcebergCacheSizeEstimator.estimateTableEntry(key, this); + }); if (sizeEstimate.isComplete()) { icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); queryIsolationPrepared = true; @@ -78,6 +81,12 @@ Table getRetainedIcebergTable() { synchronized Table newQueryScopedTable() { if (!queryIsolationPrepared) { + // A failed optional size preparation must only reject weighted cache admission. Do not + // repeat the same unsupported metadata access on the query path and turn it into a + // table-load failure; this value is not retained by the weighted cache in that case. + if (sizeEstimate != null && !sizeEstimate.isComplete()) { + return icebergTable; + } retainedCurrentSnapshotJson = IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); @@ -103,4 +112,27 @@ long getRetainedCurrentSnapshotPayloadBytes() { return IcebergSnapshotCacheValue.retainedSnapshotJsonBytes( retainedCurrentSnapshotJson); } + + Optional getTableUuid() { + TableMetadata metadata = retainedMetadata(); + return metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + ? Optional.empty() : Optional.of(metadata.uuid()); + } + + boolean isSamePhysicalGeneration(IcebergTableCacheValue other) { + if (other == null) { + return false; + } + TableMetadata left = retainedMetadata(); + TableMetadata right = other.retainedMetadata(); + return left != null && right != null + && Objects.equals(left.uuid(), right.uuid()) + && Objects.equals(left.metadataFileLocation(), right.metadataFileLocation()); + } + + private TableMetadata retainedMetadata() { + Table retainedTable = icebergTable; + return retainedTable instanceof HasTableOperations + ? ((HasTableOperations) retainedTable).operations().current() : null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 1c370222d71a83..bee37c0c37cea4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1058,6 +1058,10 @@ public static Table getIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTable(dorisTable); } + public static Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getQueryScopedIcebergTable(dorisTable); + } + public static Table getWritableIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); } @@ -1721,6 +1725,12 @@ public static IcebergSchemaCacheValue getSchemaCacheValue(ExternalTable dorisTab .getIcebergSchemaCacheValue(dorisTable.getOrBuildNameMapping(), schemaId); } + static IcebergSchemaCacheValue getSchemaCacheValue( + ExternalTable dorisTable, long schemaId, Table retainedTable) { + return icebergExternalMetaCache(dorisTable).getIcebergSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), schemaId, retainedTable); + } + public static IcebergSnapshot getLatestIcebergSnapshot(Table table) { Snapshot snapshot = table.currentSnapshot(); long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); @@ -1758,7 +1768,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T Map nameToPartitionItem = Maps.newHashMap(); long retainedPayloadBytes = 0L; - List partitionColumns = IcebergUtils.getSchemaCacheValue(dorisTable, schemaId).getPartitionColumns(); + List partitionColumns = IcebergUtils.getSchemaCacheValue( + dorisTable, schemaId, table).getPartitionColumns(); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( @@ -1996,7 +2007,10 @@ public int compare(Map.Entry p1, Map.Entry retainedTable = sv.getRetainedIcebergTable(); + return retainedTable.isPresent() + ? getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId(), retainedTable.get()) + : getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId()); } public static IcebergSnapshotCacheValue getLatestSnapshotCacheValue(ExternalTable dorisTable) { @@ -2065,9 +2079,16 @@ public static View getIcebergView(ExternalTable dorisTable) { public static Optional loadSchemaCacheValue( ExternalTable dorisTable, long schemaId, boolean isView) { + return loadSchemaCacheValue(dorisTable, schemaId, isView, null); + } + + public static Optional loadSchemaCacheValue( + ExternalTable dorisTable, long schemaId, boolean isView, Table retainedTable) { return isView ? loadViewSchemaCacheValue(dorisTable, schemaId) - : loadTableSchemaCacheValue(dorisTable, schemaId); + : retainedTable == null + ? loadTableSchemaCacheValue(dorisTable, schemaId) + : Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, retainedTable)); } private static Optional loadViewSchemaCacheValue(ExternalTable dorisTable, long schemaId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index 47f8c8cde3e7df..4d63af348324e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -41,14 +41,17 @@ public class ManifestCacheValue { private final long dataFileMetricEntryCount; private final long deleteFileMetricEntryCount; private final long retainedPayloadBytes; + private final boolean accountingComplete; private ManifestCacheValue(List dataFiles, List deleteFiles, - long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes) { + long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes, + boolean accountingComplete) { this.dataFiles = ImmutableList.copyOf(dataFiles); this.deleteFiles = ImmutableList.copyOf(deleteFiles); this.dataFileMetricEntryCount = dataFileMetricEntryCount; this.deleteFileMetricEntryCount = deleteFileMetricEntryCount; this.retainedPayloadBytes = retainedPayloadBytes; + this.accountingComplete = accountingComplete; } public static ManifestCacheValue forDataFiles(List dataFiles) { @@ -68,11 +71,19 @@ public static ManifestCacheValue forDeleteFiles(List deleteFiles) { } public static Builder dataFilesBuilder() { - return new Builder(true); + return dataFilesBuilder(true); + } + + public static Builder dataFilesBuilder(boolean accountRetainedSize) { + return new Builder(true, accountRetainedSize); } public static Builder deleteFilesBuilder() { - return new Builder(false); + return deleteFilesBuilder(true); + } + + public static Builder deleteFilesBuilder(boolean accountRetainedSize) { + return new Builder(false, accountRetainedSize); } public List getDataFiles() { @@ -95,16 +106,24 @@ public long getRetainedPayloadBytes() { return retainedPayloadBytes; } + public boolean isAccountingComplete() { + return accountingComplete; + } + /** Accumulates retained-size counters in the manifest reader's existing file loop. */ public static final class Builder { private final boolean dataContent; + private final boolean accountRetainedSize; private final List dataFiles = new ArrayList<>(); private final List deleteFiles = new ArrayList<>(); private long metricEntryCount; private long retainedPayloadBytes; + private boolean accountingComplete; - private Builder(boolean dataContent) { + private Builder(boolean dataContent, boolean accountRetainedSize) { this.dataContent = dataContent; + this.accountRetainedSize = accountRetainedSize; + this.accountingComplete = accountRetainedSize; } public void addDataFile(DataFile file) { @@ -112,7 +131,7 @@ public void addDataFile(DataFile file) { throw new IllegalStateException("delete manifest builder cannot accept a data file"); } dataFiles.add(file); - account(file); + accountSafely(file); } public void addDeleteFile(DeleteFile file) { @@ -120,14 +139,30 @@ public void addDeleteFile(DeleteFile file) { throw new IllegalStateException("data manifest builder cannot accept a delete file"); } deleteFiles.add(file); - account(file); + accountSafely(file); } public ManifestCacheValue build() { return new ManifestCacheValue(dataFiles, deleteFiles, dataContent ? metricEntryCount : 0L, dataContent ? 0L : metricEntryCount, - retainedPayloadBytes); + retainedPayloadBytes, accountingComplete); + } + + private void accountSafely(ContentFile file) { + if (!accountRetainedSize || !accountingComplete) { + return; + } + try { + account(file); + } catch (RuntimeException e) { + // A new or third-party ContentFile implementation must not turn optional cache + // accounting into a manifest-read failure. Keep the files for the current query + // and mark the value incomplete so weighted admission rejects it. + metricEntryCount = 0L; + retainedPayloadBytes = 0L; + accountingComplete = false; + } } private void account(ContentFile file) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 4cf326b9110b8d..906804e5307c21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -27,6 +27,8 @@ import org.apache.doris.datasource.SchemaCacheValue; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Collection; import java.util.Collections; @@ -47,6 +49,8 @@ * to initialize a catalog explicitly before accessing entries. */ public abstract class AbstractExternalMetaCache implements ExternalMetaCache { + private static final Logger LOG = LogManager.getLogger(AbstractExternalMetaCache.class); + protected static CacheSpec defaultEntryCacheSpec() { return CacheSpec.of( true, @@ -106,6 +110,40 @@ public void initCatalog(long catalogId, Map catalogProperties) { } Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); + safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( + safeCatalogProperties, engine, metaCacheEntryDefs, + warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); + try { + budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " + + "for engine {}, catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, + engine, catalogId, e.getMessage()); + } + OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { + if (entryDef.getSizeEstimator() == null) { + continue; + } + String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) + + entryDef.getName() + ".max-weight"; + if (!safeCatalogProperties.containsKey(maxWeightKey)) { + continue; + } + CacheSpec cacheSpec = CacheSpec.fromProperties( + safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); + try { + budgetManager.validateCatalogEntryHierarchy( + runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(maxWeightKey); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " + + "for engine {}, catalog {}: {}", + maxWeightKey, engine, catalogId, e.getMessage()); + } + } validateMappedCatalogProperties(safeCatalogProperties, false); catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); } @@ -368,7 +406,7 @@ private MetaCacheEntry newMetaCacheEntry( wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), cacheSpec, refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), - entryDef.getSizeEstimator(), entryBudget); + entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener()); } catch (RuntimeException | Error e) { if (entryBudget != null) { entryBudget.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 1cea5d20a3a591..34ccf41c718b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -23,12 +23,14 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; import java.util.Set; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,7 +41,8 @@ *
      *
    • enable=false disables cache
      • *
    • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
      • - *
    • capacity=0 disables cache; capacity is count-based
      • + *
    • capacity=0 disables cache; otherwise capacity is the count limit only when max-weight is absent
      • + *
    • when max-weight is present, Caffeine uses the weight limit instead of the positive capacity
      • *
    */ public final class CacheSpec { @@ -268,6 +271,31 @@ static void validateEngineProperties(Map properties, String engi validateEngineProperties(properties, engine, entryDefs.keySet(), weightedEntries); } + /** + * Ignore invalid persisted cache options during image/replay initialization. + * New CREATE/ALTER statements still use {@link #validateEngineProperties} and fail strictly. + */ + static Map sanitizeEnginePropertiesForRuntime( + Map properties, String engine, + Map> entryDefs, Consumer warningConsumer) { + Map sanitized = new HashMap<>(properties); + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + try { + validateEngineProperties(Collections.singletonMap(key, property.getValue()), engine, entryDefs); + } catch (IllegalArgumentException e) { + sanitized.remove(key); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + key + "': " + e.getMessage()); + } + } + return sanitized; + } + public static void validateEngineProperties(Map properties, String engine, Set entryNames, Set weightedEntryNames) { if (properties == null || properties.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java index 09376a1a129051..d37e91a8922019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; import java.util.Objects; @@ -27,6 +29,8 @@ * Catalog scoped entry container. */ public class CatalogEntryGroup { + private static final Logger LOG = LogManager.getLogger(CatalogEntryGroup.class); + private final Map> entries = new ConcurrentHashMap<>(); public MetaCacheEntry get(String entryName) { @@ -48,7 +52,14 @@ public void invalidateAll() { } public void close() { - entries.values().forEach(MetaCacheEntry::close); + entries.forEach((name, entry) -> { + try { + entry.close(); + } catch (RuntimeException e) { + LOG.error("Failed to close external metadata cache entry {}; continuing group retirement", + name, e); + } + }); // Keep the closed entries reachable from this retired group. A query may have captured the // group immediately before its catalog is removed; returning a closed entry lets that query // serve an uncached load instead of spuriously observing an uninitialized entry. The group diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java index a41e53bfb35b95..904a3fb0ad8f75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -19,12 +19,22 @@ import org.apache.doris.common.Config; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongUnaryOperator; +import java.util.stream.Collectors; /** * FE-wide admission accounting for managed external metadata caches. @@ -34,12 +44,20 @@ * operations while making global/catalog/entry reservation atomic. */ public final class ExternalMetaCacheBudgetManager { + private static final Logger LOG = LogManager.getLogger(ExternalMetaCacheBudgetManager.class); + private static final ExecutorService PEER_RECLAIM_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-peer-reclaim"); + thread.setDaemon(true); + return thread; + }); + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; private final Object lock = new Object(); private final OptionalLong globalMaxWeight; private final Map catalogBuckets = new HashMap<>(); private final Map entryBuckets = new HashMap<>(); + private final Map entryBudgets = new HashMap<>(); private long globalUsedWeight; private final AtomicLong globalRejectedCount = new AtomicLong(); @@ -114,8 +132,11 @@ public EntryBudget createEntryBudget(long catalogId, String engine, String entry throw new IllegalStateException("Duplicated external meta cache budget: " + scope); } Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + EntryBudget entryBudget = new EntryBudget( + this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); entryBuckets.put(scope, entryBucket); - return new EntryBudget(this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); + entryBudgets.put(scope, entryBudget); + return entryBudget; } } @@ -208,6 +229,11 @@ private void release(AdmissionReservation reservation) { if (!reservation.active) { return; } + if (reservation.entryBudget.closed) { + reservation.bytes = 0L; + reservation.active = false; + return; + } subtractUsed(reservation.entryBudget, reservation.bytes); reservation.bytes = 0L; reservation.active = false; @@ -220,10 +246,29 @@ private void close(EntryBudget entryBudget) { return; } if (entryBudget.entryBucket.usedWeight != 0L) { - throw new IllegalStateException("entry budget closed with active reservations: " + entryBudget.scope); + long leakedWeight = entryBudget.entryBucket.usedWeight; + LOG.error("Force-closing external metadata cache budget {} with {} bytes still reserved", + entryBudget.scope, leakedWeight); + if (leakedWeight <= globalUsedWeight + && leakedWeight <= entryBudget.catalogBucket.usedWeight) { + globalUsedWeight -= leakedWeight; + entryBudget.catalogBucket.usedWeight -= leakedWeight; + entryBudget.entryBucket.usedWeight = 0L; + } else { + LOG.error("External metadata cache accounting is inconsistent while closing {}; " + + "globalUsed={}, catalogUsed={}, entryUsed={}", + entryBudget.scope, globalUsedWeight, + entryBudget.catalogBucket.usedWeight, leakedWeight); + globalUsedWeight = Math.max(0L, globalUsedWeight - leakedWeight); + entryBudget.catalogBucket.usedWeight = Math.max( + 0L, entryBudget.catalogBucket.usedWeight - leakedWeight); + entryBudget.entryBucket.usedWeight = 0L; + } } entryBudget.closed = true; + entryBudget.reclaimer = null; entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); + entryBudgets.remove(entryBudget.scope, entryBudget); Bucket catalogBucket = entryBudget.catalogBucket; boolean catalogStillReferenced = entryBuckets.keySet().stream() .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); @@ -250,6 +295,123 @@ private void subtractUsed(EntryBudget entryBudget, long bytes) { entryBudget.entryBucket.usedWeight -= bytes; } + private void requestPeerReclaim(EntryBudget requester, long additionalBytes) { + if (additionalBytes <= 0L || requester.closed) { + return; + } + long reclaimBytes; + synchronized (lock) { + if (requester.closed) { + return; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + reclaimBytes = Math.max(globalDeficit, catalogDeficit); + } + if (reclaimBytes <= 0L) { + return; + } + // Rejected values are returned uncached; there is no queue of pending admissions to fund. + // Coalesce concurrent misses to the largest single admission instead of summing identical + // deficits and evicting an entire peer cache during a miss burst. + requester.requestedAdmissionBytes.accumulateAndGet(additionalBytes, Math::max); + schedulePeerReclaim(requester); + } + + private void schedulePeerReclaim(EntryBudget requester) { + if (!requester.reclaimScheduled.compareAndSet(false, true)) { + return; + } + try { + PEER_RECLAIM_EXECUTOR.execute(() -> drainPeerReclaim(requester)); + } catch (RejectedExecutionException e) { + requester.reclaimScheduled.set(false); + LOG.warn("Failed to schedule peer reclamation for external metadata cache budget {}", + requester.scope, e); + } + } + + private void drainPeerReclaim(EntryBudget requester) { + try { + long requestedAdmissionBytes = requester.requestedAdmissionBytes.getAndSet(0L); + if (requestedAdmissionBytes <= 0L || requester.closed) { + return; + } + List candidates; + synchronized (lock) { + candidates = entryBudgets.values().stream() + .filter(candidate -> candidate != requester && !candidate.closed) + .filter(candidate -> candidate.reclaimer != null) + .filter(candidate -> candidate.entryBucket.usedWeight > 0L) + .sorted((left, right) -> { + boolean leftSibling = left.scope.catalogId == requester.scope.catalogId; + boolean rightSibling = right.scope.catalogId == requester.scope.catalogId; + if (leftSibling != rightSibling) { + return leftSibling ? -1 : 1; + } + return Long.compare( + right.entryBucket.usedWeight, left.entryBucket.usedWeight); + }) + .collect(Collectors.toList()); + } + long remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + for (EntryBudget candidate : candidates) { + boolean sibling = candidate.scope.catalogId == requester.scope.catalogId; + if (!sibling && currentCatalogDeficit(requester, requestedAdmissionBytes) > 0L) { + // Another catalog cannot create headroom under the requester's catalog limit. + continue; + } + LongUnaryOperator reclaimer = candidate.reclaimer; + if (reclaimer == null || candidate.closed) { + continue; + } + try { + reclaimer.applyAsLong(remaining); + remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + } catch (RuntimeException e) { + LOG.warn("Failed to reclaim external metadata cache budget from peer {}", + candidate.scope, e); + } + if (remaining == 0L) { + break; + } + } + } finally { + requester.reclaimScheduled.set(false); + if (!requester.closed && requester.requestedAdmissionBytes.get() > 0L) { + schedulePeerReclaim(requester); + } + } + } + + private long currentReclaimDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + if (requester.closed) { + return 0L; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + return Math.max(globalDeficit, catalogDeficit); + } + } + + private long currentCatalogDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit( + requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes); + } + } + + private static long deficit(long maxWeight, long usedWeight, long additionalBytes) { + if (maxWeight == Long.MAX_VALUE || additionalBytes <= maxWeight - Math.min(usedWeight, maxWeight)) { + return 0L; + } + return MetaCacheWeightUtils.saturatedAdd(usedWeight, additionalBytes) - maxWeight; + } + private static boolean fits(long maxWeight, long usedWeight, long delta) { return delta >= 0 && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; } @@ -326,8 +488,11 @@ public static final class EntryBudget { private final Bucket entryBucket; private final long effectiveMaxWeight; private final AtomicLong rejectedCount = new AtomicLong(); - // Guarded by manager.lock. A closed handle must never re-enter accounting. - private boolean closed; + private final AtomicLong requestedAdmissionBytes = new AtomicLong(); + private final AtomicBoolean reclaimScheduled = new AtomicBoolean(); + private volatile LongUnaryOperator reclaimer; + // Mutated under manager.lock and read by asynchronous reclamation workers. + private volatile boolean closed; private EntryBudget(ExternalMetaCacheBudgetManager manager, EntryScope scope, Bucket catalogBucket, Bucket entryBucket, long effectiveMaxWeight) { @@ -342,6 +507,14 @@ public Optional tryReserve(long bytes) { return manager.tryReserve(this, bytes); } + void setReclaimer(LongUnaryOperator reclaimer) { + this.reclaimer = Objects.requireNonNull(reclaimer, "reclaimer"); + } + + void requestPeerReclaim(long additionalBytes) { + manager.requestPeerReclaim(this, additionalBytes); + } + public long getEffectiveMaxWeight() { return effectiveMaxWeight; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 01998d16f2d3ca..de89c7b9d8d6aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -60,6 +60,7 @@ public class MetaCacheEntry { private static final int LOAD_LOCK_STRIPES = 128; private static final int LOCAL_EVICTION_BATCH_SIZE = 16; private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; + private static final long WEIGHT_REJECT_LOG_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1L); // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced // generation map per physical entry after callbacks return; cleanup tasks never capture values. private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { @@ -83,9 +84,12 @@ public class MetaCacheEntry { private final MetaCacheSizeEstimator sizeEstimator; @Nullable private final EntryBudget entryBudget; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; private final boolean weightBounded; - // Estimator-backed entries use the same generation-fenced refresh protocol even before a - // max-weight is configured. This keeps event ordering stable when weight governance is toggled. + // Entries with publication-time work use the same generation-fenced refresh protocol even + // before a max-weight is configured. This keeps estimation and dependency retirement on every + // load/refresh path instead of letting Caffeine publish values behind those hooks. private final boolean generationFencedRefresh; // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. private final LoadingCache loadingData; @@ -122,6 +126,7 @@ public class MetaCacheEntry { private final AtomicLong localEvictionCount = new AtomicLong(0L); private final AtomicLong localEvictionWeight = new AtomicLong(0L); private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final AtomicLong lastWeightRejectLogTimeMs = new AtomicLong(0L); private final AtomicBoolean closed = new AtomicBoolean(false); public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { @@ -141,6 +146,14 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -158,11 +171,16 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); this.sizeEstimator = sizeEstimator; this.entryBudget = entryBudget; + this.replacementListener = replacementListener; this.weightBounded = this.cacheSpec.isWeightBounded(); - this.generationFencedRefresh = autoRefresh && sizeEstimator != null; + this.generationFencedRefresh = autoRefresh + && (sizeEstimator != null || replacementListener != null); if (weightBounded && (sizeEstimator == null || entryBudget == null)) { throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); } + if (weightBounded) { + entryBudget.setReclaimer(this::reclaimForPeer); + } this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); @@ -245,9 +263,9 @@ public V peekIfPresent(K key) { /** * Fence loads and refreshes that started before an event, but only while the expected value is - * still current. Estimator-backed entries retain the known-good value and advance the key's - * mutation epoch. Other count-based entries must invalidate because their legacy Caffeine-managed - * refresh path does not participate in that generation protocol. + * still current. Publication-managed entries retain the known-good value and advance the key's + * mutation epoch. Other count-based entries must invalidate because their legacy + * Caffeine-managed refresh path does not participate in that generation protocol. */ public boolean fenceInFlightLoadIfSame(K key, V expectedCurrent) { Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); @@ -328,6 +346,9 @@ public ReplaceResult tryReplace(K key, V expectedCurrent, V newValue) { && data.asMap().get(key) == newValue) { record.published = true; } + if (result.get() == ReplaceResult.REPLACED && data.asMap().get(key) == newValue) { + notifyReplacement(key, expectedCurrent, newValue); + } return result.get(); } } @@ -478,7 +499,8 @@ public MetaCacheEntryStats stats() { failureCount, totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - saturatedAdd(cacheStats.evictionCount(), localEvictionCount.get()), + MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionCount(), localEvictionCount.get()), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), @@ -486,7 +508,8 @@ public MetaCacheEntryStats stats() { weightBounded, weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, weightBounded ? entryBudget.getUsedWeight() : -1L, - weightBounded ? saturatedAdd(cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + weightBounded ? MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, weightBounded ? weightAdmissionRejectedCount.get() : -1L, weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, @@ -573,6 +596,7 @@ record = null; data.put(key, value); if (reservations.get(key) == newRecord && data.asMap().get(key) == value) { newRecord.published = true; + notifyReplacement(key, null, value); } return AdmissionResult.ADMITTED; } catch (RuntimeException | Error e) { @@ -604,6 +628,9 @@ record = null; if (retained && reservedWeight != newWeight && !newRecord.reservation.tryResize(newWeight)) { throw new IllegalStateException("failed to release cache replacement reservation delta"); } + if (retained) { + notifyReplacement(key, oldValue, value); + } return AdmissionResult.ADMITTED; } catch (RuntimeException | Error e) { if (reservations.replace(key, newRecord, previousRecord)) { @@ -627,6 +654,7 @@ private Optional reserveWithLocalEviction(K incomingKey, l while (!reservation.isPresent()) { int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); if (evicted == 0) { + entryBudget.requestPeerReclaim(bytes); break; } reservation = entryBudget.tryReserve(bytes); @@ -644,6 +672,7 @@ private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation rese while (true) { int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); if (evicted == 0) { + entryBudget.requestPeerReclaim(Math.max(0L, newBytes - reservation.getBytes())); return false; } if (reservation.tryResize(newBytes)) { @@ -677,6 +706,21 @@ private int evictLocalColdest(K incomingKey, int limit) { return evicted; } + private long reclaimForPeer(long targetBytes) { + if (targetBytes <= 0L || closed.get()) { + return 0L; + } + synchronized (admissionLock) { + long before = entryBudget.getUsedWeight(); + long reclaimed = 0L; + while (reclaimed < targetBytes + && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE) > 0) { + reclaimed = Math.max(0L, before - entryBudget.getUsedWeight()); + } + return reclaimed; + } + } + private int weigh(K key, V value) { ReservationRecord record = reservations.get(key); // Every supported write path installs the reservation record before calling data.put. @@ -829,6 +873,7 @@ private long currentOwnerGeneration(K key) { } private void putNonWeightedValue(K key, V value) { + V previousValue = data.asMap().get(key); RefreshRecord previous = refreshRecords.get(key); RefreshRecord next = publishRefreshRecord(key); try { @@ -837,6 +882,9 @@ private void putNonWeightedValue(K key, V value) { if (next != null && refreshRecords.get(key) == next && data.asMap().get(key) == value) { next.published = true; } + if (data.asMap().get(key) == value) { + notifyReplacement(key, previousValue, value); + } } catch (RuntimeException | Error e) { if (next != null) { if (previous == null) { @@ -859,13 +907,35 @@ private RefreshRecord publishRefreshRecord(K key) { return record; } + private void notifyReplacement(K key, @Nullable V previousValue, V currentValue) { + if (replacementListener == null || previousValue == currentValue) { + return; + } + try { + replacementListener.onReplacement(key, previousValue, currentValue); + } catch (RuntimeException e) { + LOG.warn("Failed to retire dependencies after replacing external metadata cache entry {}", name, e); + } + } + private long nextReservationGeneration() { return reservationGeneration.incrementAndGet(); } private void rejectWeight(String reason) { weightAdmissionRejectedCount.incrementAndGet(); - lastWeightRejectReason.set(reason == null || reason.isEmpty() ? "unknown" : reason); + String normalizedReason = reason == null || reason.isEmpty() ? "unknown" : reason; + lastWeightRejectReason.set(normalizedReason); + long now = System.currentTimeMillis(); + long previous = lastWeightRejectLogTimeMs.get(); + if (now - previous >= WEIGHT_REJECT_LOG_INTERVAL_MS + && lastWeightRejectLogTimeMs.compareAndSet(previous, now)) { + LOG.warn("Rejected external metadata cache admission for entry {}: reason={}, entryUsed={}, " + + "entryMax={}, catalogUsed={}, catalogMax={}, globalUsed={}, globalMax={}", + name, normalizedReason, entryBudget.getUsedWeight(), entryBudget.getEffectiveMaxWeight(), + entryBudget.getCatalogUsedWeight(), entryBudget.getCatalogMaxWeight(), + entryBudget.getGlobalUsedWeight(), entryBudget.getGlobalMaxWeight()); + } } private void maybeRefreshManagedValue(K key, V currentValue) { @@ -916,6 +986,9 @@ private void submitNonWeightedRefresh( advanceKeyMutation(key); putNonWeightedValue(key, refreshed); } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); } finally { endKeyMutation(key, expectedMutation); refreshesInFlight.remove(key); @@ -947,13 +1020,16 @@ private void submitWeightedRefresh( } V refreshed = loadAndTrack(key, this::applyDefaultLoader); if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { - AdmissionResult result = admitWeightedValue( + admitWeightedValue( key, refreshed, null, false, expectedMutation, expectedReservationGeneration, true); - if (result == AdmissionResult.REJECTED) { - invalidateKeyIfReservationGeneration(key, expectedReservationGeneration); - } + // Admission rejection leaves the already reserved, known-good generation + // in place. A larger refresh must not turn a transient quota shortage into + // a forced cache miss for every subsequent reader. } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); } finally { endKeyMutation(key, expectedMutation); refreshesInFlight.remove(key); @@ -975,21 +1051,6 @@ private boolean isReservationCurrent( && record.published && data.asMap().get(key) != null; } - private void invalidateKeyIfReservationGeneration(K key, long expectedReservationGeneration) { - synchronized (admissionLock) { - ReservationRecord record = reservations.get(key); - if (record == null || record.generation != expectedReservationGeneration) { - return; - } - V current = data.asMap().get(key); - if (current != null && data.asMap().remove(key, current)) { - advanceKeyMutation(key); - releaseReservation(key, expectedReservationGeneration); - invalidateCount.incrementAndGet(); - } - } - } - // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; @@ -1244,10 +1305,6 @@ private static ReplaceResult toReplaceResult(AdmissionResult result) { } } - private static long saturatedAdd(long left, long right) { - return left > Long.MAX_VALUE - right ? Long.MAX_VALUE : left + right; - } - private enum AdmissionResult { ADMITTED, NOT_CURRENT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 5963f3d8aa7b86..689d3b6dc7ca99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -103,10 +103,13 @@ public final class MetaCacheEntryDef { private final MetaCacheEntryInvalidation invalidation; @Nullable private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -126,6 +129,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); this.sizeEstimator = sizeEstimator; + this.replacementListener = replacementListener; } /** @@ -145,7 +149,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation, null); + invalidation, null, null); } /** @@ -167,7 +171,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation, null); + invalidation, null, null); } /** @@ -182,14 +186,22 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation, null); + invalidation, null, null); } /** Return a definition with a publication-time size estimator. */ public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - Objects.requireNonNull(estimator, "estimator")); + Objects.requireNonNull(estimator, "estimator"), replacementListener); + } + + /** Return a definition that synchronously retires dependencies after a value replacement. */ + public MetaCacheEntryDef withReplacementListener( + MetaCacheEntryReplacementListener listener) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, + Objects.requireNonNull(listener, "listener")); } /** @@ -247,4 +259,9 @@ public MetaCacheEntryInvalidation getInvalidation() { public MetaCacheSizeEstimator getSizeEstimator() { return sizeEstimator; } + + @Nullable + public MetaCacheEntryReplacementListener getReplacementListener() { + return replacementListener; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java new file mode 100644 index 00000000000000..1bfb0cf3990962 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import javax.annotation.Nullable; + +/** Receives a successfully published value while the entry mutation is still serialized. */ +@FunctionalInterface +public interface MetaCacheEntryReplacementListener { + void onReplacement(K key, @Nullable V previousValue, V currentValue); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java index f78fd5bc734b39..5aff6a2a1a279a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -23,10 +23,11 @@ /** * Supplies the admission weight of one key/value pair. * - *

    The callback runs after load and before admission. Implementations must use already available - * shape counters and constant-time collection sizes; they must not walk object graphs, perform IO, - * materialize lazy SDK state, or copy payloads. Caffeine's weigher reads only the admitted - * reservation record, so cache hits and eviction remain O(1). + *

    The callback runs once after load and before admission. Implementations may linearly count + * loader-owned collections needed to cover skewed payloads, but must not recursively reflect over + * arbitrary object graphs, perform additional IO, materialize lazy SDK state, or copy payloads + * solely to estimate weight. Caffeine's weigher reads only the admitted reservation record, so + * cache hits and eviction remain O(1). */ @FunctionalInterface public interface MetaCacheSizeEstimator { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java index b6f36b803b24cc..d3088942ace261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java @@ -44,7 +44,8 @@ public final class PaimonLatestSnapshotProjectionLoader { @FunctionalInterface public interface SchemaValueLoader { - PaimonSchemaCacheValue load(NameMapping nameMapping, long schemaId); + PaimonSchemaCacheValue load( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable); } private final PaimonPartitionInfoLoader partitionInfoLoader; @@ -59,7 +60,8 @@ public PaimonLatestSnapshotProjectionLoader(PaimonPartitionInfoLoader partitionI public PaimonSnapshotCacheValue load(NameMapping nameMapping, Table paimonTable) { try { PaimonSnapshot latestSnapshot = resolveLatestSnapshot(paimonTable, true); - List partitionColumns = schemaValueLoader.load(nameMapping, latestSnapshot.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, latestSnapshot.getSchemaId(), 0L, latestSnapshot.getTable()) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, latestSnapshot.getTable(), partitionColumns); @@ -85,11 +87,21 @@ public PaimonSnapshotCacheValue loadFence(NameMapping nameMapping, Table paimonT } public PaimonSnapshotCacheValue loadAtFence(NameMapping nameMapping, PaimonSnapshot fence) { - return loadEffectiveAtFence(nameMapping, fence.getTable(), fence); + return loadAtFence(nameMapping, fence, 0L); + } + + public PaimonSnapshotCacheValue loadAtFence( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return loadEffectiveAtFence(nameMapping, fence.getTable(), fence, tableGeneration); } public PaimonSnapshotCacheValue loadEffectiveAtFence( NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence) { + return loadEffectiveAtFence(nameMapping, effectiveTable, fence, 0L); + } + + public PaimonSnapshotCacheValue loadEffectiveAtFence( + NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence, long tableGeneration) { try { // The fence owns both version and table generation. Reopening the catalog here can pair // the old snapshot id with a newer schema or branch after invalidation. @@ -102,12 +114,14 @@ public PaimonSnapshotCacheValue loadEffectiveAtFence( latestSchemaTable.copyWithoutTimeTravel( PaimonScanParams.isolateSnapshotRead(fence.getSnapshotId()))); } - List partitionColumns = schemaValueLoader.load(nameMapping, fence.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, fence.getSchemaId(), tableGeneration, effectiveTable) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, snapshotTable, partitionColumns); return new PaimonSnapshotCacheValue(partitionInfo, - new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable)); + new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable), + false, tableGeneration); } catch (Exception e) { throw new CacheException("failed to load paimon snapshot at fence %s.%s.%s: %s", e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java index 0a134cfd7d7d32..fc9fbeaa755752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java @@ -25,6 +25,7 @@ import org.apache.paimon.table.Table; import java.io.IOException; +import java.util.concurrent.Callable; /** * Loads the base Paimon table handle used by cache entries and runtime projections. @@ -45,4 +46,14 @@ public PaimonExternalCatalog catalog(NameMapping nameMapping) throws IOException return (PaimonExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalogOrException(nameMapping.getCtlId(), id -> new IOException("Catalog not found: " + id)); } + + public T executeAuthenticated(NameMapping nameMapping, Callable task) { + try { + return catalog(nameMapping).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new CacheException("failed to load authenticated paimon metadata %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), + e.getMessage()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index ee5fdeebb2f793..a308896bb299e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; +import javax.annotation.Nullable; /** * Paimon engine implementation of {@link AbstractExternalMetaCache}. @@ -79,7 +80,8 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withReplacementListener(this::retireTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) @@ -101,45 +103,75 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); - PaimonSnapshot fence = tableValue.getLatestSnapshotFence().getSnapshot(); + PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()).getSnapshot(); PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( nameMapping, fence, tableValue.getGeneration()); MetaCacheEntry entry = snapshotEntry.get(nameMapping.getCtlId()); - return entry.get(key, ignored -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); + PaimonSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration()))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableValue.getGeneration()) { + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { - return latestSnapshotProjectionLoader.load(dorisTable.getOrBuildNameMapping(), effectiveTable); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.load(nameMapping, effectiveTable)); } public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotFence(); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, PaimonSnapshot fence) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, Table effectiveTable, PaimonSnapshot fence) { - return latestSnapshotProjectionLoader.loadEffectiveAtFence( - dorisTable.getOrBuildNameMapping(), effectiveTable, fence); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadEffectiveAtFence( + nameMapping, effectiveTable, fence)); } public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId) { - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()) - .get(new PaimonSchemaCacheKey(nameMapping, schemaId)); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getPaimonSchemaCacheValue( + nameMapping, schemaId, tableValue.getGeneration(), tableValue.getPaimonTable()); + } + + PaimonSchemaCacheValue getPaimonSchemaCacheValue( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { + PaimonSchemaCacheKey key = new PaimonSchemaCacheKey(nameMapping, tableGeneration, schemaId); + if (tableGeneration <= 0L) { + return (PaimonSchemaCacheValue) executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable)); + } + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableGeneration) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } return (PaimonSchemaCacheValue) schemaCacheValue; } private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - Table paimonTable = tableLoader.load(nameMapping); - PaimonSnapshotCacheValue fence = latestSnapshotProjectionLoader.loadFence(nameMapping, paimonTable); - return new PaimonTableCacheValue(paimonTable, fence); + return new PaimonTableCacheValue(tableLoader.load(nameMapping)); } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -150,6 +182,40 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } + private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + if (!(dorisTable instanceof PaimonExternalTable)) { + return loadSchemaCacheValue(key); + } + dorisTable.setUpdateTime(System.currentTimeMillis()); + return ((PaimonExternalTable) dorisTable).loadSchemaForCache(retainedTable, key.getSchemaId()); + } + + private PaimonSnapshotCacheValue loadLatestSnapshotFence(NameMapping nameMapping, Table retainedTable) { + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadFence(nameMapping, retainedTable)); + } + + private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent.Callable task) { + return tableLoader.executeAuthenticated(nameMapping, task); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable PaimonTableCacheValue previousValue, PaimonTableCacheValue currentValue) { + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { Map compatibility = new java.util.HashMap<>( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index d0a3c858f4b847..defa3dd4d00ec5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -490,6 +490,14 @@ private PaimonSchemaCacheValue loadSchema(DataTable table, long schemaId) { return loadSchema(table.schemaManager().schema(schemaId)); } + PaimonSchemaCacheValue loadSchemaForCache(Table retainedTable, long schemaId) { + if (!(retainedTable instanceof DataTable)) { + throw new CacheException("retained paimon table does not expose schema history: %s", + null, retainedTable == null ? "null" : retainedTable.getClass().getName()); + } + return loadSchema((DataTable) retainedTable, schemaId); + } + private PaimonSchemaCacheValue loadSchema(TableSchema tableSchema) { List columns = tableSchema.fields(); List dorisColumns = Lists.newArrayListWithCapacity(columns.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java index 4eccb269c2fe56..49d5847e0e0469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java @@ -23,13 +23,23 @@ import com.google.common.base.Objects; public class PaimonSchemaCacheKey extends SchemaCacheKey { + private final long tableGeneration; private final long schemaId; public PaimonSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, 0L, schemaId); + } + + public PaimonSchemaCacheKey(NameMapping nameMapping, long tableGeneration, long schemaId) { super(nameMapping); + this.tableGeneration = tableGeneration; this.schemaId = schemaId; } + public long getTableGeneration() { + return tableGeneration; + } + public long getSchemaId() { return schemaId; } @@ -46,11 +56,11 @@ public boolean equals(Object o) { return false; } PaimonSchemaCacheKey that = (PaimonSchemaCacheKey) o; - return schemaId == that.schemaId; + return tableGeneration == that.tableGeneration && schemaId == that.schemaId; } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableGeneration, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index 630db395afbfa3..e6b37d4c020b72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -25,18 +25,25 @@ public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; private final boolean schemaFromSnapshotTable; + private final long tableGeneration; private long retainedTablePayloadBytes; private MetaCacheSizeEstimate sizeEstimate; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { - this(partitionInfo, snapshot, false); + this(partitionInfo, snapshot, false, 0L); } public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, boolean schemaFromSnapshotTable) { + this(partitionInfo, snapshot, schemaFromSnapshotTable, 0L); + } + + public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, + boolean schemaFromSnapshotTable, long tableGeneration) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; this.schemaFromSnapshotTable = schemaFromSnapshotTable; + this.tableGeneration = tableGeneration; } public PaimonPartitionInfo getPartitionInfo() { @@ -51,15 +58,22 @@ public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } + public long getTableGeneration() { + return tableGeneration; + } + long getRetainedTablePayloadBytes() { return retainedTablePayloadBytes; } MetaCacheSizeEstimate prepareForCachePublication(PaimonSnapshotEntryKey key) { if (sizeEstimate == null) { - retainedTablePayloadBytes = PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_snapshot_preparation_failed", - () -> PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this)); + () -> { + retainedTablePayloadBytes = + PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); + return PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); } return sizeEstimate; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index e5fba223ea92b2..9e381602dcbc9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -24,22 +24,24 @@ /** * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry so this - * value cannot grow after admission through a memoized supplier. + * value cannot grow after admission. */ public class PaimonTableCacheValue { private static final AtomicLong NEXT_GENERATION = new AtomicLong(); private final Table paimonTable; - private final PaimonSnapshotCacheValue latestSnapshotFence; private final long generation; - public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue latestSnapshotFence) { + public PaimonTableCacheValue(Table paimonTable) { this.paimonTable = paimonTable; - this.latestSnapshotFence = Objects.requireNonNull( - latestSnapshotFence, "latestSnapshotFence can not be null"); this.generation = NEXT_GENERATION.incrementAndGet(); } + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue ignoredFence) { + this(paimonTable); + Objects.requireNonNull(ignoredFence, "latestSnapshotFence can not be null"); + } + public Table getPaimonTable() { return paimonTable; } @@ -48,7 +50,4 @@ public long getGeneration() { return generation; } - public PaimonSnapshotCacheValue getLatestSnapshotFence() { - return latestSnapshotFence; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java index efbe24a4910efc..1fdee2fbb0d2f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java @@ -64,6 +64,11 @@ public static PaimonSnapshotCacheValue getSnapshotCacheValue(Optional 0L) { + return paimonExternalMetaCache(dorisTable).getPaimonSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), snapshotValue.getSnapshot().getSchemaId(), + snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable()); + } return getSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index d01ec4de8863e2..607dde99476a5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -33,6 +33,7 @@ import mockit.MockUp; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.HashMap; @@ -61,20 +62,22 @@ public void testEngineAliasCompatibility() { public void testCatalogCachePropertiesRejectUnknownEngineEntryAndAliasNamespace() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); Map properties = new HashMap<>(); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); properties.put("meta.cache.hvie.partition_values.capacity", "10"); Assert.assertThrows(IllegalArgumentException.class, - () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); properties.clear(); properties.put("meta.cache.hms.partition_values.capacity", "10"); Assert.assertThrows(IllegalArgumentException.class, - () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); properties.clear(); properties.put("meta.cache.hive.partiton_values.capacity", "10"); Assert.assertThrows(IllegalArgumentException.class, - () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); } @Test @@ -90,6 +93,20 @@ public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); } + @Test + public void testRuntimePreparationIgnoresInvalidPersistedCacheProperties() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = new HashMap<>(); + properties.put("meta.cache.max-weight", "1.5GB"); + properties.put("meta.cache.hive.partition_values.enable", "1"); + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + + metaCacheMgr.prepareCatalogByEngine(101L, "hive", properties); + + Assert.assertFalse(metaCacheMgr.getCatalogCacheStats(101L).isEmpty()); + metaCacheMgr.removeCatalog(101L); + } + @Test public void testRouteByCatalogType() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); @@ -203,6 +220,29 @@ public void testCatalogRemovalFencesInFlightFirstInitialization() throws Excepti } } + @Test + public void testRollbackRetiresGroupInitializedFromRejectedCandidate() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + long catalogId = 14L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + Map oldProperties = Collections.singletonMap("generation", "old"); + + metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties); + + Mockito.verify(catalog).rollBackCatalogProps(oldProperties); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + Assert.assertEquals(1, hive.invalidateCatalogCalls); + } + @Test public void testGetSchemaCacheValueReturnsEmptyWhenCatalogMissing() throws Exception { MissingCatalogSchemaExternalMetaCache schemaCache = new MissingCatalogSchemaExternalMetaCache("default"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index cafeaf915ddaff..df24c0c754d6a2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -26,6 +26,7 @@ import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; @@ -245,6 +246,27 @@ public void testPartitionValuesEstimateSupportsRealLiteralGraph() throws Excepti "cache publication must not rewrite common catalog partition objects"); } + @Test + public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { + List types = Collections.singletonList(Type.STRING); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + HiveExternalMetaCache.HivePartitionValues empty = realPartitionValues(types, 0, 16); + HiveExternalMetaCache.HivePartitionValues populated = realPartitionValues(types, 32, 16); + HiveExternalMetaCache.HivePartitionValues shortTail = realPartitionValues(types, 1, 16); + HiveExternalMetaCache.HivePartitionValues longTail = realPartitionValues(types, 1, 4096); + + long emptyEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, empty).getBytes(); + long populatedEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, populated).getBytes(); + long shortTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, shortTail).getBytes(); + long longTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive partition values", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, @@ -291,6 +313,26 @@ private long partitionValueWeight( return estimate.getBytes(); } + private HiveExternalMetaCache.HivePartitionValues realPartitionValues( + List types, int partitionCount, int valueLength) throws Exception { + Map items = new HashMap<>(); + HashBiMap names = HashBiMap.create(); + Map> values = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + String.join("", Collections.nCopies(valueLength, "x")); + long id = index + 1L; + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + Collections.singletonList(new PartitionValue(value)), types, true); + items.put(id, new ListPartitionItem(Collections.singletonList(partitionKey))); + names.put("p=" + value, id); + values.put(id, Collections.singletonList(value)); + } + HiveExternalMetaCache.HivePartitionValues result = + new HiveExternalMetaCache.HivePartitionValues(items, names, values); + result.sealForPublication(); + return result; + } + @SuppressWarnings("unchecked") private Map sizeOnlyMap(int size) { Map map = Mockito.mock(Map.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index bc8a7cc7a74ff0..5d98c753b82463 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -24,12 +24,14 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileMetadata; @@ -48,6 +50,7 @@ import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.encryption.EncryptedKey; import org.apache.iceberg.hadoop.HadoopTables; @@ -159,6 +162,139 @@ public void testSnapshotKeyIncludesMetadataGeneration() { newInterfaceProxy(Table.class)).isPresent()); } + @Test + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v1.json")); + IcebergTableCacheValue second = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v2.json")); + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + tables.put(mapping, first); + IcebergSnapshotEntryKey oldSnapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, first.getRetainedIcebergTable()).get(); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); + IcebergSchemaCacheKey oldSchemaKey = new IcebergSchemaCacheKey( + mapping, first.getTableUuid().get(), 0L); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue oldTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-old.json")); + IcebergTableCacheValue newTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-new.json")); + cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, newTable); + IcebergSchemaCacheKey staleKey = new IcebergSchemaCacheKey( + mapping, oldTable.getTableUuid().get(), 0L); + IcebergSchemaCacheValue staleValue = new IcebergSchemaCacheValue( + Collections.emptyList(), Collections.emptyList()); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(staleKey, staleValue); + + Assert.assertSame(staleValue, cache.getIcebergSchemaCacheValue( + mapping, 0L, oldTable.getRetainedIcebergTable())); + Assert.assertNull(schemas.peekIfPresent(staleKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testFrozenGenerationPreservesSparseEquivalentSchemaIds() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/sparse.json", "{" + + "\"format-version\":2,\"table-uuid\":\"sparse-schema-table\"," + + "\"location\":\"file:/warehouse/sparse\",\"last-sequence-number\":0," + + "\"last-updated-ms\":1,\"last-column-id\":2,\"current-schema-id\":2," + + "\"schemas\":[" + + "{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}," + + "{\"type\":\"struct\",\"schema-id\":1,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}," + + "{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]}," + + "{\"type\":\"struct\",\"schema-id\":2,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":-1,\"refs\":{},\"snapshots\":[]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[],\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + TableMetadata retainedMetadata = ((HasTableOperations) retained).operations().current(); + + Assert.assertEquals(2, retainedMetadata.currentSchemaId()); + Assert.assertEquals(java.util.Arrays.asList(0, 1, 2), retainedMetadata.schemas().stream() + .map(Schema::schemaId).collect(Collectors.toList())); + Assert.assertEquals(1, retainedMetadata.schemas().stream() + .filter(schema -> schema.schemaId() == 2).findFirst().get().columns().size()); + } + + @Test + public void testFrozenGenerationAcceptsSnapshotCreatedBeforeV3Upgrade() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/upgraded-v3.json", "{" + + "\"format-version\":3,\"table-uuid\":\"upgraded-v3-table\"," + + "\"location\":\"file:/warehouse/v3\",\"last-sequence-number\":1," + + "\"last-updated-ms\":2,\"last-column-id\":1,\"current-schema-id\":0," + + "\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":7,\"next-row-id\":0," + + "\"refs\":{\"main\":{\"snapshot-id\":7,\"type\":\"branch\"}}," + + "\"snapshots\":[{\"sequence-number\":0,\"snapshot-id\":7," + + "\"timestamp-ms\":1,\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[],\"schema-id\":0}]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[{\"timestamp-ms\":1,\"snapshot-id\":7}]," + + "\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + + Assert.assertEquals(7L, retained.currentSnapshot().snapshotId()); + Assert.assertEquals(3, ((HasTableOperations) retained).operations().current().formatVersion()); + } + @Test public void testTableSnapshotAndManifestEstimatesArePrecomputed() { NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); @@ -192,6 +328,51 @@ public void testTableSnapshotAndManifestEstimatesArePrecomputed() { Assert.assertTrue(unsupported.getIncompleteReason().startsWith("unsupported_iceberg_table:")); } + @Test + public void testIcebergPreparationFailureIsFailClosed() { + TableMetadata brokenMetadata = Mockito.mock(TableMetadata.class); + Mockito.when(brokenMetadata.currentSnapshot()) + .thenThrow(new IllegalStateException("unsupported snapshot state")); + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(brokenMetadata); + Table brokenTable = new BaseTable(operations, "db.tbl"); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(brokenTable); + MetaCacheSizeEstimate tableEstimate = tableValue.prepareForCachePublication(mapping); + + Assert.assertFalse(tableEstimate.isComplete()); + Assert.assertTrue(tableEstimate.getIncompleteReason() + .startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(tableValue.getRetainedIcebergTable(), tableValue.newQueryScopedTable()); + + Table healthyTable = tableWithMetadataLocation("/metadata/fail-closed-key.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, healthyTable).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), brokenTable); + MetaCacheSizeEstimate snapshotEstimate = snapshotValue.prepareForCachePublication(key); + + Assert.assertFalse(snapshotEstimate.isComplete()); + Assert.assertTrue(snapshotEstimate.getIncompleteReason() + .startsWith("iceberg_snapshot_preparation_failed:")); + } + + @Test + public void testManifestAccountingFailureKeepsFilesAndRejectsWeightedAdmission() { + DataFile file = Mockito.mock(DataFile.class); + Mockito.when(file.columnSizes()).thenThrow(new IllegalStateException("new metrics representation")); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles(Collections.singletonList(file)); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/fail-closed.avro", ManifestContent.DATA), value); + + Assert.assertEquals(Collections.singletonList(file), value.getDataFiles()); + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(estimate.isComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", estimate.getIncompleteReason()); + } + @Test public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { String largePayload = repeatedCharacter('x', 64 * 1024); @@ -262,18 +443,18 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { } @Test - public void testTablePayloadExcludesQueryLocalHistoricalMetadata() { + public void testTablePayloadAccountsForRetainedHistoricalMetadata() { String largePayload = repeatedCharacter('x', 64 * 1024); long smallBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(metadataWithMaterializedPayload("x", 32))); long largeBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(metadataWithMaterializedPayload(largePayload, 64 * 1024))); - Assert.assertEquals(smallBytes, largeBytes); + Assert.assertTrue(largeBytes - smallBytes >= 64L * 1024L - 32L); } @Test - public void testTableEstimateExcludesQueryLocalBranchHistory() { + public void testTableEstimateAccountsForRetainedBranchHistory() { TableMetadata oneCommit = metadataWithSnapshotSequence(1L); TableMetadata tenThousandCommits = metadataWithSnapshotSequence(10_000L); IcebergTableCacheValue smallValue = new IcebergTableCacheValue(tableWithMetadata(oneCommit)); @@ -290,14 +471,14 @@ public void testTableEstimateExcludesQueryLocalBranchHistory() { largeValue.getSizeEstimate().isComplete()); Assert.assertEquals(smallValue.getSizeEstimate().getBytes(), largeValue.getSizeEstimate().getBytes()); - Mockito.verify(oneCommit, Mockito.never()).snapshots(); - Mockito.verify(tenThousandCommits, Mockito.never()).snapshots(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshots(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshots(); Mockito.verify(oneCommit, Mockito.never()).lastSequenceNumber(); Mockito.verify(tenThousandCommits, Mockito.never()).lastSequenceNumber(); - Mockito.verify(oneCommit, Mockito.never()).snapshotLog(); - Mockito.verify(tenThousandCommits, Mockito.never()).snapshotLog(); - Mockito.verify(oneCommit, Mockito.never()).refs(); - Mockito.verify(tenThousandCommits, Mockito.never()).refs(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).refs(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).refs(); } @Test @@ -360,37 +541,24 @@ MetaCacheSizeEstimate prepareTableForCachePublication( } @Test - public void testExactMetadataReadsRunInsideCatalogAuthenticator() throws Exception { + public void testQueryScopedMetadataReusesFrozenGenerationWithoutFileIo() throws Exception { String tableLocation = temporaryFolder.newFolder("authenticated-metadata").toURI().toString(); Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); Table liveTable = new HadoopTables(new Configuration()).create( schema, PartitionSpec.unpartitioned(), tableLocation); - AtomicBoolean authenticated = new AtomicBoolean(); AtomicInteger metadataReads = new AtomicInteger(); - ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { - @Override - public T execute(Callable task) throws Exception { - Assert.assertTrue(authenticated.compareAndSet(false, true)); - try { - return task.call(); - } finally { - authenticated.set(false); - } - } - }; FileIO trackingFileIO = Mockito.mock(FileIO.class); Mockito.when(trackingFileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { - Assert.assertTrue("metadata FileIO must retain catalog authentication", authenticated.get()); metadataReads.incrementAndGet(); return liveTable.io().newInputFile((String) invocation.getArgument(0)); }); TableMetadata metadata = ((HasTableOperations) liveTable).operations().current(); Table trackedTable = new BaseTable( new StaticTableOperations(metadata, trackingFileIO), liveTable.name()); - IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable, authenticator); + IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable); countValue.getWritableIcebergTable(liveTable); Assert.assertEquals("count-based writes must not add metadata FileIO", 0, metadataReads.get()); - IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable, authenticator); + IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable); value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); Table statementTable = value.newQueryScopedTable(); @@ -398,14 +566,14 @@ public T execute(Callable task) throws Exception { IcebergSnapshotCacheValue statementValue = new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), statementTable); - Assert.assertSame(statementTable, statementValue.getIcebergTable().get()); + Assert.assertEquals(statementTable.schema().asStruct(), + statementValue.getIcebergTable().get().schema().asStruct()); com.google.common.collect.Lists.newArrayList( statementValue.getIcebergTable().get().snapshots()); - Assert.assertEquals("statement handoff must reuse parsed metadata", 1, metadataReads.get()); + Assert.assertEquals("statement handoff must reuse frozen metadata", 0, metadataReads.get()); value.getWritableIcebergTable(liveTable); - Assert.assertEquals(2, metadataReads.get()); - Assert.assertFalse(authenticated.get()); + Assert.assertEquals(0, metadataReads.get()); } @Test @@ -474,7 +642,7 @@ public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws } @Test - public void testMissingPinnedMetadataRefreshesBeforeStatementFence() throws Exception { + public void testPinnedGenerationSurvivesMetadataFileRetirement() throws Exception { String staleLocation = temporaryFolder.newFolder("stale-metadata").toURI().toString(); String freshLocation = temporaryFolder.newFolder("fresh-metadata").toURI().toString(); Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); @@ -511,8 +679,8 @@ protected CatalogIf getCatalog(long catalogId) { Table queryTable = cache.getQueryScopedIcebergTable(table); - Assert.assertEquals(freshTable.schema().asStruct(), queryTable.schema().asStruct()); - Mockito.verify(metadataOps, Mockito.times(1)).loadTable("db", "tbl"); + Assert.assertEquals(staleTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.never()).loadTable("db", "tbl"); } finally { cache.close(); executor.shutdownNow(); @@ -546,8 +714,6 @@ public void testWeightedTablePublicationRetainsNonGrowingGeneration() { Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); Table retained = value.getRetainedIcebergTable(); Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); - Assert.assertThrows(UnsupportedOperationException.class, - () -> retained.snapshot(7L).dataManifests(retained.io())); Table firstUse = value.getIcebergTable(); Table secondUse = value.getIcebergTable(); Assert.assertNotSame(retained, firstUse); @@ -556,7 +722,6 @@ public void testWeightedTablePublicationRetainsNonGrowingGeneration() { Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); - Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(firstUse)); IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); @@ -591,8 +756,6 @@ public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Excep Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); Table retained = value.getRetainedIcebergTable(); - Assert.assertThrows(UnsupportedOperationException.class, - () -> retained.currentSnapshot().dataManifests(retained.io())); Table firstQuery = value.getIcebergTable(); Table secondQuery = value.getIcebergTable(); List firstManifests = @@ -603,8 +766,7 @@ public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Excep Assert.assertEquals(1, secondManifests.size()); Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); Assert.assertNotSame(firstManifests, secondManifests); - Assert.assertThrows(UnsupportedOperationException.class, - () -> retained.currentSnapshot().dataManifests(retained.io())); + Assert.assertNotSame(retained.currentSnapshot(), firstQuery.currentSnapshot()); } @Test @@ -646,6 +808,30 @@ public void testManifestEstimateScalesWithFileCount() { Assert.assertTrue(twoFileBytes > oneFileBytes); } + @Test + public void testManifestFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol.avro", ManifestContent.DATA); + ManifestCacheValue empty = ManifestCacheValue.forDataFiles(Collections.emptyList()); + ManifestCacheValue populated = ManifestCacheValue.forDataFiles( + IntStream.range(0, 32).mapToObj(this::dataFileWithMetrics) + .collect(Collectors.toList())); + ManifestCacheValue shortTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(16))); + ManifestCacheValue longTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(4096))); + + long emptyEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, empty).getBytes(); + long populatedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, populated).getBytes(); + long shortTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, shortTail).getBytes(); + long longTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest files", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg long-tail path", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + @Test public void testManifestEstimateAccountsForSkewedFilePaths() { String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; @@ -776,7 +962,6 @@ public void testSnapshotPublicationDoesNotMaterializeManifestLists() { value.getSizeEstimate().isComplete()); Table queryTable = value.getIcebergTable().get(); Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); - Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(queryTable)); Mockito.verifyNoInteractions(fileIO); } @@ -993,6 +1178,7 @@ private long manifestWeight(IcebergManifestEntryKey key, int fileCount) { List dataFiles = sizeOnlyList(fileCount); Mockito.when(value.getDataFiles()).thenReturn(dataFiles); Mockito.when(value.getDeleteFiles()).thenReturn(Collections.emptyList()); + Mockito.when(value.isAccountingComplete()).thenReturn(true); MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, value); Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); return estimate.getBytes(); @@ -1034,6 +1220,33 @@ private Table tableWithMetadataLocation(String metadataLocation) { return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); } + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { + Map columnSizes = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> (long) index + column)); + Map valueCounts = new java.util.HashMap<>(columnSizes); + Map nullCounts = new java.util.HashMap<>(columnSizes); + Map nanCounts = new java.util.HashMap<>(columnSizes); + Map lowerBounds = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Map upperBounds = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build(); + } + + private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/" + repeatedCharacter('x', pathLength) + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .build(); + } + private Table tableWithMetadata(TableMetadata metadata) { TableOperations operations = Mockito.mock(TableOperations.class); Mockito.when(operations.current()).thenReturn(metadata); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 03d217c25d16e5..18969f0a535ebb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -18,6 +18,9 @@ package org.apache.doris.datasource.iceberg; import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -45,4 +48,29 @@ public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { sourceTable, MetadataTableType.DATA_FILES.name()); Assertions.assertTrue(dataFiles.supportsSnapshotSelection()); } + + @Test + public void testMetadataSchemaReloadsAfterSourceEvolution() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table firstGeneration = Mockito.mock(Table.class); + Table evolvedGeneration = Mockito.mock(Table.class); + Mockito.when(firstGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()))); + Mockito.when(evolvedGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.optional(2, "evolved_partition", Types.StringType.get()))); + IcebergSysExternalTable sysTable = Mockito.spy(new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name())); + Mockito.doReturn(firstGeneration, evolvedGeneration).when(sysTable).getSysIcebergTable(); + + Assertions.assertEquals(1, sysTable.getFullSchema().size()); + Assertions.assertEquals(2, sysTable.getFullSchema().size()); + Mockito.verify(sysTable, Mockito.times(2)).getSysIcebergTable(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 704c82df183e3b..3decfbef1b0483 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -178,24 +178,25 @@ public void testGlobalWeightAutomaticallyActivatesEntriesWithEstimator() { } @Test - public void testEntryWeightWithoutEstimatorFailsCatalogInitialization() { + public void testRuntimeInitIgnoresEntryWeightWithoutEstimator() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); try { TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); Map properties = Maps.newHashMap(); properties.put("meta.cache.test_engine.schema.max-weight", "1KB"); - IllegalArgumentException exception = Assert.assertThrows( - IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); - Assert.assertTrue(exception.getMessage().contains("does not support max-weight")); - Assert.assertFalse(cache.isCatalogInitialized(1L)); + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertFalse(cache.entry(1L, "schema", SchemaCacheKey.class, SchemaCacheValue.class) + .isWeightBounded()); } finally { refreshExecutor.shutdownNow(); } } @Test - public void testCommonValidationRejectsEntryWeightAboveCatalogWeight() { + public void testRuntimeInitIgnoresEntryWeightAboveCatalogWeight() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); WeightedExternalMetaCache cache = new WeightedExternalMetaCache( refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(4L * 1024L))); @@ -204,10 +205,11 @@ public void testCommonValidationRejectsEntryWeightAboveCatalogWeight() { properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"); properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); - IllegalArgumentException exception = Assert.assertThrows( - IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); - Assert.assertTrue(exception.getMessage().contains("entry max weight")); - Assert.assertFalse(cache.isCatalogInitialized(1L)); + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertEquals(1024L, cache.entry(1L, "value", String.class, Integer.class) + .stats().getMaxWeight()); } finally { cache.close(); refreshExecutor.shutdownNow(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java new file mode 100644 index 00000000000000..be4978d33c129d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.junit.Assert; +import org.openjdk.jol.info.GraphLayout; + +/** JOL oracle used only by estimator calibration tests. */ +public final class EstimatorCalibrationAssertions { + private static final long MAX_CONSERVATIVE_FACTOR = 8L; + private static final boolean PRINT_RESULT = Boolean.getBoolean( + "metacache.estimator.calibration.print"); + + static { + // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their + // offsets through the regular instrumentation path on JDK 17, so enable its Unsafe + // fallback for these test-only retained-graph measurements. Skip all attach attempts: + // Iceberg/Paimon calibration tests share their fork with Mockito's inline mock maker. + System.setProperty("jol.magicFieldOffset", "true"); + System.setProperty("jol.skipInstallAttach", "true"); + System.setProperty("jol.skipDynamicAttach", "true"); + System.setProperty("jol.skipHotspotSAAttach", "true"); + } + + private EstimatorCalibrationAssertions() { + } + + public static void assertConservativeDelta( + String fixture, long emptyEstimate, long populatedEstimate, + Object emptyGraph, Object populatedGraph) { + long actualDelta = GraphLayout.parseInstance(populatedGraph).totalSize() + - GraphLayout.parseInstance(emptyGraph).totalSize(); + long estimatedDelta = populatedEstimate - emptyEstimate; + if (PRINT_RESULT) { + System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", + fixture, estimatedDelta, actualDelta, + actualDelta == 0L ? Double.NaN : (double) estimatedDelta / actualDelta); + } + Assert.assertTrue(fixture + " must add retained heap", actualDelta > 0L); + Assert.assertTrue(fixture + " underestimates retained heap: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta >= actualDelta); + Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta <= MetaCacheWeightUtils.saturatedMultiply( + actualDelta, MAX_CONSERVATIVE_FACTOR)); + } + + public static long graphSize(Object graph) { + return GraphLayout.parseInstance(graph).totalSize(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java index 4d35dd15fd63b4..d4b76b97c0131e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -35,6 +35,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class ExternalMetaCacheBudgetManagerTest { @@ -176,6 +177,105 @@ public void testClosedBudgetRejectsStaleHandleAndReservationResize() { Assert.assertEquals(0L, manager.getGlobalUsedWeight()); } + @Test + public void testCloseForceReleasesOutstandingAccounting() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation staleReservation = staleBudget.tryReserve(40L).get(); + + staleBudget.close(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + staleReservation.release(); + Assert.assertFalse(staleReservation.isActive()); + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + replacement.close(); + } + + @Test + public void testPeerReclaimCoalescesConcurrentMissesToLargestAdmission() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget owner = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.empty(), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 2L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = owner.tryReserve(100L).get(); + CountDownLatch firstReclaimStarted = new CountDownLatch(1); + CountDownLatch releaseFirstReclaim = new CountDownLatch(1); + CountDownLatch secondReclaimFinished = new CountDownLatch(1); + AtomicInteger invocation = new AtomicInteger(); + List targets = Collections.synchronizedList(new ArrayList<>()); + owner.setReclaimer(target -> { + targets.add(target); + if (invocation.getAndIncrement() == 0) { + firstReclaimStarted.countDown(); + await(releaseFirstReclaim); + } else { + secondReclaimFinished.countDown(); + } + return 0L; + }); + try { + requester.requestPeerReclaim(10L); + Assert.assertTrue(firstReclaimStarted.await(3L, TimeUnit.SECONDS)); + + requester.requestPeerReclaim(10L); + requester.requestPeerReclaim(20L); + requester.requestPeerReclaim(15L); + releaseFirstReclaim.countDown(); + + Assert.assertTrue(secondReclaimFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(2, targets.size()); + Assert.assertEquals(Long.valueOf(10L), targets.get(0)); + Assert.assertEquals(Long.valueOf(20L), targets.get(1)); + } finally { + releaseFirstReclaim.countDown(); + reservation.release(); + owner.close(); + requester.close(); + } + } + + @Test + public void testCatalogOnlyDeficitReclaimsSiblingWithoutTouchingOtherCatalog() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(200L); + EntryBudget sibling = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget otherCatalog = manager.createEntryBudget( + 2L, "paimon", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + AdmissionReservation siblingReservation = sibling.tryReserve(100L).get(); + AdmissionReservation otherReservation = otherCatalog.tryReserve(50L).get(); + CountDownLatch siblingReclaimed = new CountDownLatch(1); + AtomicInteger otherCatalogReclaims = new AtomicInteger(); + sibling.setReclaimer(target -> { + siblingReservation.release(); + siblingReclaimed.countDown(); + return 100L; + }); + otherCatalog.setReclaimer(target -> { + otherCatalogReclaims.incrementAndGet(); + return 0L; + }); + try { + requester.requestPeerReclaim(20L); + + Assert.assertTrue(siblingReclaimed.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, otherCatalogReclaims.get()); + Assert.assertEquals(0L, sibling.getUsedWeight()); + Assert.assertEquals(50L, manager.getGlobalUsedWeight()); + } finally { + siblingReservation.release(); + otherReservation.release(); + sibling.close(); + requester.close(); + otherCatalog.close(); + } + } + private static ExternalMetaCacheBudgetManager manager(long maxWeight) { return new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index fa013d58f4a653..41d575417654e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class MetaCacheEntryTest { @@ -392,6 +393,73 @@ public void testWeightedReplacementDoesNotQueueOldValuesOnRefreshExecutor() thro } } + @Test + public void testWeightedFirstPublicationInvokesReplacementListener() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "publication", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger publications = new AtomicInteger(); + AtomicReference previous = new AtomicReference<>(); + byte[] value = new byte[10]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> value, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, loaded) -> MetaCacheSizeEstimate.complete(loaded.length), budget, + (key, oldValue, currentValue) -> { + publications.incrementAndGet(); + previous.set(oldValue); + Assert.assertSame(value, currentValue); + }); + try { + entry.put("k", value); + + Assert.assertEquals(1, publications.get()); + Assert.assertNull(previous.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCountEntryLoadAndRefreshInvokeReplacementListener() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loads = new AtomicInteger(); + AtomicInteger publications = new AtomicInteger(); + AtomicReference refreshPrevious = new AtomicReference<>(); + AtomicReference refreshCurrent = new AtomicReference<>(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> loads.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, null, null, + (key, previousValue, currentValue) -> { + publications.incrementAndGet(); + if (previousValue != null) { + refreshPrevious.set(previousValue); + refreshCurrent.set(currentValue); + } + }); + try { + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + Assert.assertEquals(1, publications.get()); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + Assert.assertEquals(2, loads.get()); + Assert.assertEquals(2, publications.get()); + Assert.assertEquals(Integer.valueOf(1), refreshPrevious.get()); + Assert.assertEquals(Integer.valueOf(2), refreshCurrent.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testQueuedWeightedRefreshDoesNotCaptureCurrentValue() throws Exception { CountDownLatch workerBlocked = new CountDownLatch(1); @@ -1580,6 +1648,98 @@ public void testRejectedAtomicReplacementKeepsExpectedValueUntilConditionalInval } } + @Test + public void testRejectedWeightedRefreshRetainsPreviousValue() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(600L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-reject", OptionalLong.empty(), OptionalLong.empty()); + byte[] current = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-reject", key -> new byte[100], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRefreshFailureRetainsPreviousValueAndExecutorThread() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + String current = new String("current"); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-failure", key -> { + throw new IllegalStateException("temporary metastore failure"); + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + AtomicBoolean executorStillAlive = new AtomicBoolean(); + refreshExecutor.submit(() -> executorStillAlive.set(true)).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(executorStillAlive.get()); + Assert.assertEquals(1L, entry.stats().getLoadFailureCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testPeerReclamationPreventsGlobalBudgetStarvation() throws Exception { + long valueWeight = accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(2L * valueWeight)); + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager.EntryBudget firstBudget = manager.createEntryBudget( + 1L, "test", "first", OptionalLong.empty(), OptionalLong.empty()); + ExternalMetaCacheBudgetManager.EntryBudget secondBudget = manager.createEntryBudget( + 2L, "test", "second", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry first = new MetaCacheEntry<>( + "first", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), firstBudget); + MetaCacheEntry second = new MetaCacheEntry<>( + "second", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), secondBudget); + try { + first.put("a", new byte[1]); + first.put("b", new byte[1]); + second.put("c", new byte[1]); + Assert.assertNull(second.peekIfPresent("c")); + + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() > valueWeight && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + second.put("c", new byte[1]); + + Assert.assertNotNull(second.peekIfPresent("c")); + Assert.assertTrue(manager.getGlobalUsedWeight() <= 2L * valueWeight); + } finally { + first.close(); + second.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testDisabledWeightedEntrySkipsEstimator() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 066905dbfaa6ae..fa016361c9ad3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -21,12 +21,14 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; @@ -51,6 +53,8 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.StreamTableCommit; import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.TableScan; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; @@ -64,12 +68,15 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; public class PaimonExternalMetaCacheTest { @Rule @@ -146,6 +153,28 @@ public void testSnapshotWeightAccountsForTableComment() throws Exception { Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); } + @Test + public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable table = newPartitionedTable("jol_snapshot", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16); + PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions(table, 32, 16); + PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions(table, 1, 16); + PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions(table, 1, 4096); + + long emptyEstimate = empty.prepareForCachePublication(key).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(key).getBytes(); + long shortTailEstimate = shortTail.prepareForCachePublication(key).getBytes(); + long longTailEstimate = longTail.prepareForCachePublication(key).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon snapshot partitions", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + @Test public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -198,18 +227,84 @@ public void testSnapshotKeySeparatesReloadedTableGenerations() { } @Test - public void testSnapshotHitReusesFenceCapturedByTableGeneration() { + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { ExecutorService executor = Executors.newSingleThreadExecutor(); PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(Mockito.mock(Table.class)); + PaimonTableCacheValue second = new PaimonTableCacheValue(Mockito.mock(Table.class)); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + PaimonSnapshotEntryKey oldSnapshotKey = new PaimonSnapshotEntryKey( + mapping, 1L, 2L, first.getGeneration()); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, first.getPaimonTable()))); + PaimonSchemaCacheKey oldSchemaKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 2L); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotHitRefreshesFenceWithoutReloadingProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); long catalogId = 1L; cache.initCatalog(catalogId, Collections.emptyMap()); NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "remote_db", "remote_tbl"); FileStoreTable table = Mockito.mock(FileStoreTable.class); - PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, table); + FileStoreTable pinnedTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + Mockito.when(table.copyWithLatestSchema()).thenReturn(table); + Mockito.when(table.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(table.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(table.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(pinnedTable); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, pinnedTable); PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( PaimonPartitionInfo.EMPTY, fence); - PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table, snapshotValue); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table); PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( mapping, fence, tableValue.getGeneration()); cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, @@ -221,10 +316,119 @@ public void testSnapshotHitReusesFenceCapturedByTableGeneration() { Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); - Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); - Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + + Mockito.verify(table, Mockito.times(4)).copyWithLatestSchema(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testContextualSnapshotAndSchemaMissesRunAuthenticated() { + AtomicInteger authenticationDepth = new AtomicInteger(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + authenticationDepth.incrementAndGet(); + try { + return task.call(); + } finally { + authenticationDepth.decrementAndGet(); + } + } + }; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Mockito.doAnswer(invocation -> { + Assert.assertTrue("schema history must be read under authentication", + authenticationDepth.get() > 0); + Column partitionColumn = new Column("part", Type.INT); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); - Mockito.verifyNoInteractions(table); + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenAnswer(invocation -> { + Assert.assertTrue("snapshot fence must be read under authentication", + authenticationDepth.get() > 0); + return latestSchemaTable; + }); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenAnswer(invocation -> { + Assert.assertTrue("snapshot pinning must run under authentication", + authenticationDepth.get() > 0); + return snapshotTable; + }); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenAnswer(invocation -> { + Assert.assertTrue("partition enumeration must run under authentication", + authenticationDepth.get() > 0); + return readBuilder; + }); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenAnswer(invocation -> { + Assert.assertTrue("partition manifest access must run under authentication", + authenticationDepth.get() > 0); + return Collections.emptyList(); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(baseTable); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + PaimonSnapshotCacheValue snapshot = cache.getSnapshotCache(dorisTable); + + Assert.assertEquals(7L, snapshot.getSnapshot().getSnapshotId()); + Assert.assertEquals(0, authenticationDepth.get()); + + PaimonTableCacheValue second = new PaimonTableCacheValue(baseTable); + tables.put(mapping, second); + PaimonSchemaCacheKey staleKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 99L); + cache.getPaimonSchemaCacheValue(mapping, 99L, first.getGeneration(), baseTable); + Assert.assertNull("a concurrent old-generation schema load must not repopulate the cache", + cache.entry(1L, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class).peekIfPresent(staleKey)); + Assert.assertEquals(0, authenticationDepth.get()); } finally { cache.close(); executor.shutdownNow(); @@ -313,7 +517,7 @@ public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -353,7 +557,7 @@ public void testFullLatestProjectionCapsManifestParallelismBeforePartitionLoad() .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -393,7 +597,7 @@ public void testLatestFenceDoesNotLoadSchemaOrPartitions() { PaimonPartitionInfoLoader partitionLoader = Mockito.mock(PaimonPartitionInfoLoader.class); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> { + (nameMapping, schemaId, tableGeneration, retainedTable) -> { throw new AssertionError("a version-only fence must not load schema metadata"); }); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); @@ -427,7 +631,7 @@ public void testFenceHydrationKeepsCapturedTableGeneration() throws Exception { .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable captured = Mockito.mock(FileStoreTable.class); @@ -464,7 +668,7 @@ public void testTagProjectionKeepsOnlyRepinnedSnapshotSelector() throws Exceptio table, Collections.singletonMap(CoreOptions.SCAN_TAG_NAME.key(), "stable")); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); PaimonSnapshotCacheValue value = loader.load( @@ -616,6 +820,24 @@ private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, in return estimate.getBytes(); } + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength) { + Map partitionItems = new HashMap<>(); + Map partitions = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + repeatedCharacter('x', valueLength); + String name = "part=" + value; + partitionItems.put(name, new org.apache.doris.catalog.ListPartitionItem( + new ArrayList<>())); + partitions.put(name, new org.apache.paimon.partition.Partition( + Collections.singletonMap("part", value), + 100L, 1024L, 1L, 1L, 1, true)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(partitionItems, partitions); + return new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + } + @SuppressWarnings("unchecked") private Map sizeOnlyMap(int size) { Map map = Mockito.mock(Map.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java index a0f01f25cdfaeb..7f5483928c8a97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -107,7 +107,7 @@ public void testStatementContextDefersPhysicalManifestValidationUntilRelationOpt PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( Mockito.mock(PaimonPartitionInfoLoader.class), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "db", "table"); Mockito.doAnswer(ignored -> new PaimonMvccSnapshot( diff --git a/fe/pom.xml b/fe/pom.xml index 6821b6c3fce2ab..f709024e526e32 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -272,6 +272,7 @@ under the License. 3.1.0 18.3.14-doris-SNAPSHOT 1.49 + 0.17 2.18.0 1.11.0 1.1.1 @@ -1931,6 +1932,11 @@ under the License. mockito-inline ${mockito.version} + + org.openjdk.jol + jol-core + ${jol.version} + it.unimi.dsi fastutil-core diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 2e2a2ea8e9b5c9..6079a17ec402ca 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -28,8 +28,24 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" String warehouse = "${default_fs}/warehouse" - // 1. test default catalog + // DDL validation must reject misspelled memory-governance options. sql """drop catalog if exists ${catalog_name};""" + test { + sql """ + create catalog ${catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'meta.cache.iceberg.snapshot.max-weigth' = '16MB' + ); + """ + exception "Unknown external meta cache" + } + + // 1. test a catalog-level memory bound without a global bound. The existing + // create/insert/select/refresh flow below is the weighted-cache happy path. sql """ create catalog ${catalog_name} properties ( 'type'='iceberg', @@ -37,6 +53,7 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', 'fs.defaultFS' = '${default_fs}', 'warehouse' = '${warehouse}', + 'meta.cache.max-weight' = '128MB', 'meta.cache.iceberg.manifest.enable' = 'true' ); """ From 6443730e545dd58c4b8e32e7bcdc53bed106f409 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 17 Aug 2026 20:33:26 +0800 Subject: [PATCH 3/5] [fix](fe) Make external metadata cache size estimators structural and fail closed on SDK layout changes Replace calibrated slope constants in the Iceberg and Paimon cache size estimators with formulas derived from the active JVM object layout and the retained SDK structures. Iceberg: - Account Schema lookup indexes (idToName/nameToId/idToField/lowerCaseNameToId/idToAccessor), root StructType indexes and the StructType.asSchema() secondary graph structurally, including canonical/short-alias path Strings, lower-cased copies per index (Unicode aware), boxed ids and per-level accessors, for flat and nested schemas. - Account PartitionSpec lazy state (fieldList, javaClasses, partitionType, secondary schema) and reserve the O(distinctSources * fields) fieldsBySourceId graph in O(fields) work without materializing it; account SortOrder fields, transforms and identifier field sets. - Add a character budget beside the element budget, refuse lazily-loaded snapshot suppliers, pin the full instance-field layouts of the retained Iceberg classes and only account GenericDataFile/GenericDeleteFile copies with pinned BaseFile/PartitionData layouts. Paimon: - Handle Row/Array/Vector/Map/Multiset explicitly with a leaf-type whitelist, fail closed on unknown DataType implementations, reserve RowType's four lazy lookup maps at admission and pin DataType/DataField/RowType/TableSchema/FileStoreTable layouts. Tests and benchmarks: - JOL oracle cancels JVM-shared boxed caches and Class metadata; add whole-entry, uncached-id, Unicode lower-case, character/element budget, ContentFile implementation, VectorType, unknown DataType and RowType lazy-map fixtures; extend benchmarks with wide partitioned and nested tables. --- .../HivePartitionValuesSizeBenchmark.java | 7 +- .../iceberg/IcebergCacheSizeBenchmark.java | 92 +- .../hive/HiveCacheSizeEstimator.java | 22 +- .../hive/HiveExternalMetaCache.java | 48 +- .../iceberg/IcebergCacheSizeEstimator.java | 1001 ++++++++++++++- .../iceberg/IcebergSnapshotCacheValue.java | 18 +- .../iceberg/IcebergTableCacheValue.java | 6 +- .../iceberg/cache/ManifestCacheValue.java | 228 +++- .../metacache/MetaCacheSizeEstimator.java | 3 +- .../metacache/MetaCacheWeightUtils.java | 300 ++++- .../paimon/PaimonCacheSizeEstimator.java | 323 ++++- .../iceberg/IcebergExternalMetaCacheTest.java | 1121 ++++++++++++++++- .../iceberg/IcebergPartitionInfoTest.java | 8 +- .../EstimatorCalibrationAssertions.java | 65 +- .../metacache/MetaCacheEntryTest.java | 12 + .../paimon/PaimonExternalMetaCacheTest.java | 381 +++++- .../datasource/paimon/PaimonUtilTest.java | 8 +- 17 files changed, 3391 insertions(+), 252 deletions(-) diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java index 7e3acd9e2f97ac..57a2ebb0e1bdb8 100644 --- a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java @@ -31,6 +31,7 @@ import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import com.google.common.collect.HashBiMap; import com.google.common.collect.Maps; @@ -267,13 +268,13 @@ private static HivePartitionValues createPartitionValues( Map idToItem = Maps.newHashMapWithExpectedSize(count); Map> idToValues = Maps.newHashMapWithExpectedSize(count); String tailPayload = "tail_skew".equals(distribution) ? repeat('x', TAIL_PAYLOAD_BYTES) : null; - long partitionNameCharacterCount = 0L; + long partitionNamePayloadBytes = 0L; for (int i = 0; i < count; i++) { long id = i; String value = i == count - 1 && tailPayload != null ? tailPayload : "value-" + i; String name = "p=" + value; - partitionNameCharacterCount += name.length(); + partitionNamePayloadBytes += MetaCacheWeightUtils.estimatedStringPayloadBytes(name); List rawValues = Collections.singletonList(new PartitionValue(value)); PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(rawValues, types, true); List keys = new ArrayList<>(1); @@ -284,7 +285,7 @@ private static HivePartitionValues createPartitionValues( idToValues.put(id, new ArrayList<>(Collections.singletonList(value))); } return new HivePartitionValues( - idToItem, nameToId, idToValues, partitionNameCharacterCount, types.size()); + idToItem, nameToId, idToValues, partitionNamePayloadBytes, types.size()); } private static String repeat(char value, int count) { diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java index 2cca17a879b625..77235312739695 100644 --- a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java @@ -110,7 +110,11 @@ public int denseManifestValueConstruction(DenseManifestState state) { for (DataFile file : state.files) { builder.addDataFile(file.copy()); } - return builder.build().getDataFiles().size(); + ManifestCacheValue value = builder.build(); + if (value.isAccountingComplete() != state.expectedAccountingComplete()) { + throw new IllegalStateException("unexpected dense manifest accounting state"); + } + return value.getDataFiles().size(); } public long preparedWeightProvider(PreparedState state) { @@ -150,6 +154,32 @@ public static void main(String[] args) throws Exception { BenchmarkHarness.measure("iceberg.preparedTableCacheHit" + suffix, TimeUnit.MICROSECONDS, () -> benchmark.preparedTableCacheHit(prepared)); } + for (int fieldCount : new int[] {100, 1000}) { + // Wide identity-partitioned specs drive the O(fields) fieldsBySourceId reservation and + // the secondary partition Schema formula; nested schemas drive the per-field path + // and lower-case String terms. Both must stay far below the value construction cost + // of the same table. + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState partitioned = new TablePublicationState(); + partitioned.fieldCount = fieldCount; + partitioned.identityPartitioned = true; + partitioned.setup(); + BenchmarkHarness.measure("iceberg.partitionedTableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(partitioned)); + BenchmarkHarness.measure("iceberg.partitionedTablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(partitioned)); + BenchmarkHarness.measure("iceberg.partitionedTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(partitioned)); + + TablePublicationState nested = new TablePublicationState(); + nested.fieldCount = fieldCount; + nested.nestedSchema = true; + nested.setup(); + BenchmarkHarness.measure("iceberg.nestedTablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(nested)); + BenchmarkHarness.measure("iceberg.nestedTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(nested)); + } for (int snapshotCount : new int[] {1000, 10000}) { String suffix = "[snapshots=" + snapshotCount + "]"; LongHistoryTablePublicationState state = new LongHistoryTablePublicationState(); @@ -188,7 +218,9 @@ public static void main(String[] args) throws Exception { state.metricColumns = metricColumns; state.fileCount = fileCount; state.setup(); - String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + "]"; + String accounting = state.expectedAccountingComplete() ? "complete" : "rejected"; + String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + + ",accounting=" + accounting + "]"; BenchmarkHarness.measure("iceberg.denseManifestReaderBaseline" + suffix, TimeUnit.MICROSECONDS, () -> benchmark.denseManifestReaderBaseline(state)); BenchmarkHarness.measure("iceberg.denseManifestValueConstruction" + suffix, @@ -199,13 +231,15 @@ public static void main(String[] args) throws Exception { public static class TablePublicationState { public int fieldCount; + public boolean identityPartitioned; + public boolean nestedSchema; private NameMapping mapping; private Table table; public void setup() { mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); - table = newTable(fieldCount); + table = newTable(fieldCount, identityPartitioned, nestedSchema); } } @@ -286,6 +320,7 @@ public static class DenseManifestState { public int metricColumns; private List files; + private boolean accountingComplete; public void setup() { int poolSize = Math.min(fileCount, 256); @@ -315,33 +350,56 @@ public void setup() { for (int index = 0; index < fileCount; index++) { files.add(filePool.get(index % poolSize)); } + accountingComplete = ManifestCacheValue.forDataFiles(files).isAccountingComplete(); + } + + private boolean expectedAccountingComplete() { + return accountingComplete; } } private static Table newTable(int fieldCount) { + return newTable(fieldCount, false, false); + } + + private static Table newTable(int fieldCount, boolean identityPartitioned, boolean nestedSchema) { List fields = new ArrayList<>(fieldCount); for (int index = 0; index < fieldCount; index++) { fields.add(Types.NestedField.optional(index + 1, "field_" + index, Types.StringType.get())); } - Schema schema = new Schema(fields); - TableMetadata metadata = TableMetadata.newTableMetadata( - schema, PartitionSpec.unpartitioned(), "file:/benchmark/table", Collections.emptyMap()); - InMemoryFileIO fileIO = new InMemoryFileIO(); - StringBuilder snapshotJson = new StringBuilder("{\"snapshot-id\":7,\"timestamp-ms\":1,") - .append("\"summary\":{\"operation\":\"append\"},\"manifests\":["); - for (int index = 0; index < 10; index++) { - if (index > 0) { - snapshotJson.append(','); + Schema schema; + if (nestedSchema) { + List nestedFields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + nestedFields.add(Types.NestedField.optional( + 1000 + index, "Nested_" + index, Types.StringType.get())); } - String manifestPath = "/benchmark/manifest-" + index + ".avro"; - snapshotJson.append('"').append(manifestPath).append('"'); - fileIO.addFile(manifestPath, new byte[0]); + schema = new Schema( + Types.NestedField.optional(1, "payload", Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "list", Types.ListType.ofOptional(3, + Types.StructType.of(Types.NestedField.optional( + 4, "leaf", Types.StringType.get())))), + Types.NestedField.optional(5, "id", Types.LongType.get())); + } else { + schema = new Schema(fields); } - Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.append("],\"schema-id\":0}").toString()); + PartitionSpec spec = PartitionSpec.unpartitioned(); + if (identityPartitioned) { + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + for (Types.NestedField field : schema.columns()) { + specBuilder.identity(field.name()); + } + spec = specBuilder.build(); + } + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, spec, "file:/benchmark/table", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/benchmark/manifest-list.avro\",\"schema-id\":0}"); metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) .discardChanges() .withMetadataLocation("file:/benchmark/table/metadata/v1.json").build(); - return new BaseTable(new StaticTableOperations(metadata, fileIO), "benchmark.table"); + return new BaseTable(new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); } private static Table newLongHistoryTable(int snapshotCount) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java index c26a021890e880..0d5f8a8343415f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -24,18 +24,26 @@ /** Constant-time retained-weight formula for Hive partition-value cache entries. */ final class HiveCacheSizeEstimator { - // Calibrated against complete 4.1 object graphs. The per-character reserve covers the - // partition name plus derived value/literal strings and therefore remains skew-sensitive. - private static final long ENTRY_BASE_BYTES = 2L * 1024L; - private static final long PARTITION_BASE_BYTES = 4L * 1024L; - private static final long PARTITION_COLUMN_BYTES = 384L; - private static final long PARTITION_NAME_CHARACTER_BYTES = 8L; + // Calibrated against complete 4.1 object graphs. The payload reserve covers the partition + // name plus derived value/literal strings and therefore remains skew-sensitive. + private static final long ENTRY_BASE_BYTES = objectBytes(2L * 1024L); + private static final long PARTITION_BASE_BYTES = objectBytes(896L); + private static final long PARTITION_COLUMN_BYTES = objectBytes(256L); + // One copy is retained as the partition name and another in the decoded partition values. + private static final long PARTITION_NAME_PAYLOAD_COPIES = 2L; private HiveCacheSizeEstimator() { } + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + static MetaCacheSizeEstimate estimatePartitionValuesEntry( PartitionValueCacheKey key, HivePartitionValues value) { + if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { + return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } long partitionCount = value.getIdToPartitionItem() == null ? 0L : value.getIdToPartitionItem().size(); long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( @@ -48,7 +56,7 @@ static MetaCacheSizeEstimate estimatePartitionValuesEntry( MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply( - value.getPartitionNameCharacterCount(), PARTITION_NAME_CHARACTER_BYTES)); + value.getPartitionNamePayloadBytes(), PARTITION_NAME_PAYLOAD_COPIES)); return MetaCacheSizeEstimate.complete(bytes); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index dfa5b00289feb9..1e48fc0604c82f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -305,12 +305,13 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); - long partitionNameCharacterCount = 0L; + long partitionNamePayloadBytes = 0L; String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); for (String partitionName : partitionNames) { - partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( - partitionNameCharacterCount, partitionName.length()); + partitionNamePayloadBytes = MetaCacheWeightUtils.saturatedAdd( + partitionNamePayloadBytes, + MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); idToPartitionItem.put(partitionId, listPartitionItem); @@ -320,7 +321,7 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); HivePartitionValues partitionValues = new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, - partitionNameCharacterCount, key.types == null ? 0 : key.types.size()); + partitionNamePayloadBytes, key.types == null ? 0 : key.types.size()); preparePartitionValuesForPublication(partitionValues); return partitionValues; } @@ -768,7 +769,7 @@ private void addPartitionsCache(NameMapping nameMapping, allItems.put(partitionId, item); addedItems.put(partitionId, item); allNames.put(partitionName, partitionId); - copy.addPartitionNameCharacters(partitionName.length()); + copy.addPartitionNamePayload(partitionName); } if (addedItems.isEmpty()) { // Even a replay/no-op event must fence a refresh that started before the event. @@ -839,7 +840,7 @@ private void dropPartitionsCache(ExternalTable dorisTable, } allItems.remove(partitionId); allValues.remove(partitionId); - copy.removePartitionNameCharacters(partitionName.length()); + copy.removePartitionNamePayload(partitionName); changed = true; } if (!changed) { @@ -1117,7 +1118,7 @@ public static class HivePartitionValues { // Prepared once after construction/update; the cache weigher only reads this value. private transient volatile MetaCacheSizeEstimate sizeEstimate; // Maintained while the metadata is already being loaded or updated. Admission only reads it. - private long partitionNameCharacterCount; + private long partitionNamePayloadBytes; private int partitionColumnCount; private transient boolean sortedPartitionRangesPrepared; @@ -1128,19 +1129,19 @@ public HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap) { this(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, - countPartitionNameCharacters(partitionNameToIdMap), + countPartitionNamePayloadBytes(partitionNameToIdMap), inferPartitionColumnCount(partitionValuesMap)); } HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap, - long partitionNameCharacterCount, + long partitionNamePayloadBytes, int partitionColumnCount) { this.idToPartitionItem = idToPartitionItem; this.partitionNameToIdMap = partitionNameToIdMap; this.partitionValuesMap = partitionValuesMap; - this.partitionNameCharacterCount = partitionNameCharacterCount; + this.partitionNamePayloadBytes = partitionNamePayloadBytes; this.partitionColumnCount = partitionColumnCount; } @@ -1149,7 +1150,7 @@ HivePartitionValues mutableCopy() { copy.partitionNameToIdMap = partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap); copy.idToPartitionItem = idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem); copy.partitionValuesMap = partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap); - copy.partitionNameCharacterCount = partitionNameCharacterCount; + copy.partitionNamePayloadBytes = partitionNamePayloadBytes; copy.partitionColumnCount = partitionColumnCount; return copy; } @@ -1186,31 +1187,34 @@ void prepareSizeEstimate(PartitionValueCacheKey key) { sizeEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, this); } - long getPartitionNameCharacterCount() { - return partitionNameCharacterCount; + long getPartitionNamePayloadBytes() { + return partitionNamePayloadBytes; } int getPartitionColumnCount() { return partitionColumnCount; } - private void addPartitionNameCharacters(int characters) { - partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( - partitionNameCharacterCount, characters); + private void addPartitionNamePayload(String partitionName) { + partitionNamePayloadBytes = MetaCacheWeightUtils.saturatedAdd( + partitionNamePayloadBytes, + MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); } - private void removePartitionNameCharacters(int characters) { - partitionNameCharacterCount = Math.max(0L, partitionNameCharacterCount - characters); + private void removePartitionNamePayload(String partitionName) { + partitionNamePayloadBytes = Math.max(0L, partitionNamePayloadBytes + - MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); } - private static long countPartitionNameCharacters(BiMap names) { - long characters = 0L; + private static long countPartitionNamePayloadBytes(BiMap names) { + long payloadBytes = 0L; if (names != null) { for (String name : names.keySet()) { - characters = MetaCacheWeightUtils.saturatedAdd(characters, name.length()); + payloadBytes = MetaCacheWeightUtils.saturatedAdd( + payloadBytes, MetaCacheWeightUtils.estimatedStringPayloadBytes(name)); } } - return characters; + return payloadBytes; } private static int inferPartitionColumnCount(Map> values) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 8d483ecfb5153f..2b11a71f8c8d00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -22,54 +22,157 @@ import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; +import org.apache.iceberg.BlobMetadata; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortField; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.UnknownTransform; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; /** Publication-time retained-weight formulas for Iceberg cache entries. */ final class IcebergCacheSizeEstimator { - private static final long KEY_BASE_BYTES = 128L; - private static final long TABLE_BASE_BYTES = 16L * 1024L; - private static final long SCHEMA_VERSION_BYTES = 512L; - private static final long SCHEMA_FIELD_BYTES = 512L; - private static final long NESTED_SCHEMA_FIELD_BYTES = 512L; - private static final long PARTITION_SPEC_BYTES = 256L; - private static final long PARTITION_SPEC_FIELD_BYTES = 384L; - private static final long SORT_ORDER_BYTES = 256L; - private static final long SORT_FIELD_BYTES = 256L; - private static final long TABLE_PROPERTY_BYTES = 256L; - private static final long CURRENT_SNAPSHOT_BYTES = 512L; - private static final long HISTORICAL_SNAPSHOT_BYTES = 1024L; - private static final long SNAPSHOT_LOG_ENTRY_BYTES = 64L; - private static final long METADATA_LOG_ENTRY_BYTES = 128L; - private static final long SNAPSHOT_REF_BYTES = 128L; - private static final long STATISTICS_FILE_BYTES = 512L; - private static final long PARTITION_STATISTICS_FILE_BYTES = 256L; - private static final long ENCRYPTED_KEY_BYTES = 256L; - private static final long PARTITION_BYTES = 512L; - private static final long PARTITION_ALIAS_BYTES = 256L; - private static final long NAME_MAPPING_ENTRY_BYTES = 256L; - private static final long MANIFEST_ENTRY_BASE_BYTES = 256L; - private static final long DATA_FILE_BYTES = 16L * 1024L; - private static final long DELETE_FILE_BYTES = 18L * 1024L; - private static final long FILE_METRIC_ENTRY_BYTES = 160L; + // Calibrated against JOL retained-graph deltas in IcebergExternalMetaCacheTest. + // Every metadata element visited (field, type, snapshot, summary entry, ...) costs a few + // reads; the bound only guards against pathological metadata and is far above real tables + // (a 10,000-snapshot history with 15 summary keys each is 160,000 elements). Exceeding it + // rejects weighted admission, so it must not be reachable by ordinary long-lived tables. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L; + // Total name characters the estimator may lower-case while reserving case-insensitive indexes. + private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // TableMetadata-side share of one schema version: schemas list slot and schemasById entry, + // including the growth of both from their singleton to their regular immutable shapes. + private static final long SCHEMA_VERSION_BYTES = objectBytes(128L); + private static final long PARTITION_SPEC_BYTES = objectBytes(256L); + // Exact active-layout sizes of the Iceberg/Guava objects that lazy partition, sort and + // schema state allocates. Iceberg 1.10.1 field layouts are pinned by ICEBERG_LAZY_LAYOUT_SUPPORTED. + private static final long PARTITION_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L); + private static final long SORT_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + // Identity/Bucket/Truncate transforms are allocated per parsed field; time transforms are enums. + private static final long TRANSFORM_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long NESTED_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L); + private static final long STRUCT_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L); + private static final long SCHEMA_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L); + private static final long IMMUTABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long IMMUTABLE_MAP_KEY_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SINGLETON_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long REGULAR_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long LONG_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; + // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). + private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L), + MetaCacheWeightUtils.estimatedIntArrayBytes(1L)); + private static final long LIST_MULTIMAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L); + private static final long CAPTURING_SUPPLIER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long POSITION_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L); + // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor. Required ancestors + // collapse into a single Position2/3Accessor that replaces the inner accessor, which retains + // less than this per-level reservation. + private static final long WRAPPED_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L); + private static final long LIST_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L); + private static final long DECIMAL_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final long FIXED_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long GEOMETRY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long GEOGRAPHY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SORT_ORDER_BYTES = objectBytes(256L); + private static final long TABLE_PROPERTY_BYTES = objectBytes(40L); + private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L); + private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L); + private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L); + private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L); + private static final long SNAPSHOT_REF_BYTES = objectBytes(128L); + private static final long STATISTICS_FILE_BYTES = objectBytes(512L); + private static final long BLOB_METADATA_BYTES = objectBytes(128L); + private static final long BLOB_FIELD_BYTES = objectBytes(32L); + private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); + private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); + private static final long PARTITION_BYTES = objectBytes(640L); + private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); + private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); + private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); + private static final long DATA_FILE_BYTES = objectBytes(896L); + private static final long DELETE_FILE_BYTES = objectBytes(1024L); + private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L); + private static final String BASE_SNAPSHOT_CLASS_NAME = "org.apache.iceberg.BaseSnapshot"; + private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS = + loadBaseSnapshotRetainedCacheFields(); + // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog supplier + // (REST snapshot-loading-mode=refs). Publication must not perform that IO. + private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD = + loadTableMetadataField("snapshotsLoaded", boolean.class); + private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD = + loadTableMetadataField("snapshotsSupplier", null); + // The formulas above are built on the Iceberg 1.10.1 instance-field layouts of the classes a + // cached table retains. Every non-static field is pinned, not only the transient lazy ones: a + // library upgrade that adds a retained reference makes weighted admission fail closed. + private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED = checkIcebergLayout(); private IcebergCacheSizeEstimator() { } + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } Table table = value.getRetainedIcebergTable(); MetaCacheSizeEstimate support = checkSupportedTable(table); if (!support.isComplete()) { @@ -86,6 +189,10 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCac static MetaCacheSizeEstimate estimateSnapshotEntry( IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } long bytes = KEY_BASE_BYTES; bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); @@ -99,7 +206,7 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); - bytes = addCount(bytes, value.getNameMapping().map(java.util.Map::size).orElse(0), + bytes = addCount(bytes, value.getNameMapping().map(Map::size).orElse(0), NAME_MAPPING_ENTRY_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getRetainedNameMappingPayloadBytes()); @@ -121,6 +228,10 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( static MetaCacheSizeEstimate estimateManifestEntry( IcebergManifestEntryKey key, ManifestCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } if (!value.isAccountingComplete()) { return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); } @@ -135,7 +246,16 @@ static MetaCacheSizeEstimate estimateManifestEntry( return MetaCacheSizeEstimate.complete(bytes); } + private static MetaCacheSizeEstimate checkJvmObjectLayout() { + return MetaCacheWeightUtils.isSupportedJvmObjectLayout() + ? MetaCacheSizeEstimate.complete(1L) + : MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } + private static MetaCacheSizeEstimate checkSupportedTable(Table table) { + if (!ICEBERG_LAZY_LAYOUT_SUPPORTED) { + return MetaCacheSizeEstimate.incomplete("unsupported_iceberg_lazy_layout"); + } if (table == null) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); } @@ -147,6 +267,9 @@ private static MetaCacheSizeEstimate checkSupportedTable(Table table) { if (metadata == null) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata"); } + if (!areSnapshotsLoaded(metadata)) { + return MetaCacheSizeEstimate.incomplete("iceberg_snapshots_not_loaded"); + } if (metadata.metadataFileLocation() == null || metadata.metadataFileLocation().isEmpty()) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location"); @@ -171,7 +294,11 @@ private static long estimateTable(Table table) { return bytes; } - /** Captures exact historical cardinalities and skew-sensitive payload once before admission. */ + /** + * Fully accounts variable payload with a bounded amount of publication-time work. Only + * already-parsed metadata is read; the SDK state it touches on the way (StructType, ListType + * and MapType fieldList copies, the identifier field set) is small, accounted and O(N). + */ static long retainedTablePayloadBytes(Table table) { if (!(table instanceof HasTableOperations)) { return 0L; @@ -180,54 +307,65 @@ static long retainedTablePayloadBytes(Table table) { if (metadata == null) { return 0L; } + if (!areSnapshotsLoaded(metadata)) { + // snapshots()/refs() would call the catalog's lazy snapshot supplier: fail closed. + throw new IllegalStateException("Iceberg table snapshots are not loaded"); + } long bytes = 0L; - for (Schema schema : metadata.schemas()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SCHEMA_VERSION_BYTES); - for (Types.NestedField field : schema.columns()) { - bytes = addFieldPayload(bytes, field, false); - } - } + AccountingBudget budget = new AccountingBudget( + MAX_TABLE_ACCOUNTING_ELEMENTS, MAX_TABLE_ACCOUNTING_CHARACTERS); for (PartitionSpec spec : metadata.specs()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_BYTES); - for (org.apache.iceberg.PartitionField field : spec.fields()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_FIELD_BYTES); - bytes = addString(bytes, field.name()); - } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionSpecBytes(spec, budget)); } for (SortOrder sortOrder : metadata.sortOrders()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_BYTES); - bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, sortOrderBytes(sortOrder, budget)); + } + for (Schema schema : metadata.schemas()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaBytes(schema, budget)); } + budget.chargeElements(metadata.properties().size()); for (Map.Entry property : metadata.properties().entrySet()) { bytes = addString(bytes, property.getKey()); bytes = addString(bytes, property.getValue()); } for (Snapshot snapshot : metadata.snapshots()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HISTORICAL_SNAPSHOT_BYTES); - bytes = addString(bytes, snapshot.operation()); - bytes = addString(bytes, snapshot.manifestListLocation()); - bytes = addStringMap(bytes, snapshot.summary(), TABLE_PROPERTY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, snapshotBytes(snapshot, budget)); } + budget.chargeElements(metadata.snapshotLog().size()); bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + budget.chargeElements(metadata.previousFiles().size()); for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); bytes = addString(bytes, previousFile.file()); } + budget.chargeElements(metadata.refs().size()); bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); for (String refName : metadata.refs().keySet()) { bytes = addString(bytes, refName); } + budget.chargeElements(metadata.statisticsFiles().size()); for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); bytes = addString(bytes, statisticsFile.path()); - bytes = addCount(bytes, statisticsFile.blobMetadata().size(), TABLE_PROPERTY_BYTES); + for (BlobMetadata blob : statisticsFile.blobMetadata()) { + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, + MetaCacheWeightUtils.saturatedAdd( + blob.fields().size(), blob.properties().size()))); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, BLOB_METADATA_BYTES); + bytes = addString(bytes, blob.type()); + bytes = addCount(bytes, blob.fields().size(), BLOB_FIELD_BYTES); + bytes = addStringMap(bytes, blob.properties(), TABLE_PROPERTY_BYTES); + } } + budget.chargeElements(metadata.partitionStatisticsFiles().size()); for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); bytes = addString(bytes, statisticsFile.path()); } + budget.chargeElements(metadata.encryptionKeys().size()); for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { + budget.chargeElements(encryptedKey.properties().size()); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); bytes = addString(bytes, encryptedKey.keyId()); bytes = addString(bytes, encryptedKey.encryptedById()); @@ -238,34 +376,579 @@ static long retainedTablePayloadBytes(Table table) { return bytes; } + /** + * Account a PartitionSpec together with the lazy state that a normal scan materializes after + * admission: fieldList, javaClasses, partitionType() with its StructType indexes, the secondary + * Schema/Binder graph behind partitionType().asSchema() and fieldsBySourceId. Iceberg 1.10.1 + * allocates one Object[fieldCount] per distinct source id inside fieldsBySourceId, so that + * retained graph is O(distinctSourceIds * fieldCount); it is reserved here in O(fieldCount) + * publication work without materializing any of it. + */ + private static long partitionSpecBytes(PartitionSpec spec, AccountingBudget budget) { + List fields = spec.fields(); + long fieldCount = fields.size(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); + long bytes = PARTITION_SPEC_BYTES; + if (fieldCount == 0L) { + return bytes; + } + Set distinctSourceIds = new HashSet<>(); + long uncachedSourceIds = 0L; + long uncachedFieldIds = 0L; + long lowerCaseNameBytes = 0L; + for (PartitionField field : fields) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_FIELD_BYTES); + bytes = addTransformPayload(bytes, field.transform()); + bytes = addString(bytes, field.name()); + lowerCaseNameBytes = MetaCacheWeightUtils.saturatedAdd( + lowerCaseNameBytes, generatedLowerCaseNameBytes(field.name(), budget)); + if (isUncachedInteger(field.fieldId())) { + uncachedFieldIds++; + } + if (distinctSourceIds.add(field.sourceId()) && isUncachedInteger(field.sourceId())) { + uncachedSourceIds++; + } + } + // Eager PartitionField[] plus lazy fieldList and javaClasses. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + // partitionType(): the StructType itself also exists for an unpartitioned spec. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + structTypeIndexBytes(fieldCount, uncachedFieldIds, lowerCaseNameBytes)); + if (!spec.schema().idsToOriginal().isEmpty()) { + // rawPartitionType() rebuilds the struct with original ids. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); + bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); + } + // A partition filter binds against partitionType().asSchema(). + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes( + SchemaShape.flat(fieldCount, uncachedFieldIds, lowerCaseNameBytes))); + // fieldsBySourceId: HashMap. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LIST_MULTIMAP_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CAPTURING_SUPPLIER_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(distinctSourceIds.size(), uncachedSourceIds)); + return addCount(bytes, distinctSourceIds.size(), + MetaCacheWeightUtils.saturatedAdd(ARRAY_LIST_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount))); + } + + /** Account a SortOrder: SortField[] with per-field transforms plus the lazy fieldList copy. */ + private static long sortOrderBytes(SortOrder sortOrder, AccountingBudget budget) { + long fieldCount = sortOrder.fields().size(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); + long bytes = SORT_ORDER_BYTES; + if (fieldCount == 0L) { + return bytes; + } + for (SortField field : sortOrder.fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_FIELD_BYTES); + bytes = addTransformPayload(bytes, field.transform()); + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + return MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); + } + + /** Transform instance plus the payload only some transforms retain. */ + private static long addTransformPayload(long bytes, Transform transform) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TRANSFORM_BYTES); + if (transform instanceof UnknownTransform) { + return addString(bytes, transform.toString()); + } + if (transform.toString().startsWith(TRUNCATE_TRANSFORM_PREFIX)) { + // Truncate is package-private; its serialized name is the SPI contract. + return MetaCacheWeightUtils.saturatedAdd(bytes, TRUNCATE_WIDTH_BYTES); + } + return bytes; + } + + /** Lazy StructType indexes: fieldList, fieldsByName, fieldsByLowerCaseName and fieldsById. */ + private static long structTypeIndexBytes( + long fieldCount, long uncachedFieldIds, long lowerCaseNameBytes) { + long bytes = immutableListBytes(fieldCount); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, 0L, 0L)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, 0L, lowerCaseNameBytes)); + return MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, uncachedFieldIds, 0L)); + } + + /** + * The Schema created by StructType.asSchema(): its constructor materializes idToName and two + * empty id maps; Binder and projection paths add nameToId, lowerCaseNameToId, idToField and + * idToAccessor; its own StructType copy grows the same lazy indexes as the root struct. + */ + private static long secondarySchemaBytes(SchemaShape shape) { + long bytes = schemaObjectBytes(shape); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); + return MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( + shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, + shape.topLevelLowerCaseStringBytes)); + } + + /** Schema object, empty identifier int[], the two empty id maps and the eager idToName keySet. */ + private static long schemaObjectBytes(SchemaShape shape) { + long bytes = MetaCacheWeightUtils.saturatedAdd( + SCHEMA_BYTES, MetaCacheWeightUtils.estimatedIntArrayBytes(0L)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); + if (shape.fieldCount == 1L) { + return MetaCacheWeightUtils.saturatedAdd(bytes, SINGLETON_IMMUTABLE_SET_BYTES); + } + return shape.fieldCount > 1L + ? MetaCacheWeightUtils.saturatedAdd(bytes, IMMUTABLE_MAP_KEY_SET_BYTES) : bytes; + } + + /** + * idToName (eager in the constructor), nameToId and idToField. Every map boxes uncached ids + * itself; idToName and nameToId each retain their own copy of every nested canonical name and + * nameToId also retains the short aliases. + */ + private static long schemaLookupBytes(SchemaShape shape) { + long bytes = immutableNameMapBytes( + shape.fieldCount, shape.uncachedFieldIdCount, shape.pathStringBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableNameMapBytes( + shape.nameEntryCount, shape.uncachedNameIdCount, + MetaCacheWeightUtils.saturatedAdd( + shape.pathStringBytes, shape.aliasStringBytes))); + return MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(shape.fieldCount, shape.uncachedFieldIdCount)); + } + + /** lowerCaseNameToId and idToAccessor, materialized by case-insensitive lookups and Binder. */ + private static long schemaLazyIndexBytes(SchemaShape shape) { + long bytes = immutableNameMapBytes( + shape.nameEntryCount, shape.uncachedNameIdCount, shape.lowerCaseStringBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(shape.accessorFieldCount, shape.uncachedAccessorIdCount)); + bytes = addCount(bytes, shape.accessorFieldCount, POSITION_ACCESSOR_BYTES); + return addCount(bytes, shape.wrappedAccessorCount, WRAPPED_ACCESSOR_BYTES); + } + + /** + * One table schema version with every index a normal scan can materialize afterwards. Only + * metadata already parsed is read; nothing lazy is touched, and each field is visited once. + */ + private static long schemaBytes(Schema schema, AccountingBudget budget) { + budget.chargeElements(1L); + SchemaShape shape = new SchemaShape(); + long bytes = SCHEMA_VERSION_BYTES; + for (Types.NestedField field : schema.columns()) { + bytes = addFieldPayload( + bytes, field, PathState.ROOT, FieldKind.STRUCT_FIELD, budget, shape); + } + Set identifierFieldIds = schema.identifierFieldIds(); + budget.chargeElements(identifierFieldIds.size()); + bytes = addIdentifierFieldPayload(bytes, identifierFieldIds); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, shape.typeObjectBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaObjectBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); + if (shape.fieldCount == 0L) { + // Nothing can be looked up in an empty schema; its indexes stay shared singletons. + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); + // Future lazy growth: main lookups, root struct indexes and the asSchema() secondary graph. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( + shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, + shape.topLevelLowerCaseStringBytes)); + return MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes(shape)); + } + + /** ImmutableList.copyOf(array): shared empty, singleton, or a list object plus copied array. */ + private static long immutableListBytes(long elementCount) { + if (elementCount <= 0L) { + return 0L; + } + if (elementCount == 1L) { + return IMMUTABLE_LIST_BYTES; + } + return MetaCacheWeightUtils.saturatedAdd(IMMUTABLE_LIST_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount)); + } + + /** Growth of a reference array that replaces an empty array retained by the empty shape. */ + private static long objectArrayGrowthBytes(long elementCount) { + long populated = MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount); + long empty = MetaCacheWeightUtils.estimatedObjectArrayBytes(0L); + return populated == Long.MAX_VALUE ? populated : populated - empty; + } + + /** Boxed Integer keys outside the JVM Integer cache are retained per lookup map. */ + private static boolean isUncachedInteger(int value) { + return value < -128 || value > 127; + } + + /** + * Retained bytes of one lower-cased copy of a name, or 0 when the name is already lower case + * and the index reuses it. Every case-insensitive index (partition StructType, secondary + * Schema and secondary StructType) allocates its own copy, so callers add this per index. + */ + private static long generatedLowerCaseNameBytes(String name, AccountingBudget budget) { + budget.chargeCharacters(name.length()); + String lowerName = name.toLowerCase(Locale.ROOT); + if (lowerName.equals(name)) { + return 0L; + } + return MetaCacheWeightUtils.estimatedGeneratedStringBytes( + lowerName.length(), MetaCacheWeightUtils.isLatin1String(lowerName)); + } + + private static long hashIdMapBytes(long entryCount, long uncachedIds) { + long bytes = HASH_MAP_BYTES; + if (entryCount <= 0L) { + // HashMap allocates its table on the first put. + return bytes; + } + bytes = addCount(bytes, entryCount, HASH_MAP_NODE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + hashMapCapacity(entryCount))); + return addCount(bytes, uncachedIds, INTEGER_BYTES); + } + + private static long immutableNameMapBytes( + long entryCount, long uncachedIds, long generatedStringBytes) { + long bytes = 0L; + if (entryCount == 1L) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(8L, 0L)); + } else if (entryCount > 1L) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 4L)); + bytes = addCount(bytes, entryCount, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(entryCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + immutableMapTableCapacity(entryCount))); + } + bytes = addCount(bytes, uncachedIds, INTEGER_BYTES); + return MetaCacheWeightUtils.saturatedAdd(bytes, generatedStringBytes); + } + + private static long snapshotBytes(Snapshot snapshot, AccountingBudget budget) { + rejectMaterializedSnapshotPayload(snapshot); + Map summary = snapshot.summary(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( + 1L, summary == null ? 0L : summary.size())); + long bytes = HISTORICAL_SNAPSHOT_BYTES; + // The parsed parent id is never inside the Long cache; the row-id fields are boxed too + // and only tiny values would share a cached instance, so each present field is charged. + bytes = addBoxedLong(bytes, snapshot.parentId()); + bytes = addBoxedLong(bytes, snapshot.firstRowId()); + bytes = addBoxedLong(bytes, snapshot.addedRows()); + bytes = addString(bytes, snapshot.operation()); + String manifestListLocation = snapshot.manifestListLocation(); + if (manifestListLocation == null) { + // A snapshot serialized with an inline "manifests" array (legacy writers) retains a + // String[] of manifest locations that is only exposed through ManifestFile wrappers. + // Reject weighted admission instead of doing IO or admitting an underestimate. + throw new IllegalStateException( + "Iceberg snapshot with inline manifest list is unsupported"); + } + bytes = addString(bytes, manifestListLocation); + bytes = addString(bytes, snapshot.keyId()); + return addStringMap(bytes, summary, TABLE_PROPERTY_BYTES); + } + + private static void rejectMaterializedSnapshotPayload(Snapshot snapshot) { + if (!BASE_SNAPSHOT_CLASS_NAME.equals(snapshot.getClass().getName())) { + throw new IllegalStateException( + "Unsupported Iceberg snapshot implementation: " + + snapshot.getClass().getName()); + } + if (BASE_SNAPSHOT_RETAINED_CACHE_FIELDS == null) { + throw new IllegalStateException( + "Iceberg BaseSnapshot retained-cache inspection is unavailable"); + } + try { + // The field list is resolved once per process. Publication only performs a bounded + // number of O(1) reads and never walks a retained manifest/file graph. + for (Field retainedCacheField : BASE_SNAPSHOT_RETAINED_CACHE_FIELDS) { + if (retainedCacheField.get(snapshot) != null) { + throw new IllegalStateException( + "Iceberg snapshot has materialized retained payload: " + + retainedCacheField.getName()); + } + } + } catch (IllegalAccessException e) { + throw new IllegalStateException( + "Cannot inspect Iceberg BaseSnapshot retained payload", e); + } + } + + /** Iceberg marks snapshots loaded at construction unless a lazy supplier was configured. */ + private static boolean areSnapshotsLoaded(TableMetadata metadata) { + if (TABLE_METADATA_SNAPSHOTS_LOADED_FIELD == null + || TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD == null) { + return false; + } + try { + return TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD.get(metadata) == null + || TABLE_METADATA_SNAPSHOTS_LOADED_FIELD.getBoolean(metadata); + } catch (IllegalAccessException | RuntimeException e) { + return false; + } + } + + private static Field loadTableMetadataField(String name, Class expectedType) { + try { + Field field = TableMetadata.class.getDeclaredField(name); + if ((expectedType != null && field.getType() != expectedType) + || Modifier.isStatic(field.getModifiers())) { + return null; + } + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException | RuntimeException e) { + return null; + } + } + + private static Field[] loadBaseSnapshotRetainedCacheFields() { + try { + Class snapshotClass = Class.forName( + BASE_SNAPSHOT_CLASS_NAME, false, Snapshot.class.getClassLoader()); + List retainedCacheFields = new ArrayList<>(); + for (Field field : snapshotClass.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (Modifier.isTransient(modifiers) && !Modifier.isStatic(modifiers) + && !field.getType().isPrimitive()) { + field.setAccessible(true); + retainedCacheFields.add(field); + } + } + return retainedCacheFields.isEmpty() + ? null : retainedCacheFields.toArray(new Field[0]); + } catch (ReflectiveOperationException | RuntimeException e) { + return null; + } + } + + private static boolean checkIcebergLayout() { + ClassLoader loader = Snapshot.class.getClassLoader(); + return MetaCacheWeightUtils.hasExpectedInstanceFields(Schema.class, + "struct:StructType", "schemaId:int", "identifierFieldIds:int[]", + "highestFieldId:int", "aliasToId:BiMap", "idToField:Map", "nameToId:Map", + "lowerCaseNameToId:Map", "idToAccessor:Map", "idToName:Map", + "identifierFieldIdSet:Set", "idsToReassigned:Map", "idsToOriginal:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionSpec.class, + "schema:Schema", "specId:int", "fields:PartitionField[]", + "fieldsBySourceId:ListMultimap", "lazyJavaClasses:Class[]", + "lazyPartitionType:StructType", "lazyRawPartitionType:StructType", + "fieldList:List", "lastAssignedFieldId:int") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionField.class, + "sourceId:int", "fieldId:int", "name:String", "transform:Transform") + && MetaCacheWeightUtils.hasExpectedInstanceFields(SortOrder.class, + "schema:Schema", "orderId:int", "fields:SortField[]", "fieldList:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(SortField.class, + "transform:Transform", "sourceId:int", "direction:SortDirection", + "nullOrder:NullOrder") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.StructType.class, + "fields:NestedField[]", "schema:Schema", "fieldList:List", + "fieldsByName:Map", "fieldsByLowerCaseName:Map", "fieldsById:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.ListType.class, + "elementField:NestedField", "fields:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.MapType.class, + "keyField:NestedField", "valueField:NestedField", "fields:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.NestedField.class, + "isOptional:boolean", "id:int", "name:String", "type:Type", + "doc:String", "initialDefault:Literal", "writeDefault:Literal") + && MetaCacheWeightUtils.hasExpectedInstanceFields(TableMetadata.class, + "metadataFileLocation:String", "formatVersion:int", "uuid:String", + "location:String", "lastSequenceNumber:long", "lastUpdatedMillis:long", + "lastColumnId:int", "currentSchemaId:int", "schemas:List", + "defaultSpecId:int", "specs:List", "lastAssignedPartitionId:int", + "defaultSortOrderId:int", "sortOrders:List", "properties:Map", + "currentSnapshotId:long", "schemasById:Map", "specsById:Map", + "sortOrdersById:Map", "snapshotLog:List", "previousFiles:List", + "statisticsFiles:List", "partitionStatisticsFiles:List", "changes:List", + "nextRowId:long", "encryptionKeys:List", + "snapshotsSupplier:SerializableSupplier", "snapshots:List", + "snapshotsById:Map", "refs:Map", "snapshotsLoaded:boolean") + && MetaCacheWeightUtils.hasExpectedInstanceFields(BASE_SNAPSHOT_CLASS_NAME, loader, + "snapshotId:long", "parentId:Long", "sequenceNumber:long", + "timestampMillis:long", "manifestListLocation:String", + "operation:String", "summary:Map", "schemaId:Integer", + "v1ManifestLocations:String[]", "firstRowId:Long", "addedRows:Long", + "keyId:String", "allManifests:List", "dataManifests:List", + "deleteManifests:List", "addedDataFiles:List", "removedDataFiles:List", + "addedDeleteFiles:List", "removedDeleteFiles:List"); + } + + private static long addBoxedLong(long bytes, Long value) { + return value == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, LONG_BYTES); + } + private static long addBufferPayload(long bytes, ByteBuffer buffer) { return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); } - private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); - bytes = addString(bytes, field.name()); + /** + * Account one NestedField, its owned strings and its type subtree, and record the shape data + * the lookup-map formulas need. Canonical and short names follow Iceberg's IndexByName: a + * nested name joins its ancestors with '.', a struct-typed list element or map value is left + * out of its children's short names (which then become aliases), and every lower-case index + * lower-cases each entry. + */ + private static long addFieldPayload( + long bytes, Types.NestedField field, PathState ancestors, FieldKind kind, + AccountingBudget budget, SchemaShape shape) { + budget.chargeElements(1L); + budget.chargeCharacters(field.name().length()); + String name = field.name(); + String lowerName = name.toLowerCase(Locale.ROOT); + boolean nameLatin1 = MetaCacheWeightUtils.isLatin1String(name); + boolean lowerLatin1 = MetaCacheWeightUtils.isLatin1String(lowerName); + shape.addField(field.fieldId(), ancestors, name, nameLatin1, lowerName, lowerLatin1); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + if (kind == FieldKind.STRUCT_FIELD) { + // List element and map key/value fields are named by shared "element"/"key"/"value" + // literals inside Iceberg's type constructors. + bytes = addString(bytes, name); + } bytes = addString(bytes, field.doc()); bytes = addDefaultPayload(bytes, field.initialDefault()); bytes = addDefaultPayload(bytes, field.writeDefault()); - return addTypePayload(bytes, field.type()); + boolean pushShortName = kind == FieldKind.STRUCT_FIELD || kind == FieldKind.MAP_KEY + || !field.type().isStructType(); + // Only fields nested through a chain of struct fields get accessors; anything below a + // list or map does not. + boolean structChildren = kind == FieldKind.STRUCT_FIELD && field.type().isStructType(); + PathState children = ancestors.push(name.length(), nameLatin1, lowerName.length(), + lowerLatin1, pushShortName, structChildren); + return addTypePayload(bytes, field.type(), children, budget, shape); } - private static long addTypePayload(long bytes, Type type) { + private static long addTypePayload( + long bytes, Type type, PathState ancestors, AccountingBudget budget, + SchemaShape shape) { + if (ancestors.typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException( + "Iceberg cache accounting type depth exceeded"); + } + budget.chargeElements(1L); if (type.isStructType()) { - for (Types.NestedField field : type.asStructType().fields()) { - bytes = addFieldPayload(bytes, field, true); + List fields = type.asStructType().fields(); + // A nested struct's fieldList is materialized by every visitor. Its own name/id + // lookup indexes and asSchema() are not reserved: read paths resolve nested names + // through the root Schema maps and Binder binds only root and partition structs; + // nested-column DDL runs against a freshly loaded live table, not a cached one. + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(STRUCT_TYPE_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(fields.size())), + immutableListBytes(fields.size()))); + for (Types.NestedField field : fields) { + bytes = addFieldPayload( + bytes, field, ancestors, FieldKind.STRUCT_FIELD, budget, shape); } } else if (type.isListType()) { - bytes = addTypePayload(bytes, type.asListType().elementType()); + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + LIST_TYPE_BYTES, IMMUTABLE_LIST_BYTES)); + bytes = addFieldPayload(bytes, type.asListType().fields().get(0), + ancestors, FieldKind.LIST_ELEMENT, budget, shape); } else if (type.isMapType()) { - bytes = addTypePayload(bytes, type.asMapType().keyType()); - bytes = addTypePayload(bytes, type.asMapType().valueType()); + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(MAP_TYPE_BYTES, IMMUTABLE_LIST_BYTES), + MetaCacheWeightUtils.estimatedObjectArrayBytes(2L))); + bytes = addFieldPayload(bytes, type.asMapType().fields().get(0), + ancestors, FieldKind.MAP_KEY, budget, shape); + bytes = addFieldPayload(bytes, type.asMapType().fields().get(1), + ancestors, FieldKind.MAP_VALUE, budget, shape); + } else if (type instanceof Types.DecimalType) { + shape.addTypeObject(DECIMAL_TYPE_BYTES); + } else if (type instanceof Types.FixedType) { + shape.addTypeObject(FIXED_TYPE_BYTES); + } else if (type instanceof Types.GeometryType) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOMETRY_TYPE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes( + ((Types.GeometryType) type).crs()))); + } else if (type instanceof Types.GeographyType) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOGRAPHY_TYPE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes( + ((Types.GeographyType) type).crs()))); } + // Other primitive types are shared singletons. return bytes; } + /** Account the int[] and lazy ImmutableSet retained by Schema.identifierFieldIds(). */ + private static long addIdentifierFieldPayload(long bytes, Set fieldIds) { + long count = fieldIds.size(); + if (count == 0L) { + return bytes; + } + long uncachedIds = 0L; + for (int fieldId : fieldIds) { + if (isUncachedInteger(fieldId)) { + uncachedIds++; + } + } + // The int[] grows from the empty array of a schema without identifier fields. + long additions = MetaCacheWeightUtils.estimatedIntArrayPayloadBytes(count); + additions = addCount(additions, uncachedIds, INTEGER_BYTES); + if (count == 1L) { + // ImmutableSet.copyOf(one element) is a SingletonImmutableSet. + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedAdd(additions, SINGLETON_IMMUTABLE_SET_BYTES)); + } + // RegularImmutableSet: the set object, its dense elements array and open-addressing table. + // The shared empty set of an identifier-free schema stays reachable through other schemas, + // so nothing is subtracted for it. + additions = MetaCacheWeightUtils.saturatedAdd(additions, REGULAR_IMMUTABLE_SET_BYTES); + additions = MetaCacheWeightUtils.saturatedAdd( + additions, MetaCacheWeightUtils.estimatedObjectArrayBytes(count)); + additions = MetaCacheWeightUtils.saturatedAdd(additions, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + immutableSetTableCapacity(count))); + return MetaCacheWeightUtils.saturatedAdd(bytes, additions); + } + + private static long immutableSetTableCapacity(long size) { + long capacity = 2L; + while (MetaCacheWeightUtils.saturatedMultiply(size, 10L) + > MetaCacheWeightUtils.saturatedMultiply(capacity, 7L)) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long hashMapCapacity(long size) { + long capacity = 16L; + while (size > capacity - capacity / 4L) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long immutableMapTableCapacity(long size) { + long capacity = Long.highestOneBit(size); + return MetaCacheWeightUtils.saturatedMultiply(size, 5L) + > MetaCacheWeightUtils.saturatedMultiply(capacity, 6L) + ? MetaCacheWeightUtils.saturatedMultiply(capacity, 2L) : capacity; + } + private static long addDefaultPayload(long bytes, Object value) { if (value instanceof CharSequence) { return MetaCacheWeightUtils.saturatedAdd(bytes, @@ -300,4 +983,210 @@ private static long addCount(long bytes, long count, long bytesPerItem) { MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); } + /** + * Hard bound on publication-time estimator work. Elements bound the number of metadata + * objects visited; characters bound the String scanning (lower-casing) performed for + * lazy-index reservations. Exceeding either throws, which estimateSafely turns into an + * incomplete estimate: weighted admission is rejected but the load itself succeeds. + */ + private static final class AccountingBudget { + private long remainingElements; + private long remainingCharacters; + + private AccountingBudget(long elements, long characters) { + this.remainingElements = elements; + this.remainingCharacters = characters; + } + + private void chargeElements(long elements) { + if (elements < 0L || elements > remainingElements) { + throw new IllegalStateException("Iceberg cache accounting work budget exceeded"); + } + remainingElements -= elements; + } + + private void chargeCharacters(long characters) { + if (characters < 0L || characters > remainingCharacters) { + throw new IllegalStateException( + "Iceberg cache accounting character budget exceeded"); + } + remainingCharacters -= characters; + } + } + + private enum FieldKind { + STRUCT_FIELD, LIST_ELEMENT, MAP_KEY, MAP_VALUE + } + + /** + * Immutable name-stack state of a field's ancestors: character counts and Latin-1 coders of + * the joined canonical path, the short-alias path and both lower-cased forms. + */ + private static final class PathState { + private static final PathState ROOT = new PathState( + -1L, true, -1L, true, -1L, true, -1L, true, 0, 0); + + private final long pathCharacters; + private final boolean pathLatin1; + private final long shortPathCharacters; + private final boolean shortPathLatin1; + private final long lowerPathCharacters; + private final boolean lowerPathLatin1; + private final long shortLowerPathCharacters; + private final boolean shortLowerPathLatin1; + // Struct-field ancestors of the next field, or -1 inside a list or map (no accessors). + private final int structDepth; + private final int typeDepth; + + private PathState(long pathCharacters, boolean pathLatin1, long shortPathCharacters, + boolean shortPathLatin1, long lowerPathCharacters, boolean lowerPathLatin1, + long shortLowerPathCharacters, boolean shortLowerPathLatin1, int structDepth, + int typeDepth) { + this.pathCharacters = pathCharacters; + this.pathLatin1 = pathLatin1; + this.shortPathCharacters = shortPathCharacters; + this.shortPathLatin1 = shortPathLatin1; + this.lowerPathCharacters = lowerPathCharacters; + this.lowerPathLatin1 = lowerPathLatin1; + this.shortLowerPathCharacters = shortLowerPathCharacters; + this.shortLowerPathLatin1 = shortLowerPathLatin1; + this.structDepth = structDepth; + this.typeDepth = typeDepth; + } + + private boolean isRoot() { + return pathCharacters < 0L; + } + + private boolean shortPathDiffers() { + return shortPathCharacters != pathCharacters; + } + + /** + * Push a field name for its children; the short name is pushed only when requested and + * accessor depth continues only for the children of a struct-typed struct field. + */ + private PathState push(long nameCharacters, boolean nameLatin1, long lowerCharacters, + boolean lowerLatin1, boolean pushShortName, boolean structChildren) { + return new PathState( + join(pathCharacters, nameCharacters), pathLatin1 && nameLatin1, + pushShortName ? join(shortPathCharacters, nameCharacters) : shortPathCharacters, + pushShortName ? shortPathLatin1 && nameLatin1 : shortPathLatin1, + join(lowerPathCharacters, lowerCharacters), lowerPathLatin1 && lowerLatin1, + pushShortName ? join(shortLowerPathCharacters, lowerCharacters) + : shortLowerPathCharacters, + pushShortName ? shortLowerPathLatin1 && lowerLatin1 : shortLowerPathLatin1, + structChildren && structDepth >= 0 ? structDepth + 1 : -1, typeDepth + 1); + } + + private static long join(long parentCharacters, long nameCharacters) { + return parentCharacters < 0L ? nameCharacters + : MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(parentCharacters, 1L), + nameCharacters); + } + } + + /** Cardinalities and generated-String bytes that size a schema's lookup indexes. */ + private static final class SchemaShape { + private long fieldCount; + private long topLevelFieldCount; + private long uncachedFieldIdCount; + private long uncachedTopLevelFieldIdCount; + private long nameEntryCount; + private long uncachedNameIdCount; + // One copy each; the formulas add a copy per index that retains it. + private long pathStringBytes; + private long aliasStringBytes; + private long lowerCaseStringBytes; + private long topLevelLowerCaseStringBytes; + private long accessorFieldCount; + private long uncachedAccessorIdCount; + private long wrappedAccessorCount; + private long typeObjectBytes; + + /** A flat struct of {@code fieldCount} top-level fields, as used by partition types. */ + private static SchemaShape flat( + long fieldCount, long uncachedFieldIds, long lowerCaseStringBytes) { + SchemaShape shape = new SchemaShape(); + shape.fieldCount = fieldCount; + shape.topLevelFieldCount = fieldCount; + shape.uncachedFieldIdCount = uncachedFieldIds; + shape.uncachedTopLevelFieldIdCount = uncachedFieldIds; + shape.nameEntryCount = fieldCount; + shape.uncachedNameIdCount = uncachedFieldIds; + shape.lowerCaseStringBytes = lowerCaseStringBytes; + shape.topLevelLowerCaseStringBytes = lowerCaseStringBytes; + shape.accessorFieldCount = fieldCount; + shape.uncachedAccessorIdCount = uncachedFieldIds; + return shape; + } + + private void addField(int fieldId, PathState ancestors, String name, boolean nameLatin1, + String lowerName, boolean lowerLatin1) { + boolean uncached = isUncachedInteger(fieldId); + fieldCount++; + nameEntryCount++; + if (uncached) { + uncachedFieldIdCount++; + uncachedNameIdCount++; + } + if (ancestors.structDepth >= 0) { + accessorFieldCount++; + wrappedAccessorCount = MetaCacheWeightUtils.saturatedAdd( + wrappedAccessorCount, ancestors.structDepth); + if (uncached) { + uncachedAccessorIdCount++; + } + } + if (ancestors.isRoot()) { + topLevelFieldCount++; + if (uncached) { + uncachedTopLevelFieldIdCount++; + } + if (!name.equals(lowerName)) { + // Lower-case indexes only allocate when toLowerCase() changes the name. + long lowerBytes = MetaCacheWeightUtils.estimatedGeneratedStringBytes( + lowerName.length(), lowerLatin1); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( + lowerCaseStringBytes, lowerBytes); + topLevelLowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( + topLevelLowerCaseStringBytes, lowerBytes); + } + return; + } + // Nested: IndexByName joins a new canonical String, and the lower-case index keeps + // either that joined String or its lower-cased copy. + pathStringBytes = MetaCacheWeightUtils.saturatedAdd(pathStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.pathCharacters, name.length()), + ancestors.pathLatin1 && nameLatin1)); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.lowerPathCharacters, lowerName.length()), + ancestors.lowerPathLatin1 && lowerLatin1)); + if (ancestors.shortPathDiffers()) { + // A short alias exists whenever an ancestor was left out of the short path. + // Iceberg drops an alias that collides with a canonical name; counting the rare + // collision is conservative and avoids building name sets at publication. + nameEntryCount++; + if (uncached) { + uncachedNameIdCount++; + } + aliasStringBytes = MetaCacheWeightUtils.saturatedAdd(aliasStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.shortPathCharacters, name.length()), + ancestors.shortPathLatin1 && nameLatin1)); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join( + ancestors.shortLowerPathCharacters, lowerName.length()), + ancestors.shortLowerPathLatin1 && lowerLatin1)); + } + } + + private void addTypeObject(long bytes) { + typeObjectBytes = MetaCacheWeightUtils.saturatedAdd(typeObjectBytes, bytes); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 7cfc5bcc2bcd31..5d3618b21f01fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -132,12 +132,14 @@ MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { if (sizeEstimate == null) { sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", () -> { + // Account before serializing the current snapshot: v1 snapshot JSON + // materializes the transient manifest list that accounting rejects. + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); if (retainedCurrentSnapshotJson == null) { retainedCurrentSnapshotJson = icebergTable .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); } - retainedTablePayloadBytes = icebergTable - .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); return IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this); }); if (sizeEstimate.isComplete()) { @@ -414,7 +416,12 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { } } - /** Query-local operations expose exact retained metadata without shared lazy table state. */ + /** + * Query-local read-only operations over the exact retained metadata. Only snapshot state is + * isolated per query (see QueryScopedTable); TableMetadata, Schema, StructType and + * PartitionSpec are shared with the cached generation, and their lazy indexes grow inside the + * cache value. IcebergCacheSizeEstimator reserves that growth at publication. + */ private static final class QueryScopedTableOperations extends RetainedTableOperations { private QueryScopedTableOperations(TableOperations retainedOperations) { super(retainedOperations, retainedOperations.current()); @@ -426,7 +433,10 @@ public void commit(TableMetadata base, TableMetadata metadata) { } } - /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ + /** + * A per-caller view whose Iceberg lazy snapshot state (manifest lists, manifests, files) is + * never written into the cache value. It does not isolate schema/spec lazy indexes. + */ private static final class QueryScopedTable extends BaseTable { private final QueryScopedTableOperations queryOperations; private final Snapshot currentSnapshot; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index a4d4f600f66d6e..0b6bd23c634381 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -56,10 +56,12 @@ synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { if (sizeEstimate == null) { sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", () -> { - retainedCurrentSnapshotJson = - IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + // Order matters: serializing a v1 snapshot materializes its transient + // manifest list, which the payload accounting rejects, so account first. retainedTablePayloadBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); return IcebergCacheSizeEstimator.estimateTableEntry(key, this); }); if (sizeEstimate.isComplete()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index 4d63af348324e2..cb8923a5bf2e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -23,10 +23,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.StructLike; +import org.apache.iceberg.types.Types; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; @@ -34,7 +37,26 @@ * Cached manifest payload containing parsed files. */ public class ManifestCacheValue { - private static final long AUXILIARY_LIST_ENTRY_BYTES = 32L; + private static final long AUXILIARY_LIST_ENTRY_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(32L); + // A copied PartitionData instance shares its partition type, Avro schema, and serialized + // schema with the other files produced by one manifest reader. These constants are calibrated + // against the production reuseContainers()+file.copy() graph in IcebergExternalMetaCacheTest. + private static final long SHARED_PARTITION_BASE_BYTES = 1320L; + private static final long SHARED_PARTITION_FIELD_BYTES = 904L; + private static final long PARTITION_FIELD_NAME_RETENTION_COPIES = 2L; + // Bound variable-payload traversal (bound entries plus partition values) independently of + // manifest size. Values beyond this limit are rejected from a weighted cache instead of being + // admitted with an underestimate, so it sits well above ordinary wide manifests: 10,000 files + // with 200 lower/upper bounds each is 4,000,000 elements. + private static final long MAX_DEEP_ACCOUNTING_ELEMENTS = 8_000_000L; + // The per-file constants in IcebergCacheSizeEstimator describe the Iceberg 1.10.1 copies that + // ManifestReader + ContentFile.copy() produce. Only those implementations, with their pinned + // instance-field layouts, are accounted; anything else fails closed at build time. + private static final String GENERIC_DATA_FILE_CLASS_NAME = "org.apache.iceberg.GenericDataFile"; + private static final String GENERIC_DELETE_FILE_CLASS_NAME = + "org.apache.iceberg.GenericDeleteFile"; + private static final boolean CONTENT_FILE_LAYOUT_SUPPORTED = checkContentFileLayout(); private final List dataFiles; private final List deleteFiles; @@ -86,6 +108,31 @@ public static Builder deleteFilesBuilder(boolean accountRetainedSize) { return new Builder(false, accountRetainedSize); } + private static boolean checkContentFileLayout() { + ClassLoader loader = ContentFile.class.getClassLoader(); + return MetaCacheWeightUtils.hasExpectedInstanceFields(GENERIC_DATA_FILE_CLASS_NAME, loader) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + GENERIC_DELETE_FILE_CLASS_NAME, loader) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.iceberg.BaseFile", loader, + "partitionType:StructType", "fileOrdinal:Long", "manifestLocation:String", + "partitionSpecId:int", "content:FileContent", "filePath:String", + "format:FileFormat", "partitionData:PartitionData", "recordCount:Long", + "fileSizeInBytes:long", "dataSequenceNumber:Long", + "fileSequenceNumber:Long", "columnSizes:Map", "valueCounts:Map", + "nullValueCounts:Map", "nanValueCounts:Map", "lowerBounds:Map", + "upperBounds:Map", "splitOffsets:long[]", "equalityIds:int[]", + "keyMetadata:byte[]", "sortOrderId:Integer", "firstRowId:Long", + "referencedDataFile:String", "contentOffset:Long", + "contentSizeInBytes:Long", "avroSchema:Schema") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.iceberg.avro.SupportsIndexProjection", loader, + "fromProjectionPos:int[]") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionData.class, + "partitionType:StructType", "size:int", "data:Object[]", + "stringSchema:String", "schema:Schema"); + } + public List getDataFiles() { return dataFiles; } @@ -110,7 +157,7 @@ public boolean isAccountingComplete() { return accountingComplete; } - /** Accumulates retained-size counters in the manifest reader's existing file loop. */ + /** Accounts retained payload while the manifest reader builds the cached lists. */ public static final class Builder { private final boolean dataContent; private final boolean accountRetainedSize; @@ -118,7 +165,10 @@ public static final class Builder { private final List deleteFiles = new ArrayList<>(); private long metricEntryCount; private long retainedPayloadBytes; + private long deepAccountingElements; private boolean accountingComplete; + private final IdentityHashMap> + accountedPartitionSchemas = new IdentityHashMap<>(); private Builder(boolean dataContent, boolean accountRetainedSize) { this.dataContent = dataContent; @@ -131,7 +181,7 @@ public void addDataFile(DataFile file) { throw new IllegalStateException("delete manifest builder cannot accept a data file"); } dataFiles.add(file); - accountSafely(file); + recordAccounting(file); } public void addDeleteFile(DeleteFile file) { @@ -139,7 +189,7 @@ public void addDeleteFile(DeleteFile file) { throw new IllegalStateException("data manifest builder cannot accept a delete file"); } deleteFiles.add(file); - accountSafely(file); + recordAccounting(file); } public ManifestCacheValue build() { @@ -149,27 +199,106 @@ public ManifestCacheValue build() { retainedPayloadBytes, accountingComplete); } - private void accountSafely(ContentFile file) { + private void recordAccounting(ContentFile file) { if (!accountRetainedSize || !accountingComplete) { return; } try { - account(file); - } catch (RuntimeException e) { + requireSupportedContentFile(file); + StructLike partition = file.partition(); + long nextDeepElements = MetaCacheWeightUtils.saturatedAdd( + deepAccountingElements, deepAccountingElements(file, partition)); + if (nextDeepElements > MAX_DEEP_ACCOUNTING_ELEMENTS) { + rejectAccounting(); + return; + } + deepAccountingElements = nextDeepElements; + addAccounting(account(file, partition)); + accountPartitionOwnership(partition); + } catch (RuntimeException | LinkageError e) { // A new or third-party ContentFile implementation must not turn optional cache // accounting into a manifest-read failure. Keep the files for the current query // and mark the value incomplete so weighted admission rejects it. - metricEntryCount = 0L; - retainedPayloadBytes = 0L; - accountingComplete = false; + rejectAccounting(); + } + } + + private void requireSupportedContentFile(ContentFile file) { + if (!CONTENT_FILE_LAYOUT_SUPPORTED) { + throw new IllegalStateException("unsupported Iceberg content file layout"); + } + String expectedClassName = dataContent + ? GENERIC_DATA_FILE_CLASS_NAME : GENERIC_DELETE_FILE_CLASS_NAME; + if (file == null || !expectedClassName.equals(file.getClass().getName())) { + throw new IllegalStateException("unsupported Iceberg content file implementation: " + + (file == null ? "null" : file.getClass().getName())); } } - private void account(ContentFile file) { + private void addAccounting(FileAccounting accounting) { metricEntryCount = MetaCacheWeightUtils.saturatedAdd( - metricEntryCount, metricEntryCount(file)); + metricEntryCount, accounting.metricEntryCount); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( - retainedPayloadBytes, retainedPayloadBytes(file)); + retainedPayloadBytes, accounting.retainedPayloadBytes); + } + + private void rejectAccounting() { + metricEntryCount = 0L; + retainedPayloadBytes = 0L; + deepAccountingElements = 0L; + accountingComplete = false; + } + + private void accountPartitionOwnership(StructLike partition) { + if (partition == null || partition.size() == 0) { + return; + } + if (!(partition instanceof PartitionData)) { + throw new IllegalArgumentException( + "unsupported Iceberg partition container: " + + partition.getClass().getName()); + } + PartitionData partitionData = (PartitionData) partition; + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partitionInstanceBytes(partitionData)); + Object partitionTypeIdentity = partitionData.getPartitionType(); + // A reader-produced PartitionData carries its Avro schema; only a Java-deserialized + // instance would rebuild it here (CPU only, no IO). + Object schemaIdentity = partitionData.getSchema(); + IdentityHashMap schemas = accountedPartitionSchemas.computeIfAbsent( + partitionTypeIdentity, ignored -> new IdentityHashMap<>()); + if (schemas.put(schemaIdentity, Boolean.TRUE) == null) { + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, + sharedPartitionBytes(partitionData.getPartitionType())); + } + } + } + + private static FileAccounting account(ContentFile file, StructLike partition) { + return new FileAccounting( + metricEntryCount(file), retainedPayloadBytes(file, partition)); + } + + private static long deepAccountingElements(ContentFile file, StructLike partition) { + long elements = MetaCacheWeightUtils.saturatedAdd( + mapSize(file.lowerBounds()), mapSize(file.upperBounds())); + if (partition != null) { + if (partition.size() < 0) { + throw new IllegalArgumentException("negative Iceberg partition size"); + } + elements = MetaCacheWeightUtils.saturatedAdd(elements, partition.size()); + } + return elements; + } + + private static final class FileAccounting { + private final long metricEntryCount; + private final long retainedPayloadBytes; + + private FileAccounting(long metricEntryCount, long retainedPayloadBytes) { + this.metricEntryCount = metricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; } } @@ -182,7 +311,7 @@ private static long metricEntryCount(ContentFile file) { return MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.upperBounds())); } - private static long retainedPayloadBytes(ContentFile file) { + private static long retainedPayloadBytes(ContentFile file, StructLike partition) { long bytes = MetaCacheWeightUtils.estimatedCharSequenceBytes(file.path()); bytes = addBuffer(bytes, file.keyMetadata()); bytes = addBuffers(bytes, file.lowerBounds()); @@ -194,7 +323,7 @@ private static long retainedPayloadBytes(ContentFile file) { MetaCacheWeightUtils.estimatedStringBytes( ((DeleteFile) file).referencedDataFile())); } - return addPartitionPayload(bytes, file.partition()); + return addPartitionPayload(bytes, partition); } private static long addBuffers(long bytes, Map buffers) { @@ -224,21 +353,62 @@ private static long addPartitionPayload(long bytes, StructLike partition) { if (partition == null) { return bytes; } - try { - for (int index = 0; index < partition.size(); index++) { - Object value = partition.get(index, Object.class); - if (value instanceof CharSequence) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); - } else if (value instanceof ByteBuffer) { - bytes = addBuffer(bytes, (ByteBuffer) value); - } else if (value instanceof byte[]) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); - } + for (int index = 0; index < partition.size(); index++) { + Object value = partition.get(index, Object.class); + if (value instanceof CharSequence) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + bytes = addByteArray(bytes, ((ByteBuffer) value).capacity()); + } else if (value instanceof byte[]) { + bytes = addByteArray(bytes, ((byte[]) value).length); + } else if (value instanceof java.math.BigDecimal) { + int bits = ((java.math.BigDecimal) value).unscaledValue().bitLength(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(96L)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedIntArrayPayloadBytes( + (bits + 31L) / 32L)); + } else if (value instanceof java.util.UUID) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(32L)); + } else if (value instanceof Long || value instanceof Double) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(24L)); + } else if (value instanceof Number || value instanceof Boolean + || value instanceof Character) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(16L)); + } else if (value != null) { + throw new IllegalArgumentException( + "unsupported Iceberg partition value: " + value.getClass().getName()); } - } catch (RuntimeException ignored) { - // A third-party StructLike may reject Object.class. The fixed per-file allowance - // remains conservative, and cache accounting must never fail manifest loading. + } + return bytes; + } + + private static long addByteArray(long bytes, long payloadBytes) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedByteArrayBytes(payloadBytes)); + } + + private static long partitionInstanceBytes(PartitionData partition) { + return MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectBytes(32L), + MetaCacheWeightUtils.estimatedObjectArrayBytes(partition.size())); + } + + private static long sharedPartitionBytes(Types.StructType partitionType) { + long rawBytes = MetaCacheWeightUtils.saturatedAdd( + SHARED_PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + partitionType.fields().size(), SHARED_PARTITION_FIELD_BYTES)); + long bytes = MetaCacheWeightUtils.estimatedObjectBytes(rawBytes); + for (Types.NestedField field : partitionType.fields()) { + long payloadBytes = MetaCacheWeightUtils.estimatedStringPayloadBytes(field.name()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + payloadBytes, PARTITION_FIELD_NAME_RETENTION_COPIES)); } return bytes; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java index 5aff6a2a1a279a..e74c6e631b69c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -40,7 +40,8 @@ static MetaCacheSizeEstimate estimateSafely( Objects.requireNonNull(estimation, "estimation"); try { return Objects.requireNonNull(estimation.get(), "size estimate"); - } catch (RuntimeException e) { + } catch (RuntimeException | LinkageError e) { + // A missing or incompatible SDK class must reject weighted admission, not the load. return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java index 7869ac5993f48e..828bc28719d4d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -19,27 +19,207 @@ import org.apache.doris.datasource.NameMapping; -/** Constant-time helpers for conservative external metadata cache weights. */ +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.management.ManagementFactory; +import java.lang.management.PlatformManagedObject; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Overflow-safe helpers for conservative external metadata cache weights. */ public final class MetaCacheWeightUtils { - private static final long STRING_BASE_BYTES = 40L; - private static final long STRING_BYTES_PER_CHARACTER = 2L; private static final long NAME_MAPPING_BASE_BYTES = 64L; + private static final MethodHandle STRING_VALUE_GETTER; + private static final long STRING_VALUE_OFFSET; + private static final int OBJECT_ALIGNMENT_BYTES; + private static final int OBJECT_REFERENCE_BYTES; + private static final int OBJECT_HEADER_BYTES; + private static final int OBJECT_ARRAY_BASE_BYTES; + private static final int BYTE_ARRAY_BASE_BYTES; + private static final int CHAR_ARRAY_BASE_BYTES; + private static final int INT_ARRAY_BASE_BYTES; + private static final boolean SUPPORTED_OBJECT_LAYOUT; + private static final long OBJECT_LAYOUT_PERCENT; + + static { + MethodHandle stringValueGetter = null; + long stringValueOffset = -1L; + int referenceBytes = Long.BYTES; + // 24B is the safe fallback for an uncompressed class pointer. Unsafe replaces these + // values with the exact active-VM layout when access is available. + int objectArrayBaseBytes = 24; + int byteArrayBaseBytes = 24; + int charArrayBaseBytes = 24; + int intArrayBaseBytes = 24; + try { + Class unsafeClass = Class.forName("sun.misc.Unsafe"); + Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + Object unsafe = unsafeField.get(null); + stringValueOffset = (long) unsafeClass + .getMethod("objectFieldOffset", Field.class) + .invoke(unsafe, String.class.getDeclaredField("value")); + stringValueGetter = MethodHandles.lookup() + .unreflect(unsafeClass.getMethod("getObject", Object.class, long.class)) + .bindTo(unsafe) + .asType(MethodType.methodType(Object.class, Object.class, long.class)); + referenceBytes = (int) unsafeClass + .getMethod("arrayIndexScale", Class.class) + .invoke(unsafe, Object[].class); + objectArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, Object[].class); + byteArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, byte[].class); + charArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, char[].class); + intArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, int[].class); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // A conservative UTF-16 fallback is used when the VM hides String storage. + } + STRING_VALUE_GETTER = stringValueGetter; + STRING_VALUE_OFFSET = stringValueOffset; + OBJECT_REFERENCE_BYTES = referenceBytes; + OBJECT_ARRAY_BASE_BYTES = objectArrayBaseBytes; + BYTE_ARRAY_BASE_BYTES = byteArrayBaseBytes; + CHAR_ARRAY_BASE_BYTES = charArrayBaseBytes; + INT_ARRAY_BASE_BYTES = intArrayBaseBytes; + String alignmentOption = readVmOption("ObjectAlignmentInBytes"); + int objectAlignmentBytes = parseObjectAlignment(alignmentOption); + OBJECT_ALIGNMENT_BYTES = objectAlignmentBytes; + SUPPORTED_OBJECT_LAYOUT = alignmentOption != null + && (objectAlignmentBytes == 8 || objectAlignmentBytes == 16); + boolean compressedClassPointers = readBooleanVmOption( + "UseCompressedClassPointers", false); + OBJECT_HEADER_BYTES = Long.BYTES + + (compressedClassPointers ? Integer.BYTES : Long.BYTES); + long referencePercent = referenceBytes <= Integer.BYTES ? 100L : 145L; + long classPointerPercent = compressedClassPointers ? 100L : 140L; + long alignmentPercent = objectAlignmentBytes <= 8 ? 100L : 120L; + OBJECT_LAYOUT_PERCENT = (referencePercent * classPointerPercent * alignmentPercent + + 9_999L) / 10_000L; + } private MetaCacheWeightUtils() { } - /** - * Estimate a String without inspecting its contents. Two bytes per character deliberately - * avoids depending on CompactStrings or VM-private layout details. - */ public static long estimatedStringBytes(String value) { - return estimatedCharSequenceBytes(value); + if (value == null) { + return 0L; + } + Object storage = stringStorage(value); + long backingArrayBytes; + if (storage instanceof byte[]) { + backingArrayBytes = estimatedByteArrayBytes(((byte[]) storage).length); + } else if (storage instanceof char[]) { + backingArrayBytes = alignedArrayBytes( + CHAR_ARRAY_BASE_BYTES, ((char[]) storage).length, Character.BYTES); + } else { + backingArrayBytes = alignedArrayBytes( + CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES); + } + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), backingArrayBytes); } /** Estimate retained character data without materializing a String copy. */ public static long estimatedCharSequenceBytes(CharSequence value) { - return value == null ? 0L : saturatedAdd( - STRING_BASE_BYTES, saturatedMultiply(value.length(), STRING_BYTES_PER_CHARACTER)); + if (value == null) { + return 0L; + } + if (value instanceof String) { + return estimatedStringBytes((String) value); + } + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), + alignedArrayBytes(CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES)); + } + + /** Whether calibrated formulas support the active VM object alignment. */ + public static boolean isSupportedJvmObjectLayout() { + return SUPPORTED_OBJECT_LAYOUT; + } + + /** Adjust a default compressed-reference object-graph constant to the active VM layout. */ + public static long estimatedObjectBytes(long compressedReferenceBytes) { + long product = saturatedMultiply(compressedReferenceBytes, OBJECT_LAYOUT_PERCENT); + if (product == Long.MAX_VALUE) { + return product; + } + long roundedProduct = saturatedAdd(product, 99L); + return roundedProduct == Long.MAX_VALUE ? roundedProduct : roundedProduct / 100L; + } + + /** Returns the actual backing-array payload in O(1), or a conservative UTF-16 fallback. */ + public static long estimatedStringPayloadBytes(String value) { + if (value == null) { + return 0L; + } + Object storage = stringStorage(value); + if (storage instanceof byte[]) { + return alignPayload(((byte[]) storage).length); + } + if (storage instanceof char[]) { + return alignPayload(saturatedMultiply( + ((char[]) storage).length, Character.BYTES)); + } + return alignPayload(saturatedMultiply(value.length(), Character.BYTES)); + } + + /** Estimate a generated String whose encoded width is derived from its source components. */ + public static long estimatedGeneratedStringBytes(long characterCount, boolean latin1) { + long payloadBytes = saturatedMultiply(characterCount, latin1 ? 1L : Character.BYTES); + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), + estimatedByteArrayBytes(payloadBytes)); + } + + /** Whether this VM stores the String with one byte per character. */ + public static boolean isLatin1String(String value) { + if (value == null || value.isEmpty()) { + return true; + } + Object storage = stringStorage(value); + return storage instanceof byte[] && ((byte[]) storage).length == value.length(); + } + + /** VM-layout size of a retained byte array, conservatively if VM introspection is hidden. */ + public static long estimatedByteArrayBytes(long length) { + return alignedArrayBytes(BYTE_ARRAY_BASE_BYTES, length, Byte.BYTES); + } + + /** VM-layout size of an object-reference array, conservatively if introspection is hidden. */ + public static long estimatedObjectArrayBytes(long length) { + return alignedArrayBytes(OBJECT_ARRAY_BASE_BYTES, length, OBJECT_REFERENCE_BYTES); + } + + /** Size of an object with a known field layout on the active VM. */ + public static long estimatedObjectLayoutBytes(long referenceFields, long primitiveBytes) { + if (referenceFields < 0L || primitiveBytes < 0L) { + return Long.MAX_VALUE; + } + long bytes = saturatedAdd( + OBJECT_HEADER_BYTES, + saturatedMultiply(referenceFields, OBJECT_REFERENCE_BYTES)); + return alignPayload(saturatedAdd(bytes, primitiveBytes)); + } + + /** VM-layout size of a retained int array, conservatively if introspection is hidden. */ + public static long estimatedIntArrayBytes(long length) { + return alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); + } + + /** Incremental VM-layout payload of an int array whose header is accounted elsewhere. */ + public static long estimatedIntArrayPayloadBytes(long length) { + long populated = alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); + long empty = alignPayload(INT_ARRAY_BASE_BYTES); + return populated == Long.MAX_VALUE ? populated : populated - empty; } /** Estimate the fixed set of names retained by a cache key. */ @@ -47,13 +227,49 @@ public static long estimatedNameMappingBytes(NameMapping nameMapping) { if (nameMapping == null) { return 0L; } - long bytes = NAME_MAPPING_BASE_BYTES; + long bytes = estimatedObjectBytes(NAME_MAPPING_BASE_BYTES); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); } + /** + * Whether {@code type} itself declares exactly the expected non-static instance fields, each + * written as {@code name:SimpleTypeName}. Estimator formulas are calibrated against pinned SDK + * layouts; callers fail closed when a library upgrade adds, removes or retypes a field so a + * new retained reference cannot be silently undercounted. Superclasses are pinned separately. + */ + public static boolean hasExpectedInstanceFields(Class type, String... expectedFields) { + if (type == null) { + return false; + } + Set expected = new HashSet<>(Arrays.asList(expectedFields)); + Set actual = new HashSet<>(); + try { + for (Field field : type.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) || field.isSynthetic()) { + continue; + } + actual.add(field.getName() + ":" + field.getType().getSimpleName()); + } + } catch (RuntimeException | LinkageError e) { + return false; + } + return actual.equals(expected); + } + + /** Same as {@link #hasExpectedInstanceFields(Class, String...)} for a class resolved by name. */ + public static boolean hasExpectedInstanceFields( + String className, ClassLoader loader, String... expectedFields) { + try { + return hasExpectedInstanceFields( + Class.forName(className, false, loader), expectedFields); + } catch (ReflectiveOperationException | RuntimeException | LinkageError e) { + return false; + } + } + public static long saturatedAdd(long left, long right) { if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { return Long.MAX_VALUE; @@ -67,4 +283,66 @@ public static long saturatedMultiply(long left, long right) { } return left * right; } + + private static boolean readBooleanVmOption(String option, boolean fallback) { + String value = readVmOption(option); + return value == null ? fallback : Boolean.parseBoolean(value); + } + + private static String readVmOption(String option) { + try { + @SuppressWarnings("unchecked") + Class beanClass = + (Class) + Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); + Object bean = ManagementFactory.getPlatformMXBean(beanClass); + Method getVmOption = beanClass.getMethod("getVMOption", String.class); + Object vmOption = getVmOption.invoke(bean, option); + Method getValue = vmOption.getClass().getMethod("getValue"); + return (String) getValue.invoke(vmOption); + } catch (ReflectiveOperationException | RuntimeException ignored) { + return null; + } + } + + private static long alignPayload(long bytes) { + if (bytes == Long.MAX_VALUE) { + return bytes; + } + long remainder = bytes % OBJECT_ALIGNMENT_BYTES; + return remainder == 0L ? bytes + : saturatedAdd(bytes, OBJECT_ALIGNMENT_BYTES - remainder); + } + + private static long alignedArrayBytes(long baseBytes, long length, long elementBytes) { + if (length < 0L) { + return Long.MAX_VALUE; + } + return alignPayload(saturatedAdd( + baseBytes, saturatedMultiply(length, elementBytes))); + } + + private static Object stringStorage(String value) { + if (STRING_VALUE_GETTER != null && STRING_VALUE_OFFSET >= 0L) { + try { + return (Object) STRING_VALUE_GETTER.invokeExact( + (Object) value, STRING_VALUE_OFFSET); + } catch (Throwable ignored) { + // Return null so callers use the conservative UTF-16 fallback. + } + } + return null; + } + + private static int parseObjectAlignment(String value) { + if (value == null) { + return 16; + } + try { + int alignment = Integer.parseInt(value); + return alignment > 0 ? alignment : 16; + } catch (NumberFormatException ignored) { + return 16; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index b5a6178538a9c4..8484c4aa477354 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -20,39 +20,182 @@ import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; +import com.google.common.collect.ImmutableMap; import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobType; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.SmallIntType; +import org.apache.paimon.types.TimeType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; +import org.apache.paimon.types.VarBinaryType; +import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VariantType; +import org.apache.paimon.types.VectorType; import java.util.List; import java.util.Map; -/** Constant-time retained-weight formula for Paimon snapshot projections. */ +/** Publication-time retained-weight formula for Paimon snapshot projections. */ final class PaimonCacheSizeEstimator { - private static final long KEY_BASE_BYTES = 128L; - private static final long SNAPSHOT_BASE_BYTES = 4L * 1024L; - private static final long TABLE_BASE_BYTES = 16L * 1024L; - private static final long TABLE_FIELD_BYTES = 3584L; - private static final long TABLE_OPTION_BYTES = 256L; - private static final long TABLE_KEY_BYTES = 128L; - private static final long NESTED_FIELD_BYTES = 512L; - private static final long PARTITION_BYTES = 1280L; - private static final long PARTITION_ITEM_BYTES = 1024L; - private static final long WRAPPER_BYTES = 512L; + // Calibrated against JOL retained-graph deltas in PaimonExternalMetaCacheTest. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long SNAPSHOT_BASE_BYTES = objectBytes(4L * 1024L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // A top-level DataField, its list slot and shared per-field overhead; the DataType instance + // is accounted separately by addTypePayload. + private static final long TABLE_FIELD_BYTES = objectBytes(40L); + private static final long TABLE_OPTION_BYTES = objectBytes(44L); + private static final long TABLE_KEY_BYTES = objectBytes(128L); + // Exact Paimon 1.4.2 layouts, pinned by PAIMON_TYPE_LAYOUT_SUPPORTED. + private static final long DATA_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 4L); + private static final long ARRAY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); + private static final long VECTOR_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 5L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 1L); + private static final long MULTISET_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); + // RowType plus Collections.unmodifiableList(new ArrayList<>(fields)). + private static final long ROW_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 1L); + private static final long UNMODIFIABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final int ROW_TYPE_LAZY_MAP_COUNT = 4; + // Accepted leaf DataType implementations and the int fields each adds to DataType's nullable + // flag and type root. Any other class, including a future or third-party type, rejects + // weighted admission instead of being counted as an arbitrary primitive. + private static final String[] NO_LEAF_FIELDS = {}; + private static final String[] LENGTH_LEAF_FIELDS = {"length:int"}; + private static final String[] PRECISION_LEAF_FIELDS = {"precision:int"}; + private static final Map, String[]> LEAF_TYPE_FIELDS = + ImmutableMap., String[]>builder() + .put(CharType.class, LENGTH_LEAF_FIELDS) + .put(VarCharType.class, LENGTH_LEAF_FIELDS) + .put(BooleanType.class, NO_LEAF_FIELDS) + .put(BinaryType.class, LENGTH_LEAF_FIELDS) + .put(VarBinaryType.class, LENGTH_LEAF_FIELDS) + .put(DecimalType.class, new String[] {"precision:int", "scale:int"}) + .put(TinyIntType.class, NO_LEAF_FIELDS) + .put(SmallIntType.class, NO_LEAF_FIELDS) + .put(IntType.class, NO_LEAF_FIELDS) + .put(BigIntType.class, NO_LEAF_FIELDS) + .put(FloatType.class, NO_LEAF_FIELDS) + .put(DoubleType.class, NO_LEAF_FIELDS) + .put(DateType.class, NO_LEAF_FIELDS) + .put(TimeType.class, PRECISION_LEAF_FIELDS) + .put(TimestampType.class, PRECISION_LEAF_FIELDS) + .put(LocalZonedTimestampType.class, PRECISION_LEAF_FIELDS) + .put(VariantType.class, NO_LEAF_FIELDS) + .put(BlobType.class, NO_LEAF_FIELDS) + .build(); + private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED = checkPaimonTypeLayout(); + private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); + private static final long PARTITION_BYTES = objectBytes(160L); + private static final long PARTITION_ITEM_BYTES = objectBytes(640L); + private static final long WRAPPER_BYTES = objectBytes(512L); private PaimonCacheSizeEstimator() { } + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + + /** DataType: typeRoot reference plus the isNullable flag, then the subclass int fields. */ + private static long leafTypeBytes(String[] intFields) { + return MetaCacheWeightUtils.estimatedObjectLayoutBytes( + 1L, 1L + (long) Integer.BYTES * intFields.length); + } + + /** Pin the Paimon 1.4.2 DataType/DataField/RowType layouts the formulas above are built on. */ + private static boolean checkPaimonTypeLayout() { + boolean supported = MetaCacheWeightUtils.hasExpectedInstanceFields( + DataType.class, "isNullable:boolean", "typeRoot:DataTypeRoot") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + DataField.class, "id:int", "name:String", "type:DataType", + "description:String", "defaultValue:String") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + RowType.class, "fields:List", "laziedNameToField:Map", + "laziedNameToIndex:Map", "laziedFieldIdToField:Map", + "laziedFieldIdToIndex:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + ArrayType.class, "elementType:DataType") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + VectorType.class, "elementType:DataType", "length:int") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + MapType.class, "keyType:DataType", "valueType:DataType") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + MultisetType.class, "elementType:DataType"); + for (Map.Entry, String[]> leaf : LEAF_TYPE_FIELDS.entrySet()) { + supported &= MetaCacheWeightUtils.hasExpectedInstanceFields( + leaf.getKey(), leaf.getValue()); + } + return supported; + } + + /** Pin TableSchema and the two accepted FileStoreTable implementations. */ + private static boolean checkPaimonTableLayout() { + ClassLoader loader = FileStoreTable.class.getClassLoader(); + String[] abstractTableFields = { + "fileIO:FileIO", "path:Path", "tableSchema:TableSchema", + "catalogEnvironment:CatalogEnvironment", "manifestCache:SegmentsCache", + "snapshotCache:Cache", "statsCache:Cache", "dvmetaCache:DVMetaCache"}; + return MetaCacheWeightUtils.hasExpectedInstanceFields( + TableSchema.class, "version:int", "id:long", "fields:List", + "highestFieldId:int", "partitionKeys:List", "primaryKeys:List", + "bucketKeys:List", "numBucket:int", "options:Map", "comment:String", + "timeMillis:long") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.AbstractFileStoreTable", loader, + abstractTableFields) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.AppendOnlyFileStoreTable", loader, + "lazyStore:AppendOnlyFileStore") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.PrimaryKeyFileStoreTable", loader, + "lazyStore:KeyValueFileStore"); + } + static MetaCacheSizeEstimate estimateSnapshotEntry( PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { + return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } + if (!PAIMON_TYPE_LAYOUT_SUPPORTED || !PAIMON_TABLE_LAYOUT_SUPPORTED) { + return MetaCacheSizeEstimate.incomplete("unsupported_paimon_layout"); + } Table table = value.getSnapshot().getTable(); if (!isSupportedTable(table)) { return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:" @@ -118,14 +261,21 @@ private static long estimateTable(Table table) { * All collections are already materialized in TableSchema; this never opens the table store. */ static long retainedTablePayloadBytes(Table table) { + return retainedTablePayloadBytes( + table, new AccountingBudget(MAX_TABLE_ACCOUNTING_ELEMENTS)); + } + + private static long retainedTablePayloadBytes(Table table, AccountingBudget budget) { + budget.charge(1L); if (table instanceof PrivilegedFileStoreTable) { - return retainedTablePayloadBytes(((PrivilegedFileStoreTable) table).wrapped()); + return retainedTablePayloadBytes( + ((PrivilegedFileStoreTable) table).wrapped(), budget); } if (table instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; return MetaCacheWeightUtils.saturatedAdd( - retainedTablePayloadBytes(fallback.wrapped()), - retainedTablePayloadBytes(fallback.other())); + retainedTablePayloadBytes(fallback.wrapped(), budget), + retainedTablePayloadBytes(fallback.other(), budget)); } if (!(table instanceof FileStoreTable)) { return 0L; @@ -137,48 +287,137 @@ static long retainedTablePayloadBytes(Table table) { } long bytes = addString(0L, schema.comment()); for (DataField field : schema.fields()) { - bytes = addFieldPayload(bytes, field, false); + bytes = addFieldPayload(bytes, field, false, budget, 0); } + budget.charge(schema.options().size()); for (Map.Entry option : schema.options().entrySet()) { bytes = addString(bytes, option.getKey()); bytes = addString(bytes, option.getValue()); } - bytes = addStrings(bytes, schema.partitionKeys()); - bytes = addStrings(bytes, schema.primaryKeys()); - return addStrings(bytes, schema.bucketKeys()); + bytes = addStrings(bytes, schema.partitionKeys(), budget); + bytes = addStrings(bytes, schema.primaryKeys(), budget); + return addStrings(bytes, schema.bucketKeys(), budget); } - private static long addStrings(long bytes, List values) { + private static long addStrings( + long bytes, List values, AccountingBudget budget) { + budget.charge(values.size()); for (String value : values) { bytes = addString(bytes, value); } return bytes; } - private static long addFieldPayload(long bytes, DataField field, boolean nested) { + private static long addFieldPayload( + long bytes, DataField field, boolean nested, AccountingBudget budget, + int typeDepth) { + budget.charge(1L); if (nested) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, DATA_FIELD_BYTES); } bytes = addString(bytes, field.name()); bytes = addString(bytes, field.description()); bytes = addString(bytes, field.defaultValue()); - return addTypePayload(bytes, field.type()); + return addTypePayload(bytes, field.type(), budget, typeDepth); } - private static long addTypePayload(long bytes, DataType type) { - if (type instanceof RowType) { - for (DataField field : ((RowType) type).getFields()) { - bytes = addFieldPayload(bytes, field, true); + /** + * Account one DataType instance and its owned children. Every accepted implementation is + * matched explicitly; an unknown class throws so estimateSafely rejects weighted admission + * instead of counting a future composite type as a small primitive. + */ + private static long addTypePayload( + long bytes, DataType type, AccountingBudget budget, int typeDepth) { + if (typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException( + "Paimon cache accounting type depth exceeded"); + } + budget.charge(1L); + if (type == null) { + throw new IllegalStateException("Paimon field type is missing"); + } + Class typeClass = type.getClass(); + if (typeClass == RowType.class) { + RowType rowType = (RowType) type; + List fields = rowType.getFields(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fields)); + for (DataField field : fields) { + bytes = addFieldPayload(bytes, field, true, budget, typeDepth + 1); } - } else if (type instanceof ArrayType) { - bytes = addTypePayload(bytes, ((ArrayType) type).getElementType()); - } else if (type instanceof MapType) { - bytes = addTypePayload(bytes, ((MapType) type).getKeyType()); - bytes = addTypePayload(bytes, ((MapType) type).getValueType()); - } else if (type instanceof MultisetType) { - bytes = addTypePayload(bytes, ((MultisetType) type).getElementType()); + return bytes; } - return bytes; + if (typeClass == ArrayType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_TYPE_BYTES); + return addTypePayload( + bytes, ((ArrayType) type).getElementType(), budget, typeDepth + 1); + } + if (typeClass == VectorType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, VECTOR_TYPE_BYTES); + return addTypePayload( + bytes, ((VectorType) type).getElementType(), budget, typeDepth + 1); + } + if (typeClass == MapType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MAP_TYPE_BYTES); + bytes = addTypePayload( + bytes, ((MapType) type).getKeyType(), budget, typeDepth + 1); + return addTypePayload( + bytes, ((MapType) type).getValueType(), budget, typeDepth + 1); + } + if (typeClass == MultisetType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MULTISET_TYPE_BYTES); + return addTypePayload( + bytes, ((MultisetType) type).getElementType(), budget, typeDepth + 1); + } + String[] leafFields = LEAF_TYPE_FIELDS.get(typeClass); + if (leafFields == null) { + throw new IllegalStateException( + "Unsupported Paimon data type: " + typeClass.getName()); + } + return MetaCacheWeightUtils.saturatedAdd(bytes, leafTypeBytes(leafFields)); + } + + /** + * RowType, its unmodifiable ArrayList copy of the fields, and the four lazy lookup maps that + * getField/getFieldIndex materialize after admission. The maps are reserved up front in O(N) + * so a query cannot grow the retained graph past the admitted weight; nothing is materialized. + */ + private static long rowTypeBytes(List fields) { + long fieldCount = fields.size(); + long bytes = MetaCacheWeightUtils.saturatedAdd(ROW_TYPE_BYTES, UNMODIFIABLE_LIST_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_LIST_BYTES); + if (fieldCount == 0L) { + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); + long uncachedFieldIds = 0L; + for (DataField field : fields) { + if (field.id() < -128 || field.id() > 127) { + uncachedFieldIds++; + } + } + long uncachedIndexes = fieldCount > 128L ? fieldCount - 128L : 0L; + long mapBytes = MetaCacheWeightUtils.saturatedAdd(HASH_MAP_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(hashMapCapacity(fieldCount))); + mapBytes = addCount(mapBytes, fieldCount, HASH_MAP_NODE_BYTES); + bytes = addCount(bytes, ROW_TYPE_LAZY_MAP_COUNT, mapBytes); + // Boxed keys/values outside the Integer cache: nameToIndex values, fieldIdToField keys, + // and fieldIdToIndex boxes both again. + bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); + bytes = addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); + return addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + } + + private static long hashMapCapacity(long size) { + long capacity = 16L; + while (size > capacity - capacity / 4L) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; } private static long addString(long bytes, String value) { @@ -190,4 +429,20 @@ private static long addCount(long bytes, long count, long bytesPerItem) { return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); } + + private static final class AccountingBudget { + private long remaining; + + private AccountingBudget(long remaining) { + this.remaining = remaining; + } + + private void charge(long elements) { + if (elements < 0L || elements > remaining) { + throw new IllegalStateException( + "Paimon cache accounting work budget exceeded"); + } + remaining -= elements; + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 5d98c753b82463..180d2b44adf6cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; @@ -41,6 +42,7 @@ import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; @@ -48,6 +50,7 @@ import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; @@ -56,6 +59,7 @@ import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Rule; @@ -63,21 +67,27 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + // U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) lower-cases to two characters in Locale.ROOT. + private static final String DOTTED_CAPITAL_I = String.valueOf((char) 0x0130); @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -359,9 +369,45 @@ public void testIcebergPreparationFailureIsFailClosed() { } @Test + public void testManifestAccountingAcceptsOnlyGenericContentFileCopies() { + DataFile copied = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/copied.parquet").withFileSizeInBytes(10L).withRecordCount(1L) + .build().copy(); + ManifestCacheValue supported = ManifestCacheValue.forDataFiles( + Collections.singletonList(copied)); + Assert.assertTrue(supported.isAccountingComplete()); + Assert.assertTrue(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/copied.avro", ManifestContent.DATA), + supported).isComplete()); + + // A proxy, mock or third-party ContentFile has an unknown retained layout: keep the file + // for the current query but reject weighted admission. + DataFile proxy = newInterfaceProxy(DataFile.class); + ManifestCacheValue unsupported = ManifestCacheValue.forDataFiles( + Collections.singletonList(proxy)); + Assert.assertEquals(Collections.singletonList(proxy), unsupported.getDataFiles()); + Assert.assertFalse(unsupported.isAccountingComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", + IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/proxy.avro", ManifestContent.DATA), + unsupported).getIncompleteReason()); + + // A data-file implementation inside a delete manifest is equally unsupported. + ManifestCacheValue.Builder deleteBuilder = ManifestCacheValue.deleteFilesBuilder(); + deleteBuilder.addDeleteFile(newInterfaceProxy(DeleteFile.class)); + Assert.assertFalse(deleteBuilder.build().isAccountingComplete()); + } + + @Test + @SuppressWarnings("unchecked") public void testManifestAccountingFailureKeepsFilesAndRejectsWeightedAdmission() { - DataFile file = Mockito.mock(DataFile.class); - Mockito.when(file.columnSizes()).thenThrow(new IllegalStateException("new metrics representation")); + Map brokenColumnSizes = Mockito.mock(Map.class); + Mockito.when(brokenColumnSizes.size()) + .thenThrow(new IllegalStateException("new metrics representation")); + DataFile file = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/broken-metrics.parquet").withFileSizeInBytes(10L) + .withMetrics(new Metrics(1L, brokenColumnSizes, null, null, null)) + .build(); ManifestCacheValue value = ManifestCacheValue.forDataFiles(Collections.singletonList(file)); MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry( @@ -385,7 +431,8 @@ public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { smallValue.prepareForCachePublication(mapping); largeValue.prepareForCachePublication(mapping); - long expectedPayloadDelta = (largePayload.length() - 1L) * 4L; + long expectedPayloadDelta = (MetaCacheWeightUtils.estimatedStringBytes(largePayload) + - MetaCacheWeightUtils.estimatedStringBytes("x")) * 2L; Assert.assertTrue(largeValue.getSizeEstimate().getBytes() - smallValue.getSizeEstimate().getBytes() >= expectedPayloadDelta); } @@ -416,8 +463,10 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { specBuilder.identity(field.name()); sortBuilder.asc(field.name()); } + PartitionSpec populatedSpec = specBuilder.build(); + SortOrder populatedSortOrder = sortBuilder.build(); TableMetadata fieldHistory = TableMetadata.newTableMetadata( - largeSchema, specBuilder.build(), sortBuilder.build(), + largeSchema, populatedSpec, populatedSortOrder, "file:/warehouse/field-history", Collections.emptyMap()); fieldHistory = TableMetadata.buildFrom(fieldHistory) .setDefaultPartitionSpec( @@ -428,6 +477,23 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { TableMetadata emptyFields = TableMetadata.newTableMetadata( largeSchema, PartitionSpec.unpartitioned(), SortOrder.unsorted(), "file:/warehouse/field-history", Collections.emptyMap()); + TableMetadata partitionFields = TableMetadata.newTableMetadata( + largeSchema, populatedSpec, SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + TableMetadata sortFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), populatedSortOrder, + "file:/warehouse/field-history", Collections.emptyMap()); + + // Cache values come from Iceberg's parser. Round-trip builder fixtures so JOL measures + // the same canonical ownership graph used in production instead of write-side builders. + // Metadata locations of compared fixtures have equal length: that String is not part of + // retainedTablePayloadBytes and must not leak into the JOL delta. + schemaHistory = roundTripMetadata(schemaHistory, "/metadata/jol-schema-large.json"); + smallSchemaOnly = roundTripMetadata(smallSchemaOnly, "/metadata/jol-schema-small.json"); + fieldHistory = roundTripMetadata(fieldHistory, "/metadata/jol-fields-both.json"); + emptyFields = roundTripMetadata(emptyFields, "/metadata/jol-fields-none.json"); + partitionFields = roundTripMetadata(partitionFields, "/metadata/jol-fields-spec.json"); + sortFields = roundTripMetadata(sortFields, "/metadata/jol-fields-sort.json"); long schemaDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(schemaHistory)) @@ -437,9 +503,296 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { tableWithMetadata(fieldHistory)) - IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(emptyFields)); + long partitionFieldDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(partitionFields)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + long sortFieldDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(sortFields)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + + materializeAllLazyState(schemaHistory); + materializeAllLazyState(smallSchemaOnly); + materializeAllLazyState(fieldHistory); + materializeAllLazyState(emptyFields); + materializeAllLazyState(partitionFields); + materializeAllLazyState(sortFields); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg schema history", 0L, schemaDelta, + smallSchemaOnly, schemaHistory); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg partition fields", 0L, partitionFieldDelta, + emptyFields, partitionFields); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg sort fields", 0L, sortFieldDelta, + emptyFields, sortFields); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg spec and sort fields", 0L, specAndSortDelta, + emptyFields, fieldHistory); + } + + @Test + public void testSchemaLookupFormulaScalesWithSchemaWidth() { + for (int fieldCount : new int[] {3, 32, 100, 1000}) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + assertSchemaLookupFormula( + new Schema(0, fields), "schema lookup width " + fieldCount); + } + // Single-column schemas cannot grow; check their partition-spec graph instead. + assertPartitionSpecFormula(new Schema(0, Types.NestedField.optional( + 1, "field_0", Types.StringType.get())), "partition spec width 1"); + } + + @Test + public void testSchemaLookupFormulaCountsListAndMapSyntheticFields() { + List listFields = new ArrayList<>(); + listFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + listFields.add(Types.NestedField.optional( + index + 2, "list_" + index, + Types.ListType.ofOptional(10_000 + index, Types.StringType.get()))); + } + assertSchemaLookupFormula(new Schema(0, listFields), "schema lookup list synthetic fields"); + + List mapFields = new ArrayList<>(); + mapFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + mapFields.add(Types.NestedField.optional( + index + 2, "map_" + index, + Types.MapType.ofOptional( + 10_000 + index * 2, 10_001 + index * 2, + Types.StringType.get(), Types.LongType.get()))); + } + assertSchemaLookupFormula(new Schema(0, mapFields), "schema lookup map synthetic fields"); + } + + @Test + public void testSchemaLookupFormulaCountsNestedStructShortAliases() { + List listFields = new ArrayList<>(); + listFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + Types.StructType elementType = Types.StructType.of(Types.NestedField.optional( + 20_000 + index, "leaf", Types.StringType.get())); + listFields.add(Types.NestedField.optional( + index + 2, "list_" + index, + Types.ListType.ofOptional(10_000 + index, elementType))); + } + assertSchemaLookupFormula( + new Schema(0, listFields), "list struct short aliases"); + + List mapFields = new ArrayList<>(); + mapFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + Types.StructType valueType = Types.StructType.of(Types.NestedField.optional( + 30_000 + index, "leaf", Types.LongType.get())); + mapFields.add(Types.NestedField.optional( + index + 2, "map_" + index, + Types.MapType.ofOptional( + 10_000 + index * 2, 10_001 + index * 2, + Types.StringType.get(), valueType))); + } + assertSchemaLookupFormula( + new Schema(0, mapFields), "map struct short aliases"); + } + + @Test + public void testSchemaIdentifierFieldFormulaScalesWithWidth() { + for (int fieldCount : new int[] {1, 32, 100, 1000}) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.required( + index + 1, "identifier_" + index, Types.LongType.get())) + .collect(Collectors.toList()); + Set identifierIds = fields.stream() + .map(Types.NestedField::fieldId) + .collect(Collectors.toSet()); + Schema withoutIdentifiers = new Schema(0, fields); + Schema withIdentifiers = new Schema(0, fields, identifierIds); + TableMetadata empty = roundTripMetadata(TableMetadata.newTableMetadata( + withoutIdentifiers, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-identifiers", Collections.emptyMap()), + "/metadata/jol-schema-identifiers-none-" + fieldCount + ".json"); + TableMetadata populated = roundTripMetadata(TableMetadata.newTableMetadata( + withIdentifiers, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-identifiers", Collections.emptyMap()), + "/metadata/jol-schema-identifiers-with-" + fieldCount + ".json"); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(empty)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populated)); + materializeAllLazyState(empty); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg identifier fields " + fieldCount, + emptyEstimate, populatedEstimate, empty, populated); + } + } + + @Test + public void testUnicodeLowerCaseSchemaFormulaAgainstJolOwnedGraph() { + // U+0130 lower-cases to "i" plus U+0307 in Locale.ROOT: the generated lower-case index + // keys are longer than their sources and switch the String coder to UTF-16. + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithSchemaAndProperties( + unicodeNestedSchema(1), Collections.emptyMap()); + IcebergTableCacheValue populated = tableValueWithSchemaAndProperties( + unicodeNestedSchema(33), Collections.emptyMap()); + + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(small); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg unicode lower-case nested fields", + smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testUnicodeLowerCasePartitionNameFormulaAgainstJolOwnedGraph() { + List fields = new ArrayList<>(); + fields.add(Types.NestedField.optional(1, DOTTED_CAPITAL_I + "dentity_Key", Types.StringType.get())); + for (int index = 0; index < 8; index++) { + fields.add(Types.NestedField.optional(index + 2, DOTTED_CAPITAL_I + "_field_" + index, + Types.StringType.get())); + } + // The identity partition name is lower-cased three times: by the partition StructType, + // by the secondary Schema and by the secondary StructType. + assertPartitionSpecFormula(new Schema(0, fields), "unicode partition name"); + } + + @Test + public void testTableAccountingCharacterBudgetFailsClosedWithoutFailingLoad() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + String hugeName = repeatedCharacter('x', 1 << 20); + List fields = IntStream.range(0, 5) + .mapToObj(index -> Types.NestedField.optional( + index + 1, hugeName + index, Types.StringType.get())) + .collect(Collectors.toList()); + IcebergTableCacheValue value = tableValueWithSchemaAndProperties( + new Schema(0, fields), Collections.emptyMap()); + + IllegalStateException budgetFailure = Assert.assertThrows(IllegalStateException.class, + () -> IcebergCacheSizeEstimator.retainedTablePayloadBytes( + value.getRetainedIcebergTable())); + Assert.assertTrue(budgetFailure.getMessage(), + budgetFailure.getMessage().contains("character budget")); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(mapping); + + Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), + estimate.getIncompleteReason().startsWith("iceberg_table_preparation_failed:")); + Assert.assertNotNull(value.getRetainedIcebergTable()); + Assert.assertEquals(5, value.getRetainedIcebergTable().schema().columns().size()); + Assert.assertSame(value.getRetainedIcebergTable(), value.newQueryScopedTable()); + } + + @Test + public void testTableAccountingElementBudgetFailsClosedWithoutFailingLoad() { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + @SuppressWarnings("unchecked") + Map oversizedProperties = Mockito.mock(Map.class); + Mockito.when(oversizedProperties.size()).thenReturn(2_000_001); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(oversizedProperties); + Table table = tableWithMetadata(metadata); + + IllegalStateException budgetFailure = Assert.assertThrows(IllegalStateException.class, + () -> IcebergCacheSizeEstimator.retainedTablePayloadBytes(table)); + Assert.assertTrue(budgetFailure.getMessage(), + budgetFailure.getMessage().contains("work budget")); + Mockito.verify(oversizedProperties, Mockito.never()).entrySet(); + + IcebergTableCacheValue value = new IcebergTableCacheValue(table); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), + estimate.getIncompleteReason().startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(value.getRetainedIcebergTable(), value.newQueryScopedTable()); + } + + @Test + public void testAdmittedTableEstimateCoversFullyMaterializedRetainedGraph() { + // Whole-entry oracle: the admitted weight must cover the complete retained graph after + // every lazy Schema/StructType/PartitionSpec index a scan can create has materialized, + // including the O(distinctSources * fields) fieldsBySourceId graph. Component deltas can + // miss a shared baseline; this compares absolute sizes. + // The tight bound applies once the variable payload dominates the fixed per-table base + // (TABLE_BASE_BYTES); small tables are deliberately covered by that base. + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + for (int width : new int[] {1, 100, 1000}) { + assertAdmittedEstimateCoversRetainedGraph( + "flat identity-partitioned " + width, mapping, + tableValueWithIdentityPartitionedFields(width), width >= 100); + assertAdmittedEstimateCoversRetainedGraph( + "nested mixed-case " + width, mapping, + tableValueWithNestedMixedCaseFields(width), width >= 1000); + } + } + + private void assertAdmittedEstimateCoversRetainedGraph( + String fixture, NameMapping mapping, IcebergTableCacheValue value, + boolean requireTightBound) { + long estimate = value.prepareForCachePublication(mapping).getBytes(); + long before = EstimatorCalibrationAssertions.graphSize(value); + materializeAllLazyState(value); + long after = EstimatorCalibrationAssertions.graphSize(value); + Assert.assertTrue(fixture + " lazy state must grow the retained graph", after > before); + Assert.assertTrue(fixture + " underestimates the materialized entry: estimate=" + + estimate + ", retained=" + after, estimate >= after); + if (requireTightBound) { + Assert.assertTrue(fixture + " is excessively conservative: estimate=" + estimate + + ", retained=" + after, estimate <= Math.ceil(after * 1.10D)); + } + } + + @Test + public void testSchemaFormulaCountsBoxedIdsOfUncachedFieldIds() { + // TableMetadata.newTableMetadata() reassigns fresh ids from 1, which the JVM Integer + // cache serves for free. Add the schema to existing metadata instead so ids above 127 + // survive and every lookup map really boxes its keys and values. + List oneFlat = Collections.singletonList( + Types.NestedField.optional(10_000, "Field_0", Types.StringType.get())); + List manyFlat = IntStream.range(0, 32) + .mapToObj(index -> Types.NestedField.optional( + 10_000 + index, "Field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + assertRetainedPayloadDelta("uncached flat field ids", + metadataWithAddedSchema(new Schema(1, oneFlat)), + metadataWithAddedSchema(new Schema(1, manyFlat)), "jol-uncached-ids"); + + List nestedFields = IntStream.range(0, 32) + .mapToObj(index -> Types.NestedField.optional( + 20_000 + index, "Nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema oneNested = new Schema(1, Types.NestedField.optional(10_000, "payload", + Types.StructType.of(nestedFields.get(0)))); + Schema manyNested = new Schema(1, Types.NestedField.optional(10_000, "payload", + Types.StructType.of(nestedFields))); + assertRetainedPayloadDelta("uncached nested field ids", + metadataWithAddedSchema(oneNested), metadataWithAddedSchema(manyNested), + "jol-uncached-ids"); + } - Assert.assertTrue(schemaDelta >= 99L * 512L); - Assert.assertTrue(specAndSortDelta >= 100L * (384L + 256L)); + private TableMetadata metadataWithAddedSchema(Schema schema) { + Schema base = new Schema(0, Types.NestedField.optional(1, "base", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata( + base, PartitionSpec.unpartitioned(), "file:/warehouse/uncached-ids", + Collections.emptyMap()); + return TableMetadata.buildFrom(metadata).addSchema(schema) + .setCurrentSchema(schema.schemaId()).discardChanges().build(); } @Test @@ -453,6 +806,33 @@ public void testTablePayloadAccountsForRetainedHistoricalMetadata() { Assert.assertTrue(largeBytes - smallBytes >= 64L * 1024L - 32L); } + @Test + public void testStatisticsBlobFormulaAgainstJolOwnedGraph() { + GenericStatisticsFile empty = new GenericStatisticsFile( + 1L, "/stats/file.puffin", 1L, 1L, Collections.emptyList()); + List blobs = IntStream.range(0, 32) + .mapToObj(index -> new GenericBlobMetadata( + "blob-type-" + index, + 1L, + 1L, + java.util.Arrays.asList(10_000 + index, 20_000 + index), + Collections.singletonMap( + "property-" + index, "value-" + index))) + .collect(Collectors.toList()); + GenericStatisticsFile populated = new GenericStatisticsFile( + 1L, "/stats/file.puffin", 1L, 1L, blobs); + TableMetadata emptyMetadata = metadataWithStatisticsFile(empty); + TableMetadata populatedMetadata = metadataWithStatisticsFile(populated); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyMetadata)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populatedMetadata)); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg statistics blobs", emptyEstimate, populatedEstimate, empty, populated); + } + @Test public void testTableEstimateAccountsForRetainedBranchHistory() { TableMetadata oneCommit = metadataWithSnapshotSequence(1L); @@ -688,7 +1068,7 @@ protected CatalogIf getCatalog(long catalogId) { } @Test - public void testWeightedTablePublicationRetainsNonGrowingGeneration() { + public void testWeightedV1TablePublicationFailsClosedWithoutIo() { NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), @@ -711,29 +1091,13 @@ public void testWeightedTablePublicationRetainsNonGrowingGeneration() { value.prepareForCachePublication(mapping); - Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Assert.assertFalse(value.getSizeEstimate().isComplete()); Table retained = value.getRetainedIcebergTable(); Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); - Table firstUse = value.getIcebergTable(); - Table secondUse = value.getIcebergTable(); - Assert.assertNotSame(retained, firstUse); - Assert.assertNotSame(firstUse, secondUse); - Assert.assertNotSame(retained.currentSnapshot(), firstUse.currentSnapshot()); - Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); - Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); - Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); - - IcebergSnapshotEntryKey snapshotKey = - IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); - IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( - IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), - Optional.empty(), retained, value.getRetainedCurrentSnapshotJson()); - snapshotValue.prepareForCachePublication(snapshotKey); - Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), - snapshotValue.getSizeEstimate().isComplete()); - Table snapshotQuery = snapshotValue.getIcebergTable().get(); - Assert.assertEquals(2, - snapshotQuery.currentSnapshot().dataManifests(snapshotQuery.io()).size()); + Mockito.verifyNoInteractions(fileIO); + retained.currentSnapshot().allManifests(fileIO); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/current-a.avro"); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/current-b.avro"); } @Test @@ -778,7 +1142,8 @@ public void testTablePublicationDoesNotReadHistoricalManifestLists() { + "\"summary\":{\"operation\":\"append\"}," + "\"manifest-list\":\"/manifest-list/history.avro\",\"schema-id\":0}"); Snapshot current = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," - + "\"summary\":{\"operation\":\"append\"},\"manifests\":[],\"schema-id\":0}"); + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/current.avro\",\"schema-id\":0}"); metadata = TableMetadata.buildFrom(metadata) .addSnapshot(historical) .setBranchSnapshot(current, SnapshotRef.MAIN_BRANCH) @@ -820,16 +1185,288 @@ public void testManifestFormulaAgainstJolOwnedGraph() { Collections.singletonList(dataFileWithPathPayload(16))); ManifestCacheValue longTail = ManifestCacheValue.forDataFiles( Collections.singletonList(dataFileWithPathPayload(4096))); + ManifestCacheValue emptyDeletes = ManifestCacheValue.forDeleteFiles(Collections.emptyList()); + ManifestCacheValue populatedDeletes = ManifestCacheValue.forDeleteFiles( + IntStream.range(0, 32).mapToObj(this::deleteFileWithMetrics) + .collect(Collectors.toList())); long emptyEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, empty).getBytes(); long populatedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, populated).getBytes(); long shortTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, shortTail).getBytes(); long longTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, longTail).getBytes(); + IcebergManifestEntryKey deleteKey = new IcebergManifestEntryKey( + "/manifest/jol-delete.avro", ManifestContent.DELETES); + long emptyDeleteEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + deleteKey, emptyDeletes).getBytes(); + long populatedDeleteEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + deleteKey, populatedDeletes).getBytes(); EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg manifest files", emptyEstimate, populatedEstimate, empty, populated); EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg long-tail path", shortTailEstimate, longTailEstimate, shortTail, longTail); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest delete files", emptyDeleteEstimate, populatedDeleteEstimate, + emptyDeletes, populatedDeletes); + } + + @Test + public void testManifestPartitionDataFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol-partitioned.avro", ManifestContent.DATA); + for (int fieldCount : new int[] {1, 8, 32, 100}) { + ManifestCacheValue unpartitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, 0)); + ManifestCacheValue partitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, fieldCount)); + long unpartitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, unpartitioned).getBytes(); + long partitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, partitioned).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest partition fields " + fieldCount, + unpartitionedEstimate, partitionedEstimate, + unpartitioned, partitioned); + } + } + + @Test + public void testManifestVariablePartitionValuesAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol-variable-partition.avro", ManifestContent.DATA); + ManifestCacheValue unpartitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, 0)); + long unpartitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, unpartitioned).getBytes(); + + ManifestCacheValue strings = ManifestCacheValue.forDataFiles( + manifestFilesWithPartitions(32, 8, Types.StringType.get(), + (fileIndex, fieldIndex) -> repeatedCharacter('s', 64) + + fileIndex + "_" + fieldIndex)); + long stringEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, strings).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest string partitions", + unpartitionedEstimate, stringEstimate, unpartitioned, strings); + + ManifestCacheValue binary = ManifestCacheValue.forDataFiles( + manifestFilesWithPartitions(32, 8, Types.BinaryType.get(), + (fileIndex, fieldIndex) -> ByteBuffer.allocate(64))); + long binaryEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, binary).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest binary partitions", + unpartitionedEstimate, binaryEstimate, unpartitioned, binary); + } + + @Test + public void testManifestAccountingFindsPathTailAtAnyPosition() { + List baselineFiles = new ArrayList<>(); + List tailFiles = new ArrayList<>(); + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + for (int index = 0; index < 101; index++) { + baselineFiles.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/file-" + index + ".parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()); + tailFiles.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(index == 57 ? largePath : "/data/file-" + index + ".parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()); + } + ManifestCacheValue baseline = ManifestCacheValue.forDataFiles(baselineFiles); + ManifestCacheValue withTail = ManifestCacheValue.forDataFiles(tailFiles); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-tail.avro", ManifestContent.DATA); + + long baselineBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, baseline).getBytes(); + long tailBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, withTail).getBytes(); + + Assert.assertTrue(tailBytes - baselineBytes >= largePath.length() - 32L); + Assert.assertTrue(tailBytes - baselineBytes < largePath.length() * 2L); + } + + @Test + @SuppressWarnings("unchecked") + public void testManifestAccountingFailsClosedBeyondWorkBudget() { + Map oversizedBounds = Mockito.mock(Map.class); + Mockito.when(oversizedBounds.size()).thenReturn(8_000_001); + // GenericDataFile wraps the bounds map without copying it, so the oversized size is + // observed by accounting without allocating the entries. + DataFile file = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/oversized-bounds.parquet").withFileSizeInBytes(10L) + .withMetrics(new Metrics(1L, null, null, null, null, oversizedBounds, null)) + .build(); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles( + Collections.singletonList(file)); + + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/oversized.avro", ManifestContent.DATA), + value).isComplete()); + } + + @Test + public void testManifestAccountingFailsClosedForUnreadablePartition() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("id").build(); + DataFile file = DataFiles.builder(spec) + .withPath("/data/unreadable-partition.parquet").withFileSizeInBytes(10L) + .withRecordCount(1L).withPartitionPath("id=1").build(); + // A partition value of a class the accounting does not know cannot be sized. + ((PartitionData) file.partition()).set(0, new Object()); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles( + Collections.singletonList(file)); + + Assert.assertFalse(value.isAccountingComplete()); + } + + @Test + public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + // Compare two non-empty schemas: an empty schema never materializes any lookup index, + // so it is not a fair baseline for the per-field formula. + IcebergTableCacheValue emptyTable = tableValueWithFields(1); + IcebergTableCacheValue populatedTable = tableValueWithFields(32); + long emptyTableEstimate = emptyTable.prepareForCachePublication(mapping).getBytes(); + long populatedTableEstimate = populatedTable.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(emptyTable); + materializeAllLazyState(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg table fields", emptyTableEstimate, populatedTableEstimate, + emptyTable, populatedTable); + + Table keyTable = tableWithMetadataLocation("/metadata/jol-snapshot-v1.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, keyTable).get(); + IcebergSnapshotCacheValue emptySnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(0), new IcebergSnapshot(-1L, 0L)); + IcebergSnapshotCacheValue populatedSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32), new IcebergSnapshot(-1L, 0L)); + long emptySnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, emptySnapshot).getBytes(); + long populatedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, populatedSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot partitions", emptySnapshotEstimate, populatedSnapshotEstimate, + emptySnapshot, populatedSnapshot); + } + + @Test + public void testNestedSchemaFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithNestedFields(1); + IcebergTableCacheValue populated = tableValueWithNestedFields(33); + + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(small); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg nested fields", smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testTablePropertyFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue empty = tableValueWithProperties(0); + IcebergTableCacheValue populated = tableValueWithProperties(32); + + long emptyEstimate = empty.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg table properties", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testSnapshotHistoryFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithSnapshotHistory(1, false); + IcebergTableCacheValue populated = tableValueWithSnapshotHistory(33, false); + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot history", smallEstimate, populatedEstimate, small, populated); + + IcebergTableCacheValue withoutLog = tableValueWithSnapshotHistory(33, false); + IcebergTableCacheValue withLog = tableValueWithSnapshotHistory(33, true); + long withoutLogEstimate = withoutLog.prepareForCachePublication(mapping).getBytes(); + long withLogEstimate = withLog.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot log", withoutLogEstimate, withLogEstimate, withoutLog, withLog); + } + + @Test + public void testV1SnapshotAccountingFailsClosedWithoutIo() { + Snapshot snapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifests\":[\"/manifest/v1-a.avro\",\"/manifest/v1-b.avro\"]}"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(snapshot))); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertFalse(estimate.isComplete()); + } + + @Test + public void testSnapshotWithoutSummaryRemainsCacheable() { + Snapshot snapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\"}"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(snapshot))); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + } + + @Test + public void testMaterializedV2SnapshotPayloadFailsClosed() throws Exception { + String snapshotJson = "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\"}"; + Snapshot unloaded = SnapshotParser.fromJson(snapshotJson); + MetaCacheSizeEstimate unloadedEstimate = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(unloaded))) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertTrue(unloadedEstimate.getIncompleteReason(), unloadedEstimate.isComplete()); + + for (String fieldName : new String[] { + "allManifests", "dataManifests", "deleteManifests", + "addedDataFiles", "removedDataFiles", + "addedDeleteFiles", "removedDeleteFiles"}) { + Snapshot loaded = SnapshotParser.fromJson(snapshotJson); + Field retainedField = loaded.getClass().getDeclaredField(fieldName); + retainedField.setAccessible(true); + retainedField.set(loaded, Collections.emptyList()); + + MetaCacheSizeEstimate loadedEstimate = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(loaded))) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertFalse(fieldName, loadedEstimate.isComplete()); + } + } + + @Test + public void testSnapshotKeyIdPayloadIsAccounted() { + String longKeyId = repeatedCharacter('k', 64 * 1024); + Snapshot shortSnapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\",\"key-id\":\"k\"}"); + Snapshot longSnapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\",\"key-id\":\"" + + longKeyId + "\"}"); + + long shortBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithSnapshots(shortSnapshot))); + long longBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithSnapshots(longSnapshot))); + + Assert.assertTrue(longBytes - shortBytes >= longKeyId.length() - 8L); } @Test @@ -853,7 +1490,7 @@ public void testManifestEstimateAccountsForSkewedFilePaths() { long smallBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue).getBytes(); long largeBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue).getBytes(); - Assert.assertTrue(largeBytes - smallBytes >= (largePath.length() - "/data/x.parquet".length()) * 2L); + Assert.assertTrue(largeBytes - smallBytes >= largePath.length() - "/data/x.parquet".length()); } @Test @@ -940,7 +1577,7 @@ public void testManifestEstimateAccountsForDeleteFileAuxiliaryPayload() { } @Test - public void testSnapshotPublicationDoesNotMaterializeManifestLists() { + public void testV1SnapshotPublicationDoesNotPolluteManifestIo() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), "file:/warehouse/db/tbl", Collections.emptyMap()); @@ -950,6 +1587,11 @@ public void testSnapshotPublicationDoesNotMaterializeManifestLists() { metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) .discardChanges().withMetadataLocation("/metadata/v1.json").build(); FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); Table table = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), Optional.empty(), table); @@ -958,11 +1600,13 @@ public void testSnapshotPublicationDoesNotMaterializeManifestLists() { value.prepareForCachePublication(key); - Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), - value.getSizeEstimate().isComplete()); + Assert.assertFalse(value.getSizeEstimate().isComplete()); Table queryTable = value.getIcebergTable().get(); - Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); + Assert.assertSame(table.currentSnapshot(), queryTable.currentSnapshot()); Mockito.verifyNoInteractions(fileIO); + queryTable.currentSnapshot().allManifests(fileIO); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/a.avro"); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/b.avro"); } @Test @@ -1220,23 +1864,190 @@ private Table tableWithMetadataLocation(String metadataLocation) { return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); } + private IcebergTableCacheValue tableValueWithFields(int fieldCount) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + TableMetadata metadata = TableMetadata.newTableMetadata( + new Schema(fields), PartitionSpec.unpartitioned(), + "file:/warehouse/jol-table", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithNestedFields(int nestedFieldCount) { + List nestedFields = IntStream.range(0, nestedFieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 2, "nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema(Types.NestedField.optional( + 1, "payload", Types.StructType.of(nestedFields))); + return tableValueWithSchemaAndProperties(schema, Collections.emptyMap()); + } + + private Schema unicodeNestedSchema(int nestedFieldCount) { + List nestedFields = new ArrayList<>(); + for (int index = 0; index < nestedFieldCount; index++) { + nestedFields.add(Types.NestedField.optional( + index + 10, "Nested_" + DOTTED_CAPITAL_I + "_" + index, Types.StringType.get())); + } + Types.StructType element = Types.StructType.of(Types.NestedField.optional( + 3, "Leaf_" + DOTTED_CAPITAL_I, Types.StringType.get())); + return new Schema( + Types.NestedField.optional(1, "Payload_" + DOTTED_CAPITAL_I, Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "List_" + DOTTED_CAPITAL_I, + Types.ListType.ofOptional(4, element)), + Types.NestedField.optional(5, "Map_" + DOTTED_CAPITAL_I, Types.MapType.ofOptional( + 6, 7, Types.StringType.get(), Types.StringType.get()))); + } + + private IcebergTableCacheValue tableValueWithIdentityPartitionedFields(int fieldCount) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema(fields); + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + } + return tableValueWithSchemaAndSpec(schema, specBuilder.build()); + } + + private IcebergTableCacheValue tableValueWithNestedMixedCaseFields(int nestedFieldCount) { + // Uncached field ids, upper-case names and list/map synthetic fields exercise the boxed + // key, lower-case String and short-alias terms of the schema formula. + List nestedFields = IntStream.range(0, nestedFieldCount) + .mapToObj(index -> Types.NestedField.optional( + 1000 + index, "Nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema( + Types.NestedField.optional(1, "payload", Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "list", Types.ListType.ofOptional(3, + Types.StructType.of(Types.NestedField.optional( + 4, "leaf", Types.StringType.get())))), + Types.NestedField.optional(5, "map", Types.MapType.ofOptional( + 6, 7, Types.StringType.get(), Types.LongType.get())), + Types.NestedField.optional(8, "id", Types.LongType.get())); + return tableValueWithSchemaAndSpec( + schema, PartitionSpec.builderFor(schema).identity("id").build()); + } + + private IcebergTableCacheValue tableValueWithSchemaAndSpec(Schema schema, PartitionSpec spec) { + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, spec, "file:/warehouse/jol-table", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithProperties(int propertyCount) { + Map properties = IntStream.range(0, propertyCount).boxed() + .collect(Collectors.toMap(index -> "key_" + index, index -> "value_" + index)); + Schema schema = new Schema(Types.NestedField.required( + 1, "id", Types.IntegerType.get())); + return tableValueWithSchemaAndProperties(schema, properties); + } + + private IcebergTableCacheValue tableValueWithSchemaAndProperties( + Schema schema, Map properties) { + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/jol-table", properties); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithSnapshotHistory( + int snapshotCount, boolean includeSnapshotLog) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"jol-table\",") + .append("\"location\":\"file:/warehouse/jol-table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index); + if (index > 0) { + json.append(",\"parent-snapshot-id\":").append(1000L + index - 1L); + } + json.append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/jol/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + if (includeSnapshotLog) { + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "/metadata/jol-history-v1.json", json.toString()); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Exception { + Map partitionItems = new java.util.HashMap<>(); + Map partitions = new java.util.HashMap<>(); + List partitionColumns = Collections.singletonList( + new org.apache.doris.catalog.Column( + "part", org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + for (int index = 0; index < partitionCount; index++) { + String value = Integer.toString(index); + String name = "part=" + value; + partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( + IcebergUtils.getPartitionRange(value, "day", partitionColumns))); + partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, + 1L, 1L, Collections.singletonList(value), + Collections.singletonList("day"))); + } + return new IcebergPartitionInfo( + partitionItems, partitions, Collections.emptyMap()); + } + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { - Map columnSizes = IntStream.range(0, 8).boxed() - .collect(Collectors.toMap(column -> column, column -> (long) index + column)); - Map valueCounts = new java.util.HashMap<>(columnSizes); - Map nullCounts = new java.util.HashMap<>(columnSizes); - Map nanCounts = new java.util.HashMap<>(columnSizes); - Map lowerBounds = IntStream.range(0, 8).boxed() - .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); - Map upperBounds = IntStream.range(0, 8).boxed() - .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Map columnSizes = metricLongMap(index, 0); + Map valueCounts = metricLongMap(index, 1); + Map nullCounts = metricLongMap(index, 2); + Map nanCounts = metricLongMap(index, 3); + Map lowerBounds = metricBufferMap(); + Map upperBounds = metricBufferMap(); Metrics metrics = new Metrics( 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); return DataFiles.builder(PartitionSpec.unpartitioned()) .withPath("/data/jol-" + index + ".parquet") .withFileSizeInBytes(1024L) .withMetrics(metrics) - .build(); + .build() + .copy(); } private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { @@ -1247,12 +2058,198 @@ private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { .build(); } + private DeleteFile deleteFileWithMetrics(int index) { + Map columnSizes = metricLongMap(index, 0); + Map valueCounts = metricLongMap(index, 1); + Map nullCounts = metricLongMap(index, 2); + Map nanCounts = metricLongMap(index, 3); + Map lowerBounds = metricBufferMap(); + Map upperBounds = metricBufferMap(); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .withReferencedDataFile("/data/jol-" + index + ".parquet") + .withMetrics(metrics) + .build() + .copy(); + } + + private Map metricLongMap(int fileIndex, int mapIndex) { + Map values = new java.util.HashMap<>(); + for (int column = 0; column < 8; column++) { + values.put(Integer.valueOf(10_000 + column), + Long.valueOf(10_000L + fileIndex * 100L + mapIndex * 10L + column)); + } + return values; + } + + private Map metricBufferMap() { + Map values = new java.util.HashMap<>(); + for (int column = 0; column < 8; column++) { + values.put(Integer.valueOf(10_000 + column), ByteBuffer.allocate(32)); + } + return values; + } + + private List manifestFilesWithIntegerPartitions( + int fileCount, int partitionFieldCount) { + return manifestFilesWithPartitions( + fileCount, partitionFieldCount, Types.IntegerType.get(), + (fileIndex, fieldIndex) -> Integer.valueOf( + 10_000 + fileIndex * partitionFieldCount + fieldIndex)); + } + + private List manifestFilesWithPartitions( + int fileCount, int partitionFieldCount, Type.PrimitiveType partitionType, + BiFunction valueFactory) { + if (partitionFieldCount == 0) { + return IntStream.range(0, fileCount) + .mapToObj(index -> DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/partition-data/file-" + index + ".parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()) + .collect(Collectors.toList()); + } + List fields = IntStream.range(0, partitionFieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "partition_" + index, partitionType)) + .collect(Collectors.toList()); + Schema schema = new Schema(fields); + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + fields.forEach(field -> specBuilder.identity(field.name())); + PartitionSpec spec = specBuilder.build(); + DataFiles.Builder fileBuilder = DataFiles.builder(spec); + PartitionData partitionData = new PartitionData(spec.partitionType()); + List files = new ArrayList<>(fileCount); + for (int fileIndex = 0; fileIndex < fileCount; fileIndex++) { + for (int fieldIndex = 0; fieldIndex < partitionFieldCount; fieldIndex++) { + partitionData.set(fieldIndex, valueFactory.apply(fileIndex, fieldIndex)); + } + files.add(fileBuilder.withPartition(partitionData) + .withPath("/partition-data/file-" + fileIndex + ".parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()); + } + return files; + } + private Table tableWithMetadata(TableMetadata metadata) { TableOperations operations = Mockito.mock(TableOperations.class); Mockito.when(operations.current()).thenReturn(metadata); return new BaseTable(operations, "db.tbl"); } + private TableMetadata roundTripMetadata(TableMetadata metadata, String metadataLocation) { + return TableMetadataParser.fromJson(metadataLocation, TableMetadataParser.toJson(metadata)); + } + + private void materializeAllLazyState(IcebergTableCacheValue value) { + Table table = value.getRetainedIcebergTable(); + materializeAllLazyState(((HasTableOperations) table).operations().current()); + } + + private void materializeAllLazyState(TableMetadata metadata) { + for (Schema schema : metadata.schemas()) { + materializeSchemaAndStruct(schema); + } + for (PartitionSpec spec : metadata.specs()) { + spec.fields(); + spec.javaClasses(); + Types.StructType partitionType = spec.partitionType(); + spec.rawPartitionType(); + if (!spec.fields().isEmpty()) { + spec.getFieldsBySourceId(spec.fields().get(0).sourceId()); + } + materializeStructAndSecondarySchema(partitionType); + } + } + + private void materializeSchemaAndStruct(Schema schema) { + if (schema.columns().isEmpty()) { + return; + } + Types.NestedField first = schema.columns().get(0); + materializeSchemaIndexes(schema, first); + materializeStructAndSecondarySchema(schema.asStruct()); + } + + private void materializeStructAndSecondarySchema(Types.StructType struct) { + if (struct.fields().isEmpty()) { + return; + } + Types.NestedField first = struct.fields().get(0); + materializeStructIndexes(struct, first); + Schema secondary = struct.asSchema(); + materializeSchemaIndexes(secondary, first); + materializeStructIndexes(secondary.asStruct(), first); + } + + private void materializeSchemaIndexes(Schema schema, Types.NestedField first) { + schema.findField(first.name()); + schema.findField(first.fieldId()); + schema.caseInsensitiveFindField(first.name().toUpperCase(java.util.Locale.ROOT)); + schema.idToName(); + schema.identifierFieldIds(); + schema.accessorForField(first.fieldId()); + } + + private void materializeStructIndexes(Types.StructType struct, Types.NestedField first) { + struct.fields(); + struct.field(first.name()); + struct.caseInsensitiveField(first.name().toUpperCase(java.util.Locale.ROOT)); + struct.field(first.fieldId()); + } + + /** + * Delta between the first two columns and the whole schema: measures the schema graph. Both + * sides reach the same shared type singletons (StringType, ListType element names, ...) so + * only per-column growth is compared. + */ + private void assertSchemaLookupFormula(Schema schema, String fixture) { + Schema firstColumns = new Schema(schema.schemaId(), schema.columns().subList(0, 2)); + TableMetadata empty = TableMetadata.newTableMetadata( + firstColumns, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + TableMetadata populated = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + assertRetainedPayloadDelta(fixture, empty, populated, "jol-schema-lookup"); + } + + /** Delta between an unpartitioned spec and one identity field: measures the spec graph. */ + private void assertPartitionSpecFormula(Schema schema, String fixture) { + TableMetadata empty = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + TableMetadata populated = TableMetadata.newTableMetadata( + schema, PartitionSpec.builderFor(schema).identity(schema.columns().get(0).name()).build(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + assertRetainedPayloadDelta(fixture, empty, populated, "jol-partition-spec"); + } + + private void assertRetainedPayloadDelta( + String fixture, TableMetadata empty, TableMetadata populated, String locationPrefix) { + empty = roundTripMetadata(empty, + "/metadata/" + locationPrefix + "-none-" + fixture.hashCode() + ".json"); + populated = roundTripMetadata(populated, + "/metadata/" + locationPrefix + "-with-" + fixture.hashCode() + ".json"); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(empty)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populated)); + materializeAllLazyState(empty); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg " + fixture, emptyEstimate, populatedEstimate, empty, populated); + } + private TableMetadata metadataWithMaterializedPayload(String payload, int bufferBytes) { TableMetadata metadata = Mockito.mock(TableMetadata.class); Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); @@ -1289,6 +2286,42 @@ private TableMetadata metadataWithMaterializedPayload(String payload, int buffer return metadata; } + private TableMetadata metadataWithSnapshots(Snapshot snapshot) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshots()).thenReturn(Collections.singletonList(snapshot)); + Mockito.when(metadata.currentSnapshot()).thenReturn(snapshot); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.refs()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/snapshot.json"); + return metadata; + } + + private TableMetadata metadataWithStatisticsFile(StatisticsFile statisticsFile) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshots()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.refs()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.statisticsFiles()).thenReturn( + Collections.singletonList(statisticsFile)); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/statistics.json"); + return metadata; + } + private TableMetadata metadataWithSnapshotSequence(long lastSequenceNumber) { TableMetadata metadata = Mockito.mock(TableMetadata.class); Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java index 84745f815f828c..c184debd3d70ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; @@ -36,8 +38,12 @@ public void testRetainedPayloadCounterTracksSkewedPartitionValues() { IcebergPartition large = new IcebergPartition("p=" + largeValue, 0, 0, 0, 0, 1, 101, Collections.singletonList(largeValue), Collections.singletonList("identity")); + long expectedDelta = MetaCacheWeightUtils.estimatedStringBytes("p=" + largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("p=x") + + MetaCacheWeightUtils.estimatedStringBytes(largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("x"); Assertions.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() - >= (largeValue.length() - 1L) * 4L); + >= expectedDelta); IcebergPartitionInfo info = new IcebergPartitionInfo( Collections.emptyMap(), Collections.singletonMap(large.getPartitionName(), large), Collections.emptyMap()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java index be4978d33c129d..d4215518c86356 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -19,12 +19,41 @@ import org.junit.Assert; import org.openjdk.jol.info.GraphLayout; +import org.openjdk.jol.info.GraphPathRecord; + +import java.lang.reflect.Field; +import java.util.stream.IntStream; +import java.util.stream.LongStream; /** JOL oracle used only by estimator calibration tests. */ public final class EstimatorCalibrationAssertions { - private static final long MAX_CONSERVATIVE_FACTOR = 8L; + private static final double MAX_CONSERVATIVE_FACTOR = 1.10D; private static final boolean PRINT_RESULT = Boolean.getBoolean( "metacache.estimator.calibration.print"); + // Integer.valueOf/Long.valueOf serve -128..127 from JVM-wide static caches. A populated + // fixture that reaches those shared instances (field ids, list indexes, small partition + // values) must not be charged for them as retained growth, so every graph is measured + // together with the same cache roots and the shared instances cancel out of the delta. + private static final Integer[] SHARED_INTEGER_CACHE = + IntStream.rangeClosed(-128, 127).boxed().toArray(Integer[]::new); + private static final Long[] SHARED_LONG_CACHE = + LongStream.rangeClosed(-128L, 127L).boxed().toArray(Long[]::new); + // Accessor objects reference java.lang.Class instances (String.class, StructLike.class, ...). + // JOL follows them into the JVM's per-class reflection and ClassValue caches, whose size + // depends on unrelated reflective use earlier in the same JVM (Mockito, layout fingerprints, + // JOL itself). Everything reached through a Class object is shared JVM state, not retained + // cache payload, and is excluded from every measurement. + private static final Field GRAPH_PATH_PARENT = graphPathParentField(); + + private static Field graphPathParentField() { + try { + Field field = GraphPathRecord.class.getDeclaredField("parent"); + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("JOL GraphPathRecord.parent is unavailable", e); + } + } static { // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their @@ -43,8 +72,7 @@ private EstimatorCalibrationAssertions() { public static void assertConservativeDelta( String fixture, long emptyEstimate, long populatedEstimate, Object emptyGraph, Object populatedGraph) { - long actualDelta = GraphLayout.parseInstance(populatedGraph).totalSize() - - GraphLayout.parseInstance(emptyGraph).totalSize(); + long actualDelta = graphSize(populatedGraph) - graphSize(emptyGraph); long estimatedDelta = populatedEstimate - emptyEstimate; if (PRINT_RESULT) { System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", @@ -57,11 +85,36 @@ public static void assertConservativeDelta( estimatedDelta >= actualDelta); Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta + ", actual=" + actualDelta, - estimatedDelta <= MetaCacheWeightUtils.saturatedMultiply( - actualDelta, MAX_CONSERVATIVE_FACTOR)); + estimatedDelta <= Math.ceil(actualDelta * MAX_CONSERVATIVE_FACTOR)); } + /** Retained size of the graph excluding JVM-shared boxed-value caches and Class metadata. */ public static long graphSize(Object graph) { - return GraphLayout.parseInstance(graph).totalSize(); + long sharedCacheBytes = GraphLayout.parseInstance( + SHARED_INTEGER_CACHE, SHARED_LONG_CACHE).totalSize(); + GraphLayout layout = GraphLayout.parseInstance( + graph, SHARED_INTEGER_CACHE, SHARED_LONG_CACHE); + long bytes = 0L; + for (long address : layout.addresses()) { + GraphPathRecord record = layout.record(address); + if (!reachedThroughClassObject(record)) { + bytes += record.size(); + } + } + return bytes - sharedCacheBytes; + } + + private static boolean reachedThroughClassObject(GraphPathRecord record) { + try { + for (GraphPathRecord current = record; current != null; + current = (GraphPathRecord) GRAPH_PATH_PARENT.get(current)) { + if (current.klass() == Class.class) { + return true; + } + } + return false; + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 41d575417654e5..e3052fc42f31fd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -46,6 +46,18 @@ public class MetaCacheEntryTest { + @Test + public void testCompactStringPayloadEstimate() { + // Three Latin-1 bytes and two UTF-16 characters both occupy one aligned slot; the exact + // slot size follows the JVM object alignment (8 by default, 16 with large heaps). + long latin1 = MetaCacheWeightUtils.estimatedStringPayloadBytes("abc"); + long utf16 = MetaCacheWeightUtils.estimatedStringPayloadBytes("中文"); + Assert.assertEquals(latin1, utf16); + Assert.assertTrue(latin1 >= 4L && latin1 <= 16L); + Assert.assertTrue(MetaCacheWeightUtils.estimatedStringPayloadBytes("abcdefghijklmnopq") + > latin1); + } + @Test public void testRefreshUsesConfiguredLoader() throws Exception { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index fa016361c9ad3b..99bce8a3259c40 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -31,6 +31,7 @@ import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; @@ -55,10 +56,19 @@ import org.apache.paimon.table.sink.StreamTableWrite; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.DataTypeVisitor; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.FloatType; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import org.junit.Assert; import org.junit.Assume; import org.junit.Rule; @@ -115,7 +125,9 @@ public void testSnapshotWeightAccountsForSkewedTableOptions() throws Exception { long smallBytes = snapshotWeight(smallKey, smallTable, 0); long largeBytes = snapshotWeight(largeKey, largeTable, 0); - Assert.assertTrue(largeBytes - smallBytes >= (largePayload.length() - 1L) * 2L); + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largePayload) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test @@ -133,7 +145,9 @@ public void testSnapshotWeightAccountsForNestedSchemaPayload() throws Exception long smallBytes = snapshotWeight(smallKey, smallTable, 0); long largeBytes = snapshotWeight(largeKey, largeTable, 0); - Assert.assertTrue(largeBytes - smallBytes >= (largeFieldName.length() - 1L) * 2L); + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largeFieldName) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test @@ -150,19 +164,25 @@ public void testSnapshotWeightAccountsForTableComment() throws Exception { long largeBytes = snapshotWeight(new PaimonSnapshotEntryKey( mapping, 1L, largeTable.schema().id(), 1L), largeTable, 0); - Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largeComment) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { - FileStoreTable table = newPartitionedTable("jol_snapshot", Collections.emptyMap()); + FileStoreTable table = newStringPartitionedTable("jol_snapshot"); + FileStoreTable intTable = newPartitionedTable("jol_int_snapshot", Collections.emptyMap()); NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( mapping, 1L, table.schema().id(), 1L); - PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16); - PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions(table, 32, 16); - PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions(table, 1, 16); - PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions(table, 1, 4096); + PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16, Type.STRING); + PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions( + table, 32, 16, Type.STRING); + PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions( + table, 1, 16, Type.STRING); + PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions( + table, 1, 4096, Type.STRING); long emptyEstimate = empty.prepareForCachePublication(key).getBytes(); long populatedEstimate = populated.prepareForCachePublication(key).getBytes(); @@ -173,6 +193,113 @@ public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { "paimon snapshot partitions", emptyEstimate, populatedEstimate, empty, populated); EstimatorCalibrationAssertions.assertConservativeDelta( "paimon long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + + PaimonSnapshotEntryKey intKey = new PaimonSnapshotEntryKey( + mapping, 1L, intTable.schema().id(), 1L); + PaimonSnapshotCacheValue emptyInts = snapshotValueWithRealPartitions( + intTable, 0, 0, Type.INT); + PaimonSnapshotCacheValue populatedInts = snapshotValueWithRealPartitions( + intTable, 32, 0, Type.INT); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon int snapshot partitions", + emptyInts.prepareForCachePublication(intKey).getBytes(), + populatedInts.prepareForCachePublication(intKey).getBytes(), + emptyInts, populatedInts); + } + + @Test + public void testTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable emptyTable = newTableWithExtraFields("jol_empty_schema", 0); + FileStoreTable populatedTable = newTableWithExtraFields("jol_populated_schema", 32); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey emptyKey = new PaimonSnapshotEntryKey( + mapping, 1L, emptyTable.schema().id(), 1L); + PaimonSnapshotEntryKey populatedKey = new PaimonSnapshotEntryKey( + mapping, 1L, populatedTable.schema().id(), 1L); + PaimonSnapshotCacheValue empty = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, emptyTable.schema().id(), emptyTable)); + PaimonSnapshotCacheValue populated = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, populatedTable.schema().id(), populatedTable)); + + long emptyEstimate = empty.prepareForCachePublication(emptyKey).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon table fields", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testNestedTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable smallTable = newTableWithNestedFields("jol_nested_small", 1); + FileStoreTable populatedTable = newTableWithNestedFields("jol_nested_large", 33); + assertTableDeltaAgainstJol("paimon nested fields", smallTable, populatedTable); + } + + @Test + public void testTableOptionFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable emptyTable = newTableWithOptions("jol_options_empty", 0); + FileStoreTable populatedTable = newTableWithOptions("jol_options_large", 32); + assertTableDeltaAgainstJol("paimon table options", emptyTable, populatedTable); + } + + @Test + public void testCompositeTypeFormulaAgainstJolOwnedGraph() { + assertTableDeltaAgainstJol("paimon array type", + newTableWithPayloadType("array", nestedArrayType(1)), + newTableWithPayloadType("array", nestedArrayType(100))); + assertTableDeltaAgainstJol("paimon map type", + newTableWithPayloadType("map", nestedMapType(1)), + newTableWithPayloadType("map", nestedMapType(100))); + assertTableDeltaAgainstJol("paimon multiset type", + newTableWithPayloadType("multiset", nestedMultisetType(1)), + newTableWithPayloadType("multiset", nestedMultisetType(100))); + assertTableDeltaAgainstJol("paimon row type", + newTableWithPayloadType("row", nestedRowType(1)), + newTableWithPayloadType("row", nestedRowType(100))); + assertTableDeltaAgainstJol("paimon vector type", + newTableWithPayloadType("vector", rowOfLeafTypes(1, VectorType.class)), + newTableWithPayloadType("vector", rowOfLeafTypes(100, VectorType.class))); + assertTableDeltaAgainstJol("paimon decimal type", + newTableWithPayloadType("decimal", rowOfLeafTypes(1, DecimalType.class)), + newTableWithPayloadType("decimal", rowOfLeafTypes(100, DecimalType.class))); + } + + @Test + public void testUnknownDataTypeFailsClosedWithoutFailingLoad() { + DataType unknownType = new DataType(true, DataTypeRoot.INTEGER) { + @Override + public int defaultSize() { + return Integer.BYTES; + } + + @Override + public DataType copy(boolean isNullable) { + return this; + } + + @Override + public String asSQLString() { + return "UNKNOWN"; + } + + @Override + public R accept(DataTypeVisitor visitor) { + return new IntType().accept(visitor); + } + }; + FileStoreTable table = newTableWithPayloadType("unknown-type", unknownType); + Assert.assertThrows(IllegalStateException.class, + () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + + Assert.assertFalse(estimate.isComplete()); + Assert.assertSame(table, value.getSnapshot().getTable()); } @Test @@ -209,6 +336,48 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testTablePayloadAccountingWorkIsBounded() { + FileStoreTable table = Mockito.mock(FileStoreTable.class); + TableSchema schema = Mockito.mock(TableSchema.class); + @SuppressWarnings("unchecked") + Map oversizedOptions = Mockito.mock(Map.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(schema.fields()).thenReturn(Collections.emptyList()); + Mockito.when(schema.options()).thenReturn(oversizedOptions); + Mockito.when(oversizedOptions.size()).thenReturn(50_001); + + Assert.assertThrows(IllegalStateException.class, + () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + } + + @Test + public void testRowTypeLazyLookupReservationCoversPostAdmissionGrowth() throws Exception { + // Field ids above the Integer cache make every lazy map box its keys and values. + RowType small = wideRowType(1); + RowType populated = wideRowType(200); + FileStoreTable smallTable = newTableWithPayloadType("row-lazy-small", small); + FileStoreTable populatedTable = newTableWithPayloadType("row-lazy-large", populated); + for (String fieldName : ROW_TYPE_LAZY_FIELDS) { + Assert.assertNull(fieldName, readField(populated, fieldName)); + } + + // The oracle inside assertTableDeltaAgainstJol materializes the four maps after the + // estimate is taken; the estimate reserved at admission must already cover them. + assertTableDeltaAgainstJol("paimon row lazy lookup maps", smallTable, populatedTable); + for (String fieldName : ROW_TYPE_LAZY_FIELDS) { + Assert.assertNotNull(fieldName, readField(populated, fieldName)); + } + + // A RowType whose maps were materialized before admission is estimated identically. + RowType preloaded = wideRowType(200); + long unloadedEstimate = PaimonCacheSizeEstimator.retainedTablePayloadBytes( + newTableWithPayloadType("row-unloaded", wideRowType(200))); + materializeRowTypeIndexes(preloaded); + Assert.assertEquals(unloadedEstimate, PaimonCacheSizeEstimator.retainedTablePayloadBytes( + newTableWithPayloadType("row-loaded", preloaded))); + } + @Test public void testSnapshotKeySeparatesReloadedTableGenerations() { NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); @@ -786,6 +955,24 @@ private FileStoreTable newPartitionedTable( CatalogEnvironment.empty()); } + private FileStoreTable newStringPartitionedTable(String name) throws Exception { + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "part", DataTypes.STRING())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + private FileStoreTable newPartitionedTableWithNestedField( String name, String nestedFieldName) throws Exception { RowType nestedType = new RowType(Collections.singletonList( @@ -807,6 +994,173 @@ private FileStoreTable newPartitionedTableWithNestedField( CatalogEnvironment.empty()); } + private FileStoreTable newTableWithExtraFields(String name, int fieldCount) throws Exception { + ArrayList fields = new ArrayList<>(); + fields.add(new DataField(0, "part", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new IntType())); + } + TableSchema schema = new TableSchema( + 0, + fields, + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithNestedFields(String name, int nestedFieldCount) throws Exception { + ArrayList nestedFields = new ArrayList<>(); + for (int index = 0; index < nestedFieldCount; index++) { + nestedFields.add(new DataField(index + 2, "nested_" + index, new IntType())); + } + RowType nestedType = new RowType(nestedFields); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path(temporaryFolder.newFolder(name).toURI()), + schema, CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithOptions(String name, int optionCount) throws Exception { + Map options = new HashMap<>(); + for (int index = 0; index < optionCount; index++) { + options.put("key_" + index, "value_" + index); + } + return newPartitionedTable(name, options); + } + + private FileStoreTable newTableWithPayloadType(String name, DataType type) { + TableSchema schema = new TableSchema( + 0, Collections.singletonList(new DataField(0, "payload", type)), 1, + Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/paimon-composite-" + name), + schema, CatalogEnvironment.empty()); + } + + private DataType nestedArrayType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new ArrayType(type); + } + return type; + } + + private DataType nestedMapType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new MapType(new IntType(), type); + } + return type; + } + + private DataType nestedMultisetType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new MultisetType(type); + } + return type; + } + + private DataType nestedRowType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new RowType(Collections.singletonList( + new DataField(index + 1, "nested_" + index, type))); + } + return type; + } + + private void assertTableDeltaAgainstJol( + String fixture, FileStoreTable smallTable, FileStoreTable populatedTable) { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey populatedKey = new PaimonSnapshotEntryKey( + mapping, 1L, populatedTable.schema().id(), 1L); + PaimonSnapshotCacheValue small = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, smallTable.schema().id(), smallTable)); + PaimonSnapshotCacheValue populated = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, populatedTable.schema().id(), populatedTable)); + long smallEstimate = small.prepareForCachePublication(smallKey).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + // The estimate reserves the lookup maps every nested RowType can materialize after + // admission, so the JOL oracle measures the fully grown graph. + materializeRowTypeIndexes(smallTable.schema()); + materializeRowTypeIndexes(populatedTable.schema()); + EstimatorCalibrationAssertions.assertConservativeDelta( + fixture, smallEstimate, populatedEstimate, small, populated); + } + + private static final String[] ROW_TYPE_LAZY_FIELDS = { + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex"}; + + private void materializeRowTypeIndexes(TableSchema schema) { + for (DataField field : schema.fields()) { + materializeRowTypeIndexes(field.type()); + } + } + + private void materializeRowTypeIndexes(DataType type) { + if (type instanceof RowType) { + RowType rowType = (RowType) type; + if (!rowType.getFields().isEmpty()) { + DataField first = rowType.getFields().get(0); + rowType.getField(first.name()); + rowType.getFieldIndex(first.name()); + rowType.getField(first.id()); + rowType.getFieldIndexByFieldId(first.id()); + } + for (DataField field : rowType.getFields()) { + materializeRowTypeIndexes(field.type()); + } + } else if (type instanceof ArrayType) { + materializeRowTypeIndexes(((ArrayType) type).getElementType()); + } else if (type instanceof MultisetType) { + materializeRowTypeIndexes(((MultisetType) type).getElementType()); + } else if (type instanceof MapType) { + materializeRowTypeIndexes(((MapType) type).getKeyType()); + materializeRowTypeIndexes(((MapType) type).getValueType()); + } else if (type instanceof VectorType) { + materializeRowTypeIndexes(((VectorType) type).getElementType()); + } + } + + private RowType wideRowType(int fieldCount) { + ArrayList fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(1000 + index, "wide_" + index, new IntType())); + } + return new RowType(fields); + } + + private DataType rowOfLeafTypes(int fieldCount, Class leafType) { + ArrayList fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + DataType type = leafType == VectorType.class + ? new VectorType(4, new FloatType()) : new DecimalType(10, 2); + fields.add(new DataField(index + 1, "leaf_" + index, type)); + } + return new RowType(fields); + } + private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, int partitionCount) { PaimonPartitionInfo partitionInfo = Mockito.mock(PaimonPartitionInfo.class); Map partitionItems = sizeOnlyMap(partitionCount); @@ -821,14 +1175,17 @@ private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, in } private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( - FileStoreTable table, int partitionCount, int valueLength) { + FileStoreTable table, int partitionCount, int valueLength, Type partitionType) + throws AnalysisException { Map partitionItems = new HashMap<>(); Map partitions = new HashMap<>(); for (int index = 0; index < partitionCount; index++) { - String value = "p" + index + repeatedCharacter('x', valueLength); + String value = partitionType == Type.INT + ? Integer.toString(index) + : "p" + index + repeatedCharacter('x', valueLength); String name = "part=" + value; - partitionItems.put(name, new org.apache.doris.catalog.ListPartitionItem( - new ArrayList<>())); + partitionItems.put(name, PaimonUtil.toListPartitionItem( + Collections.singletonList(value), Collections.singletonList(partitionType))); partitions.put(name, new org.apache.paimon.partition.Partition( Collections.singletonMap("part", value), 100L, 1024L, 1L, 1L, 1, true)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 17fa76bb5593bc..97597d774d85ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.catalog.VariantType; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TFieldPtr; @@ -98,7 +99,7 @@ public void testCompatibilityConstructorDerivesRetainedPartitionPayload() { PaimonPartitionInfo info = new PaimonPartitionInfo( Collections.emptyMap(), Collections.singletonMap("part=" + largeValue, partition)); - Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 4L); + Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 2L); } @Test @@ -261,7 +262,7 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; - Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length() * 2L); + Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length()); Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) @@ -284,7 +285,8 @@ public void testRetainedPayloadCounterTracksSkewedPartitionValues() { Collections.singletonList(partitionEntry(stringPartitionRow(largeValue), 1L))); Assert.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() - >= (largeValue.length() - 1L) * 4L); + >= MetaCacheWeightUtils.estimatedStringBytes(largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test From 3d978950c8b545442dc2b631e7eab7f92a313b98 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 17 Aug 2026 23:36:55 +0800 Subject: [PATCH 4/5] [fix](fe) Address review findings on cache property lifecycle, statistics and partition-width accounting - Tolerate legacy engine namespaces in persisted cache properties during replay, and validate an ALTER strictly only for the keys it supplies while persisted keys are reduced to what runtime honors; CREATE stays strict for every supplied key. - Re-prepare a catalog whose cache group was retired between preparation and lookup, without blocking on the lifecycle fence. - Expose weight-bound statistics (bounds, estimated/eviction weight, rejections, reason) in information_schema.catalog_meta_cache_statistics on FE and BE. - Require the same table UUID before a retained Iceberg generation retries a commit, and bind snapshot-selectable Iceberg system tables to the relation's frozen generation for their schema. - Charge Iceberg range-partition items and Paimon list-partition items per partition column, and recalibrate the per-partition constants against realistic fixtures. --- ...chema_catalog_meta_cache_stats_scanner.cpp | 10 ++ .../org/apache/doris/catalog/SchemaTable.java | 12 ++ .../apache/doris/datasource/CatalogMgr.java | 36 ++++++ .../doris/datasource/ExternalCatalog.java | 19 +++- .../datasource/ExternalMetaCacheMgr.java | 97 ++++++++++++++++ .../iceberg/IcebergCacheSizeEstimator.java | 4 +- .../iceberg/IcebergPartitionInfo.java | 26 +++++ .../iceberg/IcebergSnapshotCacheValue.java | 9 +- .../iceberg/IcebergSysExternalTable.java | 32 +++++- .../datasource/iceberg/IcebergUtils.java | 4 + .../metacache/AbstractExternalMetaCache.java | 106 ++++++++++++------ .../metacache/ExternalMetaCache.java | 25 +++++ .../paimon/PaimonCacheSizeEstimator.java | 4 +- .../paimon/PaimonPartitionInfo.java | 17 +++ .../doris/datasource/paimon/PaimonUtil.java | 3 + .../tablefunction/MetadataGenerator.java | 18 +++ .../ExternalMetaCacheRouteResolverTest.java | 69 ++++++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 35 +++++- .../iceberg/IcebergSysExternalTableTest.java | 43 +++++++ .../iceberg/IcebergTransactionTest.java | 25 +++++ .../paimon/PaimonExternalMetaCacheTest.java | 33 +++++- .../test_iceberg_table_meta_cache.groovy | 13 +++ 22 files changed, 584 insertions(+), 56 deletions(-) diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index 7e72e6bd1a777a..bffbf40a3e0ca9 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -53,6 +53,16 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_ERROR", TYPE_STRING, sizeof(StringRef), true}, + {"WEIGHT_BOUNDED", TYPE_BOOLEAN, sizeof(bool), true}, + {"MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"EVICTION_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"WEIGHT_REJECT_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"GLOBAL_MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"GLOBAL_ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index cb4ef35eb002fd..81160fb4e382ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -635,6 +635,18 @@ public class SchemaTable extends Table { .column("LAST_LOAD_SUCCESS_TIME", ScalarType.createStringType()) .column("LAST_LOAD_FAILURE_TIME", ScalarType.createStringType()) .column("LAST_ERROR", ScalarType.createStringType()) + .column("WEIGHT_BOUNDED", ScalarType.createType(PrimitiveType.BOOLEAN)) + .column("MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ESTIMATED_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("EVICTION_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("WEIGHT_REJECT_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_ESTIMATED_WEIGHT", + ScalarType.createType(PrimitiveType.BIGINT)) + .column("GLOBAL_MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("GLOBAL_ESTIMATED_WEIGHT", + ScalarType.createType(PrimitiveType.BIGINT)) + .column("LAST_WEIGHT_REJECT_REASON", ScalarType.createStringType()) .build()) ) .put("backend_kerberos_ticket_cache", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 3e8a8f813571ea..1ab521987e08f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -539,6 +539,7 @@ private void createCatalogInternal(CatalogIf catalog, boolean isReplay) throws D try { if (!isReplay && catalog instanceof ExternalCatalog) { ((ExternalCatalog) catalog).checkProperties(); + validateSuppliedCacheProperties((ExternalCatalog) catalog, catalog.getProperties()); } Map props = catalog.getProperties(); if (props.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { @@ -623,6 +624,40 @@ public List listCatalogsWithCheckPriv(UserIdentity userIdentity) { } + /** CREATE: every supplied external meta cache property is validated strictly. */ + private static void validateSuppliedCacheProperties(ExternalCatalog catalog, Map properties) + throws DdlException { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + return; + } + try { + cacheMgr.validateCatalogCacheProperties(catalog, properties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + } + + /** + * ALTER: newly supplied external meta cache properties are validated strictly, persisted + * ones only as runtime honors them, so a legacy key cannot block an unrelated update. + */ + private static void validateSuppliedCacheProperties(ExternalCatalog catalog, + Map persistedProperties, Map updatedProperties) + throws DdlException { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + return; + } + try { + cacheMgr.validateCatalogCachePropertyUpdate(catalog, persistedProperties, updatedProperties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + } + /** * Reply for alter catalog props event. */ @@ -637,6 +672,7 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope boolean tentativelyMutated = false; try { ExternalCatalog externalCatalog = (ExternalCatalog) catalog; + validateSuppliedCacheProperties(externalCatalog, oldProperties, newProps); boolean validatedWithoutMutation = externalCatalog.validatePropertiesBeforeUpdate( oldProperties, newProps); if (!validatedWithoutMutation) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index f32c77a7015818..bc1e361a4a2c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -454,7 +454,10 @@ protected void checkProperties(CatalogProperty property) throws DdlException { // This fallback is only for isolated construction tests before Env is initialized. ExternalMetaCacheBudgetManager.fromConfig().validateCatalogMaxWeight(properties); } else { - extMetaCacheMgr.validateCatalogCacheProperties(this, properties); + // Validate what runtime will honor. Newly supplied keys are validated strictly by + // CatalogMgr for CREATE and ALTER; persisted legacy keys that initialization + // ignores must not reject an unrelated later ALTER. + extMetaCacheMgr.validateEffectiveCatalogCacheProperties(this, properties); } } catch (IllegalArgumentException e) { throw new DdlException(e.getMessage()); @@ -1393,8 +1396,18 @@ public void notifyPropertiesUpdated(Map updatedProps) { } String remainder = key.substring("meta.cache.".length()); int separator = remainder.indexOf('.'); - if (separator > 0) { - extMetaCacheMgr.removeCatalogByEngine(id, remainder.substring(0, separator)); + if (separator <= 0) { + continue; + } + String engine = remainder.substring(0, separator); + try { + extMetaCacheMgr.removeCatalogByEngine(id, engine); + } catch (IllegalArgumentException e) { + // New DDL is validated before it reaches this notification. A persisted key with + // an unknown or legacy engine namespace (edit-log replay, image load) has no cache + // group to retire and must not abort the replay; runtime sanitization ignores it. + LOG.warn("Ignoring external meta cache property '{}' with unknown engine namespace '{}' " + + "for catalog {}: {}", key, engine, id, e.getMessage()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index e58a984eacc462..316e3d3a9fd007 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -44,6 +44,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -320,6 +321,63 @@ public void validateCatalogCacheProperties(CatalogIf catalog, Map catalog, Map persistedProperties, + Map updatedProperties) { + validateCatalogCacheProperties(catalog, updatedProperties); + Map effective = runtimeEffectiveCacheProperties( + catalog, persistedProperties == null ? Collections.emptyMap() : persistedProperties); + effective.putAll(updatedProperties); + validateRuntimeCacheProperties(catalog, effective); + } + + /** + * Validate persisted properties as runtime will apply them: unknown engine namespaces and + * options no engine honors are dropped, then the remaining set is validated with the + * semantics initialization uses. + */ + public void validateEffectiveCatalogCacheProperties( + CatalogIf catalog, Map catalogProperties) { + validateRuntimeCacheProperties(catalog, runtimeEffectiveCacheProperties(catalog, catalogProperties)); + } + + private void validateRuntimeCacheProperties(CatalogIf catalog, Map effective) { + budgetManager.parseCatalogMaxWeight(effective); + for (ExternalMetaCache cache : routeResolver.resolveCatalogCaches(catalog.getId(), catalog)) { + cache.validateCatalogPropertiesForRuntime(effective); + } + } + + private Map runtimeEffectiveCacheProperties( + CatalogIf catalog, Map catalogProperties) { + Map effective = sanitizeCatalogCachePropertiesForRuntime( + catalog.getId(), catalogProperties); + List routedCaches = routeResolver.resolveCatalogCaches(catalog.getId(), catalog); + Set routedEngines = routedCaches.stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + effective.keySet().removeIf(key -> { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + return false; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + return separator <= 0 || !routedEngines.contains(remainder.substring(0, separator)); + }); + for (ExternalMetaCache cache : routedCaches) { + effective = cache.sanitizeCatalogPropertiesForRuntime(effective); + } + return effective; + } + public void invalidateCatalog(long catalogId) { routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateCatalog", @@ -431,6 +489,44 @@ public MetaCacheEntryStats getEntryStats() { private void initEngineCaches() { registerBuiltinEngineCaches(); + bindCatalogPreparers(); + } + + private void bindCatalogPreparers() { + for (ExternalMetaCache cache : cacheRegistry.allCaches()) { + String engine = cache.engine(); + cache.bindCatalogPreparer(catalogId -> tryPrepareCatalogByEngine(catalogId, engine)); + } + } + + /** + * Re-prepare a catalog whose group was retired between a caller's preparation and its lookup. + * Lookups may run inside a cache loader, and retirement holds the lifecycle lock while it + * closes groups, so this never blocks: when the fence is contended the lookup keeps its + * pre-existing failure and the next call prepares under the new policy. + */ + private void tryPrepareCatalogByEngine(long catalogId, String engine) { + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + if (!lifecycleLock.tryLock()) { + return; + } + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "tryPrepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } private void registerBuiltinEngineCaches() { @@ -545,6 +641,7 @@ public static Map getCacheStats(CacheStats cacheStats, long esti void replaceEngineCachesForTest(List caches) { cacheRegistry.resetForTest(caches); + bindCatalogPreparers(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 2b11a71f8c8d00..dc3caf2393fffe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -140,7 +140,9 @@ final class IcebergCacheSizeEstimator { private static final long BLOB_FIELD_BYTES = objectBytes(32L); private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); - private static final long PARTITION_BYTES = objectBytes(640L); + // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a + // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. + private static final long PARTITION_BYTES = objectBytes(680L); private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index 5c43cc56f8bbf5..a274bd10c29ee5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -25,6 +25,13 @@ import java.util.Set; public class IcebergPartitionInfo { + // Each RangePartitionItem endpoint holds one LiteralExpr per partition column beyond the + // first (createPartitionKey fills the vacancy with an infinity literal): literal, its lazy + // supplier, children list and array. Calibrated against JOL in IcebergExternalMetaCacheTest. + private static final long RANGE_KEY_EXTRA_COLUMN_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(208L); + private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; + private final Map nameToPartitionItem; private final Map nameToIcebergPartition; private final Map> nameToIcebergPartitionNames; @@ -85,11 +92,30 @@ private static long retainedPayloadBytes(Map partition if (partition != null) { bytes = MetaCacheWeightUtils.saturatedAdd( bytes, partition.getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionItemColumnBytes( + partition.getPartitionValues() == null + ? 0 : partition.getPartitionValues().size())); } } return bytes; } + /** + * Structural bytes a partition item retains for every partition column beyond the first; + * the fixed per-partition constants of the estimator cover a single column. The width is + * taken from the loaded metadata generation, so a spec that grew after the related-table + * check was cached is still charged for its full width. + */ + static long partitionItemColumnBytes(long partitionColumnCount) { + if (partitionColumnCount <= 1L) { + return 0L; + } + return MetaCacheWeightUtils.saturatedMultiply( + MetaCacheWeightUtils.saturatedMultiply( + partitionColumnCount - 1L, RANGE_ENDPOINTS_PER_ITEM), + RANGE_KEY_EXTRA_COLUMN_BYTES); + } + public long getLatestSnapshotId(String partitionName) { Set icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName); if (icebergPartitionNames == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 5d3618b21f01fb..cec5a47ad2a830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -392,8 +392,8 @@ public TableMetadata refresh() { // fail instead of silently committing files produced for another metadata generation. if (!isWriterCompatible(refreshedMetadata)) { throw new CommitFailedException( - "Cannot retry Iceberg commit after schema, spec, sort order, location, " - + "format version, or table properties changed"); + "Cannot retry Iceberg commit after the table UUID, schema, spec, sort " + + "order, location, format version, or table properties changed"); } currentMetadata = refreshedMetadata; return refreshedMetadata; @@ -407,7 +407,10 @@ public void commit(TableMetadata base, TableMetadata newMetadata) { } private boolean isWriterCompatible(TableMetadata refreshedMetadata) { - return retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() + // A dropped and recreated table can restart schema/spec/order ids at the same + // location; only the same table UUID may absorb a retried commit. + return Objects.equals(retainedMetadata.uuid(), refreshedMetadata.uuid()) + && retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() && retainedMetadata.currentSchemaId() == refreshedMetadata.currentSchemaId() && retainedMetadata.defaultSpecId() == refreshedMetadata.defaultSpecId() && retainedMetadata.defaultSortOrderId() == refreshedMetadata.defaultSortOrderId() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index 5bb537340ba6e0..f38d2689dfbbf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheKey; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.systable.SysTable; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; @@ -32,6 +33,7 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.Table; @@ -97,14 +99,38 @@ public boolean supportsSnapshotSelection() { } public Table getSysIcebergTable() { - Table baseTable = IcebergUtils.getQueryScopedIcebergTable(sourceTable); MetadataTableType tableType = MetadataTableType.from(sysTableType); if (tableType == null) { throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); } // Metadata tables capture their base operations. Keep them statement-local so exact // previousFiles/history state and stale-generation retry never leak into this table object. - return MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); + return MetadataTableUtils.createMetadataTableInstance(resolveBaseTable(), tableType); + } + + /** + * The base generation this statement binds the metadata table to. Snapshot-selectable + * metadata tables derive both their scan and their schema from the source relation's frozen + * snapshot (the same generation IcebergScanNode scans), so analysis and execution cannot see + * different partition specs or schemas when the table entry refreshes mid-statement. Static + * metadata tables and statements without a bound snapshot read the latest generation. + * + *

    The statement snapshot is looked up by source table (like the scan node's fallback); + * a statement that time-travels the same table under several relations resolves the default + * or, if ambiguous, the latest generation for the schema. + */ + @VisibleForTesting + Table resolveBaseTable() { + if (supportsSnapshotSelection()) { + Optional

    frozenTable = MvccUtil.getSnapshotFromContext(sourceTable) + .filter(IcebergMvccSnapshot.class::isInstance) + .map(IcebergMvccSnapshot.class::cast) + .flatMap(snapshot -> snapshot.getSnapshotCacheValue().getIcebergTable()); + if (frozenTable.isPresent()) { + return frozenTable.get(); + } + } + return IcebergUtils.getQueryScopedIcebergTable(sourceTable); } @Override @@ -172,7 +198,7 @@ private static long generateSysTableId(long sourceTableId, String sysTableType) private SchemaCacheValue loadSchemaCacheValue() { // Metadata-table schemas may change after source schema or partition-spec evolution. - // Resolve the schema from the same latest-generation path instead of permanently pairing + // Resolve the schema from the statement's bound generation instead of permanently pairing // this long-lived system-table object with its first observed generation. return new SchemaCacheValue(IcebergUtils.parseSchema(getSysIcebergTable().schema(), getCatalog().getEnableMappingVarbinary(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index bee37c0c37cea4..c19d61ba328430 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1770,10 +1770,14 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T List partitionColumns = IcebergUtils.getSchemaCacheValue( dorisTable, schemaId, table).getPartitionColumns(); + long partitionItemColumnBytes = IcebergPartitionInfo.partitionItemColumnBytes( + partitionColumns.size()); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( retainedPayloadBytes, partition.getRetainedPayloadBytes()); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partitionItemColumnBytes); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 906804e5307c21..7dd629685b12a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -37,7 +37,9 @@ import java.util.Objects; import java.util.OptionalLong; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.LongConsumer; import java.util.function.Predicate; /** @@ -75,6 +77,8 @@ protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecut this(engine, refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.empty())); } + private volatile LongConsumer catalogPreparer; + protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { this.engine = engine; @@ -99,6 +103,59 @@ public void validateCatalogProperties(Map catalogProperties) { validateMappedCatalogProperties(safeCatalogProperties, true); } + @Override + public Map sanitizeCatalogPropertiesForRuntime(Map catalogProperties) { + return sanitizeCatalogPropertiesForRuntime(catalogProperties, warning -> LOG.debug(warning)); + } + + @Override + public void validateCatalogPropertiesForRuntime(Map catalogProperties) { + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + validateMappedCatalogProperties(safeCatalogProperties, false); + } + + /** + * Exactly what initCatalog keeps: mapped legacy keys, only known entries/options in this + * engine's namespace, a parsable catalog max-weight, and entry max-weights within it. + */ + private Map sanitizeCatalogPropertiesForRuntime( + Map catalogProperties, Consumer warningConsumer) { + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( + safeCatalogProperties, engine, metaCacheEntryDefs, warningConsumer); + try { + budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY + "': " + e.getMessage()); + } + OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { + if (entryDef.getSizeEstimator() == null) { + continue; + } + String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) + + entryDef.getName() + ".max-weight"; + if (!safeCatalogProperties.containsKey(maxWeightKey)) { + continue; + } + CacheSpec cacheSpec = CacheSpec.fromProperties( + safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); + try { + budgetManager.validateCatalogEntryHierarchy( + runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(maxWeightKey); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + maxWeightKey + "': " + e.getMessage()); + } + } + return safeCatalogProperties; + } + @Override public void initCatalog(long catalogId, Map catalogProperties) { if (catalogEntries.containsKey(catalogId)) { @@ -108,42 +165,9 @@ public void initCatalog(long catalogId, Map catalogProperties) { if (catalogEntries.containsKey(catalogId)) { return; } - Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( - catalogProperties, catalogPropertyCompatibilityMap()); - safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( - safeCatalogProperties, engine, metaCacheEntryDefs, + Map safeCatalogProperties = sanitizeCatalogPropertiesForRuntime( + catalogProperties, warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); - try { - budgetManager.parseCatalogMaxWeight(safeCatalogProperties); - } catch (IllegalArgumentException e) { - safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); - LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " - + "for engine {}, catalog {}: {}", - ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, - engine, catalogId, e.getMessage()); - } - OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); - for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { - if (entryDef.getSizeEstimator() == null) { - continue; - } - String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) - + entryDef.getName() + ".max-weight"; - if (!safeCatalogProperties.containsKey(maxWeightKey)) { - continue; - } - CacheSpec cacheSpec = CacheSpec.fromProperties( - safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); - try { - budgetManager.validateCatalogEntryHierarchy( - runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); - } catch (IllegalArgumentException e) { - safeCatalogProperties.remove(maxWeightKey); - LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " - + "for engine {}, catalog {}: {}", - maxWeightKey, engine, catalogId, e.getMessage()); - } - } validateMappedCatalogProperties(safeCatalogProperties, false); catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); } @@ -308,6 +332,13 @@ protected final ExternalTable findExternalTable(NameMapping nameMapping, String private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { CatalogEntryGroup group = catalogEntries.get(catalogId); + if (group == null && catalogPreparer != null) { + // The caller prepared the catalog before capturing this engine, but a cache-policy + // ALTER retired the group in between. Re-prepare once under the lifecycle fence so + // the lookup observes the new policy instead of failing a valid catalog. + catalogPreparer.accept(catalogId); + group = catalogEntries.get(catalogId); + } if (group == null) { throw new IllegalStateException(String.format( "Catalog %d is not initialized for engine '%s'.", @@ -316,6 +347,11 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { return group; } + @Override + public void bindCatalogPreparer(LongConsumer catalogPreparer) { + this.catalogPreparer = catalogPreparer; + } + protected CatalogIf getCatalog(long catalogId) { if (Env.getCurrentEnv() == null || Env.getCurrentEnv().getCatalogMgr() == null) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 8b874fac8f659e..47e623b219c19a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.LongConsumer; /** * Engine-level abstraction for external metadata cache. @@ -45,6 +46,30 @@ public interface ExternalMetaCache { default void validateCatalogProperties(Map catalogProperties) { } + /** + * Drop the properties in this engine's namespace that runtime initialization would ignore + * (unknown entries, obsolete or unparsable options), returning what the engine will honor. + */ + default Map sanitizeCatalogPropertiesForRuntime(Map catalogProperties) { + return catalogProperties; + } + + /** + * Validate cache properties with the semantics initialization applies to persisted state: + * entry weights must fit their catalog bound, but the catalog bound is not compared with this + * FE's local global bound (runtime clamps it instead). + */ + default void validateCatalogPropertiesForRuntime(Map catalogProperties) { + } + + /** + * Bind the callback that (re)prepares a catalog group under the manager's lifecycle fence. + * A lookup that finds no group (the catalog was retired by a concurrent cache-policy ALTER + * after the caller prepared it) uses it once before failing. + */ + default void bindCatalogPreparer(LongConsumer catalogPreparer) { + } + /** * Initialize all registered entries for one catalog under current engine. * Entry instances are created eagerly at this stage. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 8484c4aa477354..85e0bed91b4959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -122,7 +122,9 @@ final class PaimonCacheSizeEstimator { .build(); private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED = checkPaimonTypeLayout(); private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); - private static final long PARTITION_BYTES = objectBytes(160L); + // One Paimon Partition record with its single-column LinkedHashMap spec plus map entry; extra + // columns are charged by PaimonPartitionInfo. + private static final long PARTITION_BYTES = objectBytes(272L); private static final long PARTITION_ITEM_BYTES = objectBytes(640L); private static final long WRAPPER_BYTES = objectBytes(512L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 004adc197061b5..f4570dccf57f6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -44,6 +44,12 @@ public enum PruningStatus { UNPRUNABLE } + // Each ListPartitionItem key holds one LiteralExpr (with lazy supplier, children list and + // array) and the Paimon Partition spec one map entry per partition column beyond the first; + // the fixed per-partition constants cover a single column. Calibrated against JOL. + private static final long PARTITION_EXTRA_COLUMN_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(216L); + public static final PaimonPartitionInfo EMPTY = new PaimonPartitionInfo(PruningStatus.PRUNABLE); public static final PaimonPartitionInfo UNPRUNABLE = new PaimonPartitionInfo(PruningStatus.UNPRUNABLE); @@ -92,6 +98,15 @@ static long addRetainedStringPayload(long bytes, String value) { return addString(bytes, value); } + /** Structural bytes one partition retains for every partition column beyond the first. */ + static long partitionColumnBytes(long partitionColumnCount) { + if (partitionColumnCount <= 1L) { + return 0L; + } + return MetaCacheWeightUtils.saturatedMultiply( + partitionColumnCount - 1L, PARTITION_EXTRA_COLUMN_BYTES); + } + private static long retainedPayloadBytes(Map partitions) { if (partitions == null) { return 0L; @@ -104,6 +119,8 @@ private static long retainedPayloadBytes(Map partitions) { continue; } bytes = addStrings(bytes, partition.spec()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionColumnBytes( + partition.spec() == null ? 0 : partition.spec().size())); bytes = addString(bytes, partition.createdBy()); bytes = addString(bytes, partition.updatedBy()); bytes = addStrings(bytes, partition.options()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index a1eaf3100cdc28..66511038e7f4c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -31,6 +31,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TArrayField; @@ -187,6 +188,8 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List partitionValues = Lists.newArrayListWithExpectedSize(partitionColumns.size()); LinkedHashMap orderedTypedSpec = new LinkedHashMap<>(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + PaimonPartitionInfo.partitionColumnBytes(partitionColumns.size())); for (Column partitionColumn : partitionColumns) { String partitionColumnName = partitionColumn.getName(); Preconditions.checkState(typedSpec.containsKey(partitionColumnName), diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6217d06b587b46..d4c3c4a0d21261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -1775,6 +1775,24 @@ private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaT trow.addToColumnValue(new TCell().setStringVal( formatMetaCacheTime(entryStats.getLastLoadFailureTimeMs(), timeZone))); trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastError())); // LAST_ERROR + // Memory governance: -1 for count-bounded entries without a weight budget. + trow.addToColumnValue(new TCell().setBoolVal(entryStats.isWeightBounded())); // WEIGHT_BOUNDED + trow.addToColumnValue(new TCell().setLongVal(entryStats.getMaxWeight())); // MAX_WEIGHT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getEstimatedWeight())); // ESTIMATED_WEIGHT + trow.addToColumnValue(new TCell().setLongVal(entryStats.getEvictionWeight())); // EVICTION_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getWeightAdmissionRejectedCount())); // WEIGHT_REJECT_COUNT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getCatalogMaxWeight())); // CATALOG_MAX_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getCatalogEstimatedWeight())); // CATALOG_ESTIMATED_WEIGHT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getGlobalMaxWeight())); // GLOBAL_MAX_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getGlobalEstimatedWeight())); // GLOBAL_ESTIMATED_WEIGHT + trow.addToColumnValue(new TCell().setStringVal( + entryStats.getLastWeightRejectReason())); // LAST_WEIGHT_REJECT_REASON dataBatch.add(trow); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 607dde99476a5f..111a8173257c3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -93,6 +93,75 @@ public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); } + @Test + public void testCatalogCachePropertyUpdateIgnoresPersistedLegacyKeys() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); + Map persisted = new HashMap<>(); + // Legacy keys admitted by image/replay: a typo, an unknown engine namespace and an + // option that is valid but stale relative to the update below. + persisted.put("meta.cache.hive.partiton_values.capacity", "10"); + persisted.put("meta.cache.hvie.partition_values.capacity", "10"); + persisted.put("meta.cache.max-weight", "64MB"); + persisted.put("meta.cache.hive.partition_values.max-weight", "16MB"); + + // The persisted map is not valid as a whole ... + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, persisted)); + // ... but runtime only honors the sane subset, so an unrelated ALTER passes ... + metaCacheMgr.validateEffectiveCatalogCacheProperties(catalog, persisted); + metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partition_values.capacity", "20")); + // ... while newly supplied keys stay strict, alone and against the honored hierarchy. + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partiton_values.enable", "true"))); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partition_values.max-weight", "128MB"))); + } + + @Test + public void testPropertyNotificationToleratesUnknownEngineNamespace() { + long catalogId = 15L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + Map replayedProperties = new HashMap<>(); + replayedProperties.put("meta.cache.hvie.partition_values.capacity", "10"); + replayedProperties.put("meta.cache.hive.partition_values.capacity", "10"); + + // Edit-log replay publishes persisted properties without DDL validation; an unknown + // legacy namespace must be ignored instead of aborting the replay. + catalog.notifyPropertiesUpdated(replayedProperties); + } + + @Test + public void testLookupRepreparesCatalogRetiredByConcurrentPolicyChange() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + long catalogId = 16L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + ExternalMetaCache hive = metaCacheMgr.hive(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + + // A cache-policy ALTER retires the group after the caller captured the engine ... + metaCacheMgr.removeCatalog(catalogId); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + // ... and the pending lookup re-prepares the catalog instead of failing. + hive.checkCatalogInitialized(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + metaCacheMgr.removeCatalog(catalogId); + + // A catalog that was really dropped is not re-created by a stale lookup. + mockCurrentCatalog(catalogId, null); + Assert.assertThrows(IllegalStateException.class, () -> hive.checkCatalogInitialized(catalogId)); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + } + @Test public void testRuntimePreparationIgnoresInvalidPersistedCacheProperties() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 180d2b44adf6cc..c850d387bca4f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -1350,6 +1350,17 @@ public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg snapshot partitions", emptySnapshotEstimate, populatedSnapshotEstimate, emptySnapshot, populatedSnapshot); + + // A spec that widened after the related-table check retains one more literal per range + // endpoint and one more value/transform per partition; the projection charges the width + // it actually loaded instead of the single field the check assumed. + IcebergSnapshotCacheValue wideSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 3), new IcebergSnapshot(-1L, 0L)); + long wideSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, wideSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg wide snapshot partitions", populatedSnapshotEstimate, wideSnapshotEstimate, + populatedSnapshot, wideSnapshot); } @Test @@ -2015,19 +2026,31 @@ private IcebergTableCacheValue tableValueWithSnapshotHistory( } private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Exception { + return realPartitionInfo(partitionCount, 1); + } + + private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partitionColumnCount) + throws Exception { Map partitionItems = new java.util.HashMap<>(); Map partitions = new java.util.HashMap<>(); - List partitionColumns = Collections.singletonList( - new org.apache.doris.catalog.Column( - "part", org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + List partitionColumns = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + partitionColumns.add(new org.apache.doris.catalog.Column( + "part" + column, org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + } for (int index = 0; index < partitionCount; index++) { String value = Integer.toString(index); String name = "part=" + value; partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( IcebergUtils.getPartitionRange(value, "day", partitionColumns))); - partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, - 1L, 1L, Collections.singletonList(value), - Collections.singletonList("day"))); + // Loaded partitions own one String per value and transform. + List values = new ArrayList<>(); + List transforms = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + values.add(new String(value)); + transforms.add(new String("day")); + } + partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); } return new IcebergPartitionInfo( partitionItems, partitions, Collections.emptyMap()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 18969f0a535ebb..5e73286fb8260d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -17,14 +17,20 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; + import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.Optional; + public class IcebergSysExternalTableTest { @Test public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { @@ -73,4 +79,41 @@ public void testMetadataSchemaReloadsAfterSourceEvolution() { Assertions.assertEquals(2, sysTable.getFullSchema().size()); Mockito.verify(sysTable, Mockito.times(2)).getSysIcebergTable(); } + + @Test + public void testSnapshotSelectableSchemaFollowsRelationSnapshot() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table frozenGeneration = Mockito.mock(Table.class); + Table latestGeneration = Mockito.mock(Table.class); + IcebergSnapshotCacheValue snapshotValue = Mockito.mock(IcebergSnapshotCacheValue.class); + Mockito.when(snapshotValue.getIcebergTable()).thenReturn(Optional.of(frozenGeneration)); + Optional relationSnapshot = Optional.of(new IcebergMvccSnapshot(snapshotValue)); + + try (MockedStatic mvccUtil = Mockito.mockStatic(MvccUtil.class); + MockedStatic icebergUtils = Mockito.mockStatic(IcebergUtils.class)) { + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(relationSnapshot); + icebergUtils.when(() -> IcebergUtils.getQueryScopedIcebergTable(sourceTable)) + .thenReturn(latestGeneration); + + // $partitions is snapshot-selectable: analysis must see the generation the scan uses. + IcebergSysExternalTable partitions = new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name()); + Assertions.assertSame(frozenGeneration, partitions.resolveBaseTable()); + + // $snapshots ignores a selected snapshot and keeps reading the latest generation. + IcebergSysExternalTable snapshots = new IcebergSysExternalTable( + sourceTable, MetadataTableType.SNAPSHOTS.name()); + Assertions.assertSame(latestGeneration, snapshots.resolveBaseTable()); + + // Without a bound relation snapshot the latest generation is used. + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(Optional.empty()); + Assertions.assertSame(latestGeneration, partitions.resolveBaseTable()); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index 2df4ad7229dbe7..897a7d3e23dcf8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -34,6 +34,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; @@ -825,6 +826,30 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User Assert.assertEquals(2, refreshedTable.history().size()); } + @Test + public void testRetainedGenerationRefusesRetryAgainstRecreatedTable() { + HadoopCatalog icebergCatalog = (HadoopCatalog) ops.getCatalog(); + TableIdentifier identifier = TableIdentifier.of(dbName, tbWithoutPartition); + Table originalTable = icebergCatalog.loadTable(identifier); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), originalTable); + Table retainedTable = cacheValue.getIcebergTable().get(); + String retainedUuid = ((HasTableOperations) retainedTable).operations().current().uuid(); + + // Drop and recreate at the same location: schema, spec and sort-order ids restart, and + // the writer contract looks identical except for the table UUID. + icebergCatalog.dropTable(identifier, true); + Table recreatedTable = icebergCatalog.createTable(identifier, originalTable.schema()); + Assert.assertNotEquals(retainedUuid, + ((HasTableOperations) recreatedTable).operations().current().uuid()); + + Table writableTable = IcebergSnapshotCacheValue.createWritableTable(retainedTable, recreatedTable); + CommitFailedException failure = Assert.assertThrows(CommitFailedException.class, + () -> ((HasTableOperations) writableTable).operations().refresh()); + Assert.assertTrue(failure.getMessage(), failure.getMessage().contains("table UUID")); + } + @Test public void testStaticPartitionFilterRejectsUnknownKey() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 99bce8a3259c40..ceac5396f18723 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -81,6 +81,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; @@ -205,6 +206,16 @@ public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { emptyInts.prepareForCachePublication(intKey).getBytes(), populatedInts.prepareForCachePublication(intKey).getBytes(), emptyInts, populatedInts); + + // Every partition column beyond the first adds a literal to each ListPartitionItem key + // and an entry to the Partition spec; the estimate scales with the loaded width. + PaimonSnapshotCacheValue wideInts = snapshotValueWithRealPartitions( + intTable, 32, 0, Type.INT, 3); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon wide snapshot partitions", + populatedInts.prepareForCachePublication(intKey).getBytes(), + wideInts.prepareForCachePublication(intKey).getBytes(), + populatedInts, wideInts); } @Test @@ -1177,6 +1188,12 @@ private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, in private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( FileStoreTable table, int partitionCount, int valueLength, Type partitionType) throws AnalysisException { + return snapshotValueWithRealPartitions(table, partitionCount, valueLength, partitionType, 1); + } + + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength, Type partitionType, + int partitionColumnCount) throws AnalysisException { Map partitionItems = new HashMap<>(); Map partitions = new HashMap<>(); for (int index = 0; index < partitionCount; index++) { @@ -1184,11 +1201,19 @@ private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( ? Integer.toString(index) : "p" + index + repeatedCharacter('x', valueLength); String name = "part=" + value; - partitionItems.put(name, PaimonUtil.toListPartitionItem( - Collections.singletonList(value), Collections.singletonList(partitionType))); + List values = new ArrayList<>(); + List types = new ArrayList<>(); + Map spec = new java.util.LinkedHashMap<>(); + for (int column = 0; column < partitionColumnCount; column++) { + // Each loaded column owns its own value String. + String columnValue = new String(value); + values.add(columnValue); + types.add(partitionType); + spec.put("part" + column, columnValue); + } + partitionItems.put(name, PaimonUtil.toListPartitionItem(values, types)); partitions.put(name, new org.apache.paimon.partition.Partition( - Collections.singletonMap("part", value), - 100L, 1024L, 1L, 1L, 1, true)); + spec, 100L, 1024L, 1L, 1L, 1, true)); } PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(partitionItems, partitions); return new PaimonSnapshotCacheValue( diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 6079a17ec402ca..0c2bebc52fed56 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -80,6 +80,19 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern sql """refresh table test_iceberg_meta_cache_db.sales""" // select 3 rows sql """select * from test_iceberg_meta_cache_db.sales""" + // The weight-bounded entries expose their budget hierarchy in the statistics view. + def weightStats = sql """ + select entry_name, weight_bounded, max_weight, estimated_weight, catalog_max_weight, + weight_reject_count, last_weight_reject_reason + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalog_name}" and engine_name = "iceberg" and weight_bounded = true + order by entry_name; + """ + assertTrue(weightStats.size() > 0) + for (row in weightStats) { + assertTrue((row[2] as long) > 0L) + assertTrue((row[3] as long) >= 0L) + } sql """drop table test_iceberg_meta_cache_db.sales""" // 2. test catalog with meta.cache.iceberg.table.ttl-second From 053a91a2b564899691480550496a1b76b8e4f28a Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 00:58:11 +0800 Subject: [PATCH 5/5] [fix](fe) Address review findings on statistics upgrade tolerance, eviction telemetry and Iceberg payload terms - BE catalog_meta_cache_statistics scanner retries with the legacy column set against an FE that does not know the weight columns and leaves them NULL. - Report the exact reservation weight of Caffeine-driven evictions instead of the int-clamped weigher value, fenced by generation against replacement races. - Charge Iceberg v3 field-default Literal wrappers and values, merged-overlap alias sets per enclosed partition name, and name-mapping alias arrays per historical name. - Document why a snapshot projection charges the frozen table generation it retains even though the table entry that produced it also charges it. --- ...chema_catalog_meta_cache_stats_scanner.cpp | 55 ++++++++---- .../schema_catalog_meta_cache_stats_scanner.h | 2 + .../iceberg/IcebergCacheSizeEstimator.java | 50 +++++++++-- .../iceberg/IcebergPartitionInfo.java | 25 +++++- .../iceberg/IcebergSnapshotCacheValue.java | 5 ++ .../datasource/iceberg/IcebergUtils.java | 2 + .../datasource/metacache/MetaCacheEntry.java | 45 +++++++++- .../metacache/MetaCacheWeightUtils.java | 20 +++++ .../iceberg/IcebergExternalMetaCacheTest.java | 83 ++++++++++++++++++- .../metacache/MetaCacheEntryTest.java | 33 ++++++++ 10 files changed, 289 insertions(+), 31 deletions(-) diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index bffbf40a3e0ca9..255813230aa266 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -65,6 +65,12 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; +// Columns that every FE knows. The weight statistics columns appended after LAST_ERROR are +// only served by FEs that carry the memory-governance change; during a rolling upgrade an older +// FE rejects a projection that names them, so the scanner falls back to this prefix and leaves +// the newer columns NULL. +static constexpr size_t kLegacyMetaCacheStatsColumnCount = 23; + SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} @@ -77,9 +83,10 @@ Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { +Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, + TFetchSchemaTableDataResult* result) { TSchemaTableRequestParams schema_table_request_params; - for (int i = 0; i < _s_tbls_columns.size(); i++) { + for (size_t i = 0; i < column_count; i++) { schema_table_request_params.__isset.columns_name = true; schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); } @@ -89,20 +96,28 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); request.__set_schema_table_params(schema_table_request_params); - TFetchSchemaTableDataResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( _fe_addr.hostname, _fe_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->fetchSchemaTableData(result, request); + [&request, result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(*result, request); }, _rpc_timeout)); + return Status::create(result->status); +} - Status status(Status::create(result.status)); +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TFetchSchemaTableDataResult result; + Status status = _fetch_from_fe(_s_tbls_columns.size(), &result); if (!status.ok()) { - LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname - << ") failed, errmsg=" << status; - return status; + LOG(INFO) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") with all columns failed, retrying with the legacy column set: " << status; + result = TFetchSchemaTableDataResult(); + status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } } std::vector result_data = result.data_batch; @@ -116,19 +131,29 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { _block->reserve(_block_rows_limit); + size_t col_size = _s_tbls_columns.size(); if (result_data.size() > 0) { - auto col_size = result_data[0].column_value.size(); - if (col_size != _s_tbls_columns.size()) { + col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size() && col_size != kLegacyMetaCacheStatsColumnCount) { return Status::InternalError( "catalog meta cache stats schema is not match for FE and BE"); } } + int available_columns = static_cast(col_size); + int total_columns = static_cast(_s_tbls_columns.size()); for (int i = 0; i < result_data.size(); i++) { TRow row = result_data[i]; - for (int j = 0; j < _s_tbls_columns.size(); j++) { - RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), - _s_tbls_columns[j].type)); + for (int j = 0; j < total_columns; j++) { + if (j < available_columns) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } else { + // Column unknown to the serving FE: NULL. + auto column_guard = _block->mutate_column_scoped(j); + column_guard.mutable_column()->insert_default(); + column_guard.restore(); + } } } return Status::OK(); diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h index 836500fd97de85..7a2339baf44335 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h @@ -25,6 +25,7 @@ namespace doris { class RuntimeState; class Block; +class TFetchSchemaTableDataResult; class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); @@ -40,6 +41,7 @@ class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { private: Status _get_meta_cache_from_fe(); + Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result); TNetworkAddress _fe_addr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index dc3caf2393fffe..94283ca6b25bb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -35,6 +35,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.UnknownTransform; import org.apache.iceberg.types.Type; @@ -42,6 +43,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashSet; @@ -100,6 +102,14 @@ final class IcebergCacheSizeEstimator { MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); private static final long LONG_BYTES = MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + // Literals.BaseLiteral: value plus the transient serialized-buffer slot. + private static final long LITERAL_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + // JDK 17 HeapByteBuffer: Buffer header fields, address, segment, hb and offset. + private static final long BYTE_BUFFER_BYTES = objectBytes(56L); + private static final long BIG_DECIMAL_BYTES = objectBytes(104L); + private static final long BOXED_DEFAULT_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 16L); private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( @@ -143,7 +153,11 @@ final class IcebergCacheSizeEstimator { // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. private static final long PARTITION_BYTES = objectBytes(680L); - private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); + // Outer map entry and table share of one merged-overlap group; the alias set itself and its + // contents are charged by IcebergPartitionInfo per enclosed partition name. + private static final long PARTITION_ALIAS_BYTES = objectBytes(144L); + // One name-mapping field: map node, boxed id and list object; alias arrays and Strings are + // charged by IcebergSnapshotCacheValue when the mapping is copied. private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); private static final long DATA_FILE_BYTES = objectBytes(896L); @@ -214,6 +228,11 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( bytes, value.getRetainedNameMappingPayloadBytes()); if (value.getRetainedIcebergTable().isPresent()) { + // The projection keeps its own reference to the frozen table generation. That graph + // is charged here as well as by the table entry that produced it: the two entries have + // independent lifetimes (TTL, weight eviction, soft collection) and either may outlive + // the other, so each must be able to carry the graph on its own. Budgets should be + // sized for the table metadata being counted once per dependent entry. Table table = value.getRetainedIcebergTable().get(); MetaCacheSizeEstimate support = checkSupportedTable(table); if (!support.isComplete()) { @@ -825,8 +844,8 @@ private static long addFieldPayload( bytes = addString(bytes, name); } bytes = addString(bytes, field.doc()); - bytes = addDefaultPayload(bytes, field.initialDefault()); - bytes = addDefaultPayload(bytes, field.writeDefault()); + bytes = addDefaultPayload(bytes, field.initialDefaultLiteral()); + bytes = addDefaultPayload(bytes, field.writeDefaultLiteral()); boolean pushShortName = kind == FieldKind.STRUCT_FIELD || kind == FieldKind.MAP_KEY || !field.type().isStructType(); // Only fields nested through a chain of struct fields get accessors; anything below a @@ -951,16 +970,33 @@ private static long immutableMapTableCapacity(long size) { ? MetaCacheWeightUtils.saturatedMultiply(capacity, 2L) : capacity; } - private static long addDefaultPayload(long bytes, Object value) { + /** + * A v3 field default is retained as an Iceberg Literal wrapper (value plus a transient + * ByteBuffer slot) around its boxed or buffer value. + */ + private static long addDefaultPayload(long bytes, Literal literal) { + if (literal == null) { + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LITERAL_BYTES); + Object value = literal.value(); if (value instanceof CharSequence) { return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); } else if (value instanceof ByteBuffer) { - return MetaCacheWeightUtils.saturatedAdd(bytes, ((ByteBuffer) value).capacity()); + return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedAdd( + BYTE_BUFFER_BYTES, + MetaCacheWeightUtils.estimatedByteArrayBytes(((ByteBuffer) value).capacity()))); } else if (value instanceof byte[]) { - return MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedByteArrayBytes(((byte[]) value).length)); + } else if (value instanceof BigDecimal) { + return MetaCacheWeightUtils.saturatedAdd(bytes, BIG_DECIMAL_BYTES); + } else if (value == null || value instanceof Boolean) { + return bytes; } - return bytes; + // Boxed numbers, UUIDs and other small immutable values. + return MetaCacheWeightUtils.saturatedAdd(bytes, BOXED_DEFAULT_BYTES); } private static long addString(long bytes, String value) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index a274bd10c29ee5..05f469dfdc5229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -31,6 +31,9 @@ public class IcebergPartitionInfo { private static final long RANGE_KEY_EXTRA_COLUMN_BYTES = MetaCacheWeightUtils.estimatedObjectBytes(208L); private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; + // A merged-overlap alias group is a HashSet of the enclosed physical partition names; the + // names themselves are shared with the partition maps. + private static final long HASH_SET_BYTES = MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); private final Map nameToPartitionItem; private final Map nameToIcebergPartition; @@ -50,7 +53,9 @@ public IcebergPartitionInfo(Map nameToPartitionItem, Map nameToIcebergPartition, Map> nameToIcebergPartitionNames) { this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, - retainedPayloadBytes(nameToIcebergPartition)); + MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes(nameToIcebergPartition), + partitionAliasBytes(nameToIcebergPartitionNames))); } public IcebergPartitionInfo(Map nameToPartitionItem, @@ -100,6 +105,24 @@ private static long retainedPayloadBytes(Map partition return bytes; } + /** + * Retained bytes of the merged-overlap alias sets: every group keeps one HashSet with one + * node per enclosed physical partition name (the estimator's per-group constant covers only + * the outer map entry and the empty set object). + */ + static long partitionAliasBytes(Map> nameToIcebergPartitionNames) { + if (nameToIcebergPartitionNames == null) { + return 0L; + } + long bytes = 0L; + for (Set aliases : nameToIcebergPartitionNames.values()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_SET_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedHashMapBytes(aliases == null ? 0L : aliases.size())); + } + return bytes; + } + /** * Structural bytes a partition item retains for every partition column beyond the first; * the fixed per-partition constants of the estimator cover a single column. The width is diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index cec5a47ad2a830..4b613fa25d1b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -96,6 +96,11 @@ private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSna for (Map.Entry> entry : nameMapping.get().entrySet()) { List names = ImmutableList.copyOf(entry.getValue()); copy.put(entry.getKey(), names); + if (names.size() > 1) { + // A field with several historical names keeps an element array per name. + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(names.size())); + } for (String name : names) { payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, MetaCacheWeightUtils.estimatedStringBytes(name)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index c19d61ba328430..ef408a81e20636 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1785,6 +1785,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, IcebergPartitionInfo.partitionAliasBytes(partitionNameMap)); return new IcebergPartitionInfo( nameToPartitionItem, nameToPartition, partitionNameMap, retainedPayloadBytes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index de89c7b9d8d6aa..e76bfc0540f6d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -125,6 +125,11 @@ public class MetaCacheEntry { private final AtomicLong weightAdmissionRejectedCount = new AtomicLong(0L); private final AtomicLong localEvictionCount = new AtomicLong(0L); private final AtomicLong localEvictionWeight = new AtomicLong(0L); + // Exact byte weight released by Caffeine-driven evictions (size, expiry, soft collection). + // Caffeine's own eviction weight is clamped to the int weigher, so it is not used for stats. + private final AtomicLong automaticEvictionWeight = new AtomicLong(0L); + // Generation reported evicted for a key, consumed by the fenced cleanup of that generation. + private final Map pendingEvictionGenerations = new ConcurrentHashMap<>(); private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); private final AtomicLong lastWeightRejectLogTimeMs = new AtomicLong(0L); private final AtomicBoolean closed = new AtomicBoolean(false); @@ -453,12 +458,14 @@ public void invalidateAll() { reservations.values().forEach(record -> record.reservation.release()); reservations.clear(); pendingRemovalGenerations.clear(); + pendingEvictionGenerations.clear(); invalidateCount.addAndGet(size); } else { long size = data.estimatedSize(); data.invalidateAll(); refreshRecords.clear(); pendingRemovalGenerations.clear(); + pendingEvictionGenerations.clear(); invalidateCount.addAndGet(size); } } @@ -509,7 +516,7 @@ public MetaCacheEntryStats stats() { weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, weightBounded ? entryBudget.getUsedWeight() : -1L, weightBounded ? MetaCacheWeightUtils.saturatedAdd( - cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + automaticEvictionWeight.get(), localEvictionWeight.get()) : -1L, weightBounded ? weightAdmissionRejectedCount.get() : -1L, weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, @@ -570,6 +577,12 @@ private AdmissionResult admitWeightedValue( return AdmissionResult.NOT_CURRENT; } if (oldValue == null && record != null) { + // The previous generation was already removed by Caffeine; if that removal was + // an eviction whose asynchronous cleanup has not run yet, account it here. + if (pendingEvictionGenerations.remove(key, record.generation)) { + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } reservations.remove(key, record); record.reservation.release(); record = null; @@ -752,6 +765,10 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (weightBounded) { ReservationRecord record = reservations.get(key); if (record != null) { + if (cause.wasEvicted()) { + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } releaseReservation(key, record.generation); } } else { @@ -770,9 +787,13 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (closed.get()) { return; } + if (cause.wasEvicted()) { + pendingEvictionGenerations.merge(key, ownerGeneration, Math::max); + } pendingRemovalGenerations.merge(key, ownerGeneration, Math::max); if (closed.get()) { pendingRemovalGenerations.remove(key, ownerGeneration); + pendingEvictionGenerations.remove(key, ownerGeneration); return; } scheduleRemovalCleanup(); @@ -808,12 +829,21 @@ private void drainRemovalCleanups() { if (!pendingRemovalGenerations.remove(key, generation)) { continue; } + boolean evicted = pendingEvictionGenerations.remove(key, generation); + if (!evicted) { + // A newer generation superseded the evicted one; its eviction can no longer + // be attributed, so drop the stale marker instead of retaining the key. + pendingEvictionGenerations.remove(key); + } try { - cleanupRemovedReservation(key, generation); + cleanupRemovedReservation(key, generation, evicted); } catch (RuntimeException e) { // Restore the generation for retry. The finally block requeues one bounded // drain instead of permanently wedging this entry's scheduled flag. pendingRemovalGenerations.merge(key, generation, Math::max); + if (evicted) { + pendingEvictionGenerations.merge(key, generation, Math::max); + } LOG.warn("Failed to clean a removal reservation for external metadata cache entry {}", name, e); } @@ -828,7 +858,7 @@ private void drainRemovalCleanups() { } } - private void cleanupRemovedReservation(K key, long expectedReservationGeneration) { + private void cleanupRemovedReservation(K key, long expectedReservationGeneration, boolean evicted) { beforeRemovalCleanupLockForTest(key); synchronized (admissionLock) { if (weightBounded) { @@ -836,6 +866,11 @@ private void cleanupRemovedReservation(K key, long expectedReservationGeneration if (record != null && record.generation == expectedReservationGeneration && data.asMap().get(key) == null && reservations.remove(key, record)) { + if (evicted) { + // The exact reservation, not Caffeine's int-clamped weigher value. + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } record.reservation.release(); } } else { @@ -849,11 +884,13 @@ private void cleanupRemovedReservation(K key, long expectedReservationGeneration afterRemovalCleanupForTest(key); } - private void releaseReservation(K key, long expectedGeneration) { + private boolean releaseReservation(K key, long expectedGeneration) { ReservationRecord record = reservations.get(key); if (record != null && record.generation == expectedGeneration && reservations.remove(key, record)) { record.reservation.release(); + return true; } + return false; } private void releaseRefreshRecord(K key, long expectedGeneration) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java index 828bc28719d4d4..8e32a989588677 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -199,6 +199,26 @@ public static long estimatedObjectArrayBytes(long length) { return alignedArrayBytes(OBJECT_ARRAY_BASE_BYTES, length, OBJECT_REFERENCE_BYTES); } + /** + * A java.util.HashMap holding {@code entries} mappings: the map object, its power-of-two + * table (allocated on the first put) and one node per entry; keys and values are separate. + */ + public static long estimatedHashMapBytes(long entries) { + long bytes = estimatedObjectLayoutBytes(4L, 16L); + if (entries <= 0L) { + return bytes; + } + long capacity = 16L; + while (entries > capacity - capacity / 4L) { + capacity = saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + break; + } + } + bytes = saturatedAdd(bytes, estimatedObjectArrayBytes(capacity)); + return saturatedAdd(bytes, saturatedMultiply(entries, estimatedObjectLayoutBytes(3L, 4L))); + } + /** Size of an object with a known field layout on the active VM. */ public static long estimatedObjectLayoutBytes(long referenceFields, long primitiveBytes) { if (referenceFields < 0L || primitiveBytes < 0L) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index c850d387bca4f9..41cdc2ca8bb966 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -55,7 +55,9 @@ import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -787,14 +789,43 @@ public void testSchemaFormulaCountsBoxedIdsOfUncachedFieldIds() { } private TableMetadata metadataWithAddedSchema(Schema schema) { + return metadataWithAddedSchema(schema, 2); + } + + private TableMetadata metadataWithAddedSchema(Schema schema, int formatVersion) { Schema base = new Schema(0, Types.NestedField.optional(1, "base", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata( - base, PartitionSpec.unpartitioned(), "file:/warehouse/uncached-ids", - Collections.emptyMap()); + base, PartitionSpec.unpartitioned(), SortOrder.unsorted(), "file:/warehouse/uncached-ids", + Collections.singletonMap(TableProperties.FORMAT_VERSION, Integer.toString(formatVersion))); return TableMetadata.buildFrom(metadata).addSchema(schema) .setCurrentSchema(schema.schemaId()).discardChanges().build(); } + @Test + public void testSchemaFormulaCountsFieldDefaultLiterals() { + // v3 field defaults are retained as Literal wrappers around boxed or String values. + assertRetainedPayloadDelta("defaulted fields", + metadataWithAddedSchema(new Schema(1, defaultedFields(1)), 3), + metadataWithAddedSchema(new Schema(1, defaultedFields(32)), 3), "jol-defaults"); + } + + private List defaultedFields(int fieldCount) { + List fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional("text_" + index).withId(10_000 + index) + .ofType(Types.StringType.get()) + .withInitialDefault(Expressions.lit("initial_" + index)) + .withWriteDefault(Expressions.lit("write_" + index)) + .build()); + fields.add(Types.NestedField.optional("number_" + index).withId(20_000 + index) + .ofType(Types.LongType.get()) + .withInitialDefault(Expressions.lit(100_000L + index)) + .withWriteDefault(Expressions.lit(200_000L + index)) + .build()); + } + return fields; + } + @Test public void testTablePayloadAccountsForRetainedHistoricalMetadata() { String largePayload = repeatedCharacter('x', 64 * 1024); @@ -1361,6 +1392,41 @@ public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg wide snapshot partitions", populatedSnapshotEstimate, wideSnapshotEstimate, populatedSnapshot, wideSnapshot); + + // Overlapping physical partitions merge into one Doris partition that keeps every + // enclosed name in a HashSet; the weight follows the set cardinality, not the group count. + IcebergSnapshotCacheValue aliasedSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 1, true), new IcebergSnapshot(-1L, 0L)); + long aliasedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, aliasedSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg partition aliases", populatedSnapshotEstimate, aliasedSnapshotEstimate, + populatedSnapshot, aliasedSnapshot); + + // A name mapping retains an element array per field once it has several historical names. + IcebergSnapshotCacheValue singleNames = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.of(nameMappingWithAliases(32, 1))); + IcebergSnapshotCacheValue manyNames = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.of(nameMappingWithAliases(32, 8))); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg name mapping aliases", + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, singleNames).getBytes(), + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, manyNames).getBytes(), + singleNames, manyNames); + } + + private Map> nameMappingWithAliases(int fieldCount, int aliasesPerField) { + Map> mapping = new java.util.HashMap<>(); + for (int field = 0; field < fieldCount; field++) { + List names = new ArrayList<>(); + for (int alias = 0; alias < aliasesPerField; alias++) { + names.add("field_" + field + "_v" + alias); + } + mapping.put(1000 + field, names); + } + return mapping; } @Test @@ -2031,6 +2097,11 @@ private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Except private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partitionColumnCount) throws Exception { + return realPartitionInfo(partitionCount, partitionColumnCount, false); + } + + private IcebergPartitionInfo realPartitionInfo( + int partitionCount, int partitionColumnCount, boolean mergeAllIntoFirst) throws Exception { Map partitionItems = new java.util.HashMap<>(); Map partitions = new java.util.HashMap<>(); List partitionColumns = new ArrayList<>(); @@ -2052,8 +2123,12 @@ private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partition } partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); } - return new IcebergPartitionInfo( - partitionItems, partitions, Collections.emptyMap()); + Map> aliases = Collections.emptyMap(); + if (mergeAllIntoFirst && partitionCount > 0) { + // mergeOverlapPartitions() shape: the surviving name owns a set of every enclosed name. + aliases = Collections.singletonMap("part=0", new java.util.HashSet<>(partitions.keySet())); + } + return new IcebergPartitionInfo(partitionItems, partitions, aliases); } private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index e3052fc42f31fd..5607d604e1b01e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1008,6 +1008,39 @@ public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() thr } } + @Test + public void testAutomaticEvictionTelemetryKeepsExactWeightAboveWeigherLimit() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long hugeEstimate = 3L << 30; // above Caffeine's int weigher limit + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(8L << 30)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "huge-eviction", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "huge-eviction", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 8L << 30), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(hugeEstimate), budget); + try { + entry.put("k", new byte[1]); + Assert.assertEquals(accountedWeight(hugeEstimate), manager.getGlobalUsedWeight()); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + // A soft-value collection is an automatic eviction reported through Caffeine, whose + // weigher saw at most Integer.MAX_VALUE; the statistics must report the reservation. + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + awaitGlobalWeight(manager, 0L); + + Assert.assertEquals(accountedWeight(hugeEstimate), entry.stats().getEvictionWeight()); + Assert.assertTrue(entry.stats().getEvictionWeight() > Integer.MAX_VALUE); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testStrongQueryReferenceSurvivesSoftValueCollectionChecks() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor();