From 308f6d90fd01582a07f2e7ef3e38fa49a89177cb Mon Sep 17 00:00:00 2001 From: cyan-zheng Date: Fri, 3 Jul 2026 19:48:22 +1000 Subject: [PATCH 1/2] HIVE-29265: Remove stale COLUMN_STATS_ACCURATE markers when column stats collection fails When column stats collection is skipped (e.g. UnsupportedDoubleException for Inf/NaN in float/double min/max stats), failed columns were still makred as true in COLUMN_STATS_ACCURATE even though stats are not present. Track per-table/partition column failures in ColStatsProcessor and remove incorrect accurate markers via alterTable and alterPartitions. --- .../hive/ql/stats/ColStatsProcessor.java | 87 ++- .../stats_col_stats_inaccurate.q | 94 +++ .../llap/columnstats_infinity.q.out | 2 +- .../llap/stats_col_stats_inaccurate.q.out | 561 ++++++++++++++++++ 4 files changed, 729 insertions(+), 15 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/stats_col_stats_inaccurate.q create mode 100644 ql/src/test/results/clientpositive/llap/stats_col_stats_inaccurate.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index be62d94019ed..5b427060016e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -38,6 +40,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -97,7 +100,8 @@ public int process(Hive db, Table tbl) throws Exception { return persistColumnStats(db, tbl); } - private boolean constructColumnStatsFromPackedRows(Table tbl, List stats, long maxNumStats) + private boolean constructColumnStatsFromPackedRows(Table tbl, List stats, + long maxNumStats, Map> failedColumnStatsByTarget) throws HiveException, MetaException, IOException { String partName = null; List colName = colStatDesc.getColName(); @@ -118,6 +122,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List failedColumns = new ArrayList<>(); for (int i = 0; i < colName.size(); i++) { String columnName = colName.get(i); String columnType = colType.get(i); @@ -133,13 +138,14 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List partColSchema = new ArrayList<>(); List partVals = new ArrayList<>(); @@ -168,14 +174,22 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List= maxNumStats) { - return false; + if (!failedColumns.isEmpty()) { + String statsTarget = isTblLevel ? tbl.getFullyQualifiedName() : partName; + failedColumnStatsByTarget.computeIfAbsent(statsTarget, k -> new ArrayList<>()) + .addAll(failedColumns); + } + + if (!statsObjs.isEmpty()) { + ColumnStatisticsDesc statsDesc = buildColumnStatsDesc(tbl, partName, isTblLevel); + ColumnStatistics colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + colStats.setEngine(Constants.HIVE_ENGINE); + stats.add(colStats); + if (numStats >= maxNumStats) { + return false; + } } } } @@ -215,12 +229,43 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce long maxNumStats = conf.getLongVar(HiveConf.ConfVars.HIVE_STATS_MAX_NUM_STATS); while (!done) { List colStats = new ArrayList<>(); + Map> failedColumnStatsByTarget = new HashMap<>(); - long start = System. currentTimeMillis(); - done = constructColumnStatsFromPackedRows(tbl, colStats, maxNumStats); + long start = System.currentTimeMillis(); + done = constructColumnStatsFromPackedRows(tbl, colStats, maxNumStats, failedColumnStatsByTarget); long end = System.currentTimeMillis(); LOG.info("Time taken to build " + colStats.size() + " stats desc : " + ((end - start)/1000F) + " seconds."); + // Remove inaccurate column stats markers + List partitionsToUpdate = new ArrayList<>(); + for (Map.Entry> entry : failedColumnStatsByTarget.entrySet()) { + List failedColumns = entry.getValue(); + if (CollectionUtils.isEmpty(failedColumns)) { + continue; + } + + if (tbl.isNonNative() && tbl.getStorageHandler().canSetColStatistics(tbl)) { + if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary())) { + setOrRemoveColumnStatsAccurateProperty(db, tbl, failedColumns, false); + } + } else { + if (colStatDesc.isTblLevel()) { + setOrRemoveColumnStatsAccurateProperty(db, tbl, failedColumns, false); + } else if (!tbl.hasNonNativePartitionSupport()) { // Native HMS partitions only + Map partSpec = Warehouse.makeSpecFromName(entry.getKey()); + Partition partition = db.getPartition(tbl, partSpec, false); + if (partition == null) { + LOG.debug("Skipping removal of column stats accurate marker for missing partition {}", + entry.getKey()); + continue; + } + StatsSetupConst.removeColumnStatsState(partition.getParameters(), failedColumns); + partitionsToUpdate.add(partition); + } + } + } + removePartitionColumnStatsAccurateProperty(db, tbl, partitionsToUpdate); + // Persist the column statistics object to the metastore // Note, this function is shared for both table and partition column stats. if (colStats.isEmpty()) { @@ -235,7 +280,7 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce } } - start = System. currentTimeMillis(); + start = System.currentTimeMillis(); if (tbl.isNonNative() && tbl.getStorageHandler().canSetColStatistics(tbl)) { boolean success = tbl.getStorageHandler().setColStatistics(tbl, colStats); if (!(tbl.isMaterializedView() || tbl.isView() || tbl.isTemporary())) { @@ -268,6 +313,20 @@ private void setOrRemoveColumnStatsAccurateProperty(Hive db, Table tbl, List partitions) + throws HiveException { + if (CollectionUtils.isEmpty(partitions)) { + return; + } + EnvironmentContext environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + try { + db.alterPartitions(tbl.getFullyQualifiedName(), partitions, environmentContext, false); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } + } + /** * Enumeration of column stats fields that can currently * be computed. Each one has a field name associated. diff --git a/ql/src/test/queries/clientpositive/stats_col_stats_inaccurate.q b/ql/src/test/queries/clientpositive/stats_col_stats_inaccurate.q new file mode 100644 index 000000000000..178150dce215 --- /dev/null +++ b/ql/src/test/queries/clientpositive/stats_col_stats_inaccurate.q @@ -0,0 +1,94 @@ +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +set hive.stats.fetch.column.stats=true; + +-- Check partitioned tables on float/double columns with Infinity/NaN on inaccurate stats. + +create table stats_t1( + c_double double, + c_float float, + c_str string) +partitioned by (p int) +stored as ORC; + +insert into table stats_t1 partition(p=1) values + (cast('Infinity' as double), cast('Infinity' as float), 'row1'), + (cast('-Infinity' as double), cast('-Infinity' as float), 'row2'), + (cast('NAN' as double), cast('NaN' as float), 'row3'), + (cast(1234 as double), 123.456, 'row4'); + +describe formatted stats_t1 partition(p=1) c_double; +describe formatted stats_t1 partition(p=1) c_float; +describe formatted stats_t1 partition(p=1); + + +-- Check non-partitioned tables on flout/double columns with Infinity/NaN on inaccurate stats. + +create table stats_t2( + c_double double, + c_float float, + c_str string) +stored as ORC; + +insert into table stats_t2 values + (cast('Infinity' as double), cast('Infinity' as float), 'row1'), + (cast('-Infinity' as double), cast('-Infinity' as float), 'row2'), + (cast('NAN' as double), cast('NaN' as float), 'row3'), + (cast(1234 as double), 123.456, 'row4'); +analyze table stats_t2 compute statistics for columns; + + +describe formatted stats_t2 c_double; +describe formatted stats_t2 c_float; +describe formatted stats_t2; + +-- All columns fail with UnsupportedDoubleException +create table stats_t3( + a double, + b float) +partitioned by (p int) +stored as ORC; + +insert into table stats_t3 partition(p=1) values + (cast('Infinity' as double), cast('Infinity' as float)), + (cast('-Infinity' as double), cast('-Infinity' as float)), + (cast('NaN' as double), cast('NaN' as float)); +describe formatted stats_t3 partition(p=1) a; +describe formatted stats_t3 partition(p=1) b; +describe formatted stats_t3 partition(p=1); + +-- Multiple partitions with different columns failed with UnsupportedDoubleException + +create table stats_t4( + a double, + b float) +partitioned by (p int) +stored as ORC; + +-- a fails with UnsupportedDoubleException, b normal +insert into table stats_t4 partition(p=1) values + (cast('Infinity' as double), 3.14), + (cast('-Infinity' as double), 2.72), + (cast('NaN' as double), 1.618); + +-- a normal, b fails with UnsupportedDoubleException +insert into table stats_t4 partition(p=2) values + (42.0, cast('Infinity' as float)), + (17.3, cast('-Infinity' as float)), + (5.4, cast('NaN' as float)); + +-- both a and b fail with UnsupportedDoubleException +insert into table stats_t4 partition(p=3) values + (cast('Infinity' as double), cast('-Infinity' as float)), + (cast('-Infinity' as double), cast('Infinity' as float)), + (cast('NaN' as double), cast('NaN' as float)); + +-- both a and b normal +insert into table stats_t4 partition(p=4) values + (1.0, 2.0), + (3.0, 4.0); + +describe formatted stats_t4 partition(p=1); +describe formatted stats_t4 partition(p=2); +describe formatted stats_t4 partition(p=3); +describe formatted stats_t4 partition(p=4); diff --git a/ql/src/test/results/clientpositive/llap/columnstats_infinity.q.out b/ql/src/test/results/clientpositive/llap/columnstats_infinity.q.out index 69ec58b6595a..bbb677f29064 100644 --- a/ql/src/test/results/clientpositive/llap/columnstats_infinity.q.out +++ b/ql/src/test/results/clientpositive/llap/columnstats_infinity.q.out @@ -279,7 +279,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"c1\":\"true\",\"c10\":\"true\",\"c11\":\"true\",\"c12\":\"true\",\"c13\":\"true\",\"c14\":\"true\",\"c15\":\"true\",\"c2\":\"true\",\"c3\":\"true\",\"c4\":\"true\",\"c5\":\"true\",\"c6\":\"true\",\"c7\":\"true\",\"c8\":\"true\",\"c9\":\"true\",\"insert_num\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"c1\":\"true\",\"c10\":\"true\",\"c12\":\"true\",\"c13\":\"true\",\"c15\":\"true\",\"c3\":\"true\",\"c4\":\"true\",\"c6\":\"true\",\"c7\":\"true\",\"c9\":\"true\",\"insert_num\":\"true\"}} bucketing_version 2 numFiles 1 numRows 5 diff --git a/ql/src/test/results/clientpositive/llap/stats_col_stats_inaccurate.q.out b/ql/src/test/results/clientpositive/llap/stats_col_stats_inaccurate.q.out new file mode 100644 index 000000000000..71cf14dc616f --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/stats_col_stats_inaccurate.q.out @@ -0,0 +1,561 @@ +PREHOOK: query: create table stats_t1( + c_double double, + c_float float, + c_str string) +partitioned by (p int) +stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_t1 +POSTHOOK: query: create table stats_t1( + c_double double, + c_float float, + c_str string) +partitioned by (p int) +stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_t1 +PREHOOK: query: insert into table stats_t1 partition(p=1) values + (cast('Infinity' as double), cast('Infinity' as float), 'row1'), + (cast('-Infinity' as double), cast('-Infinity' as float), 'row2'), + (cast('NAN' as double), cast('NaN' as float), 'row3'), + (cast(1234 as double), 123.456, 'row4') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t1@p=1 +POSTHOOK: query: insert into table stats_t1 partition(p=1) values + (cast('Infinity' as double), cast('Infinity' as float), 'row1'), + (cast('-Infinity' as double), cast('-Infinity' as float), 'row2'), + (cast('NAN' as double), cast('NaN' as float), 'row3'), + (cast(1234 as double), 123.456, 'row4') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t1@p=1 +POSTHOOK: Lineage: stats_t1 PARTITION(p=1).c_double SCRIPT [] +POSTHOOK: Lineage: stats_t1 PARTITION(p=1).c_float SCRIPT [] +POSTHOOK: Lineage: stats_t1 PARTITION(p=1).c_str SCRIPT [] +PREHOOK: query: describe formatted stats_t1 partition(p=1) c_double +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t1 +POSTHOOK: query: describe formatted stats_t1 partition(p=1) c_double +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t1 +col_name c_double +data_type double +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment from deserializer +PREHOOK: query: describe formatted stats_t1 partition(p=1) c_float +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t1 +POSTHOOK: query: describe formatted stats_t1 partition(p=1) c_float +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t1 +col_name c_float +data_type float +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment from deserializer +PREHOOK: query: describe formatted stats_t1 partition(p=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t1 +POSTHOOK: query: describe formatted stats_t1 partition(p=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t1 +# col_name data_type comment +c_double double +c_float float +c_str string + +# Partition Information +# col_name data_type comment +p int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: stats_t1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c_str\":\"true\"}} + numFiles 1 + numRows 4 + rawDataSize 392 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table stats_t2( + c_double double, + c_float float, + c_str string) +stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_t2 +POSTHOOK: query: create table stats_t2( + c_double double, + c_float float, + c_str string) +stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_t2 +PREHOOK: query: insert into table stats_t2 values + (cast('Infinity' as double), cast('Infinity' as float), 'row1'), + (cast('-Infinity' as double), cast('-Infinity' as float), 'row2'), + (cast('NAN' as double), cast('NaN' as float), 'row3'), + (cast(1234 as double), 123.456, 'row4') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t2 +POSTHOOK: query: insert into table stats_t2 values + (cast('Infinity' as double), cast('Infinity' as float), 'row1'), + (cast('-Infinity' as double), cast('-Infinity' as float), 'row2'), + (cast('NAN' as double), cast('NaN' as float), 'row3'), + (cast(1234 as double), 123.456, 'row4') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t2 +POSTHOOK: Lineage: stats_t2.c_double SCRIPT [] +POSTHOOK: Lineage: stats_t2.c_float SCRIPT [] +POSTHOOK: Lineage: stats_t2.c_str SCRIPT [] +PREHOOK: query: analyze table stats_t2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@stats_t2 +PREHOOK: Output: default@stats_t2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_t2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@stats_t2 +POSTHOOK: Output: default@stats_t2 +#### A masked pattern was here #### +PREHOOK: query: describe formatted stats_t2 c_double +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t2 +POSTHOOK: query: describe formatted stats_t2 c_double +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t2 +col_name c_double +data_type double +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c_str\":\"true\"}} +PREHOOK: query: describe formatted stats_t2 c_float +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t2 +POSTHOOK: query: describe formatted stats_t2 c_float +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t2 +col_name c_float +data_type float +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c_str\":\"true\"}} +PREHOOK: query: describe formatted stats_t2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t2 +POSTHOOK: query: describe formatted stats_t2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t2 +# col_name data_type comment +c_double double +c_float float +c_str string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c_str\":\"true\"}} + bucketing_version 2 + numFiles 1 + numRows 4 + rawDataSize 392 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table stats_t3( + a double, + b float) +partitioned by (p int) +stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_t3 +POSTHOOK: query: create table stats_t3( + a double, + b float) +partitioned by (p int) +stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_t3 +PREHOOK: query: insert into table stats_t3 partition(p=1) values + (cast('Infinity' as double), cast('Infinity' as float)), + (cast('-Infinity' as double), cast('-Infinity' as float)), + (cast('NaN' as double), cast('NaN' as float)) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t3@p=1 +POSTHOOK: query: insert into table stats_t3 partition(p=1) values + (cast('Infinity' as double), cast('Infinity' as float)), + (cast('-Infinity' as double), cast('-Infinity' as float)), + (cast('NaN' as double), cast('NaN' as float)) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t3@p=1 +POSTHOOK: Lineage: stats_t3 PARTITION(p=1).a SCRIPT [] +POSTHOOK: Lineage: stats_t3 PARTITION(p=1).b SCRIPT [] +PREHOOK: query: describe formatted stats_t3 partition(p=1) a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t3 +POSTHOOK: query: describe formatted stats_t3 partition(p=1) a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t3 +col_name a +data_type double +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment from deserializer +PREHOOK: query: describe formatted stats_t3 partition(p=1) b +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t3 +POSTHOOK: query: describe formatted stats_t3 partition(p=1) b +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t3 +col_name b +data_type float +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment from deserializer +PREHOOK: query: describe formatted stats_t3 partition(p=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t3 +POSTHOOK: query: describe formatted stats_t3 partition(p=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t3 +# col_name data_type comment +a double +b float + +# Partition Information +# col_name data_type comment +p int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: stats_t3 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 36 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table stats_t4( + a double, + b float) +partitioned by (p int) +stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_t4 +POSTHOOK: query: create table stats_t4( + a double, + b float) +partitioned by (p int) +stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_t4 +PREHOOK: query: insert into table stats_t4 partition(p=1) values + (cast('Infinity' as double), 3.14), + (cast('-Infinity' as double), 2.72), + (cast('NaN' as double), 1.618) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t4@p=1 +POSTHOOK: query: insert into table stats_t4 partition(p=1) values + (cast('Infinity' as double), 3.14), + (cast('-Infinity' as double), 2.72), + (cast('NaN' as double), 1.618) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t4@p=1 +POSTHOOK: Lineage: stats_t4 PARTITION(p=1).a SCRIPT [] +POSTHOOK: Lineage: stats_t4 PARTITION(p=1).b SCRIPT [] +PREHOOK: query: insert into table stats_t4 partition(p=2) values + (42.0, cast('Infinity' as float)), + (17.3, cast('-Infinity' as float)), + (5.4, cast('NaN' as float)) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t4@p=2 +POSTHOOK: query: insert into table stats_t4 partition(p=2) values + (42.0, cast('Infinity' as float)), + (17.3, cast('-Infinity' as float)), + (5.4, cast('NaN' as float)) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t4@p=2 +POSTHOOK: Lineage: stats_t4 PARTITION(p=2).a SCRIPT [] +POSTHOOK: Lineage: stats_t4 PARTITION(p=2).b SCRIPT [] +PREHOOK: query: insert into table stats_t4 partition(p=3) values + (cast('Infinity' as double), cast('-Infinity' as float)), + (cast('-Infinity' as double), cast('Infinity' as float)), + (cast('NaN' as double), cast('NaN' as float)) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t4@p=3 +POSTHOOK: query: insert into table stats_t4 partition(p=3) values + (cast('Infinity' as double), cast('-Infinity' as float)), + (cast('-Infinity' as double), cast('Infinity' as float)), + (cast('NaN' as double), cast('NaN' as float)) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t4@p=3 +POSTHOOK: Lineage: stats_t4 PARTITION(p=3).a SCRIPT [] +POSTHOOK: Lineage: stats_t4 PARTITION(p=3).b SCRIPT [] +PREHOOK: query: insert into table stats_t4 partition(p=4) values + (1.0, 2.0), + (3.0, 4.0) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_t4@p=4 +POSTHOOK: query: insert into table stats_t4 partition(p=4) values + (1.0, 2.0), + (3.0, 4.0) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_t4@p=4 +POSTHOOK: Lineage: stats_t4 PARTITION(p=4).a SCRIPT [] +POSTHOOK: Lineage: stats_t4 PARTITION(p=4).b SCRIPT [] +PREHOOK: query: describe formatted stats_t4 partition(p=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t4 +POSTHOOK: query: describe formatted stats_t4 partition(p=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t4 +# col_name data_type comment +a double +b float + +# Partition Information +# col_name data_type comment +p int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: stats_t4 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 36 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted stats_t4 partition(p=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t4 +POSTHOOK: query: describe formatted stats_t4 partition(p=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t4 +# col_name data_type comment +a double +b float + +# Partition Information +# col_name data_type comment +p int + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: stats_t4 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 36 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted stats_t4 partition(p=3) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t4 +POSTHOOK: query: describe formatted stats_t4 partition(p=3) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t4 +# col_name data_type comment +a double +b float + +# Partition Information +# col_name data_type comment +p int + +# Detailed Partition Information +Partition Value: [3] +Database: default +Table: stats_t4 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 36 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted stats_t4 partition(p=4) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_t4 +POSTHOOK: query: describe formatted stats_t4 partition(p=4) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_t4 +# col_name data_type comment +a double +b float + +# Partition Information +# col_name data_type comment +p int + +# Detailed Partition Information +Partition Value: [4] +Database: default +Table: stats_t4 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 24 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 From ecf2d9e331d35f99387620c77f4f111c22986f9d Mon Sep 17 00:00:00 2001 From: cyan-zheng Date: Fri, 3 Jul 2026 20:15:45 +1000 Subject: [PATCH 2/2] revert debug log wording --- .../java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 5b427060016e..3c1d2c48cec8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -139,7 +139,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List