From 40da855f74e0bbbf7fc177ae4de47cab80ef6730 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Mon, 29 Jun 2026 11:32:58 +0530 Subject: [PATCH 1/5] HIVE-29341:Iceberg: [V3] NULL initial default for STRUCT column Change-Id: Iefec29f3d2e320cd94e5944651d65e85471b5cb8 --- .../apache/iceberg/hive/HiveSchemaUtil.java | 86 ++++++++++++--- .../iceberg/hive/TestHiveSchemaUtil.java | 103 ++++++++++++++++++ .../iceberg/mr/hive/HiveIcebergMetaHook.java | 12 +- .../mr/hive/vector/HiveVectorizedReader.java | 7 ++ .../mr/mapreduce/IcebergRecordReader.java | 12 +- .../positive/iceberg_initial_default.q | 27 ++++- .../iceberg_alter_default_column.q.out | 8 +- .../positive/iceberg_default_column.q.out | 12 +- .../positive/iceberg_initial_default.q.out | 102 ++++++++++++----- .../vector/VectorizedDummyColumnReader.java | 39 +++++++ 10 files changed, 344 insertions(+), 64 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index b1563040bcb9..690cac2e3d2d 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -154,6 +154,23 @@ public static Type convert(TypeInfo typeInfo, String defaultValue) { return HiveSchemaConverter.convert(typeInfo, false, defaultValue); } + public static Type applyInitialDefaultsToStruct(Type type) { + Types.StructType struct = type.asStructType(); + return Types.StructType.of( + struct.fields().stream().map(HiveSchemaUtil::applyInitialDefaultsToField).collect(Collectors.toList())); + } + + private static Types.NestedField applyInitialDefaultsToField(Types.NestedField field) { + Types.NestedField.Builder builder = Types.NestedField.from(field); + if (field.type().isStructType()) { + builder.ofType(applyInitialDefaultsToStruct(field.type())); + } + if (field.initialDefaultLiteral() == null && field.writeDefaultLiteral() != null) { + builder.withInitialDefault(field.writeDefaultLiteral()); + } + return builder.build(); + } + /** * Returns a SchemaDifference containing those fields which are present in only one of the collections, as well as * those fields which are present in both (in terms of the name) but their type or comment has changed. @@ -429,6 +446,24 @@ public static void setDefaultValues(Record record, List missi } } + /** + * Backfills struct column that is null on read using nested {@code initialDefault} metadata. + * This applies to rows written before {@code ADD COLUMNS} added the struct. + * Spec allows struct defaults as {@code {}} (see https://iceberg.apache.org/spec/#default-values), but + * {@code UpdateSchema} add column only supports primitives today; + * if empty structs are allowed, this backfill can be removed. + */ + public static void backfillStructInitialDefaults(Record record, List columns) { + for (Types.NestedField field : columns) { + if (field.type().isStructType() && record.getField(field.name()) == null) { + Record nestedRecord = buildStructWithInitialDefaults(field.type().asStructType()); + if (nestedRecord != null) { + record.setField(field.name(), nestedRecord); + } + } + } + } + /** * Recursively builds a struct populated with write defaults. * * @return A populated Record, or null if no nested fields have defaults. @@ -458,6 +493,42 @@ private static Record buildStructWithDefaults(Types.StructType structType) { return hasAnyDefault ? nestedRecord : null; } + private static Record buildStructWithInitialDefaults(Types.StructType structType) { + Record nestedRecord = GenericRecord.create(structType); + boolean hasAnyDefault = false; + + for (Types.NestedField field : structType.fields()) { + if (field.initialDefault() != null) { + Object defaultValue = convertToWriteType(field.initialDefault(), field.type()); + nestedRecord.setField(field.name(), defaultValue); + hasAnyDefault = true; + } else if (field.type().isStructType()) { + Record deeperRecord = buildStructWithInitialDefaults(field.type().asStructType()); + if (deeperRecord != null) { + nestedRecord.setField(field.name(), deeperRecord); + hasAnyDefault = true; + } + } + } + + return hasAnyDefault ? nestedRecord : null; + } + + public static Map getStructInitialDefaults(Types.StructType structType) { + Map result = Maps.newHashMap(); + for (Types.NestedField field : structType.fields()) { + if (field.initialDefault() != null) { + result.put(field.name(), field.initialDefault()); + } else if (field.type().isStructType()) { + Map nested = getStructInitialDefaults(field.type().asStructType()); + if (!nested.isEmpty()) { + result.put(field.name(), nested); + } + } + } + return result; + } + /** * Sets a value into a {@link Record} using a struct-only field path (top-level column or nested * through structs). Intermediate struct records are created as needed. @@ -496,21 +567,6 @@ private static Record getOrCreateStructRecord( return record; } - // Special method for nested structs that always applies defaults to null fields - private static void setDefaultValuesForNestedStruct(Record record, List fields) { - for (Types.NestedField field : fields) { - Object fieldValue = record.getField(field.name()); - - if (field.writeDefault() != null) { - Object defaultValue = convertToWriteType(field.writeDefault(), field.type()); - record.setField(field.name(), defaultValue); - } else if (field.type().isStructType()) { - // Recursively process nested structs - setDefaultValuesForNestedStruct((Record) fieldValue, field.type().asStructType().fields()); - } - } - } - public static Object convertToWriteType(Object value, Type type) { if (value == null) { return null; diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index 6daf3aeca5d7..5252220a76ec 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -22,12 +22,16 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.iceberg.Schema; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; @@ -37,6 +41,7 @@ import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.InstanceOfAssertFactories.MAP; public class TestHiveSchemaUtil { private static final Schema SIMPLE_ICEBERG_SCHEMA = new Schema( @@ -240,4 +245,102 @@ private void assertEquals(Type expected, Type actual) { } } } + + @Test + public void testBackfillStructInitialDefaults() { + Schema schema = new Schema( + optional(1, "id", Types.IntegerType.get()), + optional(2, "point", Types.StructType.of( + Types.NestedField.builder() + .withId(3) + .withName("x") + .asOptional() + .ofType(Types.IntegerType.get()) + .withInitialDefault(Expressions.lit(100)) + .build(), + Types.NestedField.builder() + .withId(4) + .withName("y") + .asOptional() + .ofType(Types.IntegerType.get()) + .withInitialDefault(Expressions.lit(99)) + .build() + )) + ); + + Record record = GenericRecord.create(schema); + record.setField("id", 1); + + HiveSchemaUtil.backfillStructInitialDefaults(record, schema.columns()); + + Record point = (Record) record.getField("point"); + assertThat(point).isNotNull(); + assertThat(point.getField("x")).isEqualTo(100); + assertThat(point.getField("y")).isEqualTo(99); + } + + @Test + public void testApplyInitialDefaults() { + Type structType = Types.StructType.of( + Types.NestedField.builder() + .withId(1) + .withName("x") + .asOptional() + .ofType(Types.IntegerType.get()) + .withWriteDefault(Expressions.lit(100)) + .build(), + Types.NestedField.builder() + .withId(2) + .withName("y") + .asOptional() + .ofType(Types.IntegerType.get()) + .withWriteDefault(Expressions.lit(99)) + .build()); + + Type withInitialDefaults = HiveSchemaUtil.applyInitialDefaultsToStruct(structType); + Types.NestedField xField = withInitialDefaults.asStructType().field("x"); + Types.NestedField yField = withInitialDefaults.asStructType().field("y"); + assertThat(xField.initialDefault()).isEqualTo(100); + assertThat(yField.initialDefault()).isEqualTo(99); + } + + @Test + public void testGetStructInitialDefaults() { + Types.StructType addressStruct = Types.StructType.of( + Types.NestedField.builder() + .withId(4) + .withName("street") + .asOptional() + .ofType(Types.StringType.get()) + .withInitialDefault(Expressions.lit("Main St")) + .build(), + Types.NestedField.builder() + .withId(5) + .withName("city") + .asOptional() + .ofType(Types.StringType.get()) + .withInitialDefault(Expressions.lit("New York")) + .build()); + Types.StructType personStruct = Types.StructType.of( + Types.NestedField.builder() + .withId(3) + .withName("name") + .asOptional() + .ofType(Types.StringType.get()) + .withInitialDefault(Expressions.lit("John")) + .build(), + Types.NestedField.builder() + .withId(6) + .withName("address") + .asOptional() + .ofType(addressStruct) + .build()); + + Map personDefaults = HiveSchemaUtil.getStructInitialDefaults(personStruct); + assertThat(personDefaults) + .containsEntry("name", "John") + .extractingByKey("address", MAP) + .containsEntry("street", "Main St") + .containsEntry("city", "New York"); + } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 5a2bbbf70219..ce7b453bd92e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -744,12 +744,14 @@ private void handleAddColumns(org.apache.hadoop.hive.metastore.api.Table hmsTabl boolean isORc = isOrcFileFormat(hmsTable); for (FieldSchema addedCol : addedCols) { String defaultValue = defaultValues.get(addedCol.getName()); - Type type = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType()), defaultValue); - Literal defaultVal = Optional.ofNullable(defaultValue).filter(v -> !type.isStructType()) - .map(v -> Expressions.lit(HiveSchemaUtil.getDefaultValue(v, type))).orElse(null); - + Type baseType = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType()), defaultValue); + final Type resolvedType = (!isORc && baseType.isStructType()) ? + HiveSchemaUtil.applyInitialDefaultsToStruct(baseType) : + baseType; + Literal defaultVal = Optional.ofNullable(defaultValue).filter(v -> !resolvedType.isStructType()) + .map(v -> Expressions.lit(HiveSchemaUtil.getDefaultValue(v, resolvedType))).orElse(null); // ORC doesn't have support for initialDefault from iceberg layer, we only need to set default for writeDefault. - updateSchema.addColumn(addedCol.getName(), type, addedCol.getComment(), isORc ? null : defaultVal); + updateSchema.addColumn(addedCol.getName(), resolvedType, addedCol.getComment(), isORc ? null : defaultVal); if (isORc && defaultVal != null) { updateSchema.updateColumnDefault(addedCol.getName(), defaultVal); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java index 834d762062da..4189b3aea209 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java @@ -52,6 +52,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.mr.InputFormatConfig; @@ -187,6 +188,12 @@ static Map getInitialColumnDefaults(List colu for (Types.NestedField column : columns) { if (column.initialDefault() != null) { columnDefaults.put(column.name(), column.initialDefault()); + } else if (column.type().isStructType()) { + Map structDefaults = + HiveSchemaUtil.getStructInitialDefaults(column.type().asStructType()); + if (!structDefaults.isEmpty()) { + columnDefaults.put(column.name(), structDefaults); + } } } return columnDefaults; diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index e3d03a6bee66..26c715883340 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -53,6 +53,7 @@ import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.encryption.EncryptedFiles; import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.InputFile; @@ -173,7 +174,16 @@ private CloseableIterable openGeneric(FileScanTask task, Schema readSchema) { default -> throw new UnsupportedOperationException( String.format("Cannot read %s file: %s", file.format().name(), file.location())); }; - return applyResidualFiltering(iterable, residual, readSchema); + return applyResidualFiltering(withStructInitialDefaultBackfill(iterable, readSchema), residual, readSchema); + } + + private CloseableIterable withStructInitialDefaultBackfill(CloseableIterable iterable, Schema readSchema) { + return CloseableIterable.transform(iterable, record -> { + if (record instanceof Record) { + HiveSchemaUtil.backfillStructInitialDefaults((Record) record, readSchema.columns()); + } + return record; + }); } private CloseableIterable newAvroIterable( diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q index c0c058bcd5dc..c7007114e5be 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q @@ -13,7 +13,14 @@ ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT '{"x":100," created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general'); + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}'); INSERT INTO ice_parq (id) VALUES (2); @@ -45,7 +52,14 @@ ALTER TABLE ice_avro ADD COLUMNS (point STRUCT DEFAULT '{"x":100," created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general'); + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}'); INSERT INTO ice_avro (id) VALUES (2); @@ -77,7 +91,14 @@ ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT '{"x":100,"y created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general'); + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}'); INSERT INTO ice_orc (id) VALUES (2); diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out index bb1ae6aa0606..ee405875b382 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out @@ -60,7 +60,7 @@ POSTHOOK: query: SELECT * FROM ice_t ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_t POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general 2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general PREHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT, point STRUCT DEFAULT '{"x":100,"y":99}', @@ -94,7 +94,7 @@ POSTHOOK: query: SELECT * FROM ice_t ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_t POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general 2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT DEFAULT '{"x":100,"y":88}' PREHOOK: type: ALTERTABLE_RENAMECOL @@ -128,7 +128,7 @@ POSTHOOK: query: SELECT * FROM ice_t ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_t POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general 2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general 3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 general PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT DEFAULT '{"x":55,"y":88}' @@ -155,7 +155,7 @@ POSTHOOK: query: SELECT * FROM ice_t ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_t POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general 2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 general 3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 general 4 {"x":55,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 general diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out index 8400d21d3572..5ff787ef67bd 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out @@ -170,11 +170,11 @@ POSTHOOK: query: SELECT * FROM t3 ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@t3 POSTHOOK: Output: hdfs://### HDFS PATH ### -2 {"x":null,"y":7} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general NULL -3 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general NULL -4 {"x":100,"y":99} NULL 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general NULL -5 {"x":100,"y":99} custom_name 30 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general NULL -6 {"x":null,"y":null} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general NULL -7 NULL null NULL 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general NULL +2 {"x":null,"y":7} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +4 {"x":100,"y":99} NULL 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +5 {"x":100,"y":99} custom_name 30 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +6 {"x":null,"y":null} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +7 NULL null NULL 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} 8 NULL null NULL 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} 9 NULL null NULL 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":null,"address":{"street":null,"city":"Bangalore"}} diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out index da857354d811..d15997e7d81f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out @@ -28,7 +28,14 @@ PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEF created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general') + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}') PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@ice_parq PREHOOK: Output: default@ice_parq @@ -40,7 +47,14 @@ POSTHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DE created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general') + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}') POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@ice_parq POSTHOOK: Output: default@ice_parq @@ -60,8 +74,8 @@ POSTHOOK: query: SELECT * FROM ice_parq ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_parq POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT DEFAULT '{"x":100,"y":88}' PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@ice_parq @@ -94,9 +108,9 @@ POSTHOOK: query: SELECT * FROM ice_parq ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_parq POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: CREATE TABLE ice_avro ( id INT) STORED BY ICEBERG stored as avro @@ -127,7 +141,14 @@ PREHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point STRUCT DEF created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general') + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}') PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@ice_avro PREHOOK: Output: default@ice_avro @@ -139,7 +160,14 @@ POSTHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point STRUCT DE created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general') + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}') POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@ice_avro POSTHOOK: Output: default@ice_avro @@ -159,8 +187,8 @@ POSTHOOK: query: SELECT * FROM ice_avro ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_avro POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT DEFAULT '{"x":100,"y":88}' PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@ice_avro @@ -193,9 +221,9 @@ POSTHOOK: query: SELECT * FROM ice_avro ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_avro POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: CREATE TABLE ice_orc ( id INT) STORED BY ICEBERG stored as orc @@ -226,7 +254,14 @@ PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFA created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general') + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}') PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@ice_orc PREHOOK: Output: default@ice_orc @@ -238,7 +273,14 @@ POSTHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEF created_date DATE DEFAULT '2024-01-01', created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00', score DECIMAL(5,2) DEFAULT 100.00, - category STRING DEFAULT 'general') + category STRING DEFAULT 'general', + person STRUCT< + name: STRING, + address: STRUCT< + street: STRING, + city: STRING + > + > DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}') POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc @@ -258,8 +300,8 @@ POSTHOOK: query: SELECT * FROM ice_orc ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL NULL NULL NULL NULL NULL NULL NULL NULL -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT DEFAULT '{"x":100,"y":88}' PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@ice_orc @@ -292,9 +334,9 @@ POSTHOOK: query: SELECT * FROM ice_orc ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL NULL NULL NULL NULL NULL NULL NULL NULL -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: SELECT * FROM ice_parq ORDER BY id PREHOOK: type: QUERY PREHOOK: Input: default@ice_parq @@ -303,9 +345,9 @@ POSTHOOK: query: SELECT * FROM ice_parq ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_parq POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: SELECT * FROM ice_avro ORDER BY id PREHOOK: type: QUERY PREHOOK: Input: default@ice_avro @@ -314,9 +356,9 @@ POSTHOOK: query: SELECT * FROM ice_avro ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_avro POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} PREHOOK: query: SELECT * FROM ice_orc ORDER BY id PREHOOK: type: QUERY PREHOOK: Input: default@ice_orc @@ -325,6 +367,6 @@ POSTHOOK: query: SELECT * FROM ice_orc ORDER BY id POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL NULL NULL NULL NULL NULL NULL NULL NULL -2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general -3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general +1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} +3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01 2024-01-01 10:00:00 100.00 general {"name":"John","address":{"street":"Main St","city":"New York"}} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java index e8d95ccd1587..a31b0e69858e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java @@ -24,14 +24,18 @@ import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; +import java.util.Map; /** * A dummy vectorized parquet reader used for schema evolution. @@ -64,6 +68,41 @@ public void readBatch(int total, ColumnVector col, TypeInfo typeInfo) throws IOE if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { fillPrimitive(col, (PrimitiveTypeInfo) typeInfo, defaultValue); + } else if (typeInfo.getCategory() == ObjectInspector.Category.STRUCT) { + fillStruct(col, (StructTypeInfo) typeInfo, defaultValue); + } else { + throw new IOException("Unsupported type category in DummyColumnReader: " + typeInfo.getCategory()); + } + } + + private void fillStruct(ColumnVector col, StructTypeInfo structTypeInfo, Object defaultValue) throws IOException { + StructColumnVector structCol = (StructColumnVector) col; + List fieldNames = structTypeInfo.getAllStructFieldNames(); + List fieldTypes = structTypeInfo.getAllStructFieldTypeInfos(); + Map fieldDefaults = defaultValue instanceof Map ? (Map) defaultValue : null; + + for (int i = 0; i < fieldNames.size(); i++) { + Object fieldDefault = fieldDefaults != null ? fieldDefaults.get(fieldNames.get(i)) : null; + fillStructField(structCol.fields[i], fieldTypes.get(i), fieldDefault); + } + } + + private void fillStructField(ColumnVector col, TypeInfo typeInfo, Object fieldDefault) throws IOException { + if (fieldDefault == null) { + col.isRepeating = true; + Arrays.fill(col.isNull, true); + col.noNulls = false; + return; + } + + col.isRepeating = true; + col.noNulls = true; + col.isNull[0] = false; + + if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + fillPrimitive(col, (PrimitiveTypeInfo) typeInfo, fieldDefault); + } else if (typeInfo.getCategory() == ObjectInspector.Category.STRUCT) { + fillStruct(col, (StructTypeInfo) typeInfo, fieldDefault); } else { throw new IOException("Unsupported type category in DummyColumnReader: " + typeInfo.getCategory()); } From 2646e4e4ff7e550b05019480767d44795d39268b Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Tue, 30 Jun 2026 12:16:27 +0530 Subject: [PATCH 2/5] addressed sonar issues Change-Id: Iae36e3f24cdc1f1ced9e5cd827864e365ecc2a0b --- .../apache/iceberg/hive/HiveSchemaUtil.java | 8 ++--- .../iceberg/hive/TestHiveSchemaUtil.java | 34 +++++++++---------- .../mr/mapreduce/IcebergRecordReader.java | 8 ++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index 690cac2e3d2d..1b0014c2a44f 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -157,7 +157,7 @@ public static Type convert(TypeInfo typeInfo, String defaultValue) { public static Type applyInitialDefaultsToStruct(Type type) { Types.StructType struct = type.asStructType(); return Types.StructType.of( - struct.fields().stream().map(HiveSchemaUtil::applyInitialDefaultsToField).collect(Collectors.toList())); + struct.fields().stream().map(HiveSchemaUtil::applyInitialDefaultsToField).toList()); } private static Types.NestedField applyInitialDefaultsToField(Types.NestedField field) { @@ -453,12 +453,12 @@ public static void setDefaultValues(Record record, List missi * {@code UpdateSchema} add column only supports primitives today; * if empty structs are allowed, this backfill can be removed. */ - public static void backfillStructInitialDefaults(Record record, List columns) { + public static void backfillStructInitialDefaults(Record iceRecord, List columns) { for (Types.NestedField field : columns) { - if (field.type().isStructType() && record.getField(field.name()) == null) { + if (field.type().isStructType() && iceRecord.getField(field.name()) == null) { Record nestedRecord = buildStructWithInitialDefaults(field.type().asStructType()); if (nestedRecord != null) { - record.setField(field.name(), nestedRecord); + iceRecord.setField(field.name(), nestedRecord); } } } diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index 5252220a76ec..288438a6089b 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -96,12 +96,12 @@ public class TestHiveSchemaUtil { ); @Test - public void testSimpleSchemaConvertToIcebergSchema() { + void testSimpleSchemaConvertToIcebergSchema() { assertThat(HiveSchemaUtil.convert(SIMPLE_HIVE_SCHEMA).asStruct()).isEqualTo(SIMPLE_ICEBERG_SCHEMA.asStruct()); } @Test - public void testSimpleSchemaConvertToIcebergSchemaFromNameAndTypeLists() { + void testSimpleSchemaConvertToIcebergSchemaFromNameAndTypeLists() { List names = SIMPLE_HIVE_SCHEMA.stream().map(field -> field.getName()).collect(Collectors.toList()); List types = SIMPLE_HIVE_SCHEMA.stream() .map(field -> TypeInfoUtils.getTypeInfoFromTypeString(field.getType())) @@ -111,18 +111,18 @@ public void testSimpleSchemaConvertToIcebergSchemaFromNameAndTypeLists() { } @Test - public void testComplexSchemaConvertToIcebergSchema() { + void testComplexSchemaConvertToIcebergSchema() { assertThat(HiveSchemaUtil.convert(COMPLEX_HIVE_SCHEMA).asStruct()).isEqualTo(COMPLEX_ICEBERG_SCHEMA.asStruct()); } @Test - public void testSchemaConvertToIcebergSchemaForEveryPrimitiveType() { + void testSchemaConvertToIcebergSchemaForEveryPrimitiveType() { Schema schemaWithEveryType = HiveSchemaUtil.convert(getSupportedFieldSchemas()); assertThat(schemaWithEveryType.asStruct()).isEqualTo(getSchemaWithSupportedTypes().asStruct()); } @Test - public void testNotSupportedTypes() { + void testNotSupportedTypes() { for (FieldSchema notSupportedField : getNotSupportedFieldSchemas()) { assertThatThrownBy( () -> HiveSchemaUtil.convert( @@ -133,17 +133,17 @@ public void testNotSupportedTypes() { } @Test - public void testSimpleSchemaConvertToHiveSchema() { + void testSimpleSchemaConvertToHiveSchema() { assertThat(HiveSchemaUtil.convert(SIMPLE_ICEBERG_SCHEMA)).isEqualTo(SIMPLE_HIVE_SCHEMA); } @Test - public void testComplexSchemaConvertToHiveSchema() { + void testComplexSchemaConvertToHiveSchema() { assertThat(HiveSchemaUtil.convert(COMPLEX_ICEBERG_SCHEMA)).isEqualTo(COMPLEX_HIVE_SCHEMA); } @Test - public void testSimpleTypeAndTypeInfoConvert() { + void testSimpleTypeAndTypeInfoConvert() { // Test for every supported type List fieldSchemas = getSupportedFieldSchemas(); List nestedFields = getSchemaWithSupportedTypes().columns(); @@ -153,7 +153,7 @@ public void testSimpleTypeAndTypeInfoConvert() { } @Test - public void testComplexTypeAndTypeInfoConvert() { + void testComplexTypeAndTypeInfoConvert() { for (int i = 0; i < COMPLEX_HIVE_SCHEMA.size(); ++i) { checkConvert(TypeInfoUtils.getTypeInfoFromTypeString(COMPLEX_HIVE_SCHEMA.get(i).getType()), COMPLEX_ICEBERG_SCHEMA.columns().get(i).type()); @@ -161,7 +161,7 @@ public void testComplexTypeAndTypeInfoConvert() { } @Test - public void testConversionWithoutLastComment() { + void testConversionWithoutLastComment() { Schema expected = new Schema( optional(1, "customer_id", Types.LongType.get(), "customer comment"), optional(2, "first_name", Types.StringType.get(), null) @@ -247,7 +247,7 @@ private void assertEquals(Type expected, Type actual) { } @Test - public void testBackfillStructInitialDefaults() { + void testBackfillStructInitialDefaults() { Schema schema = new Schema( optional(1, "id", Types.IntegerType.get()), optional(2, "point", Types.StructType.of( @@ -268,19 +268,19 @@ public void testBackfillStructInitialDefaults() { )) ); - Record record = GenericRecord.create(schema); - record.setField("id", 1); + Record iceRecord = GenericRecord.create(schema); + iceRecord.setField("id", 1); - HiveSchemaUtil.backfillStructInitialDefaults(record, schema.columns()); + HiveSchemaUtil.backfillStructInitialDefaults(iceRecord, schema.columns()); - Record point = (Record) record.getField("point"); + Record point = (Record) iceRecord.getField("point"); assertThat(point).isNotNull(); assertThat(point.getField("x")).isEqualTo(100); assertThat(point.getField("y")).isEqualTo(99); } @Test - public void testApplyInitialDefaults() { + void testApplyInitialDefaults() { Type structType = Types.StructType.of( Types.NestedField.builder() .withId(1) @@ -305,7 +305,7 @@ public void testApplyInitialDefaults() { } @Test - public void testGetStructInitialDefaults() { + void testGetStructInitialDefaults() { Types.StructType addressStruct = Types.StructType.of( Types.NestedField.builder() .withId(4) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index 26c715883340..beef8ce02da6 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -178,11 +178,11 @@ private CloseableIterable openGeneric(FileScanTask task, Schema readSchema) { } private CloseableIterable withStructInitialDefaultBackfill(CloseableIterable iterable, Schema readSchema) { - return CloseableIterable.transform(iterable, record -> { - if (record instanceof Record) { - HiveSchemaUtil.backfillStructInitialDefaults((Record) record, readSchema.columns()); + return CloseableIterable.transform(iterable, row -> { + if (row instanceof Record curIceRecord) { + HiveSchemaUtil.backfillStructInitialDefaults(curIceRecord, readSchema.columns()); } - return record; + return row; }); } From 9426be37956e0d112dffb157a8f78cf0caa3adb6 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Tue, 30 Jun 2026 15:52:24 +0530 Subject: [PATCH 3/5] refractor and early return Change-Id: Id2df9e65a64f876600882acffb6df2f628559e1e --- .../main/java/org/apache/iceberg/hive/HiveSchemaUtil.java | 4 ++-- .../apache/iceberg/mr/mapreduce/IcebergRecordReader.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index 1b0014c2a44f..36adbe8ffd53 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -157,10 +157,10 @@ public static Type convert(TypeInfo typeInfo, String defaultValue) { public static Type applyInitialDefaultsToStruct(Type type) { Types.StructType struct = type.asStructType(); return Types.StructType.of( - struct.fields().stream().map(HiveSchemaUtil::applyInitialDefaultsToField).toList()); + struct.fields().stream().map(HiveSchemaUtil::applyInitialDefaultsToStructField).toList()); } - private static Types.NestedField applyInitialDefaultsToField(Types.NestedField field) { + private static Types.NestedField applyInitialDefaultsToStructField(Types.NestedField field) { Types.NestedField.Builder builder = Types.NestedField.from(field); if (field.type().isStructType()) { builder.ofType(applyInitialDefaultsToStruct(field.type())); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index beef8ce02da6..f3e6d27514b1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -178,6 +178,12 @@ private CloseableIterable openGeneric(FileScanTask task, Schema readSchema) { } private CloseableIterable withStructInitialDefaultBackfill(CloseableIterable iterable, Schema readSchema) { + boolean needsBackfill = readSchema.columns().stream() + .filter(field -> field.type().isStructType()) + .anyMatch(field -> !HiveSchemaUtil.getStructInitialDefaults(field.type().asStructType()).isEmpty()); + if (!needsBackfill) { + return iterable; + } return CloseableIterable.transform(iterable, row -> { if (row instanceof Record curIceRecord) { HiveSchemaUtil.backfillStructInitialDefaults(curIceRecord, readSchema.columns()); From 59bb37c4a47621b26c0c235a3040d570d00832a5 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Wed, 1 Jul 2026 12:36:57 +0530 Subject: [PATCH 4/5] refractored and optimized Change-Id: If7fc697834eda9924177d648a9e96e8e149d7c7d --- .../apache/iceberg/hive/HiveSchemaUtil.java | 60 ++++------ .../iceberg/hive/TestHiveSchemaUtil.java | 103 ------------------ .../mr/mapreduce/IcebergRecordReader.java | 22 ++-- .../vector/VectorizedDummyColumnReader.java | 36 ++---- 4 files changed, 43 insertions(+), 178 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index 36adbe8ffd53..48ddbd965ae1 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -435,7 +436,8 @@ public static void setDefaultValues(Record record, List missi for (Types.NestedField field : missingFields) { if (field.type().isStructType()) { // Attempt to build the nested struct with its defaults - Record nestedRecord = buildStructWithDefaults(field.type().asStructType()); + Record nestedRecord = buildStructFromDefaults( + field.type().asStructType(), Types.NestedField::writeDefault); if (nestedRecord != null) { record.setField(field.name(), nestedRecord); } @@ -450,60 +452,36 @@ public static void setDefaultValues(Record record, List missi * Backfills struct column that is null on read using nested {@code initialDefault} metadata. * This applies to rows written before {@code ADD COLUMNS} added the struct. * Spec allows struct defaults as {@code {}} (see https://iceberg.apache.org/spec/#default-values), but - * {@code UpdateSchema} add column only supports primitives today; + * {@code UpdateSchema} add column only supports defaults of underlying primitives and keeping the + * struct default as null due to which we need to backfill that nested default record while reading; * if empty structs are allowed, this backfill can be removed. */ - public static void backfillStructInitialDefaults(Record iceRecord, List columns) { - for (Types.NestedField field : columns) { - if (field.type().isStructType() && iceRecord.getField(field.name()) == null) { - Record nestedRecord = buildStructWithInitialDefaults(field.type().asStructType()); - if (nestedRecord != null) { - iceRecord.setField(field.name(), nestedRecord); - } + public static void backfillStructInitialDefaults( + Record iceRecord, Map initialDefaultStructsByColumn) { + for (Map.Entry columnAndInitialDefaultStruct : initialDefaultStructsByColumn.entrySet()) { + String columnName = columnAndInitialDefaultStruct.getKey(); + if (iceRecord.getField(columnName) == null) { + iceRecord.setField(columnName, columnAndInitialDefaultStruct.getValue()); } } } /** - * Recursively builds a struct populated with write defaults. - * * @return A populated Record, or null if no nested fields have defaults. + * Recursively builds a struct populated with underlying field defaults. + * @return A populated Record, or null if no nested fields have defaults. */ - private static Record buildStructWithDefaults(Types.StructType structType) { + public static Record buildStructFromDefaults( + Types.StructType structType, Function defaultForField) { Record nestedRecord = GenericRecord.create(structType); boolean hasAnyDefault = false; for (Types.NestedField field : structType.fields()) { - if (field.writeDefault() != null) { - Object defaultValue = convertToWriteType(field.writeDefault(), field.type()); - nestedRecord.setField(field.name(), defaultValue); - hasAnyDefault = true; - } else if (field.type().isStructType()) { - // Recursively process deeper nested structs - Record deeperRecord = buildStructWithDefaults(field.type().asStructType()); - - // If the deeper struct has defaults, attach it and flag this current struct as populated - if (deeperRecord != null) { - nestedRecord.setField(field.name(), deeperRecord); - hasAnyDefault = true; - } - } - } - - // If no fields (or nested fields) had defaults, return null to avoid an empty struct - return hasAnyDefault ? nestedRecord : null; - } - - private static Record buildStructWithInitialDefaults(Types.StructType structType) { - Record nestedRecord = GenericRecord.create(structType); - boolean hasAnyDefault = false; - - for (Types.NestedField field : structType.fields()) { - if (field.initialDefault() != null) { - Object defaultValue = convertToWriteType(field.initialDefault(), field.type()); - nestedRecord.setField(field.name(), defaultValue); + Object defaultValue = defaultForField.apply(field); + if (defaultValue != null) { + nestedRecord.setField(field.name(), convertToWriteType(defaultValue, field.type())); hasAnyDefault = true; } else if (field.type().isStructType()) { - Record deeperRecord = buildStructWithInitialDefaults(field.type().asStructType()); + Record deeperRecord = buildStructFromDefaults(field.type().asStructType(), defaultForField); if (deeperRecord != null) { nestedRecord.setField(field.name(), deeperRecord); hasAnyDefault = true; diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index 288438a6089b..d949ff83a3cc 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -22,16 +22,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.iceberg.Schema; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; @@ -41,7 +37,6 @@ import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.InstanceOfAssertFactories.MAP; public class TestHiveSchemaUtil { private static final Schema SIMPLE_ICEBERG_SCHEMA = new Schema( @@ -245,102 +240,4 @@ private void assertEquals(Type expected, Type actual) { } } } - - @Test - void testBackfillStructInitialDefaults() { - Schema schema = new Schema( - optional(1, "id", Types.IntegerType.get()), - optional(2, "point", Types.StructType.of( - Types.NestedField.builder() - .withId(3) - .withName("x") - .asOptional() - .ofType(Types.IntegerType.get()) - .withInitialDefault(Expressions.lit(100)) - .build(), - Types.NestedField.builder() - .withId(4) - .withName("y") - .asOptional() - .ofType(Types.IntegerType.get()) - .withInitialDefault(Expressions.lit(99)) - .build() - )) - ); - - Record iceRecord = GenericRecord.create(schema); - iceRecord.setField("id", 1); - - HiveSchemaUtil.backfillStructInitialDefaults(iceRecord, schema.columns()); - - Record point = (Record) iceRecord.getField("point"); - assertThat(point).isNotNull(); - assertThat(point.getField("x")).isEqualTo(100); - assertThat(point.getField("y")).isEqualTo(99); - } - - @Test - void testApplyInitialDefaults() { - Type structType = Types.StructType.of( - Types.NestedField.builder() - .withId(1) - .withName("x") - .asOptional() - .ofType(Types.IntegerType.get()) - .withWriteDefault(Expressions.lit(100)) - .build(), - Types.NestedField.builder() - .withId(2) - .withName("y") - .asOptional() - .ofType(Types.IntegerType.get()) - .withWriteDefault(Expressions.lit(99)) - .build()); - - Type withInitialDefaults = HiveSchemaUtil.applyInitialDefaultsToStruct(structType); - Types.NestedField xField = withInitialDefaults.asStructType().field("x"); - Types.NestedField yField = withInitialDefaults.asStructType().field("y"); - assertThat(xField.initialDefault()).isEqualTo(100); - assertThat(yField.initialDefault()).isEqualTo(99); - } - - @Test - void testGetStructInitialDefaults() { - Types.StructType addressStruct = Types.StructType.of( - Types.NestedField.builder() - .withId(4) - .withName("street") - .asOptional() - .ofType(Types.StringType.get()) - .withInitialDefault(Expressions.lit("Main St")) - .build(), - Types.NestedField.builder() - .withId(5) - .withName("city") - .asOptional() - .ofType(Types.StringType.get()) - .withInitialDefault(Expressions.lit("New York")) - .build()); - Types.StructType personStruct = Types.StructType.of( - Types.NestedField.builder() - .withId(3) - .withName("name") - .asOptional() - .ofType(Types.StringType.get()) - .withInitialDefault(Expressions.lit("John")) - .build(), - Types.NestedField.builder() - .withId(6) - .withName("address") - .asOptional() - .ofType(addressStruct) - .build()); - - Map personDefaults = HiveSchemaUtil.getStructInitialDefaults(personStruct); - assertThat(personDefaults) - .containsEntry("name", "John") - .extractingByKey("address", MAP) - .containsEntry("street", "Main St") - .containsEntry("city", "New York"); - } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java index f3e6d27514b1..96f8705a3f7e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java @@ -64,6 +64,7 @@ import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -174,19 +175,26 @@ private CloseableIterable openGeneric(FileScanTask task, Schema readSchema) { default -> throw new UnsupportedOperationException( String.format("Cannot read %s file: %s", file.format().name(), file.location())); }; - return applyResidualFiltering(withStructInitialDefaultBackfill(iterable, readSchema), residual, readSchema); + return applyResidualFiltering(withStructInitialDefaultsBackfill(iterable, readSchema), residual, readSchema); } - private CloseableIterable withStructInitialDefaultBackfill(CloseableIterable iterable, Schema readSchema) { - boolean needsBackfill = readSchema.columns().stream() - .filter(field -> field.type().isStructType()) - .anyMatch(field -> !HiveSchemaUtil.getStructInitialDefaults(field.type().asStructType()).isEmpty()); - if (!needsBackfill) { + private CloseableIterable withStructInitialDefaultsBackfill(CloseableIterable iterable, Schema readSchema) { + Map initialDefaultStructsByColumn = Maps.newHashMap(); + for (Types.NestedField column : readSchema.columns()) { + if (column.type().isStructType()) { + Record initialDefaultStruct = HiveSchemaUtil + .buildStructFromDefaults(column.type().asStructType(), Types.NestedField::initialDefault); + if (initialDefaultStruct != null) { + initialDefaultStructsByColumn.put(column.name(), initialDefaultStruct); + } + } + } + if (initialDefaultStructsByColumn.isEmpty()) { return iterable; } return CloseableIterable.transform(iterable, row -> { if (row instanceof Record curIceRecord) { - HiveSchemaUtil.backfillStructInitialDefaults(curIceRecord, readSchema.columns()); + HiveSchemaUtil.backfillStructInitialDefaults(curIceRecord, initialDefaultStructsByColumn); } return row; }); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java index a31b0e69858e..87da6cc0dda6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -53,10 +54,13 @@ public VectorizedDummyColumnReader(Object defaultValue) { @Override public void readBatch(int total, ColumnVector col, TypeInfo typeInfo) throws IOException { + fillColumnWithDefault(col, typeInfo, defaultValue); + } + private void fillColumnWithDefault(ColumnVector col, TypeInfo typeInfo, Object value) throws IOException { col.isRepeating = true; // Case 1: No default → (all nulls) - if (defaultValue == null) { + if (value == null) { Arrays.fill(col.isNull, true); col.noNulls = false; return; @@ -67,9 +71,9 @@ public void readBatch(int total, ColumnVector col, TypeInfo typeInfo) throws IOE col.isNull[0] = false; if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { - fillPrimitive(col, (PrimitiveTypeInfo) typeInfo, defaultValue); + fillPrimitive(col, (PrimitiveTypeInfo) typeInfo, value); } else if (typeInfo.getCategory() == ObjectInspector.Category.STRUCT) { - fillStruct(col, (StructTypeInfo) typeInfo, defaultValue); + fillStruct(col, (StructTypeInfo) typeInfo, value); } else { throw new IOException("Unsupported type category in DummyColumnReader: " + typeInfo.getCategory()); } @@ -79,32 +83,10 @@ private void fillStruct(ColumnVector col, StructTypeInfo structTypeInfo, Object StructColumnVector structCol = (StructColumnVector) col; List fieldNames = structTypeInfo.getAllStructFieldNames(); List fieldTypes = structTypeInfo.getAllStructFieldTypeInfos(); - Map fieldDefaults = defaultValue instanceof Map ? (Map) defaultValue : null; + Map fieldDefaults = defaultValue instanceof Map ? (Map) defaultValue : Collections.emptyMap(); for (int i = 0; i < fieldNames.size(); i++) { - Object fieldDefault = fieldDefaults != null ? fieldDefaults.get(fieldNames.get(i)) : null; - fillStructField(structCol.fields[i], fieldTypes.get(i), fieldDefault); - } - } - - private void fillStructField(ColumnVector col, TypeInfo typeInfo, Object fieldDefault) throws IOException { - if (fieldDefault == null) { - col.isRepeating = true; - Arrays.fill(col.isNull, true); - col.noNulls = false; - return; - } - - col.isRepeating = true; - col.noNulls = true; - col.isNull[0] = false; - - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { - fillPrimitive(col, (PrimitiveTypeInfo) typeInfo, fieldDefault); - } else if (typeInfo.getCategory() == ObjectInspector.Category.STRUCT) { - fillStruct(col, (StructTypeInfo) typeInfo, fieldDefault); - } else { - throw new IOException("Unsupported type category in DummyColumnReader: " + typeInfo.getCategory()); + fillColumnWithDefault(structCol.fields[i], fieldTypes.get(i), fieldDefaults.get(fieldNames.get(i))); } } From 34aff7cb0136fad77280076ce020ed15690dd475 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Fri, 3 Jul 2026 01:22:31 +0530 Subject: [PATCH 5/5] refractor Change-Id: Ia7939110bf55576cef6f7cdca329bc49454dfd48 --- .../iceberg/hive/HiveSchemaConverter.java | 31 ++++++++++++------- .../apache/iceberg/hive/HiveSchemaUtil.java | 22 ++----------- .../iceberg/hive/TestHiveSchemaUtil.java | 22 ++++++------- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 18 +++++------ .../vector/VectorizedDummyColumnReader.java | 4 ++- 5 files changed, 44 insertions(+), 53 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java index 1b743343163e..07631516cd49 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java @@ -58,21 +58,22 @@ private HiveSchemaConverter(boolean autoConvert) { static Schema convert(List names, List typeInfos, List comments, boolean autoConvert, Map defaultValues) { HiveSchemaConverter converter = new HiveSchemaConverter(autoConvert); - return new Schema(converter.convertInternal(names, typeInfos, defaultValues, comments)); + return new Schema(converter.convertInternal(names, typeInfos, defaultValues, comments, false)); } - public static Type convert(TypeInfo typeInfo, boolean autoConvert, String defaultValue) { + public static Type convert(TypeInfo typeInfo, boolean autoConvert, String defaultValue, + boolean shouldAddInitialDefault) { HiveSchemaConverter converter = new HiveSchemaConverter(autoConvert); - return converter.convertType(typeInfo, defaultValue); + return converter.convertType(typeInfo, defaultValue, shouldAddInitialDefault); } List convertInternal(List names, List typeInfos, - Map defaultValues, List comments) { + Map defaultValues, List comments, boolean shouldAddInitialDefault) { List result = Lists.newArrayListWithExpectedSize(names.size()); int outerId = id + names.size(); id = outerId; for (int i = 0; i < names.size(); ++i) { - Type type = convertType(typeInfos.get(i), defaultValues.get(names.get(i))); + Type type = convertType(typeInfos.get(i), defaultValues.get(names.get(i)), shouldAddInitialDefault); String columnName = names.get(i); Types.NestedField.Builder fieldBuilder = Types.NestedField.builder() @@ -87,6 +88,9 @@ List convertInternal(List names, List typeI Object icebergDefaultValue = HiveSchemaUtil.getDefaultValue(defaultValues.get(columnName), type); if (icebergDefaultValue != null) { fieldBuilder.withWriteDefault(Expressions.lit(icebergDefaultValue)); + if (shouldAddInitialDefault) { + fieldBuilder.withInitialDefault(Expressions.lit(icebergDefaultValue)); + } } } else if (!type.isStructType()) { throw new UnsupportedOperationException( @@ -99,7 +103,7 @@ List convertInternal(List names, List typeI return result; } - Type convertType(TypeInfo typeInfo, String defaultValue) { + Type convertType(TypeInfo typeInfo, String defaultValue, boolean shouldAddInitialDefault) { switch (typeInfo.getCategory()) { case PRIMITIVE: switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) { @@ -161,21 +165,24 @@ Type convertType(TypeInfo typeInfo, String defaultValue) { } case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; - List fields = - convertInternal(structTypeInfo.getAllStructFieldNames(), structTypeInfo.getAllStructFieldTypeInfos(), - HiveSchemaUtil.getDefaultValuesMap(null, defaultValue), Collections.emptyList()); + List fields = convertInternal( + structTypeInfo.getAllStructFieldNames(), + structTypeInfo.getAllStructFieldTypeInfos(), + HiveSchemaUtil.getDefaultValuesMap(null, defaultValue), + Collections.emptyList(), + shouldAddInitialDefault); return Types.StructType.of(fields); case MAP: MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; int keyId = id++; - Type keyType = convertType(mapTypeInfo.getMapKeyTypeInfo(), defaultValue); + Type keyType = convertType(mapTypeInfo.getMapKeyTypeInfo(), defaultValue, shouldAddInitialDefault); int valueId = id++; - Type valueType = convertType(mapTypeInfo.getMapValueTypeInfo(), defaultValue); + Type valueType = convertType(mapTypeInfo.getMapValueTypeInfo(), defaultValue, shouldAddInitialDefault); return Types.MapType.ofOptional(keyId, valueId, keyType, valueType); case LIST: ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; int listId = id++; - Type listType = convertType(listTypeInfo.getListElementTypeInfo(), defaultValue); + Type listType = convertType(listTypeInfo.getListElementTypeInfo(), defaultValue, shouldAddInitialDefault); return Types.ListType.ofOptional(listId, listType); case VARIANT: return Types.VariantType.get(); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index 48ddbd965ae1..3f18deab4eaf 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -149,27 +149,11 @@ public static TypeInfo convert(Type type) { * * @param typeInfo The Hive type * @param defaultValue the default value for the column, if any + * @param shouldAddInitialDefault whether to set initial default for the column or not * @return The Iceberg type */ - public static Type convert(TypeInfo typeInfo, String defaultValue) { - return HiveSchemaConverter.convert(typeInfo, false, defaultValue); - } - - public static Type applyInitialDefaultsToStruct(Type type) { - Types.StructType struct = type.asStructType(); - return Types.StructType.of( - struct.fields().stream().map(HiveSchemaUtil::applyInitialDefaultsToStructField).toList()); - } - - private static Types.NestedField applyInitialDefaultsToStructField(Types.NestedField field) { - Types.NestedField.Builder builder = Types.NestedField.from(field); - if (field.type().isStructType()) { - builder.ofType(applyInitialDefaultsToStruct(field.type())); - } - if (field.initialDefaultLiteral() == null && field.writeDefaultLiteral() != null) { - builder.withInitialDefault(field.writeDefaultLiteral()); - } - return builder.build(); + public static Type convert(TypeInfo typeInfo, String defaultValue, boolean shouldAddInitialDefault) { + return HiveSchemaConverter.convert(typeInfo, false, defaultValue, shouldAddInitialDefault); } /** diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index d949ff83a3cc..6234a6f8db50 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -91,12 +91,12 @@ public class TestHiveSchemaUtil { ); @Test - void testSimpleSchemaConvertToIcebergSchema() { + public void testSimpleSchemaConvertToIcebergSchema() { assertThat(HiveSchemaUtil.convert(SIMPLE_HIVE_SCHEMA).asStruct()).isEqualTo(SIMPLE_ICEBERG_SCHEMA.asStruct()); } @Test - void testSimpleSchemaConvertToIcebergSchemaFromNameAndTypeLists() { + public void testSimpleSchemaConvertToIcebergSchemaFromNameAndTypeLists() { List names = SIMPLE_HIVE_SCHEMA.stream().map(field -> field.getName()).collect(Collectors.toList()); List types = SIMPLE_HIVE_SCHEMA.stream() .map(field -> TypeInfoUtils.getTypeInfoFromTypeString(field.getType())) @@ -106,18 +106,18 @@ void testSimpleSchemaConvertToIcebergSchemaFromNameAndTypeLists() { } @Test - void testComplexSchemaConvertToIcebergSchema() { + public void testComplexSchemaConvertToIcebergSchema() { assertThat(HiveSchemaUtil.convert(COMPLEX_HIVE_SCHEMA).asStruct()).isEqualTo(COMPLEX_ICEBERG_SCHEMA.asStruct()); } @Test - void testSchemaConvertToIcebergSchemaForEveryPrimitiveType() { + public void testSchemaConvertToIcebergSchemaForEveryPrimitiveType() { Schema schemaWithEveryType = HiveSchemaUtil.convert(getSupportedFieldSchemas()); assertThat(schemaWithEveryType.asStruct()).isEqualTo(getSchemaWithSupportedTypes().asStruct()); } @Test - void testNotSupportedTypes() { + public void testNotSupportedTypes() { for (FieldSchema notSupportedField : getNotSupportedFieldSchemas()) { assertThatThrownBy( () -> HiveSchemaUtil.convert( @@ -128,17 +128,17 @@ void testNotSupportedTypes() { } @Test - void testSimpleSchemaConvertToHiveSchema() { + public void testSimpleSchemaConvertToHiveSchema() { assertThat(HiveSchemaUtil.convert(SIMPLE_ICEBERG_SCHEMA)).isEqualTo(SIMPLE_HIVE_SCHEMA); } @Test - void testComplexSchemaConvertToHiveSchema() { + public void testComplexSchemaConvertToHiveSchema() { assertThat(HiveSchemaUtil.convert(COMPLEX_ICEBERG_SCHEMA)).isEqualTo(COMPLEX_HIVE_SCHEMA); } @Test - void testSimpleTypeAndTypeInfoConvert() { + public void testSimpleTypeAndTypeInfoConvert() { // Test for every supported type List fieldSchemas = getSupportedFieldSchemas(); List nestedFields = getSchemaWithSupportedTypes().columns(); @@ -148,7 +148,7 @@ void testSimpleTypeAndTypeInfoConvert() { } @Test - void testComplexTypeAndTypeInfoConvert() { + public void testComplexTypeAndTypeInfoConvert() { for (int i = 0; i < COMPLEX_HIVE_SCHEMA.size(); ++i) { checkConvert(TypeInfoUtils.getTypeInfoFromTypeString(COMPLEX_HIVE_SCHEMA.get(i).getType()), COMPLEX_ICEBERG_SCHEMA.columns().get(i).type()); @@ -156,7 +156,7 @@ void testComplexTypeAndTypeInfoConvert() { } @Test - void testConversionWithoutLastComment() { + public void testConversionWithoutLastComment() { Schema expected = new Schema( optional(1, "customer_id", Types.LongType.get(), "customer comment"), optional(2, "first_name", Types.StringType.get(), null) @@ -220,7 +220,7 @@ private void checkConvert(TypeInfo typeInfo, Type type) { // Convert to TypeInfo assertThat(HiveSchemaUtil.convert(type)).isEqualTo(typeInfo); // Convert to Type - assertEquals(type, HiveSchemaUtil.convert(typeInfo, null)); + assertEquals(type, HiveSchemaUtil.convert(typeInfo, null, true)); } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index ce7b453bd92e..271ce74bfb7e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -741,18 +741,16 @@ private void handleAddColumns(org.apache.hadoop.hive.metastore.api.Table hmsTabl (List) SessionStateUtil.getResource(conf, SessionStateUtil.COLUMN_DEFAULTS).orElse(null); Map defaultValues = Stream.ofNullable(sqlDefaultConstraints).flatMap(Collection::stream) .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value)); - boolean isORc = isOrcFileFormat(hmsTable); + boolean isOrc = isOrcFileFormat(hmsTable); for (FieldSchema addedCol : addedCols) { String defaultValue = defaultValues.get(addedCol.getName()); - Type baseType = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType()), defaultValue); - final Type resolvedType = (!isORc && baseType.isStructType()) ? - HiveSchemaUtil.applyInitialDefaultsToStruct(baseType) : - baseType; - Literal defaultVal = Optional.ofNullable(defaultValue).filter(v -> !resolvedType.isStructType()) - .map(v -> Expressions.lit(HiveSchemaUtil.getDefaultValue(v, resolvedType))).orElse(null); + Type type = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType()), defaultValue, + !isOrc); + Literal defaultVal = Optional.ofNullable(defaultValue).filter(v -> !type.isStructType()) + .map(v -> Expressions.lit(HiveSchemaUtil.getDefaultValue(v, type))).orElse(null); // ORC doesn't have support for initialDefault from iceberg layer, we only need to set default for writeDefault. - updateSchema.addColumn(addedCol.getName(), resolvedType, addedCol.getComment(), isORc ? null : defaultVal); - if (isORc && defaultVal != null) { + updateSchema.addColumn(addedCol.getName(), type, addedCol.getComment(), isOrc ? null : defaultVal); + if (isOrc && defaultVal != null) { updateSchema.updateColumnDefault(addedCol.getName(), defaultVal); } } @@ -935,7 +933,7 @@ private void handlePartitionRename(HiveSchemaUtil.SchemaDifference schemaDiffere } private Type.PrimitiveType getPrimitiveTypeOrThrow(FieldSchema field) throws MetaException { - Type newType = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(field.getType()), null); + Type newType = HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(field.getType()), null, true); if (!(newType instanceof Type.PrimitiveType)) { throw new MetaException(String.format("Cannot promote type of column: '%s' to a non-primitive type: %s.", field.getName(), newType)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java index 87da6cc0dda6..e4a2e931ea0b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedDummyColumnReader.java @@ -83,7 +83,9 @@ private void fillStruct(ColumnVector col, StructTypeInfo structTypeInfo, Object StructColumnVector structCol = (StructColumnVector) col; List fieldNames = structTypeInfo.getAllStructFieldNames(); List fieldTypes = structTypeInfo.getAllStructFieldTypeInfos(); - Map fieldDefaults = defaultValue instanceof Map ? (Map) defaultValue : Collections.emptyMap(); + Map fieldDefaults = defaultValue instanceof Map ? + (Map) defaultValue : + Collections.emptyMap(); for (int i = 0; i < fieldNames.size(); i++) { fillColumnWithDefault(structCol.fields[i], fieldTypes.get(i), fieldDefaults.get(fieldNames.get(i)));