Skip to content

HIVE-29695: ClassCastException in MapJoin when Parquet String is mapped to Date #6578

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Aggarwal-Raghav wants to merge 1 commit into
base: master
Choose a base branch
from
+150 −0
Open

HIVE-29695: ClassCastException in MapJoin when Parquet String is mapped to Date #6578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.ArrayList;
Expand All @@ -24,6 +25,7 @@
import java.util.TimeZone;

import com.google.common.base.MoreObjects;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
Expand Down Expand Up @@ -525,6 +527,18 @@
return new HiveVarcharWritable(binary.getBytes(), ((VarcharTypeInfo) hiveTypeInfo).getLength());
}
};
case DATE:

Check warning on line 530 in ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'case' child has incorrect indentation level 10, expected level should be 8. 


See more on https://sonarcloud.io/project/issues?id=apache_hive&issues=AZ8pVx0N3-Z9uIIJayj6&open=AZ8pVx0N3-Z9uIIJayj6&pullRequest=6578
return new BinaryConverter<DateWritableV2>(type, parent, index) {

Check warning on line 531 in ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

'block' child has incorrect indentation level 12, expected level should be 10. 


See more on https://sonarcloud.io/project/issues?id=apache_hive&issues=AZ8pVx0N3-Z9uIIJayj7&open=AZ8pVx0N3-Z9uIIJayj7&pullRequest=6578
@Override
protected DateWritableV2 convert(Binary binary) {
String s = new String(binary.getBytes(), StandardCharsets.UTF_8);
try {
return new DateWritableV2(Date.valueOf(s));
} catch (IllegalArgumentException e) {
return null;
}
}
};
}
}
// STRING type
Expand Down
30 changes: 30 additions & 0 deletions ql/src/test/org/apache/hadoop/hive/ql/io/parquet/convert/TestETypeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo;
import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.stringTypeInfo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.nio.ByteBuffer;
Expand All @@ -33,6 +34,7 @@
import org.apache.hadoop.hive.ql.io.parquet.convert.ETypeConverter.BinaryConverter;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
Expand Down Expand Up @@ -341,6 +343,34 @@ public void testGetTextConverterNoHiveTypeInfo() {
assertEquals(value, textWritable.toString());
}

@Test
public void testGetTextConverterForDate() {
PrimitiveType primitiveType =
Types.optional(PrimitiveTypeName.BINARY)
.as(LogicalTypeAnnotation.stringType())
.named("value");
String value = "2026-01-01";
DateWritableV2 dateWritable =
(DateWritableV2)
getWritableFromBinaryConverter(
TypeInfoFactory.dateTypeInfo, primitiveType, Binary.fromString(value));
assertEquals(org.apache.hadoop.hive.common.type.Date.valueOf(value), dateWritable.get());
}

@Test
public void testGetTextConverterForDateInvalid() {
PrimitiveType primitiveType =
Types.optional(PrimitiveTypeName.BINARY)
.as(LogicalTypeAnnotation.stringType())
.named("value");
String value = "invalid_date";
DateWritableV2 dateWritable =
(DateWritableV2)
getWritableFromBinaryConverter(
TypeInfoFactory.dateTypeInfo, primitiveType, Binary.fromString(value));
assertNull(dateWritable);
}

@Test
public void testGetIntConverterForTinyInt() throws Exception {
PrimitiveType primitiveType = Types.optional(PrimitiveTypeName.INT32)
Expand Down
25 changes: 25 additions & 0 deletions ql/src/test/queries/clientpositive/parquet_string_to_date_mapjoin.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
CREATE TABLE repro_parquet_string (
id INT,
col1 STRING
) STORED AS PARQUET;

INSERT INTO repro_parquet_string VALUES (1, '2026-01-01');

CREATE EXTERNAL TABLE repro_parquet_date (
id INT,
col1 DATE
) STORED AS PARQUET
LOCATION '${hiveconf:hive.metastore.warehouse.dir}/repro_parquet_string';

CREATE TABLE small_table (
id INT,
date_col DATE
);
INSERT INTO small_table VALUES (1, '2026-01-01');

SET hive.auto.convert.join=true;
SET hive.vectorized.execution.enabled=false;

SELECT /*+ MAPJOIN(small_table) */ a.col1
FROM repro_parquet_date a
JOIN small_table b ON a.id = b.id;
81 changes: 81 additions & 0 deletions ql/src/test/results/clientpositive/llap/parquet_string_to_date_mapjoin.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
PREHOOK: query: CREATE TABLE repro_parquet_string (
id INT,
col1 STRING
) STORED AS PARQUET
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@repro_parquet_string
POSTHOOK: query: CREATE TABLE repro_parquet_string (
id INT,
col1 STRING
) STORED AS PARQUET
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@repro_parquet_string
PREHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@repro_parquet_string
POSTHOOK: query: INSERT INTO repro_parquet_string VALUES (1, '2026-01-01')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@repro_parquet_string
POSTHOOK: Lineage: repro_parquet_string.col1 SCRIPT []
POSTHOOK: Lineage: repro_parquet_string.id SCRIPT []
PREHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date (
id INT,
col1 DATE
) STORED AS PARQUET
#### A masked pattern was here ####
PREHOOK: type: CREATETABLE
#### A masked pattern was here ####
PREHOOK: Output: database:default
PREHOOK: Output: default@repro_parquet_date
POSTHOOK: query: CREATE EXTERNAL TABLE repro_parquet_date (
id INT,
col1 DATE
) STORED AS PARQUET
#### A masked pattern was here ####
POSTHOOK: type: CREATETABLE
#### A masked pattern was here ####
POSTHOOK: Output: database:default
POSTHOOK: Output: default@repro_parquet_date
PREHOOK: query: CREATE TABLE small_table (
id INT,
date_col DATE
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@small_table
POSTHOOK: query: CREATE TABLE small_table (
id INT,
date_col DATE
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_table
PREHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@small_table
POSTHOOK: query: INSERT INTO small_table VALUES (1, '2026-01-01')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@small_table
POSTHOOK: Lineage: small_table.date_col SCRIPT []
POSTHOOK: Lineage: small_table.id SCRIPT []
PREHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1
FROM repro_parquet_date a
JOIN small_table b ON a.id = b.id
PREHOOK: type: QUERY
PREHOOK: Input: default@repro_parquet_date
PREHOOK: Input: default@small_table
#### A masked pattern was here ####
POSTHOOK: query: SELECT /*+ MAPJOIN(small_table) */ a.col1
FROM repro_parquet_date a
JOIN small_table b ON a.id = b.id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@repro_parquet_date
POSTHOOK: Input: default@small_table
#### A masked pattern was here ####
2026-01-01
Loading