HIVE-19629: Enable Decimal64 reader after orc version upgrade (Prasanth Jayachandran...
authorPrasanth Jayachandran <prasanthj@apache.org>
Sat, 16 Jun 2018 18:23:55 +0000 (11:23 -0700)
committerPrasanth Jayachandran <prasanthj@apache.org>
Sat, 16 Jun 2018 18:23:55 +0000 (11:23 -0700)
451 files changed:
itests/src/test/resources/testconfiguration.properties
llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/GenericColumnVectorProducer.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java
llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java
ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedInputFormatInterface.java
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorInBloomFilterColDynamicValue.java
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFAdaptor.java
ql/src/java/org/apache/hadoop/hive/ql/io/BatchToRowReader.java
ql/src/java/org/apache/hadoop/hive/ql/io/NullRowsInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
ql/src/test/queries/clientpositive/explainanalyze_3.q
ql/src/test/queries/clientpositive/llap_acid2.q
ql/src/test/queries/clientpositive/llap_decimal64_reader.q [new file with mode: 0644]
ql/src/test/queries/clientpositive/llap_uncompressed.q
ql/src/test/queries/clientpositive/orc_create.q
ql/src/test/queries/clientpositive/orc_llap_counters.q
ql/src/test/queries/clientpositive/orc_llap_counters1.q
ql/src/test/queries/clientpositive/orc_merge11.q
ql/src/test/queries/clientpositive/orc_merge5.q
ql/src/test/queries/clientpositive/orc_merge6.q
ql/src/test/queries/clientpositive/orc_merge7.q
ql/src/test/queries/clientpositive/orc_merge_incompat1.q
ql/src/test/queries/clientpositive/orc_merge_incompat2.q
ql/src/test/queries/clientpositive/orc_ppd_basic.q
ql/src/test/queries/clientpositive/orc_ppd_schema_evol_3a.q
ql/src/test/queries/clientpositive/orc_schema_evolution_float.q
ql/src/test/queries/clientpositive/orc_split_elimination.q
ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_part_all_primitive.q
ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_part_all_primitive_llap_io.q
ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive.q
ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive_llap_io.q
ql/src/test/queries/clientpositive/type_change_test_int.q
ql/src/test/queries/clientpositive/type_change_test_int_vectorized.q
ql/src/test/queries/clientpositive/vector_case_when_1.q
ql/src/test/queries/clientpositive/vector_decimal_5.q
ql/src/test/queries/clientpositive/vector_decimal_mapjoin.q
ql/src/test/queries/clientpositive/vectorized_dynamic_semijoin_reduction2.q
ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out
ql/src/test/results/clientpositive/llap/llap_acid.q.out
ql/src/test/results/clientpositive/llap/llap_acid2.q.out
ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out
ql/src/test/results/clientpositive/llap/llap_decimal64_reader.q.out [new file with mode: 0644]
ql/src/test/results/clientpositive/llap/llap_partitioned.q.out
ql/src/test/results/clientpositive/llap/llap_text.q.out [new file with mode: 0644]
ql/src/test/results/clientpositive/llap/llap_uncompressed.q.out [new file with mode: 0644]
ql/src/test/results/clientpositive/llap/llap_vector_nohybridgrace.q.out
ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out
ql/src/test/results/clientpositive/llap/mergejoin.q.out
ql/src/test/results/clientpositive/llap/orc_create.q.out
ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out
ql/src/test/results/clientpositive/llap/orc_merge11.q.out
ql/src/test/results/clientpositive/llap/orc_merge5.q.out
ql/src/test/results/clientpositive/llap/orc_merge6.q.out
ql/src/test/results/clientpositive/llap/orc_merge7.q.out
ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out
ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out
ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
ql/src/test/results/clientpositive/llap/orc_split_elimination.q.out
ql/src/test/results/clientpositive/llap/orc_struct_type_vectorization.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_table.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_table_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out
ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_llap_io.q.out
ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table_llap_io.q.out
ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out
ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out
ql/src/test/results/clientpositive/llap/vector_annotate_stats_select.q.out
ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
ql/src/test/results/clientpositive/llap/vector_between_in.q.out
ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out
ql/src/test/results/clientpositive/llap/vector_bround.q.out
ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out
ql/src/test/results/clientpositive/llap/vector_char_2.q.out
ql/src/test/results/clientpositive/llap/vector_char_4.q.out
ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
ql/src/test/results/clientpositive/llap/vector_char_simple.q.out
ql/src/test/results/clientpositive/llap/vector_char_varchar_1.q.out
ql/src/test/results/clientpositive/llap/vector_coalesce.q.out
ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out
ql/src/test/results/clientpositive/llap/vector_coalesce_4.q.out
ql/src/test/results/clientpositive/llap/vector_complex_all.q.out
ql/src/test/results/clientpositive/llap/vector_complex_join.q.out
ql/src/test/results/clientpositive/llap/vector_count.q.out
ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out
ql/src/test/results/clientpositive/llap/vector_create_struct_table.q.out
ql/src/test/results/clientpositive/llap/vector_data_types.q.out
ql/src/test/results/clientpositive/llap/vector_date_1.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_5.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_6.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_cast.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_math_funcs.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_trailing.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out
ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out
ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out
ql/src/test/results/clientpositive/llap/vector_elt.q.out
ql/src/test/results/clientpositive/llap/vector_groupby4.q.out
ql/src/test/results/clientpositive/llap/vector_groupby6.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id1.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id3.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets2.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets3_dec.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets5.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets6.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_grouping_window.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_sort_11.q.out
ql/src/test/results/clientpositive/llap/vector_groupby_sort_8.q.out
ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out
ql/src/test/results/clientpositive/llap/vector_if_expr.q.out
ql/src/test/results/clientpositive/llap/vector_if_expr_2.q.out
ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out
ql/src/test/results/clientpositive/llap/vector_inner_join.q.out
ql/src/test/results/clientpositive/llap/vector_interval_1.q.out
ql/src/test/results/clientpositive/llap/vector_interval_2.q.out
ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out
ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out
ql/src/test/results/clientpositive/llap/vector_join30.q.out
ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out
ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out
ql/src/test/results/clientpositive/llap/vector_leftsemi_mapjoin.q.out
ql/src/test/results/clientpositive/llap/vector_like_2.q.out
ql/src/test/results/clientpositive/llap/vector_llap_text_1.q.out
ql/src/test/results/clientpositive/llap/vector_map_order.q.out
ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out
ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out
ql/src/test/results/clientpositive/llap/vector_null_map.q.out
ql/src/test/results/clientpositive/llap/vector_null_projection.q.out
ql/src/test/results/clientpositive/llap/vector_nullsafe_join.q.out
ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out
ql/src/test/results/clientpositive/llap/vector_nvl.q.out
ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out
ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out
ql/src/test/results/clientpositive/llap/vector_orc_null_check.q.out
ql/src/test/results/clientpositive/llap/vector_order_null.q.out
ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out
ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out
ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out
ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out
ql/src/test/results/clientpositive/llap/vector_outer_reference_windowed.q.out
ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out
ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out
ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
ql/src/test/results/clientpositive/llap/vector_reduce1.q.out
ql/src/test/results/clientpositive/llap/vector_reduce2.q.out
ql/src/test/results/clientpositive/llap/vector_reduce3.q.out
ql/src/test/results/clientpositive/llap/vector_reduce_groupby_decimal.q.out
ql/src/test/results/clientpositive/llap/vector_reduce_groupby_duplicate_cols.q.out
ql/src/test/results/clientpositive/llap/vector_retry_failure.q.out
ql/src/test/results/clientpositive/llap/vector_reuse_scratchcols.q.out
ql/src/test/results/clientpositive/llap/vector_string_concat.q.out
ql/src/test/results/clientpositive/llap/vector_string_decimal.q.out
ql/src/test/results/clientpositive/llap/vector_struct_in.q.out
ql/src/test/results/clientpositive/llap/vector_udf1.q.out
ql/src/test/results/clientpositive/llap/vector_udf2.q.out
ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out
ql/src/test/results/clientpositive/llap/vector_varchar_mapjoin1.q.out
ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out
ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out
ql/src/test/results/clientpositive/llap/vector_windowing.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_expressions.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_gby.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_multipartitioning.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_navfn.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_order_null.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_range_multiorder.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_streaming.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_windowspec.q.out
ql/src/test/results/clientpositive/llap/vector_windowing_windowspec4.q.out
ql/src/test/results/clientpositive/llap/vectorization_0.q.out
ql/src/test/results/clientpositive/llap/vectorization_1.q.out
ql/src/test/results/clientpositive/llap/vectorization_10.q.out
ql/src/test/results/clientpositive/llap/vectorization_11.q.out
ql/src/test/results/clientpositive/llap/vectorization_12.q.out
ql/src/test/results/clientpositive/llap/vectorization_13.q.out
ql/src/test/results/clientpositive/llap/vectorization_14.q.out
ql/src/test/results/clientpositive/llap/vectorization_15.q.out
ql/src/test/results/clientpositive/llap/vectorization_16.q.out
ql/src/test/results/clientpositive/llap/vectorization_17.q.out
ql/src/test/results/clientpositive/llap/vectorization_2.q.out
ql/src/test/results/clientpositive/llap/vectorization_3.q.out
ql/src/test/results/clientpositive/llap/vectorization_4.q.out
ql/src/test/results/clientpositive/llap/vectorization_5.q.out
ql/src/test/results/clientpositive/llap/vectorization_6.q.out
ql/src/test/results/clientpositive/llap/vectorization_7.q.out
ql/src/test/results/clientpositive/llap/vectorization_8.q.out
ql/src/test/results/clientpositive/llap/vectorization_9.q.out
ql/src/test/results/clientpositive/llap/vectorization_decimal_date.q.out
ql/src/test/results/clientpositive/llap/vectorization_div0.q.out
ql/src/test/results/clientpositive/llap/vectorization_limit.q.out
ql/src/test/results/clientpositive/llap/vectorization_nested_udf.q.out
ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out
ql/src/test/results/clientpositive/llap/vectorization_pushdown.q.out
ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
ql/src/test/results/clientpositive/llap/vectorized_case.q.out
ql/src/test/results/clientpositive/llap/vectorized_casts.q.out
ql/src/test/results/clientpositive/llap/vectorized_context.q.out
ql/src/test/results/clientpositive/llap/vectorized_date_funcs.q.out
ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out
ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
ql/src/test/results/clientpositive/llap/vectorized_mapjoin.q.out
ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out
ql/src/test/results/clientpositive/llap/vectorized_math_funcs.q.out
ql/src/test/results/clientpositive/llap/vectorized_nested_mapjoin.q.out
ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out
ql/src/test/results/clientpositive/llap/vectorized_shufflejoin.q.out
ql/src/test/results/clientpositive/llap/vectorized_string_funcs.q.out
ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
ql/src/test/results/clientpositive/llap/vectorized_timestamp_funcs.q.out
ql/src/test/results/clientpositive/llap/vectorized_timestamp_ints_casts.q.out
ql/src/test/results/clientpositive/mergejoin.q.out
ql/src/test/results/clientpositive/orc_file_dump.q.out
ql/src/test/results/clientpositive/orc_merge11.q.out
ql/src/test/results/clientpositive/orc_merge5.q.out
ql/src/test/results/clientpositive/orc_merge6.q.out
ql/src/test/results/clientpositive/orc_merge_incompat1.q.out
ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
ql/src/test/results/clientpositive/orc_struct_type_vectorization.q.out
ql/src/test/results/clientpositive/spark/orc_merge5.q.out
ql/src/test/results/clientpositive/spark/orc_merge6.q.out
ql/src/test/results/clientpositive/spark/orc_merge7.q.out
ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out
ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out
ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out
ql/src/test/results/clientpositive/spark/vector_between_in.q.out
ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
ql/src/test/results/clientpositive/spark/vector_char_4.q.out
ql/src/test/results/clientpositive/spark/vector_count_distinct.q.out
ql/src/test/results/clientpositive/spark/vector_data_types.q.out
ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out
ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
ql/src/test/results/clientpositive/spark/vector_elt.q.out
ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
ql/src/test/results/clientpositive/spark/vector_inner_join.q.out
ql/src/test/results/clientpositive/spark/vector_left_outer_join.q.out
ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
ql/src/test/results/clientpositive/spark/vector_outer_join0.q.out
ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out
ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out
ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out
ql/src/test/results/clientpositive/spark/vectorization_0.q.out
ql/src/test/results/clientpositive/spark/vectorization_1.q.out
ql/src/test/results/clientpositive/spark/vectorization_10.q.out
ql/src/test/results/clientpositive/spark/vectorization_11.q.out
ql/src/test/results/clientpositive/spark/vectorization_12.q.out
ql/src/test/results/clientpositive/spark/vectorization_13.q.out
ql/src/test/results/clientpositive/spark/vectorization_14.q.out
ql/src/test/results/clientpositive/spark/vectorization_15.q.out
ql/src/test/results/clientpositive/spark/vectorization_16.q.out
ql/src/test/results/clientpositive/spark/vectorization_17.q.out
ql/src/test/results/clientpositive/spark/vectorization_2.q.out
ql/src/test/results/clientpositive/spark/vectorization_3.q.out
ql/src/test/results/clientpositive/spark/vectorization_4.q.out
ql/src/test/results/clientpositive/spark/vectorization_5.q.out
ql/src/test/results/clientpositive/spark/vectorization_6.q.out
ql/src/test/results/clientpositive/spark/vectorization_9.q.out
ql/src/test/results/clientpositive/spark/vectorization_decimal_date.q.out
ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
ql/src/test/results/clientpositive/spark/vectorization_nested_udf.q.out
ql/src/test/results/clientpositive/spark/vectorization_part_project.q.out
ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
ql/src/test/results/clientpositive/spark/vectorized_case.q.out
ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
ql/src/test/results/clientpositive/spark/vectorized_math_funcs.q.out
ql/src/test/results/clientpositive/spark/vectorized_nested_mapjoin.q.out
ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
ql/src/test/results/clientpositive/spark/vectorized_string_funcs.q.out
ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out
ql/src/test/results/clientpositive/vector_aggregate_9.q.out
ql/src/test/results/clientpositive/vector_aggregate_without_gby.q.out
ql/src/test/results/clientpositive/vector_between_columns.q.out
ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out
ql/src/test/results/clientpositive/vector_bround.q.out
ql/src/test/results/clientpositive/vector_case_when_1.q.out
ql/src/test/results/clientpositive/vector_case_when_2.q.out
ql/src/test/results/clientpositive/vector_cast_constant.q.out
ql/src/test/results/clientpositive/vector_char_2.q.out
ql/src/test/results/clientpositive/vector_char_4.q.out
ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
ql/src/test/results/clientpositive/vector_char_simple.q.out
ql/src/test/results/clientpositive/vector_coalesce.q.out
ql/src/test/results/clientpositive/vector_coalesce_2.q.out
ql/src/test/results/clientpositive/vector_coalesce_3.q.out
ql/src/test/results/clientpositive/vector_coalesce_4.q.out
ql/src/test/results/clientpositive/vector_count.q.out
ql/src/test/results/clientpositive/vector_data_types.q.out
ql/src/test/results/clientpositive/vector_date_1.q.out
ql/src/test/results/clientpositive/vector_decimal_1.q.out
ql/src/test/results/clientpositive/vector_decimal_10_0.q.out
ql/src/test/results/clientpositive/vector_decimal_5.q.out
ql/src/test/results/clientpositive/vector_decimal_6.q.out
ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out
ql/src/test/results/clientpositive/vector_decimal_cast.q.out
ql/src/test/results/clientpositive/vector_decimal_expressions.q.out
ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out
ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out
ql/src/test/results/clientpositive/vector_decimal_precision.q.out
ql/src/test/results/clientpositive/vector_decimal_round.q.out
ql/src/test/results/clientpositive/vector_decimal_round_2.q.out
ql/src/test/results/clientpositive/vector_decimal_trailing.q.out
ql/src/test/results/clientpositive/vector_decimal_udf2.q.out
ql/src/test/results/clientpositive/vector_delete_orig_table.q.out
ql/src/test/results/clientpositive/vector_distinct_2.q.out
ql/src/test/results/clientpositive/vector_elt.q.out
ql/src/test/results/clientpositive/vector_empty_where.q.out
ql/src/test/results/clientpositive/vector_groupby4.q.out
ql/src/test/results/clientpositive/vector_groupby6.q.out
ql/src/test/results/clientpositive/vector_groupby_3.q.out
ql/src/test/results/clientpositive/vector_groupby_reduce.q.out
ql/src/test/results/clientpositive/vector_grouping_sets.q.out
ql/src/test/results/clientpositive/vector_if_expr.q.out
ql/src/test/results/clientpositive/vector_include_no_sel.q.out
ql/src/test/results/clientpositive/vector_interval_1.q.out
ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
ql/src/test/results/clientpositive/vector_left_outer_join.q.out
ql/src/test/results/clientpositive/vector_left_outer_join2.q.out
ql/src/test/results/clientpositive/vector_multi_insert.q.out
ql/src/test/results/clientpositive/vector_non_string_partition.q.out
ql/src/test/results/clientpositive/vector_null_projection.q.out
ql/src/test/results/clientpositive/vector_nvl.q.out
ql/src/test/results/clientpositive/vector_orderby_5.q.out
ql/src/test/results/clientpositive/vector_outer_join0.q.out
ql/src/test/results/clientpositive/vector_outer_join1.q.out
ql/src/test/results/clientpositive/vector_outer_join2.q.out
ql/src/test/results/clientpositive/vector_outer_join3.q.out
ql/src/test/results/clientpositive/vector_outer_join4.q.out
ql/src/test/results/clientpositive/vector_outer_join6.q.out
ql/src/test/results/clientpositive/vector_outer_join_no_keys.q.out
ql/src/test/results/clientpositive/vector_reduce1.q.out
ql/src/test/results/clientpositive/vector_reduce2.q.out
ql/src/test/results/clientpositive/vector_reduce3.q.out
ql/src/test/results/clientpositive/vector_reduce_groupby_decimal.q.out
ql/src/test/results/clientpositive/vector_string_concat.q.out
ql/src/test/results/clientpositive/vector_struct_in.q.out
ql/src/test/results/clientpositive/vector_tablesample_rows.q.out
ql/src/test/results/clientpositive/vector_udf3.q.out
ql/src/test/results/clientpositive/vector_varchar_4.q.out
ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
ql/src/test/results/clientpositive/vector_varchar_simple.q.out
ql/src/test/results/clientpositive/vector_when_case_null.q.out
ql/src/test/results/clientpositive/vectorization_1.q.out
ql/src/test/results/clientpositive/vectorization_10.q.out
ql/src/test/results/clientpositive/vectorization_11.q.out
ql/src/test/results/clientpositive/vectorization_12.q.out
ql/src/test/results/clientpositive/vectorization_13.q.out
ql/src/test/results/clientpositive/vectorization_14.q.out
ql/src/test/results/clientpositive/vectorization_15.q.out
ql/src/test/results/clientpositive/vectorization_16.q.out
ql/src/test/results/clientpositive/vectorization_17.q.out
ql/src/test/results/clientpositive/vectorization_2.q.out
ql/src/test/results/clientpositive/vectorization_3.q.out
ql/src/test/results/clientpositive/vectorization_4.q.out
ql/src/test/results/clientpositive/vectorization_5.q.out
ql/src/test/results/clientpositive/vectorization_6.q.out
ql/src/test/results/clientpositive/vectorization_7.q.out
ql/src/test/results/clientpositive/vectorization_8.q.out
ql/src/test/results/clientpositive/vectorization_9.q.out
ql/src/test/results/clientpositive/vectorization_decimal_date.q.out
ql/src/test/results/clientpositive/vectorization_limit.q.out
ql/src/test/results/clientpositive/vectorization_nested_udf.q.out
ql/src/test/results/clientpositive/vectorization_offset_limit.q.out
ql/src/test/results/clientpositive/vectorization_part_project.q.out
ql/src/test/results/clientpositive/vectorization_pushdown.q.out
ql/src/test/results/clientpositive/vectorized_case.q.out
ql/src/test/results/clientpositive/vectorized_casts.q.out
ql/src/test/results/clientpositive/vectorized_context.q.out
ql/src/test/results/clientpositive/vectorized_date_funcs.q.out
ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out
ql/src/test/results/clientpositive/vectorized_mapjoin3.q.out
ql/src/test/results/clientpositive/vectorized_math_funcs.q.out
ql/src/test/results/clientpositive/vectorized_string_funcs.q.out
ql/src/test/results/clientpositive/vectorized_timestamp.q.out
ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
ql/src/test/results/clientpositive/vectorized_timestamp_ints_casts.q.out

index c7d2285..aeb6211 100644 (file)
@@ -539,6 +539,9 @@ minillaplocal.query.files=\
   llap_partitioned.q,\
   llap_smb.q,\
   llap_vector_nohybridgrace.q,\
+  llap_uncompressed.q,\
+  llap_decimal64_reader.q,\
+  llap_text.q,\
   load_data_acid_rename.q,\
   load_data_using_job.q,\
   load_dyn_part5.q,\
index 40f7c83..ac1aca8 100644 (file)
@@ -19,6 +19,7 @@
 
 package org.apache.hadoop.hive.llap.io.api.impl;
 
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
 import org.apache.hadoop.hive.ql.io.BatchToRowInputFormat;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -232,4 +233,9 @@ public class LlapInputFormat implements InputFormat<NullWritable, VectorizedRowB
     }
     return tableScanOperator;
   }
+
+  @Override
+  public VectorizedSupport.Support[] getSupportedFeatures() {
+    return new VectorizedSupport.Support[] {VectorizedSupport.Support.DECIMAL_64};
+  }
 }
index 7af1b05..32f3bed 100644 (file)
@@ -31,13 +31,11 @@ import org.apache.hadoop.hive.llap.cache.SerDeLowLevelCacheImpl;
 import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
 import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
 import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
-import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes;
 import org.apache.hadoop.hive.llap.io.encoded.SerDeEncodedDataReader;
 import org.apache.hadoop.hive.llap.io.metadata.ConsumerFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.ConsumerStripeMetadata;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
 import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
 import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
 import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace;
@@ -52,8 +50,8 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hive.common.util.FixedSizedObjectPool;
 import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
 import org.apache.orc.OrcProto;
-import org.apache.orc.OrcUtils;
 import org.apache.orc.OrcProto.ColumnEncoding;
 import org.apache.orc.OrcProto.RowIndex;
 import org.apache.orc.OrcProto.RowIndexEntry;
@@ -289,5 +287,10 @@ public class GenericColumnVectorProducer implements ColumnVectorProducer {
     public TypeDescription getSchema() {
       return schema;
     }
+
+    @Override
+    public OrcFile.Version getFileVersion() {
+      return null;
+    }
   }
 }
index feccb87..0d7435c 100644 (file)
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.llap.io.metadata.ConsumerStripeMetadata;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
@@ -73,10 +74,11 @@ public class OrcEncodedDataConsumer
   private IoTrace trace;
   private final Includes includes;
   private TypeDescription[] batchSchemas;
+  private boolean useDecimal64ColumnVectors;
 
   public OrcEncodedDataConsumer(
-      Consumer<ColumnVectorBatch> consumer, Includes includes, boolean skipCorrupt,
-      QueryFragmentCounters counters, LlapDaemonIOMetrics ioMetrics) {
+    Consumer<ColumnVectorBatch> consumer, Includes includes, boolean skipCorrupt,
+    QueryFragmentCounters counters, LlapDaemonIOMetrics ioMetrics) {
     super(consumer, includes.getPhysicalColumnIds().size(), ioMetrics);
     this.includes = includes;
     // TODO: get rid of this
@@ -84,6 +86,10 @@ public class OrcEncodedDataConsumer
     this.counters = counters;
   }
 
+  public void setUseDecimal64ColumnVectors(final boolean useDecimal64ColumnVectors) {
+    this.useDecimal64ColumnVectors = useDecimal64ColumnVectors;
+  }
+
   public void setFileMetadata(ConsumerFileMetadata f) {
     assert fileMetadata == null;
     fileMetadata = f;
@@ -153,7 +159,7 @@ public class OrcEncodedDataConsumer
           if (cvb.cols[idx] == null) {
             // Orc store rows inside a root struct (hive writes it this way).
             // When we populate column vectors we skip over the root struct.
-            cvb.cols[idx] = createColumn(batchSchemas[idx], VectorizedRowBatch.DEFAULT_SIZE);
+            cvb.cols[idx] = createColumn(batchSchemas[idx], VectorizedRowBatch.DEFAULT_SIZE, useDecimal64ColumnVectors);
           }
           trace.logTreeReaderNextVector(idx);
 
@@ -217,10 +223,10 @@ public class OrcEncodedDataConsumer
     TreeReaderFactory.Context context = new TreeReaderFactory.ReaderContext()
             .setSchemaEvolution(evolution).skipCorrupt(skipCorrupt)
             .writerTimeZone(stripeMetadata.getWriterTimezone())
-        ;
+            .fileFormat(fileMetadata == null ? null : fileMetadata.getFileVersion());
     this.batchSchemas = includes.getBatchReaderTypes(fileSchema);
     StructTreeReader treeReader = EncodedTreeReaderFactory.createRootTreeReader(
-        batchSchemas, stripeMetadata.getEncodings(), batch, codec, context);
+        batchSchemas, stripeMetadata.getEncodings(), batch, codec, context, useDecimal64ColumnVectors);
     this.columnReaders = treeReader.getChildReaders();
 
     if (LlapIoImpl.LOG.isDebugEnabled()) {
@@ -232,7 +238,7 @@ public class OrcEncodedDataConsumer
     positionInStreams(columnReaders, batch.getBatchKey(), stripeMetadata);
   }
 
-  private ColumnVector createColumn(TypeDescription type, int batchSize) {
+  private ColumnVector createColumn(TypeDescription type, int batchSize, final boolean useDecimal64ColumnVectors) {
     switch (type.getCategory()) {
       case BOOLEAN:
       case BYTE:
@@ -252,30 +258,34 @@ public class OrcEncodedDataConsumer
       case TIMESTAMP:
         return new TimestampColumnVector(batchSize);
       case DECIMAL:
-        return new DecimalColumnVector(batchSize, type.getPrecision(),
-            type.getScale());
+        if (useDecimal64ColumnVectors && type.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION) {
+          return new Decimal64ColumnVector(batchSize, type.getPrecision(), type.getScale());
+        } else {
+          return new DecimalColumnVector(batchSize, type.getPrecision(), type.getScale());
+        }
       case STRUCT: {
         List<TypeDescription> subtypeIdxs = type.getChildren();
         ColumnVector[] fieldVector = new ColumnVector[subtypeIdxs.size()];
-        for(int i = 0; i < fieldVector.length; ++i) {
-          fieldVector[i] = createColumn(subtypeIdxs.get(i), batchSize);
+        for (int i = 0; i < fieldVector.length; ++i) {
+          fieldVector[i] = createColumn(subtypeIdxs.get(i), batchSize, useDecimal64ColumnVectors);
         }
         return new StructColumnVector(batchSize, fieldVector);
       }
       case UNION: {
         List<TypeDescription> subtypeIdxs = type.getChildren();
         ColumnVector[] fieldVector = new ColumnVector[subtypeIdxs.size()];
-        for(int i=0; i < fieldVector.length; ++i) {
-          fieldVector[i] = createColumn(subtypeIdxs.get(i), batchSize);
+        for (int i = 0; i < fieldVector.length; ++i) {
+          fieldVector[i] = createColumn(subtypeIdxs.get(i), batchSize, useDecimal64ColumnVectors);
         }
         return new UnionColumnVector(batchSize, fieldVector);
       }
       case LIST:
-        return new ListColumnVector(batchSize, createColumn(type.getChildren().get(0), batchSize));
+        return new ListColumnVector(batchSize, createColumn(type.getChildren().get(0), batchSize,
+          useDecimal64ColumnVectors));
       case MAP:
         List<TypeDescription> subtypeIdxs = type.getChildren();
-        return new MapColumnVector(batchSize, createColumn(subtypeIdxs.get(0), batchSize),
-            createColumn(subtypeIdxs.get(1), batchSize));
+        return new MapColumnVector(batchSize, createColumn(subtypeIdxs.get(0), batchSize, useDecimal64ColumnVectors),
+          createColumn(subtypeIdxs.get(1), batchSize, useDecimal64ColumnVectors));
       default:
         throw new IllegalArgumentException("LLAP does not support " + type.getCategory());
     }
index 2947c16..b76b0de 100644 (file)
@@ -24,37 +24,17 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.hadoop.hive.llap.counters.LlapIOCounters;
-import org.apache.orc.CompressionCodec;
-import org.apache.orc.OrcProto.BloomFilterIndex;
-import org.apache.orc.OrcProto.FileTail;
-import org.apache.orc.OrcProto.RowIndex;
-import org.apache.orc.OrcProto.Stream;
-import org.apache.orc.OrcProto.StripeStatistics;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.impl.BufferChunk;
-import org.apache.orc.impl.DataReaderProperties;
-import org.apache.orc.impl.InStream;
-import org.apache.orc.impl.OrcCodecPool;
-import org.apache.orc.impl.OrcIndex;
-import org.apache.orc.impl.OrcTail;
-import org.apache.orc.impl.ReaderImpl;
-import org.apache.orc.impl.SchemaEvolution;
-import org.apache.orc.impl.WriterImpl;
-import org.apache.tez.common.counters.TezCounters;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.Pool;
 import org.apache.hadoop.hive.common.Pool.PoolObjectHelper;
+import org.apache.hadoop.hive.common.io.Allocator;
 import org.apache.hadoop.hive.common.io.Allocator.BufferObjectFactory;
 import org.apache.hadoop.hive.common.io.DataCache;
-import org.apache.hadoop.hive.common.io.Allocator;
-import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
 import org.apache.hadoop.hive.common.io.DiskRange;
 import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
 import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -65,38 +45,58 @@ import org.apache.hadoop.hive.llap.cache.BufferUsageManager;
 import org.apache.hadoop.hive.llap.cache.LlapDataBuffer;
 import org.apache.hadoop.hive.llap.cache.LowLevelCache;
 import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.llap.counters.LlapIOCounters;
 import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
 import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
 import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes;
 import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.SchemaEvolutionFactory;
 import org.apache.hadoop.hive.llap.io.decode.OrcEncodedDataConsumer;
-import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.MetadataCache;
 import org.apache.hadoop.hive.llap.io.metadata.MetadataCache.LlapBufferOrBuffers;
+import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
 import org.apache.hadoop.hive.ql.io.HdfsUtils;
-import org.apache.orc.CompressionKind;
-import org.apache.orc.DataReader;
 import org.apache.hadoop.hive.ql.io.orc.OrcFile;
 import org.apache.hadoop.hive.ql.io.orc.OrcFile.ReaderOptions;
-import org.apache.orc.OrcConf;
 import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
-import org.apache.hadoop.hive.ql.io.orc.encoded.Reader;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
 import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedOrcFile;
 import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReader;
 import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace;
 import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Reader;
 import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
 import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.PoolFactory;
-import org.apache.orc.impl.RecordReaderUtils;
-import org.apache.orc.StripeInformation;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hive.common.util.FixedSizedObjectPool;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.DataReader;
+import org.apache.orc.OrcConf;
 import org.apache.orc.OrcProto;
+import org.apache.orc.OrcProto.BloomFilterIndex;
+import org.apache.orc.OrcProto.FileTail;
+import org.apache.orc.OrcProto.RowIndex;
+import org.apache.orc.OrcProto.Stream;
+import org.apache.orc.OrcProto.StripeStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.impl.BufferChunk;
+import org.apache.orc.impl.DataReaderProperties;
+import org.apache.orc.impl.InStream;
+import org.apache.orc.impl.OrcCodecPool;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.impl.OrcTail;
+import org.apache.orc.impl.ReaderImpl;
+import org.apache.orc.impl.RecordReaderUtils;
+import org.apache.orc.impl.SchemaEvolution;
+import org.apache.orc.impl.WriterImpl;
 import org.apache.tez.common.CallableWithNdc;
+import org.apache.tez.common.counters.TezCounters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
@@ -231,6 +231,8 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
     this.jobConf = jobConf;
     // TODO: setFileMetadata could just create schema. Called in two places; clean up later.
     this.evolution = sef.createSchemaEvolution(fileMetadata.getSchema());
+    consumer.setUseDecimal64ColumnVectors(HiveConf.getVar(jobConf,
+      ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"));
     consumer.setFileMetadata(fileMetadata);
     consumer.setSchemaEvolution(evolution);
   }
@@ -569,7 +571,8 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
             stripes.add(new ReaderImpl.StripeInformationImpl(stripeProto));
           }
           return new OrcFileMetadata(
-              fileKey, tail.getFooter(), tail.getPostscript(), stats, stripes);
+              fileKey, tail.getFooter(), tail.getPostscript(), stats, stripes,
+            ReaderImpl.getFileVersion(tail.getPostscript().getVersionList()));
         } finally {
           // We don't need the buffer anymore.
           metadataCache.decRefBuffer(tailBuffers);
@@ -586,7 +589,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
     }
     FileTail ft = orcReader.getFileTail();
     return new OrcFileMetadata(fileKey, ft.getFooter(), ft.getPostscript(),
-        orcReader.getOrcProtoStripeStatistics(), orcReader.getStripes());
+        orcReader.getOrcProtoStripeStatistics(), orcReader.getStripes(), orcReader.getFileVersion());
   }
 
   private OrcProto.StripeFooter buildStripeFooter(
index bed5887..5b54af5 100644 (file)
@@ -221,6 +221,9 @@ public class SerDeEncodedDataReader extends CallableWithNdc<Void>
     this.sourceSerDe = sourceSerDe;
     this.reporter = reporter;
     this.jobConf = jobConf;
+    final boolean useDecimal64ColumnVectors = HiveConf.getVar(jobConf, ConfVars
+      .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64");
+    consumer.setUseDecimal64ColumnVectors(useDecimal64ColumnVectors);
     this.schema = schema;
     this.writerIncludes = OrcInputFormat.genIncludedColumns(schema, columnIds);
     SchemaEvolution evolution = new SchemaEvolution(schema, null,
index de19b1d..ca6d696 100644 (file)
@@ -20,14 +20,19 @@ package org.apache.hadoop.hive.llap.io.encoded;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.DebugUtils;
@@ -35,10 +40,12 @@ import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
 import org.apache.hadoop.hive.llap.io.encoded.SerDeEncodedDataReader.CacheWriter;
 import org.apache.hadoop.hive.llap.io.encoded.SerDeEncodedDataReader.DeserializerOrcWriter;
 import org.apache.hadoop.hive.llap.io.encoded.SerDeEncodedDataReader.EncodingWriter;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorDeserializeRow;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.orc.Writer;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -46,14 +53,20 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.io.BinaryComparable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.InputFormat;
@@ -113,7 +126,7 @@ class VectorDeserializeOrcWriter extends EncodingWriter implements Runnable {
     }
     LlapIoImpl.LOG.info("Creating VertorDeserializeOrcWriter for " + path);
     return new VectorDeserializeOrcWriter(
-        daemonConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize);
+        jobConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize);
   }
 
   private VectorDeserializeOrcWriter(Configuration conf, Properties tblProps,
@@ -121,12 +134,12 @@ class VectorDeserializeOrcWriter extends EncodingWriter implements Runnable {
       int allocSize) throws IOException {
     super(sourceOi, allocSize);
     // See also: the usage of VectorDeserializeType, for binary. For now, we only want text.
-    this.vrbCtx = createVrbCtx(sourceOi);
+    this.vrbCtx = createVrbCtx(sourceOi, tblProps, conf);
     this.sourceIncludes = sourceIncludes;
     this.cacheIncludes = cacheIncludes;
     this.sourceBatch = vrbCtx.createVectorizedRowBatch();
     deserializeRead = new LazySimpleDeserializeRead(vrbCtx.getRowColumnTypeInfos(),
-        /* useExternalBuffer */ true, createSerdeParams(conf, tblProps));
+      vrbCtx.getRowdataTypePhysicalVariations(),/* useExternalBuffer */ true, createSerdeParams(conf, tblProps));
     vectorDeserializeRow = new VectorDeserializeRow<LazySimpleDeserializeRead>(deserializeRead);
     int colCount = vrbCtx.getRowColumnTypeInfos().length;
     boolean[] includes = null;
@@ -192,13 +205,41 @@ class VectorDeserializeOrcWriter extends EncodingWriter implements Runnable {
     this.orcThread.start();
   }
 
-  private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi) throws IOException {
+  private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, final Properties tblProps,
+    final Configuration conf) throws IOException {
+    final boolean useDecimal64ColumnVectors = HiveConf.getVar(conf, ConfVars
+      .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64");
+    final String serde = tblProps.getProperty(serdeConstants.SERIALIZATION_LIB);
+    final String inputFormat = tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT);
+    final boolean isTextFormat = inputFormat != null && inputFormat.equals(TextInputFormat.class.getName()) &&
+      serde != null && serde.equals(LazySimpleSerDe.class.getName());
+    List<DataTypePhysicalVariation> dataTypePhysicalVariations = new ArrayList<>();
+    if (isTextFormat) {
+      StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(oi);
+      int dataColumnCount = structTypeInfo.getAllStructFieldTypeInfos().size();
+      for (int i = 0; i < dataColumnCount; i++) {
+        DataTypePhysicalVariation dataTypePhysicalVariation = DataTypePhysicalVariation.NONE;
+        if (useDecimal64ColumnVectors) {
+          TypeInfo typeInfo = structTypeInfo.getAllStructFieldTypeInfos().get(i);
+          if (typeInfo instanceof DecimalTypeInfo) {
+            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
+            if (HiveDecimalWritable.isPrecisionDecimal64(decimalTypeInfo.precision())) {
+              dataTypePhysicalVariation = DataTypePhysicalVariation.DECIMAL_64;
+            }
+          }
+        }
+        dataTypePhysicalVariations.add(dataTypePhysicalVariation);
+      }
+    }
     VectorizedRowBatchCtx vrbCtx = new VectorizedRowBatchCtx();
     try {
       vrbCtx.init(oi, new String[0]);
     } catch (HiveException e) {
       throw new IOException(e);
     }
+    if (!dataTypePhysicalVariations.isEmpty()) {
+      vrbCtx.setRowDataTypePhysicalVariations(dataTypePhysicalVariations.toArray(new DataTypePhysicalVariation[0]));
+    }
     return vrbCtx;
   }
 
index 89ad4aa..d6b16ef 100644 (file)
@@ -21,6 +21,7 @@ import java.util.List;
 
 import org.apache.orc.CompressionKind;
 import org.apache.orc.FileFormatException;
+import org.apache.orc.OrcFile;
 import org.apache.orc.OrcProto.Type;
 import org.apache.orc.TypeDescription;
 
@@ -29,4 +30,5 @@ public interface ConsumerFileMetadata {
   CompressionKind getCompressionKind();
   List<Type> getTypes();
   TypeDescription getSchema() throws FileFormatException;
+  OrcFile.Version getFileVersion();
 }
index 5cd6f9f..5eb713c 100644 (file)
@@ -50,9 +50,10 @@ public final class OrcFileMetadata implements FileMetadata, ConsumerFileMetadata
   private final long contentLength;
   private final long numberOfRows;
   private final boolean isOriginalFormat;
+  private final OrcFile.Version fileVersion;
 
   public OrcFileMetadata(Object fileKey, OrcProto.Footer footer, OrcProto.PostScript ps,
-      List<StripeStatistics> stats, List<StripeInformation> stripes) {
+    List<StripeStatistics> stats, List<StripeInformation> stripes, final OrcFile.Version fileVersion) {
     this.stripeStats = stats;
     this.compressionKind = CompressionKind.valueOf(ps.getCompression().name());
     this.compressionBufferSize = (int)ps.getCompressionBlockSize();
@@ -67,6 +68,7 @@ public final class OrcFileMetadata implements FileMetadata, ConsumerFileMetadata
     this.numberOfRows = footer.getNumberOfRows();
     this.fileStats = footer.getStatisticsList();
     this.fileKey = fileKey;
+    this.fileVersion = fileVersion;
   }
 
   // FileMetadata
@@ -163,4 +165,9 @@ public final class OrcFileMetadata implements FileMetadata, ConsumerFileMetadata
   public TypeDescription getSchema() throws FileFormatException {
     return OrcUtils.convertTypeFromProtobuf(this.types, 0);
   }
+
+  @Override
+  public OrcFile.Version getFileVersion() {
+    return fileVersion;
+  }
 }
index 2246901..183fae5 100644 (file)
@@ -214,8 +214,8 @@ public class FetchOperator implements Serializable {
   private static final Map<String, InputFormat> inputFormats = new HashMap<String, InputFormat>();
 
   @SuppressWarnings("unchecked")
-  static InputFormat getInputFormatFromCache(
-    Class<? extends InputFormat> inputFormatClass, JobConf conf) throws IOException {
+  public static InputFormat getInputFormatFromCache(
+    Class<? extends InputFormat> inputFormatClass, Configuration conf) throws IOException {
     if (Configurable.class.isAssignableFrom(inputFormatClass) ||
         JobConfigurable.class.isAssignableFrom(inputFormatClass)) {
       return ReflectionUtil.newInstance(inputFormatClass, conf);
index 9ddb136..84a0a3a 100644 (file)
@@ -584,8 +584,8 @@ public class VectorizationContext {
 
         // Re-use an existing, available column of the same required type.
         if (usedOutputColumns.contains(i) ||
-            !(scratchVectorTypeNames)[i].equalsIgnoreCase(columnType) &&
-              scratchDataTypePhysicalVariations[i] == dataTypePhysicalVariation) {
+            !(scratchVectorTypeNames[i].equalsIgnoreCase(columnType) &&
+              scratchDataTypePhysicalVariations[i] == dataTypePhysicalVariation)) {
           continue;
         }
         //Use i
@@ -874,6 +874,7 @@ public class VectorizationContext {
       LOG.debug("Input Expression = " + exprDesc.toString()
           + ", Vectorized Expression = " + ve.toString());
     }
+
     return ve;
   }
 
@@ -1965,7 +1966,7 @@ public class VectorizationContext {
     return cleaned;
   }
 
-  private VectorExpression instantiateExpression(Class<?> vclass, TypeInfo returnTypeInfo,
+  public VectorExpression instantiateExpression(Class<?> vclass, TypeInfo returnTypeInfo,
       DataTypePhysicalVariation returnDataTypePhysicalVariation, Object...args)
       throws HiveException {
     VectorExpression ve = null;
index 6588385..ffbfb6f 100644 (file)
@@ -163,6 +163,11 @@ public class VectorizedRowBatchCtx {
     return rowDataTypePhysicalVariations;
   }
 
+  public void setRowDataTypePhysicalVariations(
+    final DataTypePhysicalVariation[] rowDataTypePhysicalVariations) {
+    this.rowDataTypePhysicalVariations = rowDataTypePhysicalVariations;
+  }
+
   public int[] getDataColumnNums() {
     return dataColumnNums;
   }
index d8a3cac..8bf990a 100644 (file)
@@ -82,6 +82,7 @@ public class VectorInBloomFilterColDynamicValue extends VectorExpression {
     // Instantiate BloomFilterCheck based on input column type
     switch (colVectorType) {
     case LONG:
+    case DECIMAL_64:
       bfCheck = new LongBloomFilterCheck();
       break;
     case DOUBLE:
index 18bacc5..fdb067f 100644 (file)
@@ -107,6 +107,7 @@ public class VectorUDAFBloomFilter extends VectorAggregateExpression {
     }
     switch (colVectorType) {
     case LONG:
+    case DECIMAL_64:
       valueProcessor = new ValueProcessorLong();
       break;
     case DOUBLE:
index 5b2cb4c..e011657 100644 (file)
@@ -235,4 +235,12 @@ public class VectorUDFAdaptor extends VectorExpression {
   public VectorExpressionDescriptor.Descriptor getDescriptor() {
     return (new VectorExpressionDescriptor.Builder()).build();
   }
+
+  public VectorUDFArgDesc[] getArgDescs() {
+    return argDescs;
+  }
+
+  public void setArgDescs(final VectorUDFArgDesc[] argDescs) {
+    this.argDescs = argDescs;
+  }
 }
index 2b005c4..c88ee99 100644 (file)
@@ -24,6 +24,7 @@ import org.apache.hadoop.hive.llap.DebugUtils;
 
 import java.util.Arrays;
 
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -416,7 +417,12 @@ public abstract class BatchToRowReader<StructType, UnionType>
       } else {
         result = (HiveDecimalWritable) previous;
       }
-      result.set(((DecimalColumnVector) vector).vector[row]);
+      if (vector instanceof Decimal64ColumnVector) {
+        long value = ((Decimal64ColumnVector) vector).vector[row];
+        result.deserialize64(value, ((Decimal64ColumnVector) vector).scale);
+      } else {
+        result.set(((DecimalColumnVector) vector).vector[row]);
+      }
       return result;
     } else {
       return null;
index e632d43..6434414 100644 (file)
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
 
 import java.io.IOException;
 
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.io.NullWritable;
@@ -50,6 +51,11 @@ public class NullRowsInputFormat implements InputFormat<NullWritable, NullWritab
   static final int MAX_ROW = 100; // to prevent infinite loop
   static final Logger LOG = LoggerFactory.getLogger(NullRowsRecordReader.class.getName());
 
+  @Override
+  public VectorizedSupport.Support[] getSupportedFeatures() {
+    return null;
+  }
+
   public static class DummyInputSplit extends FileSplit {
     @SuppressWarnings("unused")  // Serialization ctor.
     private DummyInputSplit() {
index b6f92e3..3c11847 100644 (file)
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
 import org.apache.hadoop.hive.ql.io.AcidInputFormat;
 import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -161,6 +162,11 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
   SelfDescribingInputFormatInterface, AcidInputFormat<NullWritable, OrcStruct>,
   CombineHiveInputFormat.AvoidSplitCombination, BatchToRowInputFormat {
 
+  @Override
+  public VectorizedSupport.Support[] getSupportedFeatures() {
+    return new VectorizedSupport.Support[] {VectorizedSupport.Support.DECIMAL_64};
+  }
+
   static enum SplitStrategyKind {
     HYBRID,
     BI,
@@ -328,7 +334,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
     List<OrcProto.Type> types = OrcUtils.getOrcTypes(schema);
     options.include(genIncludedColumns(schema, conf));
     setSearchArgument(options, types, conf, isOriginal);
-    return file.rowsOptions(options);
+    return file.rowsOptions(options, conf);
   }
 
   public static boolean isOriginal(Reader file) {
index 9d954ca..6571a24 100644 (file)
@@ -23,6 +23,7 @@ import java.util.Map;
 import java.util.TreeMap;
 
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -231,16 +232,17 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
      * @param maxKey only return keys less than or equal to maxKey if it is
      *               non-null
      * @param options options to provide to read the rows.
+     * @param conf
      * @throws IOException
      */
     @VisibleForTesting
     ReaderPairAcid(ReaderKey key, Reader reader,
-                   RecordIdentifier minKey, RecordIdentifier maxKey,
-                   ReaderImpl.Options options) throws IOException {
+      RecordIdentifier minKey, RecordIdentifier maxKey,
+      ReaderImpl.Options options, final Configuration conf) throws IOException {
       this.reader = reader;
       this.key = key;
       // TODO use stripe statistics to jump over stripes
-      recordReader = reader.rowsOptions(options);
+      recordReader = reader.rowsOptions(options, conf);
       this.minKey = minKey;
       this.maxKey = maxKey;
       // advance the reader until we reach the minimum key
@@ -440,7 +442,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
 
       RecordIdentifier newMinKey = minKey;
       RecordIdentifier newMaxKey = maxKey;
-      recordReader = reader.rowsOptions(options);
+      recordReader = reader.rowsOptions(options, conf);
       /**
        * Logically each bucket consists of 0000_0, 0000_0_copy_1... 0000_0_copy_N. etc  We don't
        * know N a priori so if this is true, then the current split is from 0000_0_copy_N file.
@@ -589,7 +591,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
         throw new IllegalStateException("No 'original' files found for bucketId=" + this.bucketId +
           " in " + mergerOptions.getRootPath());
       }
-      recordReader = getReader().rowsOptions(options);
+      recordReader = getReader().rowsOptions(options, conf);
       next(nextRecord());//load 1st row
     }
     @Override public RecordReader getRecordReader() {
@@ -623,7 +625,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
               nextRecord = null;
               return;
             }
-            recordReader = reader.rowsOptions(options);
+            recordReader = reader.rowsOptions(options, conf);
           }
         }
       }
@@ -1043,7 +1045,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
             //required (on Tez) that base_x/ doesn't have a file for 'bucket'
             reader = OrcFile.createReader(bucketPath, OrcFile.readerOptions(conf));
             pair = new ReaderPairAcid(baseKey, reader, keyInterval.getMinKey(), keyInterval.getMaxKey(),
-              eventOptions);
+              eventOptions, conf);
           }
           else {
             pair = new EmptyReaderPair();
@@ -1053,7 +1055,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
         else {
           assert reader != null : "no reader? " + mergerOptions.getRootPath();
           pair = new ReaderPairAcid(baseKey, reader, keyInterval.getMinKey(), keyInterval.getMaxKey(),
-            eventOptions);
+            eventOptions, conf);
         }
       }
       minKey = pair.getMinKey();
@@ -1113,7 +1115,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
             //HIVE-17320: we should compute a SARG to push down min/max key to delete_delta
             Reader deltaReader = OrcFile.createReader(deltaFile, OrcFile.readerOptions(conf));
             ReaderPair deltaPair = new ReaderPairAcid(key, deltaReader, minKey, maxKey,
-                deltaEventOptions);
+                deltaEventOptions, conf);
             if (deltaPair.nextRecord() != null) {
               ensurePutReader(key, deltaPair);
               key = new ReaderKey();
@@ -1128,7 +1130,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
           assert length >= 0;
           Reader deltaReader = OrcFile.createReader(deltaFile, OrcFile.readerOptions(conf).maxLength(length));
           //must get statementId from file name since Acid 1.0 doesn't write it into bucketProperty
-          ReaderPairAcid deltaPair = new ReaderPairAcid(key, deltaReader, minKey, maxKey, deltaEventOptions);
+          ReaderPairAcid deltaPair = new ReaderPairAcid(key, deltaReader, minKey, maxKey, deltaEventOptions, conf);
           if (deltaPair.nextRecord() != null) {
             ensurePutReader(key, deltaPair);
             key = new ReaderKey();
index 7485e60..8fd9b90 100644 (file)
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.io.orc;
 
 import java.io.IOException;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 
@@ -55,7 +56,16 @@ public interface Reader extends org.apache.orc.Reader {
    * @throws IOException
    */
   RecordReader rowsOptions(Options options) throws IOException;
-  
+
+  /**
+   * Create a RecordReader that reads everything with the given options.
+   * @param options the options to use
+   * @param conf conf object
+   * @return a new RecordReader
+   * @throws IOException
+   */
+  RecordReader rowsOptions(Options options, Configuration conf) throws IOException;
+
   /**
    * Create a RecordReader that will scan the entire file.
    * This is a legacy method and rowsOptions is preferred.
index 1a6db1f..171b02b 100644 (file)
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.io.orc;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -73,11 +74,17 @@ public class ReaderImpl extends org.apache.orc.impl.ReaderImpl
 
   @Override
   public RecordReader rowsOptions(Options options) throws IOException {
+    return rowsOptions(options, null);
+  }
+
+  @Override
+  public RecordReader rowsOptions(Options options, Configuration conf) throws IOException {
     LOG.info("Reading ORC rows from " + path + " with " + options);
-    return new RecordReaderImpl(this, options);
+    return new RecordReaderImpl(this, options, conf);
   }
 
 
+
   @Override
   public RecordReader rows(boolean[] include) throws IOException {
     return rowsOptions(new Options().include(include));
index 5b001a0..c6fe4fc 100644 (file)
@@ -23,8 +23,11 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
@@ -48,6 +51,7 @@ import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.orc.OrcFile;
 import org.apache.orc.TypeDescription;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,9 +64,15 @@ public class RecordReaderImpl extends org.apache.orc.impl.RecordReaderImpl
   private long baseRow;
 
   protected RecordReaderImpl(ReaderImpl fileReader,
-                             Reader.Options options) throws IOException {
+    Reader.Options options, final Configuration conf) throws IOException {
     super(fileReader, options);
-    batch = this.schema.createRowBatch();
+    final boolean useDecimal64ColumnVectors = conf != null && HiveConf.getVar(conf,
+      HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64");
+    if (useDecimal64ColumnVectors){
+      batch = this.schema.createRowBatchV2();
+    } else {
+      batch = this.schema.createRowBatch();
+    }
     rowInBatch = 0;
   }
 
@@ -80,8 +90,8 @@ public class RecordReaderImpl extends org.apache.orc.impl.RecordReaderImpl
     return true;
   }
 
-  public VectorizedRowBatch createRowBatch() {
-    return this.schema.createRowBatch();
+  public VectorizedRowBatch createRowBatch(boolean useDecimal64) {
+    return useDecimal64 ? this.schema.createRowBatchV2() : this.schema.createRowBatch();
   }
 
   @Override
@@ -393,7 +403,12 @@ public class RecordReaderImpl extends org.apache.orc.impl.RecordReaderImpl
       } else {
         result = (HiveDecimalWritable) previous;
       }
-      result.set(((DecimalColumnVector) vector).vector[row]);
+      if (vector instanceof Decimal64ColumnVector) {
+        long value = ((Decimal64ColumnVector) vector).vector[row];
+        result.deserialize64(value, ((Decimal64ColumnVector) vector).scale);
+      } else {
+        result.set(((DecimalColumnVector) vector).vector[row]);
+      }
       return result;
     } else {
       return null;
index 66ffcae..1a91d4a 100644 (file)
@@ -112,7 +112,7 @@ public class VectorizedOrcAcidRowBatchReader
 
     final Reader reader = OrcInputFormat.createOrcReaderForSplit(conf, (OrcSplit) inputSplit);
     // Careful with the range here now, we do not want to read the whole base file like deltas.
-    innerReader = reader.rowsOptions(readerOptions.range(offset, length));
+    innerReader = reader.rowsOptions(readerOptions.range(offset, length), conf);
     baseReader = new org.apache.hadoop.mapred.RecordReader<NullWritable, VectorizedRowBatch>() {
 
       @Override
@@ -145,7 +145,13 @@ public class VectorizedOrcAcidRowBatchReader
         return innerReader.getProgress();
       }
     };
-    this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch();
+    final boolean useDecimal64ColumnVectors = HiveConf
+      .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64");
+    if (useDecimal64ColumnVectors) {
+      this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch(true);
+    } else {
+      this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch(false);
+    }
   }
 
   /**
@@ -864,11 +870,17 @@ public class VectorizedOrcAcidRowBatchReader
       private final Reader reader;
 
       DeleteReaderValue(Reader deleteDeltaReader, Reader.Options readerOptions, int bucket,
-          ValidWriteIdList validWriteIdList, boolean isBucketedTable) throws IOException {
+        ValidWriteIdList validWriteIdList, boolean isBucketedTable, final JobConf conf) throws IOException {
         this.reader = deleteDeltaReader;
-        this.recordReader  = deleteDeltaReader.rowsOptions(readerOptions);
+        this.recordReader  = deleteDeltaReader.rowsOptions(readerOptions, conf);
         this.bucketForSplit = bucket;
-        this.batch = deleteDeltaReader.getSchema().createRowBatch();
+        final boolean useDecimal64ColumnVector = HiveConf.getVar(conf, ConfVars
+          .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64");
+        if (useDecimal64ColumnVector) {
+          this.batch = deleteDeltaReader.getSchema().createRowBatchV2();
+        } else {
+          this.batch = deleteDeltaReader.getSchema().createRowBatch();
+        }
         if (!recordReader.nextBatch(batch)) { // Read the first batch.
           this.batch = null; // Oh! the first batch itself was null. Close the reader.
         }
@@ -1067,7 +1079,7 @@ public class VectorizedOrcAcidRowBatchReader
                   throw new DeleteEventsOverflowMemoryException();
                 }
                 DeleteReaderValue deleteReaderValue = new DeleteReaderValue(deleteDeltaReader,
-                    readerOptions, bucket, validWriteIdList, isBucketedTable);
+                    readerOptions, bucket, validWriteIdList, isBucketedTable, conf);
                 DeleteRecordKey deleteRecordKey = new DeleteRecordKey();
                 if (deleteReaderValue.next(deleteRecordKey)) {
                   sortMerger.put(deleteRecordKey, deleteReaderValue);
index c581bba..892fcc0 100644 (file)
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.InputFormatChecker;
 import org.apache.hadoop.hive.ql.io.SelfDescribingInputFormatInterface;
@@ -99,7 +100,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
       options.include(OrcInputFormat.genIncludedColumns(schema, conf));
       OrcInputFormat.setSearchArgument(options, types, conf, true);
 
-      this.reader = file.rowsOptions(options);
+      this.reader = file.rowsOptions(options, conf);
 
       int partitionColumnCount = rbCtx.getPartitionColumnCount();
       if (partitionColumnCount > 0) {
@@ -204,4 +205,9 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
     }
     return true;
   }
+
+  @Override
+  public VectorizedSupport.Support[] getSupportedFeatures() {
+    return new VectorizedSupport.Support[] {VectorizedSupport.Support.DECIMAL_64};
+  }
 }
index 71682af..91a01e9 100644 (file)
@@ -24,6 +24,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -63,6 +65,7 @@ import org.apache.hadoop.io.Text;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.orc.PhysicalWriter;
+import org.apache.orc.TypeDescription;
 
 /**
  * An ORC file writer. The file is divided into stripes, which is the natural
@@ -93,7 +96,15 @@ public class WriterImpl extends org.apache.orc.impl.WriterImpl implements Writer
              OrcFile.WriterOptions opts) throws IOException {
     super(fs, path, opts);
     this.inspector = opts.getInspector();
-    this.internalBatch = opts.getSchema().createRowBatch(opts.getBatchSize());
+    boolean useDecimal64ColumnVectors = opts.getConfiguration() != null &&
+      HiveConf.getVar(opts.getConfiguration(), HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED)
+        .equalsIgnoreCase("decimal_64");
+    if (useDecimal64ColumnVectors) {
+      this.internalBatch = opts.getSchema().createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,
+        opts.getBatchSize());
+    } else {
+      this.internalBatch = opts.getSchema().createRowBatch(opts.getBatchSize());
+    }
     this.fields = initializeFieldsFromOi(inspector);
   }
 
@@ -207,9 +218,15 @@ public class WriterImpl extends org.apache.orc.impl.WriterImpl implements Writer
               break;
             }
             case DECIMAL: {
-              DecimalColumnVector vector = (DecimalColumnVector) column;
-              vector.set(rowId, ((HiveDecimalObjectInspector) inspector)
+              if (column instanceof Decimal64ColumnVector) {
+                Decimal64ColumnVector vector = (Decimal64ColumnVector) column;
+                vector.set(rowId, ((HiveDecimalObjectInspector) inspector)
+                  .getPrimitiveWritableObject(obj));
+              } else {
+                DecimalColumnVector vector = (DecimalColumnVector) column;
+                vector.set(rowId, ((HiveDecimalObjectInspector) inspector)
                   .getPrimitiveWritableObject(obj));
+              }
               break;
             }
           }
index c9078be..9302791 100644 (file)
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.ql.io.orc.encoded;
 
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
+import org.apache.orc.OrcFile;
 import org.apache.orc.impl.RunLengthByteReader;
 
 import java.io.IOException;
@@ -1200,6 +1202,147 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
     }
   }
 
+  protected static class Decimal64StreamReader extends Decimal64TreeReader implements SettableTreeReader {
+    private boolean _isFileCompressed;
+    private SettableUncompressedStream _presentStream;
+    private SettableUncompressedStream _valueStream;
+    private List<ColumnVector> vectors;
+    private int vectorIndex = 0;
+
+    private Decimal64StreamReader(int columnId, int precision, int scale,
+      SettableUncompressedStream presentStream,
+      SettableUncompressedStream valueStream,
+      boolean isFileCompressed,
+      OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context,
+      List<ColumnVector> vectors) throws IOException {
+      super(columnId, presentStream, valueStream, encoding,
+        precision, scale, context);
+      this._isFileCompressed = isFileCompressed;
+      this._presentStream = presentStream;
+      this._valueStream = valueStream;
+      this.vectors = vectors;
+    }
+
+    @Override
+    public void seek(PositionProvider index) throws IOException {
+      if (vectors != null) return;
+      if (present != null) {
+        if (_isFileCompressed) {
+          index.getNext();
+        }
+        present.seek(index);
+      }
+
+      // data stream could be empty stream or already reached end of stream before present stream.
+      // This can happen if all values in stream are nulls or last row group values are all null.
+      skipCompressedIndex(_isFileCompressed, index);
+      if (_valueStream.available() > 0) {
+        valueReader.seek(index);
+      } else {
+        skipSeek(index);
+      }
+    }
+
+    @Override
+    public void nextVector(
+      ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException {
+      if (vectors == null) {
+        super.nextVector(previousVector, isNull, batchSize);
+        return;
+      }
+      vectors.get(vectorIndex++).shallowCopyTo(previousVector);
+      if (vectorIndex == vectors.size()) {
+        vectors = null;
+      }
+    }
+
+    @Override
+    public void setBuffers(EncodedColumnBatch<OrcBatchKey> batch, boolean sameStripe) {
+      assert vectors == null; // See the comment in TimestampStreamReader.setBuffers.
+      ColumnStreamData[] streamsData = batch.getColumnData(columnId);
+      if (_presentStream != null) {
+        _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
+      }
+      if (_valueStream != null) {
+        _valueStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
+      }
+    }
+
+    public static class StreamReaderBuilder {
+      private int columnIndex;
+      private ColumnStreamData presentStream;
+      private ColumnStreamData valueStream;
+      private int scale;
+      private int precision;
+      private CompressionCodec compressionCodec;
+      private OrcProto.ColumnEncoding columnEncoding;
+      private List<ColumnVector> vectors;
+      private TreeReaderFactory.Context context;
+
+      public StreamReaderBuilder setColumnIndex(int columnIndex) {
+        this.columnIndex = columnIndex;
+        return this;
+      }
+
+      public StreamReaderBuilder setPrecision(int precision) {
+        this.precision = precision;
+        return this;
+      }
+
+      public StreamReaderBuilder setScale(int scale) {
+        this.scale = scale;
+        return this;
+      }
+
+      public StreamReaderBuilder setContext(TreeReaderFactory.Context context) {
+        this.context = context;
+        return this;
+      }
+
+      public StreamReaderBuilder setPresentStream(ColumnStreamData presentStream) {
+        this.presentStream = presentStream;
+        return this;
+      }
+
+      public StreamReaderBuilder setValueStream(ColumnStreamData valueStream) {
+        this.valueStream = valueStream;
+        return this;
+      }
+
+
+      public StreamReaderBuilder setCompressionCodec(CompressionCodec compressionCodec) {
+        this.compressionCodec = compressionCodec;
+        return this;
+      }
+
+      public StreamReaderBuilder setColumnEncoding(OrcProto.ColumnEncoding encoding) {
+        this.columnEncoding = encoding;
+        return this;
+      }
+
+      public Decimal64StreamReader build() throws IOException {
+        SettableUncompressedStream presentInStream = StreamUtils.createSettableUncompressedStream(
+          OrcProto.Stream.Kind.PRESENT.name(), presentStream);
+
+        SettableUncompressedStream valueInStream = StreamUtils.createSettableUncompressedStream(
+          OrcProto.Stream.Kind.DATA.name(), valueStream);
+
+        boolean isFileCompressed = compressionCodec != null;
+        return new Decimal64StreamReader(columnIndex, precision, scale, presentInStream,
+          valueInStream, isFileCompressed, columnEncoding, context, vectors);
+      }
+
+      public StreamReaderBuilder setVectors(List<ColumnVector> vectors) {
+        this.vectors = vectors;
+        return this;
+      }
+    }
+
+    public static StreamReaderBuilder builder() {
+      return new StreamReaderBuilder();
+    }
+  }
+
   protected static class DateStreamReader extends DateTreeReader implements SettableTreeReader {
     private boolean isFileCompressed;
     private SettableUncompressedStream _presentStream;
@@ -2101,8 +2244,8 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
   }
 
   public static StructTreeReader createRootTreeReader(TypeDescription[] batchSchemas,
-       List<OrcProto.ColumnEncoding> encodings, OrcEncodedColumnBatch batch,
-       CompressionCodec codec, TreeReaderFactory.Context context) throws IOException {
+    List<OrcProto.ColumnEncoding> encodings, OrcEncodedColumnBatch batch,
+    CompressionCodec codec, Context context, final boolean useDecimal64ColumnVectors) throws IOException {
     // Note: we only look at the schema here to deal with complex types. Somebody has set up the
     //       reader with whatever ideas they had to the schema and we just trust the reader to
     //       produce the CVBs that was asked for. However, we only need to look at top level columns.
@@ -2117,7 +2260,7 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
       if (!batch.hasData(batchColIx) && !batch.hasVectors(batchColIx)) {
         throw new AssertionError("No data for column " + batchColIx + ": " + batchSchemas[i]);
       }
-      childReaders[i] = createEncodedTreeReader(batchSchemas[i], encodings, batch, codec, context);
+      childReaders[i] = createEncodedTreeReader(batchSchemas[i], encodings, batch, codec, context, useDecimal64ColumnVectors);
     }
 
     // TODO: do we actually need this reader? the caller just extracts child readers.
@@ -2138,8 +2281,8 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
 
 
   private static TreeReader createEncodedTreeReader(TypeDescription schema,
-      List<OrcProto.ColumnEncoding> encodings, OrcEncodedColumnBatch batch,
-      CompressionCodec codec, TreeReaderFactory.Context context) throws IOException {
+    List<OrcProto.ColumnEncoding> encodings, OrcEncodedColumnBatch batch,
+    CompressionCodec codec, Context context, final boolean useDecimal64ColumnVectors) throws IOException {
     int columnIndex = schema.getId();
     ColumnStreamData[] streamBuffers = null;
     List<ColumnVector> vectors = null;
@@ -2200,12 +2343,12 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
       case TIMESTAMP:
       case DATE:
         return getPrimitiveTreeReader(columnIndex, schema, codec, columnEncoding,
-            present, data, dictionary, lengths, secondary, context, vectors);
+            present, data, dictionary, lengths, secondary, context, vectors, useDecimal64ColumnVectors);
       case LIST:
         assert vectors == null; // Not currently supported.
         TypeDescription elementType = schema.getChildren().get(0);
         TreeReader elementReader = createEncodedTreeReader(
-            elementType, encodings, batch, codec, context);
+            elementType, encodings, batch, codec, context, useDecimal64ColumnVectors);
         return ListStreamReader.builder()
             .setColumnIndex(columnIndex)
             .setColumnEncoding(columnEncoding)
@@ -2220,9 +2363,9 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
         TypeDescription keyType = schema.getChildren().get(0);
         TypeDescription valueType = schema.getChildren().get(1);
         TreeReader keyReader = createEncodedTreeReader(
-            keyType, encodings, batch, codec, context);
+            keyType, encodings, batch, codec, context, useDecimal64ColumnVectors);
         TreeReader valueReader = createEncodedTreeReader(
-            valueType, encodings, batch, codec, context);
+            valueType, encodings, batch, codec, context, useDecimal64ColumnVectors);
         return MapStreamReader.builder()
             .setColumnIndex(columnIndex)
             .setColumnEncoding(columnEncoding)
@@ -2240,7 +2383,7 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
         for (int i = 0; i < childCount; i++) {
           TypeDescription childType = schema.getChildren().get(i);
           childReaders[i] = createEncodedTreeReader(
-              childType, encodings, batch, codec, context);
+              childType, encodings, batch, codec, context, useDecimal64ColumnVectors);
         }
         return StructStreamReader.builder()
             .setColumnIndex(columnIndex)
@@ -2258,7 +2401,7 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
         for (int i = 0; i < childCount; i++) {
           TypeDescription childType = schema.getChildren().get(i);
           childReaders[i] = createEncodedTreeReader(
-              childType, encodings, batch, codec, context);
+              childType, encodings, batch, codec, context, useDecimal64ColumnVectors);
         }
         return UnionStreamReader.builder()
               .setColumnIndex(columnIndex)
@@ -2276,10 +2419,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
   }
 
   private static TreeReader getPrimitiveTreeReader(final int columnIndex,
-      TypeDescription columnType, CompressionCodec codec, OrcProto.ColumnEncoding columnEncoding,
-      ColumnStreamData present, ColumnStreamData data, ColumnStreamData dictionary,
-      ColumnStreamData lengths, ColumnStreamData secondary, TreeReaderFactory.Context context,
-      List<ColumnVector> vectors) throws IOException {
+    TypeDescription columnType, CompressionCodec codec, OrcProto.ColumnEncoding columnEncoding,
+    ColumnStreamData present, ColumnStreamData data, ColumnStreamData dictionary,
+    ColumnStreamData lengths, ColumnStreamData secondary, Context context,
+    List<ColumnVector> vectors, final boolean useDecimal64ColumnVectors) throws IOException {
     switch (columnType.getCategory()) {
       case BINARY:
         return BinaryStreamReader.builder()
@@ -2390,7 +2533,36 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
             .setVectors(vectors)
             .build();
       case DECIMAL:
-        return DecimalStreamReader.builder()
+        // special handling for serde reader (text) in llap IO.
+        // if file format version is null, then we are processing text IF in LLAP IO, in which case
+        // we get vectors instead of streams. If vectors contain instance of Decimal64ColumnVector we
+        // should use Decimal64StreamReader (which acts as a wrapper around vectors)
+        boolean useDecimal64Reader = context.getFileFormat() == null && vectors != null && useDecimal64ColumnVectors;
+        if (useDecimal64Reader) {
+          boolean containDecimal64CV = false;
+          for (ColumnVector vector : vectors) {
+            if (vector instanceof Decimal64ColumnVector) {
+              containDecimal64CV = true;
+              break;
+            }
+          }
+          useDecimal64Reader &= containDecimal64CV;
+        }
+        if ((context.getFileFormat() == OrcFile.Version.UNSTABLE_PRE_2_0 || useDecimal64Reader) &&
+          columnType.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION) {
+          return Decimal64StreamReader.builder()
+            .setColumnIndex(columnIndex)
+            .setPrecision(columnType.getPrecision())
+            .setScale(columnType.getScale())
+            .setPresentStream(present)
+            .setValueStream(data)
+            .setCompressionCodec(codec)
+            .setColumnEncoding(columnEncoding)
+            .setVectors(vectors)
+            .setContext(context)
+            .build();
+        } else {
+          return DecimalStreamReader.builder()
             .setColumnIndex(columnIndex)
             .setPrecision(columnType.getPrecision())
             .setScale(columnType.getScale())
@@ -2402,6 +2574,7 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
             .setVectors(vectors)
             .setContext(context)
             .build();
+        }
       case TIMESTAMP:
         return TimestampStreamReader.builder()
             .setColumnIndex(columnIndex)
index ed6d577..5e70a05 100644 (file)
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.common.io.DataCache;
 import org.apache.hadoop.hive.common.io.FileMetadataCache;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.InputFormatChecker;
 import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface;
@@ -115,4 +116,9 @@ public class MapredParquetInputFormat extends FileInputFormat<NullWritable, Arra
 
     return true;
   }
+
+  @Override
+  public VectorizedSupport.Support[] getSupportedFeatures() {
+    return null;
+  }
 }
index 394f826..f4e8207 100644 (file)
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimizer.physical;
 
 import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.lang.annotation.Annotation;
 import java.util.ArrayList;
@@ -40,7 +41,9 @@ import java.util.regex.Pattern;
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
 import org.apache.hadoop.hive.ql.exec.vector.reducesink.*;
+import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFArgDesc;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator;
 import org.slf4j.Logger;
@@ -129,7 +132,6 @@ import org.apache.hadoop.hive.ql.plan.VectorPTFDesc;
 import org.apache.hadoop.hive.ql.plan.VectorPTFInfo;
 import org.apache.hadoop.hive.ql.plan.VectorPTFDesc.SupportedFunctionType;
 import org.apache.hadoop.hive.ql.plan.VectorTableScanDesc;
-import org.apache.hadoop.hive.ql.plan.VectorizationCondition;
 import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode;
 import org.apache.hadoop.hive.ql.plan.VectorSparkHashTableSinkDesc;
 import org.apache.hadoop.hive.ql.plan.VectorSparkPartitionPruningSinkDesc;
@@ -1207,6 +1209,14 @@ public class Vectorizer implements PhysicalPlanResolver {
     private Support[] getVectorizedInputFormatSupports(
       Class<? extends InputFormat> inputFileFormatClass) {
 
+      try {
+        InputFormat inputFormat = FetchOperator.getInputFormatFromCache(inputFileFormatClass, hiveConf);
+        if (inputFormat instanceof VectorizedInputFormatInterface) {
+          return ((VectorizedInputFormatInterface) inputFormat).getSupportedFeatures();
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to instantiate {} input format class. Cannot determine vectorization support.", e);
+      }
       // FUTURE: Decide how to ask an input file format what vectorization features it supports.
       return null;
     }
@@ -1830,14 +1840,6 @@ public class Vectorizer implements PhysicalPlanResolver {
         supportRemovedReasons.add(removeString);
       }
 
-      // And, if LLAP is enabled for now, disable DECIMAL_64;
-      if (isLlapIoEnabled && supportSet.contains(Support.DECIMAL_64)) {
-        supportSet.remove(Support.DECIMAL_64);
-        String removeString =
-            "DECIMAL_64 disabled because LLAP is enabled";
-        supportRemovedReasons.add(removeString);
-      }
-
       // Now rememember what is supported for this query and any support that was
       // removed.
       vectorTaskColumnInfo.setSupportSetInUse(supportSet);
@@ -2246,6 +2248,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   @Override
   public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
 
+    physicalContext = physicalContext;
     hiveConf = physicalContext.getConf();
     planMapper = physicalContext.getContext().getPlanMapper();
 
@@ -4265,6 +4268,13 @@ public class Vectorizer implements PhysicalPlanResolver {
                 vecAggrClasses, aggregateName, inputColVectorType,
                 outputColVectorType, udafEvaluatorMode);
         if (vecAggrClass != null) {
+          // for now, disable operating on decimal64 column vectors for semijoin reduction as
+          // we have to make sure same decimal type should be used during bloom filter creation
+          // and bloom filter probing
+          if (aggregateName.equals("bloom_filter")) {
+            inputExpression = vContext.wrapWithDecimal64ToDecimalConversion(inputExpression);
+            inputColVectorType = ColumnVector.Type.DECIMAL;
+          }
           final VectorAggregationDesc vecAggrDesc =
               new VectorAggregationDesc(
                   aggrDesc, evaluator, inputTypeInfo, inputColVectorType, inputExpression,
@@ -4359,8 +4369,6 @@ public class Vectorizer implements PhysicalPlanResolver {
     return new ImmutablePair<Operator<? extends OperatorDesc>, String>(vectorOp, null);
   }
 
-  static int fake;
-
   public static Operator<? extends OperatorDesc> vectorizeSelectOperator(
       Operator<? extends OperatorDesc> selectOp, VectorizationContext vContext,
       VectorSelectDesc vectorSelectDesc)
@@ -4386,6 +4394,13 @@ public class Vectorizer implements PhysicalPlanResolver {
     if (index < size) {
       vectorSelectExprs = Arrays.copyOf(vectorSelectExprs, index);
     }
+
+    // Fix up the case where parent expression's output data type physical variations is DECIMAL whereas
+    // at least one of its children is DECIMAL_64. Some expressions like x % y for example only accepts DECIMAL
+    // for x and y (at this time there is only DecimalColModuloDecimalColumn so both x and y has to be DECIMAL).
+    // The following method introduces a cast if x or y is DECIMAL_64 and parent expression (x % y) is DECIMAL.
+    fixDecimalDataTypePhysicalVariations(vContext, vectorSelectExprs);
+
     vectorSelectDesc.setSelectExpressions(vectorSelectExprs);
     vectorSelectDesc.setProjectedOutputColumns(projectedOutputColumns);
 
@@ -4394,6 +4409,84 @@ public class Vectorizer implements PhysicalPlanResolver {
         vContext, vectorSelectDesc);
   }
 
+  private static void fixDecimalDataTypePhysicalVariations(final VectorizationContext vContext,
+    final VectorExpression[] vectorSelectExprs) throws HiveException {
+    for (int i = 0; i < vectorSelectExprs.length; i++) {
+      VectorExpression parent = vectorSelectExprs[i];
+      VectorExpression newParent = fixDecimalDataTypePhysicalVariations(parent, parent.getChildExpressions(),
+        vContext);
+      if (parent.getClass() == newParent.getClass() && parent != newParent) {
+        vectorSelectExprs[i] = newParent;
+      }
+    }
+  }
+
+  private static VectorExpression fixDecimalDataTypePhysicalVariations(final VectorExpression parent,
+    final VectorExpression[] children, final VectorizationContext vContext) throws HiveException {
+    if (children == null || children.length == 0) {
+      return parent;
+    }
+
+    for (int i = 0; i < children.length; i++) {
+      VectorExpression child = children[i];
+      VectorExpression newChild = fixDecimalDataTypePhysicalVariations(child, child.getChildExpressions(), vContext);
+      if (child.getClass() == newChild.getClass() && child != newChild) {
+        children[i] = newChild;
+      }
+    }
+    if (parent.getOutputDataTypePhysicalVariation() == DataTypePhysicalVariation.NONE) {
+      boolean inputArgsChanged = false;
+      DataTypePhysicalVariation[] dataTypePhysicalVariations = parent.getInputDataTypePhysicalVariations();
+      VectorExpression oldExpression = null;
+      VectorExpression newExpression = null;
+      for (int i = 0; i < children.length; i++) {
+        oldExpression = children[i];
+        // we found at least one children with mismatch
+        if (oldExpression.getOutputDataTypePhysicalVariation() == DataTypePhysicalVariation.DECIMAL_64) {
+          newExpression = vContext.wrapWithDecimal64ToDecimalConversion(oldExpression);
+          children[i] = newExpression;
+          inputArgsChanged = true;
+          dataTypePhysicalVariations[i] = DataTypePhysicalVariation.NONE;
+        }
+      }
+      // fix up the input column numbers and output column numbers
+      if (inputArgsChanged) {
+        if (parent instanceof VectorUDFAdaptor) {
+          VectorUDFAdaptor parentAdaptor = (VectorUDFAdaptor) parent;
+          VectorUDFArgDesc[] argDescs = parentAdaptor.getArgDescs();
+          for (VectorUDFArgDesc argDesc : argDescs) {
+            if (argDesc.getColumnNum() == oldExpression.getOutputColumnNum()) {
+              argDesc.setColumnNum(newExpression.getOutputColumnNum());
+              break;
+            }
+          }
+        } else {
+          int argumentCount = children.length + (parent.getOutputColumnNum() == -1 ? 0 : 1);
+          Object[] arguments = new Object[argumentCount];
+          // new input column numbers
+          for (int i = 0; i < children.length; i++) {
+            VectorExpression vce = children[i];
+            arguments[i] = vce.getOutputColumnNum();
+          }
+          // retain output column number from parent
+          if (parent.getOutputColumnNum() != -1) {
+            arguments[arguments.length - 1] = parent.getOutputColumnNum();
+          }
+          // re-instantiate the parent expression with new arguments
+          VectorExpression newParent = vContext.instantiateExpression(parent.getClass(), parent.getOutputTypeInfo(),
+            parent.getOutputDataTypePhysicalVariation(), arguments);
+          newParent.setOutputTypeInfo(parent.getOutputTypeInfo());
+          newParent.setOutputDataTypePhysicalVariation(parent.getOutputDataTypePhysicalVariation());
+          newParent.setInputTypeInfos(parent.getInputTypeInfos());
+          newParent.setInputDataTypePhysicalVariations(dataTypePhysicalVariations);
+          newParent.setChildExpressions(parent.getChildExpressions());
+          return newParent;
+        }
+      }
+    }
+    return parent;
+  }
+
   private static void fillInPTFEvaluators(
       List<WindowFunctionDef> windowsFunctions,
       String[] evaluatorFunctionNames,
index dc58ad1..fe475f6 100644 (file)
@@ -229,7 +229,7 @@ public class TestInputOutputFormat {
       return "booleanValue,byteValue,shortValue,intValue,longValue,floatValue,doubleValue,stringValue,decimalValue,dateValue,timestampValue";
     }
     static String getColumnTypesProperty() {
-      return "boolean:tinyint:smallint:int:bigint:float:double:string:decimal:date:timestamp";
+      return "boolean:tinyint:smallint:int:bigint:float:double:string:decimal(38,18):date:timestamp";
     }
   }
 
@@ -3847,9 +3847,10 @@ public class TestInputOutputFormat {
    * Test schema evolution when using the reader directly.
    */
   @Test
-  public void testSchemaEvolution() throws Exception {
+  public void testSchemaEvolutionOldDecimal() throws Exception {
     TypeDescription fileSchema =
         TypeDescription.fromString("struct<a:int,b:struct<c:int>,d:string>");
+    conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, "decimal_64");
     Writer writer = OrcFile.createWriter(testFilePath,
         OrcFile.writerOptions(conf)
             .fileSystem(fs)
@@ -3915,6 +3916,78 @@ public class TestInputOutputFormat {
   }
 
   /**
+   * Test schema evolution when using the reader directly.
+   */
+  @Test
+  public void testSchemaEvolutionDecimal64() throws Exception {
+    TypeDescription fileSchema =
+      TypeDescription.fromString("struct<a:int,b:struct<c:int>,d:string>");
+    conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, "decimal_64");
+    Writer writer = OrcFile.createWriter(testFilePath,
+      OrcFile.writerOptions(conf)
+        .fileSystem(fs)
+        .setSchema(fileSchema)
+        .compress(org.apache.orc.CompressionKind.NONE));
+    VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000);
+    batch.size = 1000;
+    LongColumnVector lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]);
+    for(int r=0; r < 1000; r++) {
+      ((LongColumnVector) batch.cols[0]).vector[r] = r * 42;
+      lcv.vector[r] = r * 10001;
+      ((BytesColumnVector) batch.cols[2]).setVal(r,
+        Integer.toHexString(r).getBytes(StandardCharsets.UTF_8));
+    }
+    writer.addRowBatch(batch);
+    writer.close();
+    TypeDescription readerSchema = TypeDescription.fromString(
+      "struct<a:int,b:struct<c:int,future1:int>,d:string,future2:int>");
+    Reader reader = OrcFile.createReader(testFilePath,
+      OrcFile.readerOptions(conf).filesystem(fs));
+    RecordReader rows = reader.rowsOptions(new Reader.Options()
+      .schema(readerSchema));
+    batch = readerSchema.createRowBatchV2();
+    lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]);
+    LongColumnVector future1 = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[1]);
+    assertEquals(true, rows.nextBatch(batch));
+    assertEquals(1000, batch.size);
+    assertEquals(true, future1.isRepeating);
+    assertEquals(true, future1.isNull[0]);
+    assertEquals(true, batch.cols[3].isRepeating);
+    assertEquals(true, batch.cols[3].isNull[0]);
+    for(int r=0; r < batch.size; ++r) {
+      assertEquals("row " + r, r * 42, ((LongColumnVector) batch.cols[0]).vector[r]);
+      assertEquals("row " + r, r * 10001, lcv.vector[r]);
+      assertEquals("row " + r, r * 10001, lcv.vector[r]);
+      assertEquals("row " + r, Integer.toHexString(r),
+        ((BytesColumnVector) batch.cols[2]).toString(r));
+    }
+    assertEquals(false, rows.nextBatch(batch));
+    rows.close();
+
+    // try it again with an include vector
+    rows = reader.rowsOptions(new Reader.Options()
+      .schema(readerSchema)
+      .include(new boolean[]{false, true, true, true, false, false, true}));
+    batch = readerSchema.createRowBatchV2();
+    lcv = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[0]);
+    future1 = ((LongColumnVector) ((StructColumnVector) batch.cols[1]).fields[1]);
+    assertEquals(true, rows.nextBatch(batch));
+    assertEquals(1000, batch.size);
+    assertEquals(true, future1.isRepeating);
+    assertEquals(true, future1.isNull[0]);
+    assertEquals(true, batch.cols[3].isRepeating);
+    assertEquals(true, batch.cols[3].isNull[0]);
+    assertEquals(true, batch.cols[2].isRepeating);
+    assertEquals(true, batch.cols[2].isNull[0]);
+    for(int r=0; r < batch.size; ++r) {
+      assertEquals("row " + r, r * 42, ((LongColumnVector) batch.cols[0]).vector[r]);
+      assertEquals("row " + r, r * 10001, lcv.vector[r]);
+    }
+    assertEquals(false, rows.nextBatch(batch));
+    rows.close();
+  }
+
+  /**
    * Test column projection when using ACID.
    */
   @Test
@@ -3933,7 +4006,7 @@ public class TestInputOutputFormat {
             .fileSystem(fs)
             .setSchema(fileSchema)
             .compress(org.apache.orc.CompressionKind.NONE));
-    VectorizedRowBatch batch = fileSchema.createRowBatch(1000);
+    VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000);
     batch.size = 1000;
     StructColumnVector scv = (StructColumnVector)batch.cols[5];
     // operation
@@ -4047,7 +4120,7 @@ public class TestInputOutputFormat {
         .stripeSize(128);
     // Create ORC file with small stripe size so we can write multiple stripes.
     Writer writer = OrcFile.createWriter(testFilePath, options);
-    VectorizedRowBatch batch = fileSchema.createRowBatch(1000);
+    VectorizedRowBatch batch = fileSchema.createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64,1000);
     batch.size = 1000;
     StructColumnVector scv = (StructColumnVector)batch.cols[5];
     // operation
index d8a7af8..cc29384 100644 (file)
@@ -168,7 +168,7 @@ public class TestOrcRawRecordMerger {
     setRow(row4, OrcRecordUpdater.INSERT_OPERATION, 40, 50, 60, 130, "fourth");
     OrcStruct row5 = new OrcStruct(OrcRecordUpdater.FIELDS);
     setRow(row5, OrcRecordUpdater.INSERT_OPERATION, 40, 50, 61, 140, "fifth");
-    Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class)))
+    Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class), Mockito.any(HiveConf.class)))
         .thenReturn(recordReader);
 
     Mockito.when(recordReader.hasNext()).
@@ -192,7 +192,7 @@ public class TestOrcRawRecordMerger {
     RecordIdentifier minKey = new RecordIdentifier(10, 20, 30);
     RecordIdentifier maxKey = new RecordIdentifier(40, 50, 60);
     ReaderPair pair = new OrcRawRecordMerger.ReaderPairAcid(key, reader, minKey, maxKey,
-        new Reader.Options());
+        new Reader.Options(), new HiveConf());
     RecordReader recordReader = pair.getRecordReader();
     assertEquals(10, key.getWriteId());
     assertEquals(20, key.getBucketProperty());
@@ -218,7 +218,7 @@ public class TestOrcRawRecordMerger {
     Reader reader = createMockReader();
 
     ReaderPair pair = new OrcRawRecordMerger.ReaderPairAcid(key, reader, null, null,
-        new Reader.Options());
+        new Reader.Options(), new HiveConf());
     RecordReader recordReader = pair.getRecordReader();
     assertEquals(10, key.getWriteId());
     assertEquals(20, key.getBucketProperty());
@@ -274,7 +274,7 @@ public class TestOrcRawRecordMerger {
     OrcStruct row4 = createOriginalRow("fourth");
     OrcStruct row5 = createOriginalRow("fifth");
 
-    Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class)))
+    Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class), Mockito.any(HiveConf.class)))
         .thenReturn(recordReader);
     Mockito.when(recordReader.hasNext()).
         thenReturn(true, true, true, true, true, false);
@@ -410,7 +410,7 @@ public class TestOrcRawRecordMerger {
     types.add(typeBuilder.build());
 
     Mockito.when(reader.getTypes()).thenReturn(types);
-    Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class)))
+    Mockito.when(reader.rowsOptions(Mockito.any(Reader.Options.class), Mockito.any(HiveConf.class)))
         .thenReturn(recordReader);
 
     OrcStruct row1 = new OrcStruct(OrcRecordUpdater.FIELDS);
index 0c9c95d..c23f00e 100644 (file)
@@ -49,6 +49,7 @@ import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.orc.TypeDescription;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -151,7 +152,7 @@ public class TestVectorizedORCReader {
         OrcFile.readerOptions(conf));
     RecordReaderImpl vrr = (RecordReaderImpl) vreader.rows();
     RecordReaderImpl rr = (RecordReaderImpl) reader.rows();
-    VectorizedRowBatch batch = reader.getSchema().createRowBatch();
+    VectorizedRowBatch batch = reader.getSchema().createRowBatchV2();
     OrcStruct row = null;
 
     // Check Vectorized ORC reader against ORC row reader
index e478371..551e5ca 100644 (file)
@@ -226,7 +226,7 @@ public class TestVectorizedOrcAcidRowBatchReader {
       assertTrue(vectorizedReader.getDeleteEventRegistry() instanceof SortMergedDeleteEventRegistry);
     }
     TypeDescription schema = OrcInputFormat.getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE);
-    VectorizedRowBatch vectorizedRowBatch = schema.createRowBatch();
+    VectorizedRowBatch vectorizedRowBatch = schema.createRowBatchV2();
     vectorizedRowBatch.setPartitionInfo(1, 0); // set data column count as 1.
     long previousPayload = Long.MIN_VALUE;
     while (vectorizedReader.next(null, vectorizedRowBatch)) {
index 3d5b3a8..1f31218 100644 (file)
@@ -110,7 +110,7 @@ select * from cte;
 explain analyze with cte as (select * from src  order by key limit 5)
 select * from cte;
 
-create table orc_merge5_n1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+create table orc_merge5_n1 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5_n1;
 
index a409c26..cd06d31 100644 (file)
@@ -29,18 +29,27 @@ CREATE TABLE orc_llap_n2 (
     cfloat1 FLOAT,
     cdouble1 DOUBLE,
     cstring1 string,
-    cfloat2 float
-)  stored as orc TBLPROPERTIES ('transactional'='true');
+    cfloat2 float,
+    cdecimal1 decimal(10,3),
+    cdecimal2 decimal(38,10)
+)  stored as orc TBLPROPERTIES ('transactional'='true','orc.write.format'='UNSTABLE-PRE-2.0');
 
 
 insert into table orc_llap_n2
 select cint, cbigint, cfloat, cdouble,
  cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
  cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
- cstring1, cfloat as c9 from alltypesorc order by cdouble asc  limit 30;
-
+ cstring1, cfloat as c9, cast("1.123" as decimal(10,3))as c10,
+ cast("1.123456789" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30;
 
+alter table orc_llap_n2 set TBLPROPERTIES ('transactional'='true','orc.write.format'='0.12');
 
+insert into table orc_llap_n2
+select cint, cbigint, cfloat, cdouble,
+ cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
+ cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
+ cstring1, cfloat as c9, cast("3.321" as decimal(10,3))as c10,
+ cast("9.987654321" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30;
 
 
 CREATE TABLE orc_llap2 (
@@ -57,18 +66,22 @@ CREATE TABLE orc_llap2 (
     cfloat1 FLOAT,
     cdouble1 DOUBLE,
     cstring1 string,
-    cfloat2 float
-)  stored as orc TBLPROPERTIES ('transactional'='false');
+    cfloat2 float,
+    cdecimal1 decimal(10,3),
+    cdecimal2 decimal(38,10)
+)  stored as orc TBLPROPERTIES ('transactional'='false', 'orc.write.format'='UNSTABLE-PRE-2.0');
 
 insert into table orc_llap2
 select cint, cbigint, cfloat, cdouble,
  cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
  cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
- cstring1, cfloat as c9 from alltypesorc order by cdouble asc  limit 30;
+ cstring1, cfloat as c9, cast("1.123" as decimal(10,3))as c10,
+ cast("1.123456789" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30;
 
-alter table orc_llap2 set TBLPROPERTIES ('transactional'='true');
+alter table orc_llap2 set TBLPROPERTIES ('transactional'='true','orc.write.format'='0.12');
 
-update orc_llap2 set cstring1 = 'testvalue' where cstring1 = 'N016jPED08o';
+update orc_llap2 set cstring1 = 'testvalue', cdecimal1 = cast("3.321" as decimal(10,3)),
+cdecimal2 = cast("9.987654321" as decimal(38,18))  where cstring1 = 'N016jPED08o';
 
 
 SET hive.llap.io.enabled=true;
diff --git a/ql/src/test/queries/clientpositive/llap_decimal64_reader.q b/ql/src/test/queries/clientpositive/llap_decimal64_reader.q
new file mode 100644 (file)
index 0000000..a81feba
--- /dev/null
@@ -0,0 +1,54 @@
+--! qt:dataset:alltypesorc
+SET hive.vectorized.execution.enabled=true;
+
+SET hive.llap.io.enabled=false;
+
+SET hive.exec.orc.default.row.index.stride=1000;
+SET hive.optimize.index.filter=true;
+set hive.auto.convert.join=false;
+
+DROP TABLE orc_llap_n0;
+
+-- this test mix and matches orc versions and flips config to use decimal64 column vectors
+set hive.auto.convert.join=true;
+SET hive.llap.io.enabled=true;
+CREATE TABLE orc_llap_n0(
+    ctinyint TINYINT,
+    csmallint SMALLINT,
+    cint INT,
+    cbigint BIGINT,
+    cfloat FLOAT,
+    cdouble DOUBLE,
+    cstring1 STRING,
+    cstring2 STRING,
+    ctimestamp1 TIMESTAMP,
+    ctimestamp2 TIMESTAMP,
+    cboolean1 BOOLEAN,
+    cboolean2 BOOLEAN,
+    cdecimal1 decimal(10,2),
+    cdecimal2 decimal(38,5))
+    STORED AS ORC tblproperties ("orc.compress"="NONE");
+
+insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("3.345" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc;
+
+alter table orc_llap_n0 set tblproperties ("orc.compress"="NONE", 'orc.write.format'='UNSTABLE-PRE-2.0');
+
+insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("4.456" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc;
+
+set hive.vectorized.input.format.supports.enabled=decimal_64;
+explain vectorization select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2;
+select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2;
+
+set hive.vectorized.input.format.supports.enabled=none;
+explain vectorization select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2;
+select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2;
+
+DROP TABLE orc_llap_n0;
index 875356c..de3cdc6 100644 (file)
@@ -24,13 +24,20 @@ CREATE TABLE orc_llap_n0(
     ctimestamp1 TIMESTAMP,
     ctimestamp2 TIMESTAMP,
     cboolean1 BOOLEAN,
-    cboolean2 BOOLEAN)
+    cboolean2 BOOLEAN,
+    cdecimal1 decimal(10,2),
+    cdecimal2 decimal(38,5))
     STORED AS ORC tblproperties ("orc.compress"="NONE");
 
 insert into table orc_llap_n0
-select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2
-from alltypesorc;
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("3.345" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc;
 
+alter table orc_llap_n0 set tblproperties ("orc.compress"="NONE", 'orc.write.format'='UNSTABLE-PRE-2.0');
+
+insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("3.345" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc;
 
 SET hive.llap.io.enabled=true;
 
index 6d41009..dfae138 100644 (file)
@@ -78,7 +78,7 @@ CREATE TABLE orc_create_people_staging (
   first_name string,
   last_name string,
   address string,
-  salary decimal,
+  salary decimal(38,0),
   start_date timestamp,
   state string);
 
@@ -90,7 +90,7 @@ CREATE TABLE orc_create_people (
   first_name string,
   last_name string,
   address string,
-  salary decimal,
+  salary decimal(38,0),
   start_date timestamp)
 PARTITIONED BY (state string)
 STORED AS orc;
index 9f8e3bb..f3ee76c 100644 (file)
@@ -40,7 +40,8 @@ CREATE TABLE orc_ppd_staging_n0(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_staging_n0 select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), `dec`, bin from staging_n6 order by t, s;
+insert overwrite table orc_ppd_staging_n0 select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, cast(ts as date) as da, `dec`, bin from staging_n6 order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
 
 -- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values
 -- which makes it hard to test bloom filters
@@ -62,7 +63,8 @@ CREATE TABLE orc_ppd_n1(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_n1 select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, `dec`, bin from orc_ppd_staging_n0 order by t, s;
+insert overwrite table orc_ppd_n1 select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, da, `dec`, bin from orc_ppd_staging_n0 order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
 
 describe formatted orc_ppd_n1;
 
index 16df96c..f12870c 100644 (file)
@@ -39,7 +39,8 @@ CREATE TABLE orc_ppd_staging(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), `dec`, bin from staging order by t, s;
+insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, cast(ts as date) as da, `dec`, bin from staging order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
 
 -- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values
 -- which makes it hard to test bloom filters
@@ -61,7 +62,9 @@ CREATE TABLE orc_ppd(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, `dec`, bin from orc_ppd_staging order by t, s;
+insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, da, `dec`, bin from orc_ppd_staging order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
+
 
 describe formatted orc_ppd;
 
index d5add84..208c5b7 100644 (file)
@@ -3,15 +3,15 @@ set hive.vectorized.execution.enabled=false;
 DROP TABLE orcfile_merge1_n2;
 DROP TABLE orc_split_elim_n0;
 
-create table orc_split_elim_n0 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+create table orc_split_elim_n0 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_n0;
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_n0;
 
-create table orcfile_merge1_n2 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc tblproperties("orc.compress.size"="4096");
+create table orcfile_merge1_n2 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc tblproperties("orc.compress.size"="4096");
 
-insert overwrite table orcfile_merge1_n2 select * from orc_split_elim_n0;
-insert into table orcfile_merge1_n2 select * from orc_split_elim_n0;
+insert overwrite table orcfile_merge1_n2 select * from orc_split_elim_n0 order by userid;
+insert into table orcfile_merge1_n2 select * from orc_split_elim_n0 order by userid;
 
 dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orcfile_merge1_n2/;
 
index 190c6e0..4ae5ba6 100644 (file)
@@ -3,8 +3,8 @@ set hive.explain.user=false;
 
 -- SORT_QUERY_RESULTS
 
-create table orc_merge5_n5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
-create table orc_merge5b_n0 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+create table orc_merge5_n5 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
+create table orc_merge5b_n0 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5_n5;
 
index fabe656..1c7ab08 100644 (file)
@@ -4,8 +4,8 @@ set hive.explain.user=false;
 -- SORT_QUERY_RESULTS
 
 -- orc file merge tests for static partitions
-create table orc_merge5_n4 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
-create table orc_merge5a_n1 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (year string, hour int) stored as orc;
+create table orc_merge5_n4 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
+create table orc_merge5a_n1 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) partitioned by (year string, hour int) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5_n4;
 
index 2558797..6504989 100644 (file)
@@ -5,8 +5,8 @@ set hive.explain.user=false;
 
 -- orc merge file tests for dynamic partition case
 
-create table orc_merge5_n2 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
-create table orc_merge5a_n0 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (st double) stored as orc;
+create table orc_merge5_n2 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
+create table orc_merge5a_n0 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) partitioned by (st double) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5_n2;
 
index aba4617..2b768ea 100644 (file)
@@ -3,8 +3,8 @@ set hive.explain.user=false;
 
 -- SORT_QUERY_RESULTS
 
-create table orc_merge5_n3 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
-create table orc_merge5b (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+create table orc_merge5_n3 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
+create table orc_merge5b (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5_n3;
 
index ef66522..6281c96 100644 (file)
@@ -6,8 +6,8 @@ set hive.explain.user=false;
 
 -- orc merge file tests for dynamic partition case
 
-create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
-create table orc_merge5a (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (st double) stored as orc;
+create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
+create table orc_merge5a (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) partitioned by (st double) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5;
 
index f0b0b96..b3f166a 100644 (file)
@@ -40,7 +40,8 @@ CREATE TABLE orc_ppd_staging_n1(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_staging_n1 select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), `dec`, bin from staging_n7 order by t, s;
+insert overwrite table orc_ppd_staging_n1 select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, cast(ts as date) as da, `dec`, bin from staging_n7 order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
 
 -- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values
 -- which makes it hard to test bloom filters
@@ -62,7 +63,9 @@ CREATE TABLE orc_ppd_n2(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_n2 select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, `dec`, bin from orc_ppd_staging_n1 order by t, s;
+insert overwrite table orc_ppd_n2 select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, da, `dec`, bin from orc_ppd_staging_n1 order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
+
 
 SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
 
index 4235c2c..9d79b11 100644 (file)
@@ -37,7 +37,8 @@ CREATE TABLE orc_ppd_staging_n2(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_staging_n2 select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), `dec`, bin from staging_n8 order by t, s;
+insert overwrite table orc_ppd_staging_n2 select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, cast(ts as date) as da, `dec`, bin from staging_n8 order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
 
 -- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values
 -- which makes it hard to test bloom filters
@@ -59,7 +60,8 @@ CREATE TABLE orc_ppd_n3(t tinyint,
            bin binary)
 STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*");
 
-insert overwrite table orc_ppd_n3 select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, `dec`, bin from orc_ppd_staging_n2 order by t, s;
+insert overwrite table orc_ppd_n3 select t, si, i, b, f, d, bo, s, cast(s as char(50)) as c,
+cast(s as varchar(50)) as v, da, `dec`, bin from orc_ppd_staging_n2 order by t, si, i, b, f, d, bo, s, c, v, da, `dec`, bin;
 
 SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter;
 SET hive.optimize.index.filter=false;
index c2d9840..ca5dc6f 100644 (file)
@@ -1,6 +1,8 @@
 set hive.vectorized.execution.enabled=false;
 set hive.optimize.index.filter=false;
 set hive.metastore.disallow.incompatible.col.type.changes=false;
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
 
 drop table float_text;
 create table float_text(f float);
index 719b21c..03e0e73 100644 (file)
@@ -2,7 +2,7 @@ set hive.vectorized.execution.enabled=false;
 
 -- SORT_QUERY_RESULTS
 
-create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+create table orc_split_elim (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc;
 
 load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim;
 
@@ -105,7 +105,7 @@ select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=70;
 SET hive.optimize.index.filter=false;
 
 -- partitioned table
-create table orc_split_elim_part (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (country string, year int) stored as orc;
+create table orc_split_elim_part (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) partitioned by (country string, year int) stored as orc;
 
 alter table orc_split_elim_part add partition(country='us', year=2000);
 alter table orc_split_elim_part add partition(country='us', year=2001);
index 427734f..53c16e0 100644 (file)
@@ -12,6 +12,8 @@ set hive.exec.dynamic.partition.mode=nonstrict;
 set hive.metastore.disallow.incompatible.col.type.changes=false;
 set hive.default.fileformat=orc;
 set hive.llap.io.enabled=false;
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
 
 -- SORT_QUERY_RESULTS
 --
index 1eca9e3..f2fb2f0 100644 (file)
@@ -13,6 +13,8 @@ set hive.metastore.disallow.incompatible.col.type.changes=false;
 set hive.default.fileformat=orc;
 set hive.llap.io.enabled=true;
 set hive.llap.io.encode.enabled=true;
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
 
 -- SORT_QUERY_RESULTS
 --
index 6e35f5a..e811f1d 100644 (file)
@@ -12,6 +12,8 @@ set hive.exec.dynamic.partition.mode=nonstrict;
 set hive.metastore.disallow.incompatible.col.type.changes=false;
 set hive.default.fileformat=orc;
 set hive.llap.io.enabled=false;
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
 
 -- SORT_QUERY_RESULTS
 --
index 576f994..bae6cc8 100644 (file)
@@ -13,6 +13,8 @@ set hive.metastore.disallow.incompatible.col.type.changes=false;
 set hive.default.fileformat=orc;
 set hive.llap.io.enabled=true;
 set hive.llap.io.encode.enabled=true;
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
 
 -- SORT_QUERY_RESULTS
 --
index 112a674..2a49871 100644 (file)
@@ -1,3 +1,6 @@
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
+
 -- Create a base table to be used for loading data: Begin
 drop table if exists testAltCol_n1;
 create table testAltCol_n1
index 9e93a2f..6a940ac 100644 (file)
@@ -1,3 +1,5 @@
+-- set this to 'decimal_64' after resolving HIVE-19792
+set hive.vectorized.input.format.supports.enabled=none;
 -- Create a base table to be used for loading data: Begin
 drop table if exists testAltCol;
 create table testAltCol
index 0ba17da..8614087 100644 (file)
@@ -5,6 +5,8 @@ set hive.explain.user=false;
 set hive.fetch.task.conversion=none;
 set hive.vectorized.execution.enabled=true;
 
+-- SORT_QUERY_RESULTS
+
 CREATE TABLE lineitem_test_txt (L_ORDERKEY      INT,
                                 L_PARTKEY       INT,
                                 L_SUPPKEY       INT,
@@ -69,8 +71,7 @@ SELECT
    IF(L_SUPPKEY > 10000, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE), NULL) AS Field_10,
    IF(L_SUPPKEY > 10000, NULL, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE)) AS Field_11,
    IF(L_SUPPKEY % 500 > 100, DATE_ADD('2008-12-31', 1), DATE_ADD('2008-12-31', 365)) AS Field_12
-FROM lineitem_test
-ORDER BY Quantity;
+FROM lineitem_test;
 SELECT
    L_QUANTITY as Quantity,
    CASE
@@ -109,8 +110,7 @@ SELECT
    IF(L_SUPPKEY > 10000, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE), NULL) AS Field_10,
    IF(L_SUPPKEY > 10000, NULL, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE)) AS Field_11,
    IF(L_SUPPKEY % 500 > 100, DATE_ADD('2008-12-31', 1), DATE_ADD('2008-12-31', 365)) AS Field_12
-FROM lineitem_test
-ORDER BY Quantity;
+FROM lineitem_test;
 
 SET hive.vectorized.if.expr.mode=good;
 
@@ -153,8 +153,7 @@ SELECT
    IF(L_SUPPKEY > 10000, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE), NULL) AS Field_10,
    IF(L_SUPPKEY > 10000, NULL, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE)) AS Field_11,
    IF(L_SUPPKEY % 500 > 100, DATE_ADD('2008-12-31', 1), DATE_ADD('2008-12-31', 365)) AS Field_12
-FROM lineitem_test
-ORDER BY Quantity;
+FROM lineitem_test;
 SELECT
    L_QUANTITY as Quantity,
    CASE
@@ -193,8 +192,7 @@ SELECT
    IF(L_SUPPKEY > 10000, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE), NULL) AS Field_10,
    IF(L_SUPPKEY > 10000, NULL, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE)) AS Field_11,
    IF(L_SUPPKEY % 500 > 100, DATE_ADD('2008-12-31', 1), DATE_ADD('2008-12-31', 365)) AS Field_12
-FROM lineitem_test
-ORDER BY Quantity;
+FROM lineitem_test;
 
 SET hive.vectorized.if.expr.mode=better;
 
@@ -237,8 +235,7 @@ SELECT
    IF(L_SUPPKEY > 10000, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE), NULL) AS Field_10,
    IF(L_SUPPKEY > 10000, NULL, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE)) AS Field_11,
    IF(L_SUPPKEY % 500 > 100, DATE_ADD('2008-12-31', 1), DATE_ADD('2008-12-31', 365)) AS Field_12
-FROM lineitem_test
-ORDER BY Quantity;
+FROM lineitem_test;
 SELECT
    L_QUANTITY as Quantity,
    CASE
@@ -277,6 +274,5 @@ SELECT
    IF(L_SUPPKEY > 10000, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE), NULL) AS Field_10,
    IF(L_SUPPKEY > 10000, NULL, DATEDIFF(L_RECEIPTDATE, L_COMMITDATE)) AS Field_11,
    IF(L_SUPPKEY % 500 > 100, DATE_ADD('2008-12-31', 1), DATE_ADD('2008-12-31', 365)) AS Field_12
-FROM lineitem_test
-ORDER BY Quantity;
\ No newline at end of file
+FROM lineitem_test;
index f5de13b..e0956e4 100644 (file)
@@ -21,9 +21,10 @@ SELECT key FROM DECIMAL_5 ORDER BY key;
 
 SELECT DISTINCT key FROM DECIMAL_5 ORDER BY key;
 
+explain SELECT cast(key as decimal) FROM DECIMAL_5;
 SELECT cast(key as decimal) FROM DECIMAL_5;
 
 SELECT cast(key as decimal(6,3)) FROM DECIMAL_5;
 
 DROP TABLE DECIMAL_5_txt;
-DROP TABLE DECIMAL_5;
\ No newline at end of file
+DROP TABLE DECIMAL_5;
index 6e5b5b6..ef769fb 100644 (file)
@@ -35,11 +35,13 @@ select t1_n48.`dec`, t2_n29.`dec` from t1_n48 join t2_n29 on (t1_n48.`dec`=t2_n2
 -- SORT_QUERY_RESULTS
 
 select t1_n48.`dec`, t2_n29.`dec` from t1_n48 join t2_n29 on (t1_n48.`dec`=t2_n29.`dec`);
+select count(*) from (select t1_n48.`dec`, t2_n29.`dec` from t1_n48 join t2_n29 on (t1_n48.`dec`=t2_n29.`dec`)) as t;
 
 explain vectorization detail
 select t1_n48.`dec`, t1_n48.value_dec, t2_n29.`dec`, t2_n29.value_dec from t1_n48 join t2_n29 on (t1_n48.`dec`=t2_n29.`dec`);
 
 select t1_n48.`dec`, t1_n48.value_dec, t2_n29.`dec`, t2_n29.value_dec from t1_n48 join t2_n29 on (t1_n48.`dec`=t2_n29.`dec`);
+select count(*) from (select t1_n48.`dec`, t1_n48.value_dec, t2_n29.`dec`, t2_n29.value_dec from t1_n48 join t2_n29 on (t1_n48.`dec`=t2_n29.`dec`)) as t;
 
 
 
@@ -72,11 +74,13 @@ select t1_small.`dec`, t2_small.`dec` from t1_small join t2_small on (t1_small.`
 -- SORT_QUERY_RESULTS
 
 select t1_small.`dec`, t2_small.`dec` from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`);
+select count(*) from (select t1_small.`dec`, t2_small.`dec` from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`)) as t;
 
 explain vectorization detail
 select t1_small.`dec`, t1_small.value_dec, t2_small.`dec`, t2_small.value_dec from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`);
 
 select t1_small.`dec`, t1_small.value_dec, t2_small.`dec`, t2_small.value_dec from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`);
+select count(*) from (select t1_small.`dec`, t1_small.value_dec, t2_small.`dec`, t2_small.value_dec from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`)) as t;
 
 
 set hive.vectorized.input.format.supports.enabled=none;
@@ -87,9 +91,11 @@ select t1_small.`dec`, t2_small.`dec` from t1_small join t2_small on (t1_small.`
 -- SORT_QUERY_RESULTS
 
 select t1_small.`dec`, t2_small.`dec` from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`);
+select count(*) from (select t1_small.`dec`, t2_small.`dec` from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`)) as t;
 
 explain vectorization detail
 select t1_small.`dec`, t1_small.value_dec, t2_small.`dec`, t2_small.value_dec from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`);
 
 select t1_small.`dec`, t1_small.value_dec, t2_small.`dec`, t2_small.value_dec from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`);
+select count(*) from (select t1_small.`dec`, t1_small.value_dec, t2_small.`dec`, t2_small.value_dec from t1_small join t2_small on (t1_small.`dec`=t2_small.`dec`)) as t;
 
index 7998035..743e8db 100644 (file)
@@ -39,7 +39,7 @@ EXPLAIN select count(*) from dsrv2_big a join dsrv2_small b on (a.partkey_bigint
 select count(*) from dsrv2_big a join dsrv2_small b on (a.partkey_bigint = b.partkey_bigint);
 
 -- single key (decimal)
-EXPLAIN select count(*) from dsrv2_big a join dsrv2_small b on (a.partkey_decimal = b.partkey_decimal);
+EXPLAIN VECTORIZATION DETAIL select count(*) from dsrv2_big a join dsrv2_small b on (a.partkey_decimal = b.partkey_decimal);
 select count(*) from dsrv2_big a join dsrv2_small b on (a.partkey_decimal = b.partkey_decimal);
 
 -- single key (double)
index eb4a8cb..80bbba4 100644 (file)
@@ -1144,8 +1144,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -1342,8 +1342,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -1511,8 +1511,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: false
                 usesVectorUDFAdaptor: false
@@ -1541,8 +1541,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -1929,8 +1929,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -2128,8 +2128,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -2298,8 +2298,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: false
                 usesVectorUDFAdaptor: false
@@ -2328,8 +2328,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
index 957dfd8..66bb2db 100644 (file)
@@ -665,22 +665,22 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: over10k_orc_bucketed
-                  Statistics: Num rows: 1247 Data size: 713720 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE
                   Select Operator
                     expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                     outputColumnNames: ROW__ID
-                    Statistics: Num rows: 1247 Data size: 713720 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       aggregations: count()
                       keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                       mode: hash
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 623 Data size: 52332 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 618 Data size: 51912 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: +
                         Map-reduce partition columns: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
-                        Statistics: Num rows: 623 Data size: 52332 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 618 Data size: 51912 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col1 (type: bigint)
             Execution mode: llap
             LLAP IO: may be used (ACID table)
@@ -692,13 +692,13 @@ STAGE PLANS:
                 keys: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 623 Data size: 52332 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 618 Data size: 51912 Basic stats: COMPLETE Column stats: COMPLETE
                 Filter Operator
                   predicate: (_col1 > 1L) (type: boolean)
-                  Statistics: Num rows: 207 Data size: 17388 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 206 Data size: 17304 Basic stats: COMPLETE Column stats: COMPLETE
                   File Output Operator
                     compressed: false
-                    Statistics: Num rows: 207 Data size: 17388 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 206 Data size: 17304 Basic stats: COMPLETE Column stats: COMPLETE
                     table:
                         input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                         output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
index 84477c3..7a880dd 100644 (file)
@@ -3233,19 +3233,19 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: acid_uami_n1
-                  Statistics: Num rows: 267 Data size: 83640 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 281 Data size: 87904 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
                     predicate: (((de = 109.23) or (de = 119.23)) and enforce_constraint(vc is not null)) (type: boolean)
-                    Statistics: Num rows: 5 Data size: 1566 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), i (type: int), vc (type: varchar(128))
                       outputColumnNames: _col0, _col1, _col3
-                      Statistics: Num rows: 5 Data size: 1566 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
                       Reduce Output Operator
                         key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: +
                         Map-reduce partition columns: UDFToInteger(_col0) (type: int)
-                        Statistics: Num rows: 5 Data size: 1566 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col1 (type: int), _col3 (type: varchar(128))
             Execution mode: vectorized, llap
             LLAP IO: may be used (ACID table)
@@ -3255,10 +3255,10 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 3.14 (type: decimal(5,2)), VALUE._col1 (type: varchar(128))
                 outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 5 Data size: 1566 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
-                  Statistics: Num rows: 5 Data size: 1566 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 5 Data size: 1564 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -3326,7 +3326,7 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: acid_uami_n1
-                  Statistics: Num rows: 305 Data size: 95448 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 320 Data size: 100040 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
                     predicate: ((de = 3.14) and enforce_constraint((i is not null and vc is not null))) (type: boolean)
                     Statistics: Num rows: 2 Data size: 625 Basic stats: COMPLETE Column stats: NONE
index 6196efe..635f928 100644 (file)
@@ -124,8 +124,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -269,8 +269,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -378,8 +378,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
index 4d74a17..c3e9c2a 100644 (file)
@@ -16,8 +16,10 @@ PREHOOK: query: CREATE TABLE orc_llap_n2 (
     cfloat1 FLOAT,
     cdouble1 DOUBLE,
     cstring1 string,
-    cfloat2 float
-)  stored as orc TBLPROPERTIES ('transactional'='true')
+    cfloat2 float,
+    cdecimal1 decimal(10,3),
+    cdecimal2 decimal(38,10)
+)  stored as orc TBLPROPERTIES ('transactional'='true','orc.write.format'='UNSTABLE-PRE-2.0')
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@orc_llap_n2
@@ -35,8 +37,10 @@ POSTHOOK: query: CREATE TABLE orc_llap_n2 (
     cfloat1 FLOAT,
     cdouble1 DOUBLE,
     cstring1 string,
-    cfloat2 float
-)  stored as orc TBLPROPERTIES ('transactional'='true')
+    cfloat2 float,
+    cdecimal1 decimal(10,3),
+    cdecimal2 decimal(38,10)
+)  stored as orc TBLPROPERTIES ('transactional'='true','orc.write.format'='UNSTABLE-PRE-2.0')
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@orc_llap_n2
@@ -44,7 +48,8 @@ PREHOOK: query: insert into table orc_llap_n2
 select cint, cbigint, cfloat, cdouble,
  cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
  cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
- cstring1, cfloat as c9 from alltypesorc order by cdouble asc  limit 30
+ cstring1, cfloat as c9, cast("1.123" as decimal(10,3))as c10,
+ cast("1.123456789" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30
 PREHOOK: type: QUERY
 PREHOOK: Input: default@alltypesorc
 PREHOOK: Output: default@orc_llap_n2
@@ -52,13 +57,58 @@ POSTHOOK: query: insert into table orc_llap_n2
 select cint, cbigint, cfloat, cdouble,
  cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
  cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
- cstring1, cfloat as c9 from alltypesorc order by cdouble asc  limit 30
+ cstring1, cfloat as c9, cast("1.123" as decimal(10,3))as c10,
+ cast("1.123456789" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@alltypesorc
 POSTHOOK: Output: default@orc_llap_n2
 POSTHOOK: Lineage: orc_llap_n2.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
 POSTHOOK: Lineage: orc_llap_n2.cbigint0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
 POSTHOOK: Lineage: orc_llap_n2.cbigint1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cdecimal1 SIMPLE []
+POSTHOOK: Lineage: orc_llap_n2.cdecimal2 EXPRESSION []
+POSTHOOK: Lineage: orc_llap_n2.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cdouble0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cdouble1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cfloat0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cfloat1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cfloat2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cint0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cint1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+PREHOOK: query: alter table orc_llap_n2 set TBLPROPERTIES ('transactional'='true','orc.write.format'='0.12')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@orc_llap_n2
+PREHOOK: Output: default@orc_llap_n2
+POSTHOOK: query: alter table orc_llap_n2 set TBLPROPERTIES ('transactional'='true','orc.write.format'='0.12')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@orc_llap_n2
+POSTHOOK: Output: default@orc_llap_n2
+PREHOOK: query: insert into table orc_llap_n2
+select cint, cbigint, cfloat, cdouble,
+ cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
+ cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
+ cstring1, cfloat as c9, cast("3.321" as decimal(10,3))as c10,
+ cast("9.987654321" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@orc_llap_n2
+POSTHOOK: query: insert into table orc_llap_n2
+select cint, cbigint, cfloat, cdouble,
+ cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
+ cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
+ cstring1, cfloat as c9, cast("3.321" as decimal(10,3))as c10,
+ cast("9.987654321" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@orc_llap_n2
+POSTHOOK: Lineage: orc_llap_n2.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cbigint0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cbigint1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n2.cdecimal1 SIMPLE []
+POSTHOOK: Lineage: orc_llap_n2.cdecimal2 EXPRESSION []
 POSTHOOK: Lineage: orc_llap_n2.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
 POSTHOOK: Lineage: orc_llap_n2.cdouble0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
 POSTHOOK: Lineage: orc_llap_n2.cdouble1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
@@ -84,8 +134,10 @@ PREHOOK: query: CREATE TABLE orc_llap2 (
     cfloat1 FLOAT,
     cdouble1 DOUBLE,
     cstring1 string,
-    cfloat2 float
-)  stored as orc TBLPROPERTIES ('transactional'='false')
+    cfloat2 float,
+    cdecimal1 decimal(10,3),
+    cdecimal2 decimal(38,10)
+)  stored as orc TBLPROPERTIES ('transactional'='false', 'orc.write.format'='UNSTABLE-PRE-2.0')
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@orc_llap2
@@ -103,8 +155,10 @@ POSTHOOK: query: CREATE TABLE orc_llap2 (
     cfloat1 FLOAT,
     cdouble1 DOUBLE,
     cstring1 string,
-    cfloat2 float
-)  stored as orc TBLPROPERTIES ('transactional'='false')
+    cfloat2 float,
+    cdecimal1 decimal(10,3),
+    cdecimal2 decimal(38,10)
+)  stored as orc TBLPROPERTIES ('transactional'='false', 'orc.write.format'='UNSTABLE-PRE-2.0')
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@orc_llap2
@@ -112,7 +166,8 @@ PREHOOK: query: insert into table orc_llap2
 select cint, cbigint, cfloat, cdouble,
  cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
  cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
- cstring1, cfloat as c9 from alltypesorc order by cdouble asc  limit 30
+ cstring1, cfloat as c9, cast("1.123" as decimal(10,3))as c10,
+ cast("1.123456789" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30
 PREHOOK: type: QUERY
 PREHOOK: Input: default@alltypesorc
 PREHOOK: Output: default@orc_llap2
@@ -120,13 +175,16 @@ POSTHOOK: query: insert into table orc_llap2
 select cint, cbigint, cfloat, cdouble,
  cint as c1, cbigint as c2, cfloat as c3, cdouble as c4,
  cint as c8, cbigint as c7, cfloat as c6, cdouble as c5,
- cstring1, cfloat as c9 from alltypesorc order by cdouble asc  limit 30
+ cstring1, cfloat as c9, cast("1.123" as decimal(10,3))as c10,
+ cast("1.123456789" as decimal(38,18)) as c11 from alltypesorc order by cdouble asc  limit 30
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@alltypesorc
 POSTHOOK: Output: default@orc_llap2
 POSTHOOK: Lineage: orc_llap2.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
 POSTHOOK: Lineage: orc_llap2.cbigint0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
 POSTHOOK: Lineage: orc_llap2.cbigint1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap2.cdecimal1 SIMPLE []
+POSTHOOK: Lineage: orc_llap2.cdecimal2 EXPRESSION []
 POSTHOOK: Lineage: orc_llap2.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
 POSTHOOK: Lineage: orc_llap2.cdouble0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
 POSTHOOK: Lineage: orc_llap2.cdouble1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
@@ -138,19 +196,21 @@ POSTHOOK: Lineage: orc_llap2.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(n
 POSTHOOK: Lineage: orc_llap2.cint0 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
 POSTHOOK: Lineage: orc_llap2.cint1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
 POSTHOOK: Lineage: orc_llap2.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
-PREHOOK: query: alter table orc_llap2 set TBLPROPERTIES ('transactional'='true')
+PREHOOK: query: alter table orc_llap2 set TBLPROPERTIES ('transactional'='true','orc.write.format'='0.12')
 PREHOOK: type: ALTERTABLE_PROPERTIES
 PREHOOK: Input: default@orc_llap2
 PREHOOK: Output: default@orc_llap2
-POSTHOOK: query: alter table orc_llap2 set TBLPROPERTIES ('transactional'='true')
+POSTHOOK: query: alter table orc_llap2 set TBLPROPERTIES ('transactional'='true','orc.write.format'='0.12')
 POSTHOOK: type: ALTERTABLE_PROPERTIES
 POSTHOOK: Input: default@orc_llap2
 POSTHOOK: Output: default@orc_llap2
-PREHOOK: query: update orc_llap2 set cstring1 = 'testvalue' where cstring1 = 'N016jPED08o'
+PREHOOK: query: update orc_llap2 set cstring1 = 'testvalue', cdecimal1 = cast("3.321" as decimal(10,3)),
+cdecimal2 = cast("9.987654321" as decimal(38,18))  where cstring1 = 'N016jPED08o'
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_llap2
 PREHOOK: Output: default@orc_llap2
-POSTHOOK: query: update orc_llap2 set cstring1 = 'testvalue' where cstring1 = 'N016jPED08o'
+POSTHOOK: query: update orc_llap2 set cstring1 = 'testvalue', cdecimal1 = cast("3.321" as decimal(10,3)),
+cdecimal2 = cast("9.987654321" as decimal(38,18))  where cstring1 = 'N016jPED08o'
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_llap2
 POSTHOOK: Output: default@orc_llap2
@@ -192,6 +252,36 @@ xTlDv24JYv4s
 7wH3hBKdO55Xq3gEEe0
 5QLs0LVK1g
 ET3d4F2I4lV
+N016jPED08o
+Q1JAdUlCVORmR0Q5X5Vf5u6
+eNsh5tYa
+5j7GJ8OCXgMVIcK7
+uJGHsW3cd073NGFITyQ
+G1u0pUmU6ehCm
+mk6lShdOa8kXT8i7mLd3fK
+u5C7glqT5XqtO0JE2686lk1
+h4omSc1jcLLwW
+tFY2ng51v
+vmAT10eeE47fgH20pLi
+uN803aW
+qqbDw46IgGds4
+32v414p63Jv1B4tO1xy
+73xdw4X
+d3o1712a03n20qvi62U7
+eQ80MW0h728I204P87YXc
+KHtD2A2hp6OjFgS73gdgE
+nI30tm7U55O0gI
+LSJtFA66
+mby00c
+meGb5
+pM6Gt05s1YJeii
+LR2AKy0dPt8vFdIV5760jriw
+1B3WMD5LSk65B2Moa
+xTlDv24JYv4s
+28Oe6r21yux7Lk47
+7wH3hBKdO55Xq3gEEe0
+5QLs0LVK1g
+ET3d4F2I4lV
 PREHOOK: query: select cfloat2, cint from orc_llap_n2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_llap_n2
@@ -230,6 +320,36 @@ NULL       -899422227
 11.0   385623629
 11.0   681126962
 11.0   25892751
+NULL   -838810013
+NULL   246423894
+NULL   708885482
+NULL   186967185
+NULL   -595277064
+NULL   584923170
+NULL   518213127
+NULL   -334595454
+NULL   241008004
+NULL   185212032
+NULL   -738747840
+NULL   -971543377
+NULL   940448896
+NULL   -324030556
+NULL   -899422227
+11.0   835111400
+11.0   -775326158
+11.0   653630202
+11.0   779427499
+11.0   797003983
+11.0   31832752
+11.0   783790031
+11.0   -898241885
+11.0   NULL
+11.0   -646295381
+11.0   130912195
+11.0   -391573084
+11.0   385623629
+11.0   681126962
+11.0   25892751
 PREHOOK: query: select * from orc_llap_n2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_llap_n2
@@ -238,36 +358,66 @@ POSTHOOK: query: select * from orc_llap_n2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_llap_n2
 #### A masked pattern was here ####
--838810013     1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    N016jPED08o     NULL
-246423894      -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    Q1JAdUlCVORmR0Q5X5Vf5u6 NULL
-708885482      -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    eNsh5tYa        NULL
-186967185      -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    5j7GJ8OCXgMVIcK7        NULL
--595277064     -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    uJGHsW3cd073NGFITyQ     NULL
-584923170      -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    G1u0pUmU6ehCm   NULL
-518213127      -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    mk6lShdOa8kXT8i7mLd3fK  NULL
--334595454     -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    u5C7glqT5XqtO0JE2686lk1 NULL
-241008004      -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    h4omSc1jcLLwW   NULL
-185212032      -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    tFY2ng51v       NULL
--738747840     -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    vmAT10eeE47fgH20pLi     NULL
--971543377     -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    uN803aW NULL
-940448896      -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    qqbDw46IgGds4   NULL
--324030556     -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    32v414p63Jv1B4tO1xy     NULL
--899422227     -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    73xdw4X NULL
-835111400      1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    d3o1712a03n20qvi62U7    11.0
--775326158     -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    eQ80MW0h728I204P87YXc   11.0
-653630202      1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    KHtD2A2hp6OjFgS73gdgE   11.0
-779427499      1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    nI30tm7U55O0gI  11.0
-797003983      1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    LSJtFA66        11.0
-31832752       1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    mby00c  11.0
-783790031      -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    meGb5   11.0
--898241885     -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    pM6Gt05s1YJeii  11.0
-NULL   -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    LR2AKy0dPt8vFdIV5760jriw        11.0
--646295381     -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    1B3WMD5LSk65B2Moa       11.0
-130912195      -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    xTlDv24JYv4s    11.0
--391573084     -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    28Oe6r21yux7Lk47        11.0
-385623629      236101682       11.0    NULL    385623629       236101682       11.0    NULL    385623629       236101682       11.0    NULL    7wH3hBKdO55Xq3gEEe0     11.0
-681126962      993392163       11.0    NULL    681126962       993392163       11.0    NULL    681126962       993392163       11.0    NULL    5QLs0LVK1g      11.0
-25892751       -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    ET3d4F2I4lV     11.0
+-838810013     1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    N016jPED08o     NULL    3.321   9.9876543210
+246423894      -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    Q1JAdUlCVORmR0Q5X5Vf5u6 NULL    3.321   9.9876543210
+708885482      -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    eNsh5tYa        NULL    3.321   9.9876543210
+186967185      -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    5j7GJ8OCXgMVIcK7        NULL    3.321   9.9876543210
+-595277064     -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    uJGHsW3cd073NGFITyQ     NULL    3.321   9.9876543210
+584923170      -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    G1u0pUmU6ehCm   NULL    3.321   9.9876543210
+518213127      -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    mk6lShdOa8kXT8i7mLd3fK  NULL    3.321   9.9876543210
+-334595454     -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    u5C7glqT5XqtO0JE2686lk1 NULL    3.321   9.9876543210
+241008004      -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    h4omSc1jcLLwW   NULL    3.321   9.9876543210
+185212032      -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    tFY2ng51v       NULL    3.321   9.9876543210
+-738747840     -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    vmAT10eeE47fgH20pLi     NULL    3.321   9.9876543210
+-971543377     -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    uN803aW NULL    3.321   9.9876543210
+940448896      -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    qqbDw46IgGds4   NULL    3.321   9.9876543210
+-324030556     -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    32v414p63Jv1B4tO1xy     NULL    3.321   9.9876543210
+-899422227     -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    73xdw4X NULL    3.321   9.9876543210
+835111400      1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    d3o1712a03n20qvi62U7    11.0    3.321   9.9876543210
+-775326158     -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    eQ80MW0h728I204P87YXc   11.0    3.321   9.9876543210
+653630202      1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    KHtD2A2hp6OjFgS73gdgE   11.0    3.321   9.9876543210
+779427499      1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    nI30tm7U55O0gI  11.0    3.321   9.9876543210
+797003983      1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    LSJtFA66        11.0    3.321   9.9876543210
+31832752       1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    mby00c  11.0    3.321   9.9876543210
+783790031      -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    meGb5   11.0    3.321   9.9876543210
+-898241885     -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    pM6Gt05s1YJeii  11.0    3.321   9.9876543210
+NULL   -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    LR2AKy0dPt8vFdIV5760jriw        11.0    3.321   9.9876543210
+-646295381     -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    1B3WMD5LSk65B2Moa       11.0    3.321   9.9876543210
+130912195      -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    xTlDv24JYv4s    11.0    3.321   9.9876543210
+-391573084     -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    28Oe6r21yux7Lk47        11.0    3.321   9.9876543210
+385623629      236101682       11.0    NULL    385623629       236101682       11.0    NULL    385623629       236101682       11.0    NULL    7wH3hBKdO55Xq3gEEe0     11.0    3.321   9.9876543210
+681126962      993392163       11.0    NULL    681126962       993392163       11.0    NULL    681126962       993392163       11.0    NULL    5QLs0LVK1g      11.0    3.321   9.9876543210
+25892751       -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    ET3d4F2I4lV     11.0    3.321   9.9876543210
+-838810013     1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    N016jPED08o     NULL    1.123   1.1234567890
+246423894      -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    Q1JAdUlCVORmR0Q5X5Vf5u6 NULL    1.123   1.1234567890
+708885482      -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    eNsh5tYa        NULL    1.123   1.1234567890
+186967185      -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    5j7GJ8OCXgMVIcK7        NULL    1.123   1.1234567890
+-595277064     -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    uJGHsW3cd073NGFITyQ     NULL    1.123   1.1234567890
+584923170      -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    G1u0pUmU6ehCm   NULL    1.123   1.1234567890
+518213127      -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    mk6lShdOa8kXT8i7mLd3fK  NULL    1.123   1.1234567890
+-334595454     -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    u5C7glqT5XqtO0JE2686lk1 NULL    1.123   1.1234567890
+241008004      -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    h4omSc1jcLLwW   NULL    1.123   1.1234567890
+185212032      -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    tFY2ng51v       NULL    1.123   1.1234567890
+-738747840     -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    vmAT10eeE47fgH20pLi     NULL    1.123   1.1234567890
+-971543377     -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    uN803aW NULL    1.123   1.1234567890
+940448896      -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    qqbDw46IgGds4   NULL    1.123   1.1234567890
+-324030556     -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    32v414p63Jv1B4tO1xy     NULL    1.123   1.1234567890
+-899422227     -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    73xdw4X NULL    1.123   1.1234567890
+835111400      1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    d3o1712a03n20qvi62U7    11.0    1.123   1.1234567890
+-775326158     -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    eQ80MW0h728I204P87YXc   11.0    1.123   1.1234567890
+653630202      1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    KHtD2A2hp6OjFgS73gdgE   11.0    1.123   1.1234567890
+779427499      1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    nI30tm7U55O0gI  11.0    1.123   1.1234567890
+797003983      1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    LSJtFA66        11.0    1.123   1.1234567890
+31832752       1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    mby00c  11.0    1.123   1.1234567890
+783790031      -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    meGb5   11.0    1.123   1.1234567890
+-898241885     -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    pM6Gt05s1YJeii  11.0    1.123   1.1234567890
+NULL   -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    LR2AKy0dPt8vFdIV5760jriw        11.0    1.123   1.1234567890
+-646295381     -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    1B3WMD5LSk65B2Moa       11.0    1.123   1.1234567890
+130912195      -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    xTlDv24JYv4s    11.0    1.123   1.1234567890
+-391573084     -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    28Oe6r21yux7Lk47        11.0    1.123   1.1234567890
+385623629      236101682       11.0    NULL    385623629       236101682       11.0    NULL    385623629       236101682       11.0    NULL    7wH3hBKdO55Xq3gEEe0     11.0    1.123   1.1234567890
+681126962      993392163       11.0    NULL    681126962       993392163       11.0    NULL    681126962       993392163       11.0    NULL    5QLs0LVK1g      11.0    1.123   1.1234567890
+25892751       -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    ET3d4F2I4lV     11.0    1.123   1.1234567890
 PREHOOK: query: select cstring1 from orc_llap2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_llap2
@@ -352,36 +502,36 @@ POSTHOOK: query: select * from orc_llap2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_llap2
 #### A masked pattern was here ####
-246423894      -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    Q1JAdUlCVORmR0Q5X5Vf5u6 NULL
-708885482      -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    eNsh5tYa        NULL
-186967185      -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    5j7GJ8OCXgMVIcK7        NULL
--595277064     -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    uJGHsW3cd073NGFITyQ     NULL
-584923170      -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    G1u0pUmU6ehCm   NULL
-518213127      -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    mk6lShdOa8kXT8i7mLd3fK  NULL
--334595454     -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    u5C7glqT5XqtO0JE2686lk1 NULL
-241008004      -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    h4omSc1jcLLwW   NULL
-185212032      -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    tFY2ng51v       NULL
--738747840     -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    vmAT10eeE47fgH20pLi     NULL
--971543377     -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    uN803aW NULL
-940448896      -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    qqbDw46IgGds4   NULL
--324030556     -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    32v414p63Jv1B4tO1xy     NULL
--899422227     -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    73xdw4X NULL
-835111400      1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    d3o1712a03n20qvi62U7    11.0
--775326158     -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    eQ80MW0h728I204P87YXc   11.0
-653630202      1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    KHtD2A2hp6OjFgS73gdgE   11.0
-779427499      1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    nI30tm7U55O0gI  11.0
-797003983      1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    LSJtFA66        11.0
-31832752       1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    mby00c  11.0
-783790031      -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    meGb5   11.0
--898241885     -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    pM6Gt05s1YJeii  11.0
-NULL   -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    LR2AKy0dPt8vFdIV5760jriw        11.0
--646295381     -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    1B3WMD5LSk65B2Moa       11.0
-130912195      -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    xTlDv24JYv4s    11.0
--391573084     -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    28Oe6r21yux7Lk47        11.0
-385623629      236101682       11.0    NULL    385623629       236101682       11.0    NULL    385623629       236101682       11.0    NULL    7wH3hBKdO55Xq3gEEe0     11.0
-681126962      993392163       11.0    NULL    681126962       993392163       11.0    NULL    681126962       993392163       11.0    NULL    5QLs0LVK1g      11.0
-25892751       -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    ET3d4F2I4lV     11.0
--838810013     1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    testvalue       NULL
+246423894      -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    246423894       -1645852809     NULL    NULL    Q1JAdUlCVORmR0Q5X5Vf5u6 NULL    1.123   1.1234567890
+708885482      -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    708885482       -1645852809     NULL    NULL    eNsh5tYa        NULL    1.123   1.1234567890
+186967185      -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    186967185       -1645852809     NULL    NULL    5j7GJ8OCXgMVIcK7        NULL    1.123   1.1234567890
+-595277064     -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    -595277064      -1645852809     NULL    NULL    uJGHsW3cd073NGFITyQ     NULL    1.123   1.1234567890
+584923170      -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    584923170       -1645852809     NULL    NULL    G1u0pUmU6ehCm   NULL    1.123   1.1234567890
+518213127      -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    518213127       -1645852809     NULL    NULL    mk6lShdOa8kXT8i7mLd3fK  NULL    1.123   1.1234567890
+-334595454     -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    -334595454      -1645852809     NULL    NULL    u5C7glqT5XqtO0JE2686lk1 NULL    1.123   1.1234567890
+241008004      -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    241008004       -1645852809     NULL    NULL    h4omSc1jcLLwW   NULL    1.123   1.1234567890
+185212032      -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    185212032       -1645852809     NULL    NULL    tFY2ng51v       NULL    1.123   1.1234567890
+-738747840     -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    -738747840      -1645852809     NULL    NULL    vmAT10eeE47fgH20pLi     NULL    1.123   1.1234567890
+-971543377     -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    -971543377      -1645852809     NULL    NULL    uN803aW NULL    1.123   1.1234567890
+940448896      -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    940448896       -1645852809     NULL    NULL    qqbDw46IgGds4   NULL    1.123   1.1234567890
+-324030556     -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    -324030556      -1645852809     NULL    NULL    32v414p63Jv1B4tO1xy     NULL    1.123   1.1234567890
+-899422227     -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    -899422227      -1645852809     NULL    NULL    73xdw4X NULL    1.123   1.1234567890
+835111400      1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    835111400       1964238982      11.0    NULL    d3o1712a03n20qvi62U7    11.0    1.123   1.1234567890
+-775326158     -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    -775326158      -1289793978     11.0    NULL    eQ80MW0h728I204P87YXc   11.0    1.123   1.1234567890
+653630202      1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    653630202       1281184487      11.0    NULL    KHtD2A2hp6OjFgS73gdgE   11.0    1.123   1.1234567890
+779427499      1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    779427499       1326393090      11.0    NULL    nI30tm7U55O0gI  11.0    1.123   1.1234567890
+797003983      1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    797003983       1186689849      11.0    NULL    LSJtFA66        11.0    1.123   1.1234567890
+31832752       1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    31832752        1854212271      11.0    NULL    mby00c  11.0    1.123   1.1234567890
+783790031      -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    783790031       -1482854823     11.0    NULL    meGb5   11.0    1.123   1.1234567890
+-898241885     -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    -898241885      -1785664982     11.0    NULL    pM6Gt05s1YJeii  11.0    1.123   1.1234567890
+NULL   -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    NULL    -1083386085     11.0    NULL    LR2AKy0dPt8vFdIV5760jriw        11.0    1.123   1.1234567890
+-646295381     -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    -646295381      -1654635859     11.0    NULL    1B3WMD5LSk65B2Moa       11.0    1.123   1.1234567890
+130912195      -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    130912195       -1286145901     11.0    NULL    xTlDv24JYv4s    11.0    1.123   1.1234567890
+-391573084     -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    -391573084      -236100834      11.0    NULL    28Oe6r21yux7Lk47        11.0    1.123   1.1234567890
+385623629      236101682       11.0    NULL    385623629       236101682       11.0    NULL    385623629       236101682       11.0    NULL    7wH3hBKdO55Xq3gEEe0     11.0    1.123   1.1234567890
+681126962      993392163       11.0    NULL    681126962       993392163       11.0    NULL    681126962       993392163       11.0    NULL    5QLs0LVK1g      11.0    1.123   1.1234567890
+25892751       -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    25892751        -1978674520     11.0    NULL    ET3d4F2I4lV     11.0    1.123   1.1234567890
+-838810013     1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    -838810013      1864027286      NULL    NULL    testvalue       NULL    3.321   9.9876543210
 PREHOOK: query: DROP TABLE orc_llap_n2
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@orc_llap_n2
index 37c213b..c4dc6f7 100644 (file)
@@ -118,8 +118,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -263,8 +263,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
@@ -372,8 +372,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: true
                 usesVectorUDFAdaptor: false
diff --git a/ql/src/test/results/clientpositive/llap/llap_decimal64_reader.q.out b/ql/src/test/results/clientpositive/llap/llap_decimal64_reader.q.out
new file mode 100644 (file)
index 0000000..0041206
--- /dev/null
@@ -0,0 +1,303 @@
+PREHOOK: query: DROP TABLE orc_llap_n0
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_llap_n0
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE orc_llap_n0(
+    ctinyint TINYINT,
+    csmallint SMALLINT,
+    cint INT,
+    cbigint BIGINT,
+    cfloat FLOAT,
+    cdouble DOUBLE,
+    cstring1 STRING,
+    cstring2 STRING,
+    ctimestamp1 TIMESTAMP,
+    ctimestamp2 TIMESTAMP,
+    cboolean1 BOOLEAN,
+    cboolean2 BOOLEAN,
+    cdecimal1 decimal(10,2),
+    cdecimal2 decimal(38,5))
+    STORED AS ORC tblproperties ("orc.compress"="NONE")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_llap_n0
+POSTHOOK: query: CREATE TABLE orc_llap_n0(
+    ctinyint TINYINT,
+    csmallint SMALLINT,
+    cint INT,
+    cbigint BIGINT,
+    cfloat FLOAT,
+    cdouble DOUBLE,
+    cstring1 STRING,
+    cstring2 STRING,
+    ctimestamp1 TIMESTAMP,
+    ctimestamp2 TIMESTAMP,
+    cboolean1 BOOLEAN,
+    cboolean2 BOOLEAN,
+    cdecimal1 decimal(10,2),
+    cdecimal2 decimal(38,5))
+    STORED AS ORC tblproperties ("orc.compress"="NONE")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_llap_n0
+PREHOOK: query: insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("3.345" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@orc_llap_n0
+POSTHOOK: query: insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("3.345" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@orc_llap_n0
+POSTHOOK: Lineage: orc_llap_n0.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cdecimal1 SIMPLE []
+POSTHOOK: Lineage: orc_llap_n0.cdecimal2 SIMPLE []
+POSTHOOK: Lineage: orc_llap_n0.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+PREHOOK: query: alter table orc_llap_n0 set tblproperties ("orc.compress"="NONE", 'orc.write.format'='UNSTABLE-PRE-2.0')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@orc_llap_n0
+PREHOOK: Output: default@orc_llap_n0
+POSTHOOK: query: alter table orc_llap_n0 set tblproperties ("orc.compress"="NONE", 'orc.write.format'='UNSTABLE-PRE-2.0')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@orc_llap_n0
+POSTHOOK: Output: default@orc_llap_n0
+PREHOOK: query: insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("4.456" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@orc_llap_n0
+POSTHOOK: query: insert into table orc_llap_n0
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2,
+ cast("4.456" as decimal(10,2)), cast("5.56789" as decimal(38,5)) from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@orc_llap_n0
+POSTHOOK: Lineage: orc_llap_n0.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cdecimal1 SIMPLE []
+POSTHOOK: Lineage: orc_llap_n0.cdecimal2 SIMPLE []
+POSTHOOK: Lineage: orc_llap_n0.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: orc_llap_n0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+PREHOOK: query: explain vectorization select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: orc_llap_n0
+                  filterExpr: ((cdecimal1 = 3.35) or (cdecimal1 = 4.46)) (type: boolean)
+                  Statistics: Num rows: 24576 Data size: 5505024 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: ((cdecimal1 = 3.35) or (cdecimal1 = 4.46)) (type: boolean)
+                    Statistics: Num rows: 24576 Data size: 5505024 Basic stats: COMPLETE Column stats: COMPLETE
+                    Group By Operator
+                      keys: cdecimal1 (type: decimal(10,2)), cdecimal2 (type: decimal(38,5))
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: decimal(10,2)), _col1 (type: decimal(38,5))
+                        sort order: ++
+                        Map-reduce partition columns: _col0 (type: decimal(10,2)), _col1 (type: decimal(38,5))
+                        Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                        TopN Hash Memory Usage: 0.1
+            Execution mode: vectorized, llap
+            LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+        Reducer 2 
+            Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: decimal(10,2)), KEY._col1 (type: decimal(38,5))
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                Limit
+                  Number of rows: 2
+                  Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 2
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_llap_n0
+#### A masked pattern was here ####
+POSTHOOK: query: select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_llap_n0
+#### A masked pattern was here ####
+4.46   5.56789
+3.35   5.56789
+PREHOOK: query: explain vectorization select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+  enabled: true
+  enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: orc_llap_n0
+                  filterExpr: ((cdecimal1 = 3.35) or (cdecimal1 = 4.46)) (type: boolean)
+                  Statistics: Num rows: 24576 Data size: 5505024 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: ((cdecimal1 = 3.35) or (cdecimal1 = 4.46)) (type: boolean)
+                    Statistics: Num rows: 24576 Data size: 5505024 Basic stats: COMPLETE Column stats: COMPLETE
+                    Group By Operator
+                      keys: cdecimal1 (type: decimal(10,2)), cdecimal2 (type: decimal(38,5))
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: decimal(10,2)), _col1 (type: decimal(38,5))
+                        sort order: ++
+                        Map-reduce partition columns: _col0 (type: decimal(10,2)), _col1 (type: decimal(38,5))
+                        Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                        TopN Hash Memory Usage: 0.1
+            Execution mode: vectorized, llap
+            LLAP IO: all inputs
+            Map Vectorization:
+                enabled: true
+                enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+                inputFormatFeatureSupport: [DECIMAL_64]
+                vectorizationSupportRemovedReasons: [[] is disabled because it is not in hive.vectorized.input.format.supports.enabled []]
+                featureSupportInUse: []
+                inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+        Reducer 2 
+            Execution mode: vectorized, llap
+            Reduce Vectorization:
+                enabled: true
+                enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+                allNative: false
+                usesVectorUDFAdaptor: false
+                vectorized: true
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: decimal(10,2)), KEY._col1 (type: decimal(38,5))
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                Limit
+                  Number of rows: 2
+                  Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 2
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_llap_n0
+#### A masked pattern was here ####
+POSTHOOK: query: select cdecimal1,cdecimal2 from orc_llap_n0 where cdecimal1 = cast("3.345" as decimal(10,2)) or cdecimal1 = cast("4.456" as decimal(10,2))
+ group by cdecimal1,cdecimal2 limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_llap_n0
+#### A masked pattern was here ####
+4.46   5.56789
+3.35   5.56789
+PREHOOK: query: DROP TABLE orc_llap_n0
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_llap_n0
+PREHOOK: Output: default@orc_llap_n0
+POSTHOOK: query: DROP TABLE orc_llap_n0
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_llap_n0
+POSTHOOK: Output: default@orc_llap_n0
index faab23c..e6fa1ac 100644 (file)
@@ -1679,8 +1679,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: false
                 usesVectorUDFAdaptor: false
@@ -1745,8 +1745,8 @@ STAGE PLANS:
             Map Vectorization:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
-                inputFormatFeatureSupport: []
-                featureSupportInUse: []
+                inputFormatFeatureSupport: [DECIMAL_64]
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 allNative: false
                 usesVectorUDFAdaptor: false
@@ -2127,8 +2127,7 @@ STAGE PLANS:
                 enabled: true
                 enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
                 inputFormatFeatureSupport: [DECIMAL_64]
-                vectorizationSupportRemovedReasons: [DECIMAL_64 disabled because LLAP is enabled]
-                featureSupportInUse: []
+                featureSupportInUse: [DECIMAL_64]
                 inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
                 allNative: false
                 usesVectorUDFAdaptor: true
diff --git a/ql/src/test/results/clientpositive/llap/llap_text.q.out b/ql/src/test/results/clientpositive/llap/llap_text.q.out
new file mode 100644 (file)
index 0000000..40d08d3
--- /dev/null
@@ -0,0 +1,1082 @@
+PREHOOK: query: DROP TABLE text_llap
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE text_llap
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE text_llap(
+    ctinyint TINYINT,
+    csmallint SMALLINT,
+    cint INT,
+    cbigint BIGINT,
+    cfloat FLOAT,
+    cdouble DOUBLE,
+    cstring1 STRING,
+    cstring2 STRING,
+    ctimestamp1 TIMESTAMP,
+    ctimestamp2 TIMESTAMP,
+    cboolean1 BOOLEAN,
+    cboolean2 BOOLEAN)
+row format serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+stored as inputformat "org.apache.hadoop.mapred.TextInputFormat" 
+
+ outputformat "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@text_llap
+POSTHOOK: query: CREATE TABLE text_llap(
+    ctinyint TINYINT,
+    csmallint SMALLINT,
+    cint INT,
+    cbigint BIGINT,
+    cfloat FLOAT,
+    cdouble DOUBLE,
+    cstring1 STRING,
+    cstring2 STRING,
+    ctimestamp1 TIMESTAMP,
+    ctimestamp2 TIMESTAMP,
+    cboolean1 BOOLEAN,
+    cboolean2 BOOLEAN)
+row format serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+stored as inputformat "org.apache.hadoop.mapred.TextInputFormat" 
+
+ outputformat "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@text_llap
+PREHOOK: query: insert into table text_llap
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc 
+where cboolean2 is not null or cstring1 is not null or ctinyint is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@text_llap
+POSTHOOK: query: insert into table text_llap
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc 
+where cboolean2 is not null or cstring1 is not null or ctinyint is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@text_llap
+POSTHOOK: Lineage: text_llap.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: text_llap.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: text_llap.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ]
+POSTHOOK: Lineage: text_llap.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: text_llap.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: text_llap.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: text_llap.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: text_llap.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: text_llap.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
+POSTHOOK: Lineage: text_llap.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: text_llap.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: text_llap.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+PREHOOK: query: create table text_llap2(
+          t tinyint,
+          si smallint,
+          i int,
+          b bigint,
+          f float,
+          d double,
+          bo boolean,
+          s string,
+          ts timestamp, 
+          `dec` decimal,  
+          bin binary)
+row format delimited fields terminated by '|'
+stored as inputformat "org.apache.hadoop.mapred.TextInputFormat" 
+
+outputformat "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@text_llap2
+POSTHOOK: query: create table text_llap2(
+          t tinyint,
+          si smallint,
+          i int,
+          b bigint,
+          f float,
+          d double,
+          bo boolean,
+          s string,
+          ts timestamp, 
+          `dec` decimal,  
+          bin binary)
+row format delimited fields terminated by '|'
+stored as inputformat "org.apache.hadoop.mapred.TextInputFormat" 
+
+outputformat "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@text_llap2
+PREHOOK: query: load data local inpath '../../data/files/over10k.gz' into table text_llap2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@text_llap2
+POSTHOOK: query: load data local inpath '../../data/files/over10k.gz' into table text_llap2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@text_llap2
+PREHOOK: query: create table text_llap1 like text_llap
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@text_llap1
+POSTHOOK: query: create table text_llap1 like text_llap
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@text_llap1
+PREHOOK: query: create table text_llap100 like text_llap
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@text_llap100
+POSTHOOK: query: create table text_llap100 like text_llap
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@text_llap100
+PREHOOK: query: create table text_llap1000 like text_llap
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@text_llap1000
+POSTHOOK: query: create table text_llap1000 like text_llap
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@text_llap1000
+PREHOOK: query: insert into table text_llap1
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc 
+where cboolean2 is not null or cstring1 is not null or ctinyint is not null limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@text_llap1
+POSTHOOK: query: insert into table text_llap1
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc 
+where cboolean2 is not null or cstring1 is not null or ctinyint is not null limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@text_llap1
+POSTHOOK: Lineage: text_llap1.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: text_llap1.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: text_llap1.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
+POSTHOOK: Lineage: text_llap1.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: text_llap1.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: text_llap1.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+PREHOOK: query: insert into table text_llap100
+select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboo