HIVE-18192: Introduce WriteID per table rather than using global transaction ID ...
authorSankar Hariappan <sankarh@apache.org>
Fri, 23 Feb 2018 16:30:23 +0000 (22:00 +0530)
committerSankar Hariappan <sankarh@apache.org>
Fri, 23 Feb 2018 16:30:23 +0000 (22:00 +0530)
commitcbb9233a3b39ab8489d777fc76f0758c49b69bef
treebaa74539df373ba9ac4000da2e88153741a32881
parentf9768af0e147539c52311643c8bcaa778483e051
HIVE-18192: Introduce WriteID per table rather than using global transaction ID (Sankar Hariappan, reviewed by Eugene Koifman)
169 files changed:
common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/RecordWriter.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictJsonWriter.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/StrictRegexWriter.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/TransactionBatch.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/AcidTable.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/AcidTableSerializer.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/client/MutatorClient.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorCoordinator.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorFactory.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorImpl.java
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/SequenceValidator.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/ReflectiveMutatorFactory.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/StreamingAssert.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/TestAcidTableSerializer.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/client/TestMutatorClient.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestMutatorCoordinator.java
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestMutatorImpl.java
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
metastore/scripts/upgrade/derby/044-HIVE-16997.derby.sql
metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql [new file with mode: 0644]
metastore/scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql
metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql
ql/src/java/org/apache/hadoop/hive/ql/Driver.java
ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitTask.java
ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitWork.java
ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/RecordIdentifier.java
ql/src/java/org/apache/hadoop/hive/ql/io/RecordUpdater.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
ql/src/java/org/apache/hadoop/hive/ql/plan/FileMergeDesc.java
ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java
ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java
ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java
ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
ql/src/test/results/clientpositive/acid_nullscan.q.out
ql/src/test/results/clientpositive/acid_table_stats.q.out
ql/src/test/results/clientpositive/autoColumnStats_4.q.out
ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
ql/src/test/results/clientpositive/row__id.q.out
standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddDynamicPartitions.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsRequest.java [new file with mode: 0644]
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsResponse.java [new file with mode: 0644]
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClearFileMetadataRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapabilities.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CreationMetadata.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FireEventRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetAllFunctionsResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetFileMetadataByExprRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetFileMetadataByExprResult.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetFileMetadataRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetFileMetadataResult.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetTablesRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetTablesResult.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetValidWriteIdsRequest.java [new file with mode: 0644]
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetValidWriteIdsResponse.java [new file with mode: 0644]
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/HeartbeatTxnRangeResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/InsertEventRequestData.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Materialization.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PutFileMetadataRequest.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowLocksResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TableValidWriteIds.java [new file with mode: 0644]
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnToWriteId.java [new file with mode: 0644]
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMGetAllResourcePlanResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMGetTriggersForResourePlanResponse.java
standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMValidateResourcePlanResponse.java
standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql
standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql
standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql
standalone-metastore/src/main/thrift/hive_metastore.thrift
storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorWriteIdList.java [moved from storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java with 52% similarity]
storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
storage-api/src/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java [new file with mode: 0644]
storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java [new file with mode: 0644]
storage-api/src/java/org/apache/hadoop/hive/common/ValidWriteIdList.java [new file with mode: 0644]
storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java [deleted file]
storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorWriteIdList.java [new file with mode: 0644]
storage-api/src/test/org/apache/hadoop/hive/common/TestValidReaderWriteIdList.java [new file with mode: 0644]