AMBARI-21581 - Replace Hard Coded conf-select Structures (jonathanhurley)
[ambari.git] / ambari-server / src / main / resources / common-services / HIVE / 0.12.0.2.0 / package / scripts / hive_metastore.py
1 #!/usr/bin/env python
2 """
3 Licensed to the Apache Software Foundation (ASF) under one
4 or more contributor license agreements. See the NOTICE file
5 distributed with this work for additional information
6 regarding copyright ownership. The ASF licenses this file
7 to you under the Apache License, Version 2.0 (the
8 "License"); you may not use this file except in compliance
9 with the License. You may obtain a copy of the License at
10
11 http://www.apache.org/licenses/LICENSE-2.0
12
13 Unless required by applicable law or agreed to in writing, software
14 distributed under the License is distributed on an "AS IS" BASIS,
15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 See the License for the specific language governing permissions and
17 limitations under the License.
18
19 """
20 import os
21
22 from resource_management.core.logger import Logger
23 from resource_management.core.resources.system import Execute, Directory
24 from resource_management.libraries.script import Script
25 from resource_management.libraries.functions import stack_select
26 from resource_management.libraries.functions.constants import Direction
27 from resource_management.libraries.functions.format import format
28 from resource_management.libraries.functions.version import format_stack_version
29 from resource_management.libraries.functions import StackFeature
30 from resource_management.libraries.functions.stack_features import check_stack_feature
31 from resource_management.libraries.functions.security_commons import build_expectations
32 from resource_management.libraries.functions.security_commons import cached_kinit_executor
33 from resource_management.libraries.functions.security_commons import get_params_from_filesystem
34 from resource_management.libraries.functions.security_commons import validate_security_config_properties
35 from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
36 from resource_management.core.resources.system import File
37 from setup_ranger_hive import setup_ranger_hive_metastore_service
38
39 from hive import create_metastore_schema, hive, jdbc_connector
40 from hive_service import hive_service
41 from ambari_commons.os_family_impl import OsFamilyImpl
42 from ambari_commons import OSConst
43
44 # the legacy conf.server location in previous stack versions
45 LEGACY_HIVE_SERVER_CONF = "/etc/hive/conf.server"
46
47 class HiveMetastore(Script):
48 def install(self, env):
49 import params
50 self.install_packages(env)
51
52
53 def start(self, env, upgrade_type=None):
54 import params
55 env.set_params(params)
56
57 # writing configurations on start required for securtity
58 self.configure(env)
59 if params.init_metastore_schema:
60 create_metastore_schema() # execute without config lock
61
62 hive_service('metastore', action='start', upgrade_type=upgrade_type)
63
64 # below function call is used for cluster depolyed in cloud env to create ranger hive service in ranger admin.
65 setup_ranger_hive_metastore_service()
66
67 def stop(self, env, upgrade_type=None):
68 import params
69 env.set_params(params)
70 hive_service('metastore', action='stop', upgrade_type=upgrade_type)
71
72
73 def configure(self, env):
74 import params
75 env.set_params(params)
76 hive(name = 'metastore')
77
78
79 @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
80 class HiveMetastoreWindows(HiveMetastore):
81 def status(self, env):
82 import status_params
83 from resource_management.libraries.functions import check_windows_service_status
84 check_windows_service_status(status_params.hive_metastore_win_service_name)
85
86
87 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
88 class HiveMetastoreDefault(HiveMetastore):
89 def status(self, env):
90 import status_params
91 from resource_management.libraries.functions import check_process_status
92 env.set_params(status_params)
93
94 # Recursively check all existing gmetad pid files
95 check_process_status(status_params.hive_metastore_pid)
96
97
98 def pre_upgrade_restart(self, env, upgrade_type=None):
99 Logger.info("Executing Metastore Stack Upgrade pre-restart")
100 import params
101
102 env.set_params(params)
103
104 is_upgrade = params.upgrade_direction == Direction.UPGRADE
105
106 if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
107 stack_select.select_packages(params.version)
108
109 if is_upgrade and params.stack_version_formatted_major and \
110 check_stack_feature(StackFeature.HIVE_METASTORE_UPGRADE_SCHEMA, params.stack_version_formatted_major):
111 self.upgrade_schema(env)
112
113 def upgrade_schema(self, env):
114 """
115 Executes the schema upgrade binary. This is its own function because it could
116 be called as a standalone task from the upgrade pack, but is safe to run it for each
117 metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
118
119 The metastore schema upgrade requires a database driver library for most
120 databases. During an upgrade, it's possible that the library is not present,
121 so this will also attempt to copy/download the appropriate driver.
122
123 This function will also ensure that configurations are written out to disk before running
124 since the new configs will most likely not yet exist on an upgrade.
125
126 Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
127 """
128 Logger.info("Upgrading Hive Metastore Schema")
129 import status_params
130 import params
131 env.set_params(params)
132
133 # ensure that configurations are written out before trying to upgrade the schema
134 # since the schematool needs configs and doesn't know how to use the hive conf override
135 self.configure(env)
136
137 if params.security_enabled:
138 cached_kinit_executor(status_params.kinit_path_local,
139 status_params.hive_user,
140 params.hive_metastore_keytab_path,
141 params.hive_metastore_principal,
142 status_params.hostname,
143 status_params.tmp_dir)
144
145 # ensure that the JDBC drive is present for the schema tool; if it's not
146 # present, then download it first
147 if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
148 target_directory = format("{stack_root}/{version}/hive/lib")
149
150 # download it if it does not exist
151 if not os.path.exists(params.source_jdbc_file):
152 jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar)
153
154 target_directory_and_filename = os.path.join(target_directory, os.path.basename(params.source_jdbc_file))
155
156 if params.sqla_db_used:
157 target_native_libs_directory = format("{target_directory}/native/lib64")
158
159 Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}"))
160
161 Directory(target_native_libs_directory, create_parents = True)
162
163 Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"))
164
165 Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
166 else:
167 # copy the JDBC driver from the older metastore location to the new location only
168 # if it does not already exist
169 if not os.path.exists(target_directory_and_filename):
170 Execute(('cp', params.source_jdbc_file, target_directory),
171 path=["/bin", "/usr/bin/"], sudo = True)
172
173 File(target_directory_and_filename, mode = 0644)
174
175 # build the schema tool command
176 binary = format("{hive_schematool_ver_bin}/schematool")
177
178 # the conf.server directory changed locations between stack versions
179 # since the configurations have not been written out yet during an upgrade
180 # we need to choose the original legacy location
181 schematool_hive_server_conf_dir = params.hive_server_conf_dir
182 if params.current_version is not None:
183 current_version = format_stack_version(params.current_version)
184 if not(check_stack_feature(StackFeature.CONFIG_VERSIONING, current_version)):
185 schematool_hive_server_conf_dir = LEGACY_HIVE_SERVER_CONF
186
187 env_dict = {
188 'HIVE_CONF_DIR': schematool_hive_server_conf_dir
189 }
190
191 command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
192 Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
193
194 def get_log_folder(self):
195 import params
196 return params.hive_log_dir
197
198 def get_user(self):
199 import params
200 return params.hive_user
201
202 def get_pid_files(self):
203 import status_params
204 return [status_params.hive_metastore_pid]
205
206
207 if __name__ == "__main__":
208 HiveMetastore().execute()