AMBARI-21581 - Replace Hard Coded conf-select Structures (jonathanhurley)
[ambari.git] / ambari-common / src / main / python / resource_management / libraries / functions / stack_select.py
1 #!/usr/bin/env python
2 """
3 Licensed to the Apache Software Foundation (ASF) under one
4 or more contributor license agreements. See the NOTICE file
5 distributed with this work for additional information
6 regarding copyright ownership. The ASF licenses this file
7 to you under the Apache License, Version 2.0 (the
8 "License"); you may not use this file except in compliance
9 with the License. You may obtain a copy of the License at
10
11 http://www.apache.org/licenses/LICENSE-2.0
12
13 Unless required by applicable law or agreed to in writing, software
14 distributed under the License is distributed on an "AS IS" BASIS,
15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 See the License for the specific language governing permissions and
17 limitations under the License.
18
19 """
20
21 # Python Imports
22 import os
23 import sys
24 import re
25 import ambari_simplejson as json
26
27 # Local Imports
28 from resource_management.core.logger import Logger
29 from resource_management.core.exceptions import Fail
30 from resource_management.core.resources.system import Execute
31 from resource_management.libraries.functions.default import default
32 from resource_management.libraries.functions.get_stack_version import get_stack_version
33 from resource_management.libraries.functions.format import format
34 from resource_management.libraries.script.script import Script
35 from resource_management.libraries.functions import stack_tools
36 from resource_management.core.shell import call
37 from resource_management.libraries.functions.version import format_stack_version
38 from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
39 from resource_management.libraries.functions.stack_features import check_stack_feature
40 from resource_management.libraries.functions import StackFeature
41
42 STACK_SELECT_PREFIX = 'ambari-python-wrap'
43
44 # mapping of service check to <stack-selector-tool> component
45 SERVICE_CHECK_DIRECTORY_MAP = {
46 "HDFS_SERVICE_CHECK" : "hadoop-client",
47 "TEZ_SERVICE_CHECK" : "hadoop-client",
48 "PIG_SERVICE_CHECK" : "hadoop-client",
49 "HIVE_SERVICE_CHECK" : "hadoop-client",
50 "OOZIE_SERVICE_CHECK" : "hadoop-client",
51 "MAHOUT_SERVICE_CHECK" : "mahout-client",
52 "MAPREDUCE2_SERVICE_CHECK" : "hadoop-client",
53 "YARN_SERVICE_CHECK" : "hadoop-client",
54 "SLIDER_SERVICE_CHECK" : "slider-client"
55 }
56
57 # <stack-root>/current/hadoop-client/[bin|sbin|libexec|lib]
58 # <stack-root>/2.3.0.0-1234/hadoop/[bin|sbin|libexec|lib]
59 HADOOP_DIR_TEMPLATE = "{0}/{1}/{2}/{3}"
60
61 # <stack-root>/current/hadoop-client
62 # <stack-root>/2.3.0.0-1234/hadoop
63 HADOOP_HOME_DIR_TEMPLATE = "{0}/{1}/{2}"
64
65 HADOOP_DIR_DEFAULTS = {
66 "home": "/usr/lib/hadoop",
67 "libexec": "/usr/lib/hadoop/libexec",
68 "sbin": "/usr/lib/hadoop/sbin",
69 "bin": "/usr/bin",
70 "lib": "/usr/lib/hadoop/lib"
71 }
72
73 PACKAGE_SCOPE_INSTALL = "INSTALL"
74 PACKAGE_SCOPE_STANDARD = "STANDARD"
75 PACKAGE_SCOPE_PATCH = "PATCH"
76 PACKAGE_SCOPE_STACK_SELECT = "STACK-SELECT-PACKAGE"
77 _PACKAGE_SCOPES = (PACKAGE_SCOPE_INSTALL, PACKAGE_SCOPE_STANDARD, PACKAGE_SCOPE_PATCH, PACKAGE_SCOPE_STACK_SELECT)
78
79
80 def get_package_name(default_package = None):
81 """
82 Gets the stack-select package name for the service name and
83 component from the current command. Not all services/components are used with the
84 stack-select tools, so those will return no packages.
85
86 :return: the stack-select package name for the command's component or None
87 """
88 config = Script.get_config()
89
90 if 'role' not in config or 'serviceName' not in config:
91 raise Fail("Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool")
92
93 service_name = config['serviceName']
94 component_name = config['role']
95
96 # should return a single item
97 try:
98 package = get_packages(PACKAGE_SCOPE_STACK_SELECT, service_name, component_name)
99 if package is None:
100 package = default_package
101
102 return package
103 except:
104 if default_package is not None:
105 return default_package
106 else:
107 raise
108
109
110
111 def get_packages(scope, service_name = None, component_name = None):
112 """
113 Gets the packages which should be used with the stack's stack-select tool for the
114 specified service/component. Not all services/components are used with the stack-select tools,
115 so those will return no packages.
116
117 :param scope: the scope of the command
118 :param service_name: the service name, such as ZOOKEEPER
119 :param component_name: the component name, such as ZOOKEEPER_SERVER
120 :return: the packages to use with stack-select or None
121 """
122 from resource_management.libraries.functions.default import default
123
124 if scope not in _PACKAGE_SCOPES:
125 raise Fail("The specified scope of {0} is not valid".format(scope))
126
127 config = Script.get_config()
128
129 if service_name is None or component_name is None:
130 if 'role' not in config or 'serviceName' not in config:
131 raise Fail("Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool")
132
133 service_name = config['serviceName']
134 component_name = config['role']
135
136
137 stack_name = default("/hostLevelParams/stack_name", None)
138 if stack_name is None:
139 raise Fail("The stack name is not present in the command. Packages for stack-select tool cannot be loaded.")
140
141 stack_packages_config = default("/configurations/cluster-env/stack_packages", None)
142 if stack_packages_config is None:
143 raise Fail("The stack packages are not defined on the command. Unable to load packages for the stack-select tool")
144
145 data = json.loads(stack_packages_config)
146
147 if stack_name not in data:
148 raise Fail(
149 "Cannot find stack-select packages for the {0} stack".format(stack_name))
150
151 stack_select_key = "stack-select"
152 data = data[stack_name]
153 if stack_select_key not in data:
154 raise Fail(
155 "There are no stack-select packages defined for this command for the {0} stack".format(stack_name))
156
157 # this should now be the dictionary of role name to package name
158 data = data[stack_select_key]
159 service_name = service_name.upper()
160 component_name = component_name.upper()
161
162 if service_name not in data:
163 Logger.info("Skipping stack-select on {0} because it does not exist in the stack-select package structure.".format(service_name))
164 return None
165
166 data = data[service_name]
167
168 if component_name not in data:
169 Logger.info("Skipping stack-select on {0} because it does not exist in the stack-select package structure.".format(component_name))
170 return None
171
172 return data[component_name][scope]
173
174
175 def select_all(version_to_select):
176 """
177 Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a
178 stack version such as "2.3", then this will find the latest installed version which
179 could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will use
180 that exact version.
181 :param version_to_select: the version to <stack-selector-tool> on, such as "2.3" or "2.3.0.0-1234"
182 """
183 stack_root = Script.get_stack_root()
184 (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
185 # it's an error, but it shouldn't really stop anything from working
186 if version_to_select is None:
187 Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified"))
188 return
189
190 Logger.info("Executing {0} set all on {1}".format(stack_selector_name, version_to_select))
191
192 command = format('{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`')
193 only_if_command = format('ls -d {stack_root}/{version_to_select}*')
194 Execute(command, only_if = only_if_command)
195
196
197 def select_packages(version):
198 """
199 Uses the command's service and role to determine the stack-select packages which need to be invoked.
200 :param version: the version to select
201 :return: None
202 """
203 stack_select_packages = get_packages(PACKAGE_SCOPE_STANDARD)
204 if stack_select_packages is None:
205 return
206
207 for stack_select_package_name in stack_select_packages:
208 select(stack_select_package_name, version)
209
210
211 def select(component, version):
212 """
213 Executes <stack-selector-tool> on the specific component and version. Some global
214 variables that are imported via params/status_params/params_linux will need
215 to be recalcuated after the <stack-selector-tool>. However, python does not re-import
216 existing modules. The only way to ensure that the configuration variables are
217 recalculated is to call reload(...) on each module that has global parameters.
218 After invoking <stack-selector-tool>, this function will also reload params, status_params,
219 and params_linux.
220 :param component: the <stack-selector-tool> component, such as oozie-server. If "all", then all components
221 will be updated.
222 :param version: the version to set the component to, such as 2.2.0.0-1234
223 """
224 stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
225 command = (STACK_SELECT_PREFIX, stack_selector_path, "set", component, version)
226 Execute(command, sudo=True)
227
228 # don't trust the ordering of modules:
229 # 1) status_params
230 # 2) params_linux
231 # 3) params
232 modules = sys.modules
233 param_modules = "status_params", "params_linux", "params"
234 for moduleName in param_modules:
235 if moduleName in modules:
236 module = modules.get(moduleName)
237 reload(module)
238 Logger.info("After {0}, reloaded module {1}".format(command, moduleName))
239
240
241 def get_role_component_current_stack_version():
242 """
243 Gets the current HDP version of the component that this role command is for.
244 :return: the current HDP version of the specified component or None
245 """
246 role = default("/role", "")
247 role_command = default("/roleCommand", "")
248
249 stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
250 Logger.info("Checking version for {0} via {1}".format(role, stack_selector_name))
251 if role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
252 stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]
253 else:
254 stack_select_component = get_package_name()
255
256 if stack_select_component is None:
257 if not role:
258 Logger.error("No role information available.")
259 elif not role.lower().endswith("client"):
260 Logger.error("Mapping unavailable for role {0}. Skip checking its version.".format(role))
261 return None
262
263 current_stack_version = get_stack_version(stack_select_component)
264
265 if current_stack_version is None:
266 Logger.warning("Unable to determine {0} version for {1}".format(
267 stack_selector_name, stack_select_component))
268 else:
269 Logger.info("{0} is currently at version {1}".format(
270 stack_select_component, current_stack_version))
271
272 return current_stack_version
273
274
275 def get_hadoop_dir(target, force_latest_on_upgrade=False):
276 """
277 Return the hadoop shared directory in the following override order
278 1. Use default for 2.1 and lower
279 2. If 2.2 and higher, use <stack-root>/current/hadoop-client/{target}
280 3. If 2.2 and higher AND for an upgrade, use <stack-root>/<version>/hadoop/{target}.
281 However, if the upgrade has not yet invoked <stack-selector-tool>, return the current
282 version of the component.
283 :target: the target directory
284 :force_latest_on_upgrade: if True, then this will return the "current" directory
285 without the stack version built into the path, such as <stack-root>/current/hadoop-client
286 """
287 stack_root = Script.get_stack_root()
288 stack_version = Script.get_stack_version()
289
290 if not target in HADOOP_DIR_DEFAULTS:
291 raise Fail("Target {0} not defined".format(target))
292
293 hadoop_dir = HADOOP_DIR_DEFAULTS[target]
294
295 formatted_stack_version = format_stack_version(stack_version)
296 if formatted_stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, formatted_stack_version):
297 # home uses a different template
298 if target == "home":
299 hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client")
300 else:
301 hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client", target)
302
303 # if we are not forcing "current" for HDP 2.2, then attempt to determine
304 # if the exact version needs to be returned in the directory
305 if not force_latest_on_upgrade:
306 stack_info = _get_upgrade_stack()
307
308 if stack_info is not None:
309 stack_version = stack_info[1]
310
311 # determine if <stack-selector-tool> has been run and if not, then use the current
312 # hdp version until this component is upgraded
313 current_stack_version = get_role_component_current_stack_version()
314 if current_stack_version is not None and stack_version != current_stack_version:
315 stack_version = current_stack_version
316
317 if target == "home":
318 # home uses a different template
319 hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop")
320 else:
321 hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop", target)
322
323 return hadoop_dir
324
325 def get_hadoop_dir_for_stack_version(target, stack_version):
326 """
327 Return the hadoop shared directory for the provided stack version. This is necessary
328 when folder paths of downgrade-source stack-version are needed after <stack-selector-tool>.
329 :target: the target directory
330 :stack_version: stack version to get hadoop dir for
331 """
332
333 stack_root = Script.get_stack_root()
334 if not target in HADOOP_DIR_DEFAULTS:
335 raise Fail("Target {0} not defined".format(target))
336
337 hadoop_dir = HADOOP_DIR_DEFAULTS[target]
338
339 formatted_stack_version = format_stack_version(stack_version)
340 if formatted_stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, formatted_stack_version):
341 # home uses a different template
342 if target == "home":
343 hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop")
344 else:
345 hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop", target)
346
347 return hadoop_dir
348
349
350 def _get_upgrade_stack():
351 """
352 Gets the stack name and stack version if an upgrade is currently in progress.
353 :return: the stack name and stack version as a tuple, or None if an
354 upgrade is not in progress.
355 """
356 from resource_management.libraries.functions.default import default
357 direction = default("/commandParams/upgrade_direction", None)
358 stack_name = default("/hostLevelParams/stack_name", None)
359 stack_version = default("/commandParams/version", None)
360
361 if direction and stack_name and stack_version:
362 return (stack_name, stack_version)
363
364 return None
365
366 def unsafe_get_stack_versions():
367 """
368 Gets list of stack versions installed on the host.
369 By default a call to <stack-selector-tool> versions is made to get the list of installed stack versions.
370 DO NOT use a fall-back since this function is called by alerts in order to find potential errors.
371 :return: Returns a tuple of (exit code, output, list of installed stack versions).
372 """
373 stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
374 code, out = call((STACK_SELECT_PREFIX, stack_selector_path, 'versions'))
375 versions = []
376 if 0 == code:
377 for line in out.splitlines():
378 versions.append(line.rstrip('\n'))
379 return (code, out, versions)
380
381 def get_stack_versions(stack_root):
382 """
383 Gets list of stack versions installed on the host.
384 By default a call to <stack-selector-tool> versions is made to get the list of installed stack versions.
385 As a fallback list of installed versions is collected from stack version directories in stack install root.
386 :param stack_root: Stack install root
387 :return: Returns list of installed stack versions.
388 """
389 stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
390 code, out = call((STACK_SELECT_PREFIX, stack_selector_path, 'versions'))
391 versions = []
392 if 0 == code:
393 for line in out.splitlines():
394 versions.append(line.rstrip('\n'))
395 if not versions:
396 versions = get_versions_from_stack_root(stack_root)
397 return versions
398
399 def get_stack_version_before_install(component_name):
400 """
401 Works in the similar way to '<stack-selector-tool> status component',
402 but also works for not yet installed packages.
403
404 Note: won't work if doing initial install.
405 """
406 stack_root = Script.get_stack_root()
407 component_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, "current", component_name)
408 stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
409 if os.path.islink(component_dir):
410 stack_version = os.path.basename(os.path.dirname(os.readlink(component_dir)))
411 match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', stack_version)
412 if match is None:
413 Logger.info('Failed to get extracted version with {0} in method get_stack_version_before_install'.format(stack_selector_name))
414 return None # lazy fail
415 return stack_version
416 else:
417 return None