HBASE-19901 Up yetus proclimit on nightlies; ADDENDUM -- remove docker mem sizing...
[hbase.git] / dev-support / hbase-personality.sh
1 #!/usr/bin/env bash
2 # Licensed to the Apache Software Foundation (ASF) under one or more
3 # contributor license agreements. See the NOTICE file distributed with
4 # this work for additional information regarding copyright ownership.
5 # The ASF licenses this file to You under the Apache License, Version 2.0
6 # (the "License"); you may not use this file except in compliance with
7 # the License. You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16
17 # You'll need a local installation of
18 # [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
19 # to use this personality.
20 #
21 # Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
22 # build from it, or use the convenience binaries provided on that download page.
23 #
24 # To run against, e.g. HBASE-15074 you'd then do
25 # ```bash
26 # test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
27 # ```
28 #
29 # If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
30 # ```bash
31 # test-patch --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
32 # ````
33 #
34 # pass the `--jenkins` flag if you want to allow test-patch to destructively alter local working
35 # directory / branch in order to have things match what the issue patch requests.
36
37 personality_plugins "all"
38
39 if ! declare -f "yetus_info" >/dev/null; then
40
41 function yetus_info
42 {
43 echo "[$(date) INFO]: $*" 1>&2
44 }
45
46 fi
47
48 ## @description Globals specific to this personality
49 ## @audience private
50 ## @stability evolving
51 function personality_globals
52 {
53 BUILDTOOL=maven
54 #shellcheck disable=SC2034
55 PROJECT_NAME=hbase
56 #shellcheck disable=SC2034
57 PATCH_BRANCH_DEFAULT=master
58 #shellcheck disable=SC2034
59 JIRA_ISSUE_RE='^HBASE-[0-9]+$'
60 #shellcheck disable=SC2034
61 GITHUB_REPO="apache/hbase"
62
63 # TODO use PATCH_BRANCH to select jdk versions to use.
64
65 # Override the maven options
66 MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
67
68 # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
69 # Up it. See HBASE-19902 for how we arrived at this number.
70 PROCLIMIT=10000
71
72 # Set docker container to run with 20g. Default is 4g in yetus.
73 # See HBASE-19902 for how we arrived at 20g.
74 # UNUSED AT MOMENT DOCKERMEMLIMIT=20g
75 }
76
77 ## @description Parse extra arguments required by personalities, if any.
78 ## @audience private
79 ## @stability evolving
80 function personality_parse_args
81 {
82 declare i
83
84 for i in "$@"; do
85 case ${i} in
86 --exclude-tests-url=*)
87 EXCLUDE_TESTS_URL=${i#*=}
88 ;;
89 --include-tests-url=*)
90 INCLUDE_TESTS_URL=${i#*=}
91 ;;
92 --hadoop-profile=*)
93 HADOOP_PROFILE=${i#*=}
94 ;;
95 esac
96 done
97 }
98
99 ## @description Queue up modules for this personality
100 ## @audience private
101 ## @stability evolving
102 ## @param repostatus
103 ## @param testtype
104 function personality_modules
105 {
106 local repostatus=$1
107 local testtype=$2
108 local extra=""
109 local MODULES=(${CHANGED_MODULES[@]})
110
111 yetus_info "Personality: ${repostatus} ${testtype}"
112
113 clear_personality_queue
114
115 extra="-DHBasePatchProcess"
116
117 if [[ -n "${HADOOP_PROFILE}" ]]; then
118 extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
119 fi
120
121 # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
122 # full source code instead. In this case, do full compiles, tests, etc instead of per
123 # module.
124 # Used in nightly runs.
125 # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
126 # modules if root is included. HBASE-18505
127 if [[ "${BUILDMODE}" == "full" ]] || \
128 [[ ( "${testtype}" == unit || "${testtype}" == compile ) && "${MODULES[*]}" =~ \. ]]; then
129 MODULES=(.)
130 fi
131
132 if [[ ${testtype} == mvninstall ]]; then
133 # shellcheck disable=SC2086
134 personality_enqueue_module . ${extra}
135 return
136 fi
137
138 if [[ ${testtype} == findbugs ]]; then
139 # Run findbugs on each module individually to diff pre-patch and post-patch results and
140 # report new warnings for changed modules only.
141 # For some reason, findbugs on root is not working, but running on individual modules is
142 # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
143 for module in "${CHANGED_MODULES[@]}"; do
144 # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
145 # in src/main/java where findbugs goes to look
146 if [[ ${module} == hbase-shell ]]; then
147 continue
148 elif [[ ${module} == hbase-it ]]; then
149 continue
150 else
151 # shellcheck disable=SC2086
152 personality_enqueue_module ${module} ${extra}
153 fi
154 done
155 return
156 fi
157
158 # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
159 # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
160 # tests respectively.
161 if [[ ${testtype} == unit ]]; then
162 local tests_arg=""
163 get_include_exclude_tests_arg tests_arg
164 extra="${extra} -PrunAllTests ${tests_arg}"
165
166 # Inject the jenkins build-id for our surefire invocations
167 # Used by zombie detection stuff, even though we're not including that yet.
168 if [ -n "${BUILD_ID}" ]; then
169 extra="${extra} -Dbuild.id=${BUILD_ID}"
170 fi
171 fi
172
173 for module in "${MODULES[@]}"; do
174 # shellcheck disable=SC2086
175 personality_enqueue_module ${module} ${extra}
176 done
177 }
178
179 ## @description Uses relevant include/exclude env variable to fetch list of included/excluded
180 # tests and sets given variable to arguments to be passes to maven command.
181 ## @audience private
182 ## @stability evolving
183 ## @param name of variable to set with maven arguments
184 function get_include_exclude_tests_arg
185 {
186 local __resultvar=$1
187 yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
188 yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
189 if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
190 if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
191 excludes=$(cat excludes)
192 yetus_debug "excludes=${excludes}"
193 if [[ -n "${excludes}" ]]; then
194 eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
195 fi
196 rm excludes
197 else
198 yetus_error "Wget error $? in fetching excludes file from url" \
199 "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
200 fi
201 elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
202 if wget "$INCLUDE_TESTS_URL" -O "includes"; then
203 includes=$(cat includes)
204 yetus_debug "includes=${includes}"
205 if [[ -n "${includes}" ]]; then
206 eval "${__resultvar}='-Dtest=${includes}'"
207 fi
208 rm includes
209 else
210 yetus_error "Wget error $? in fetching includes file from url" \
211 "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
212 fi
213 fi
214 }
215
216 ###################################################
217 # Below here are our one-off tests specific to hbase.
218 # TODO break them into individual files so it's easier to maintain them?
219
220 # TODO line length check? could ignore all java files since checkstyle gets them.
221
222 ###################################################
223
224 add_test_type shadedjars
225
226
227 function shadedjars_initialize
228 {
229 yetus_debug "initializing shaded client checks."
230 maven_add_install shadedjars
231 }
232
233 ## @description only run the test if java changes.
234 ## @audience private
235 ## @stability evolving
236 ## @param filename
237 function shadedjars_filefilter
238 {
239 local filename=$1
240
241 if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
242 add_test shadedjars
243 fi
244 }
245
246 ## @description test the shaded client artifacts
247 ## @audience private
248 ## @stability evolving
249 ## @param repostatus
250 function shadedjars_rebuild
251 {
252 local repostatus=$1
253 local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
254
255 if ! verify_needed_test shadedjars; then
256 return 0
257 fi
258
259 big_console_header "Checking shaded client builds on ${repostatus}"
260
261 echo_and_redirect "${logfile}" \
262 "${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \
263 -pl hbase-shaded/hbase-shaded-check-invariants -am \
264 -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
265 -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
266
267 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
268 if [[ ${count} -gt 0 ]]; then
269 add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded downstream artifacts."
270 return 1
271 fi
272
273 add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream artifacts."
274 return 0
275 }
276
277 ###################################################
278
279 add_test_type hadoopcheck
280
281 ## @description hadoopcheck file filter
282 ## @audience private
283 ## @stability evolving
284 ## @param filename
285 function hadoopcheck_filefilter
286 {
287 local filename=$1
288
289 if [[ ${filename} =~ \.java$ ]] || [[ ${filename} =~ pom.xml$ ]]; then
290 add_test hadoopcheck
291 fi
292 }
293
294 ## @description Parse args to detect if QUICK_HADOOPCHECK mode is enabled.
295 ## @audience private
296 ## @stability evolving
297 function hadoopcheck_parse_args
298 {
299 declare i
300
301 for i in "$@"; do
302 case ${i} in
303 --quick-hadoopcheck)
304 QUICK_HADOOPCHECK=true
305 ;;
306 esac
307 done
308 }
309
310 ## @description Adds QUICK_HADOOPCHECK env variable to DOCKER_EXTRAARGS.
311 ## @audience private
312 ## @stability evolving
313 function hadoopcheck_docker_support
314 {
315 DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "--env=QUICK_HADOOPCHECK=${QUICK_HADOOPCHECK}")
316 }
317
318 ## @description hadoopcheck test
319 ## @audience private
320 ## @stability evolving
321 ## @param repostatus
322 function hadoopcheck_rebuild
323 {
324 local repostatus=$1
325 local hadoopver
326 local logfile
327 local count
328 local result=0
329 local hbase_hadoop2_versions
330 local hbase_hadoop3_versions
331
332 if [[ "${repostatus}" = branch ]]; then
333 return 0
334 fi
335
336 if ! verify_needed_test hadoopcheck; then
337 return 0
338 fi
339
340 big_console_header "Compiling against various Hadoop versions"
341
342 # All supported Hadoop versions that we want to test the compilation with
343 # See the Hadoop section on prereqs in the HBase Reference Guide
344 hbase_common_hadoop2_versions="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3 2.7.4"
345 if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
346 yetus_info "Setting Hadoop versions to test based on branch-1-ish rules."
347 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
348 hbase_hadoop2_versions="2.4.1 2.5.2 2.6.5 2.7.4"
349 else
350 hbase_hadoop2_versions="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 ${hbase_common_hadoop2_versions}"
351 fi
352 hbase_hadoop3_versions=""
353 else # master or a feature branch
354 yetus_info "Setting Hadoop versions to test based on branch-2/master/feature branch rules."
355 if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
356 hbase_hadoop2_versions="2.6.5 2.7.4"
357 else
358 hbase_hadoop2_versions="${hbase_common_hadoop2_versions}"
359 fi
360 hbase_hadoop3_versions="3.0.0"
361 fi
362
363 export MAVEN_OPTS="${MAVEN_OPTS}"
364 for hadoopver in ${hbase_hadoop2_versions}; do
365 logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
366 echo_and_redirect "${logfile}" \
367 "${MAVEN}" clean install \
368 -DskipTests -DHBasePatchProcess \
369 -Dhadoop-two.version="${hadoopver}"
370 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
371 if [[ ${count} -gt 0 ]]; then
372 add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
373 ((result=result+1))
374 fi
375 done
376
377 for hadoopver in ${hbase_hadoop3_versions}; do
378 logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
379 echo_and_redirect "${logfile}" \
380 "${MAVEN}" clean install \
381 -DskipTests -DHBasePatchProcess \
382 -Dhadoop-three.version="${hadoopver}" \
383 -Dhadoop.profile=3.0
384 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
385 if [[ ${count} -gt 0 ]]; then
386 add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
387 ((result=result+1))
388 fi
389 done
390
391 if [[ ${result} -gt 0 ]]; then
392 return 1
393 fi
394
395 if [[ -n "${hbase_hadoop3_versions}" ]]; then
396 add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
397 else
398 add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
399 fi
400 return 0
401 }
402
403 ######################################
404
405 # TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
406 add_test_type hbaseprotoc
407
408 ## @description hbaseprotoc file filter
409 ## @audience private
410 ## @stability evolving
411 ## @param filename
412 function hbaseprotoc_filefilter
413 {
414 local filename=$1
415
416 if [[ ${filename} =~ \.proto$ ]]; then
417 add_test hbaseprotoc
418 fi
419 }
420
421 ## @description check hbase proto compilation
422 ## @audience private
423 ## @stability evolving
424 ## @param repostatus
425 function hbaseprotoc_rebuild
426 {
427 declare repostatus=$1
428 declare i=0
429 declare fn
430 declare module
431 declare logfile
432 declare count
433 declare result
434
435 if [[ "${repostatus}" = branch ]]; then
436 return 0
437 fi
438
439 if ! verify_needed_test hbaseprotoc; then
440 return 0
441 fi
442
443 big_console_header "HBase protoc plugin: ${BUILDMODE}"
444
445 start_clock
446
447 personality_modules patch hbaseprotoc
448 # Need to run 'install' instead of 'compile' because shading plugin
449 # is hooked-up to 'install'; else hbase-protocol-shaded is left with
450 # half of its process done.
451 modules_workers patch hbaseprotoc install -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
452
453 # shellcheck disable=SC2153
454 until [[ $i -eq "${#MODULE[@]}" ]]; do
455 if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
456 ((result=result+1))
457 ((i=i+1))
458 continue
459 fi
460 module=${MODULE[$i]}
461 fn=$(module_file_fragment "${module}")
462 logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
463
464 count=$(${GREP} -c '\[ERROR\]' "${logfile}")
465
466 if [[ ${count} -gt 0 ]]; then
467 module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
468 "${count} new protoc errors in ${module}."
469 ((result=result+1))
470 fi
471 ((i=i+1))
472 done
473
474 modules_messages patch hbaseprotoc true
475 if [[ ${result} -gt 0 ]]; then
476 return 1
477 fi
478 return 0
479 }
480
481 ######################################
482
483 add_test_type hbaseanti
484
485 ## @description hbaseanti file filter
486 ## @audience private
487 ## @stability evolving
488 ## @param filename
489 function hbaseanti_filefilter
490 {
491 local filename=$1
492
493 if [[ ${filename} =~ \.java$ ]]; then
494 add_test hbaseanti
495 fi
496 }
497
498 ## @description hbaseanti patch file check
499 ## @audience private
500 ## @stability evolving
501 ## @param filename
502 function hbaseanti_patchfile
503 {
504 local patchfile=$1
505 local warnings
506 local result
507
508 if [[ "${BUILDMODE}" = full ]]; then
509 return 0
510 fi
511
512 if ! verify_needed_test hbaseanti; then
513 return 0
514 fi
515
516 big_console_header "Checking for known anti-patterns"
517
518 start_clock
519
520 warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
521 if [[ ${warnings} -gt 0 ]]; then
522 add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
523 ((result=result+1))
524 fi
525
526 warnings=$(${GREP} -c 'import org.apache.hadoop.classification' "${patchfile}")
527 if [[ ${warnings} -gt 0 ]]; then
528 add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase."
529 ((result=result+1))
530 fi
531
532 warnings=$(${GREP} -c 'import org.codehaus.jackson' "${patchfile}")
533 if [[ ${warnings} -gt 0 ]]; then
534 add_vote_table -1 hbaseanti "" "The patch appears use Jackson 1 classes/annotations."
535 ((result=result+1))
536 fi
537
538 if [[ ${result} -gt 0 ]]; then
539 return 1
540 fi
541
542 add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
543 return 0
544 }
545
546
547 ## @description hbase custom mvnsite file filter. See HBASE-15042
548 ## @audience private
549 ## @stability evolving
550 ## @param filename
551 function mvnsite_filefilter
552 {
553 local filename=$1
554
555 if [[ ${BUILDTOOL} = maven ]]; then
556 if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
557 yetus_debug "tests/mvnsite: ${filename}"
558 add_test mvnsite
559 fi
560 fi
561 }
562
563 ## This is named so that yetus will check us right after running tests.
564 ## Essentially, we check for normal failures and then we look for zombies.
565 #function hbase_unit_logfilter
566 #{
567 # declare testtype="unit"
568 # declare input=$1
569 # declare output=$2
570 # declare processes
571 # declare process_output
572 # declare zombies
573 # declare zombie_count=0
574 # declare zombie_process
575 #
576 # yetus_debug "in hbase-specific unit logfilter."
577 #
578 # # pass-through to whatever is counting actual failures
579 # if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
580 # "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
581 # elif declare -f ${testtype}_logfilter >/dev/null; then
582 # "${testtype}_logfilter" "${input}" "${output}"
583 # fi
584 #
585 # start_clock
586 # if [ -n "${BUILD_ID}" ]; then
587 # yetus_debug "Checking for zombie test processes."
588 # processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
589 # if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
590 # yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
591 # yetus_debug "${processes}"
592 # sleep 30
593 # #shellcheck disable=SC2016
594 # for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
595 # # Test our zombie still running (and that it still an hbase build item)
596 # process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
597 # if [[ -n "${process_output}" ]]; then
598 # yetus_error "Zombie: ${process_output}"
599 # ((zombie_count = zombie_count + 1))
600 # zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
601 # zombies="${zombies} ${zombie_process}"
602 # fi
603 # done
604 # fi
605 # if [ "${zombie_count}" -ne 0 ]; then
606 # add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
607 # populate_test_table "zombie unit tests" "${zombies}"
608 # else
609 # yetus_info "Zombie check complete. All test runs exited normally."
610 # stop_clock
611 # fi
612 # else
613 # add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
614 # fi
615 #
616 #}