#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # shell function library for Bookkeeper CI builds # lists all available functions in this tool function ci_list_functions() { declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//' } # prints thread dumps for all running JVMs # used in CI when a job gets cancelled because of a job timeout function ci_print_thread_dumps() { for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do echo "----------------------- pid $java_pid -----------------------" cat /proc/$java_pid/cmdline | xargs -0 echo jcmd $java_pid Thread.print -l jcmd $java_pid GC.heap_info done return 0 } # copies test reports into test-reports and surefire-reports directory # subsequent runs of tests might overwrite previous reports. This ensures that all test runs get reported. function ci_move_test_reports() { ( if [ -n "${GITHUB_WORKSPACE}" ]; then cd "${GITHUB_WORKSPACE}" mkdir -p test-reports mkdir -p surefire-reports fi # aggregate all junit xml reports in a single directory if [ -d test-reports ]; then # copy test reports to single directory, rename duplicates find . -path '*/target/surefire-reports/junitreports/TEST-*.xml' -print0 | xargs -0 -r -n 1 mv -t test-reports --backup=numbered # rename possible duplicates to have ".xml" extension ( for f in test-reports/*~; do mv -- "$f" "${f}.xml" done 2>/dev/null ) || true fi # aggregate all surefire-reports in a single directory if [ -d surefire-reports ]; then ( find . -type d -path '*/target/surefire-reports' -not -path './surefire-reports/*' | while IFS=$'\n' read -r directory; do echo "Copying reports from $directory" target_dir="surefire-reports/${directory}" if [ -d "$target_dir" ]; then # rotate backup directory names *~3 -> *~2, *~2 -> *~3, *~1 -> *~2, ... ( command ls -vr1d "${target_dir}~"* 2> /dev/null | awk '{print "mv "$0" "substr($0,0,length-1)substr($0,length,1)+1}' | sh ) || true # backup existing target directory, these are the results of the previous test run mv "$target_dir" "${target_dir}~1" fi # copy files cp -R --parents "$directory" surefire-reports # remove the original directory rm -rf "$directory" done ) fi ) } # Finds fastest up-to-date ubuntu mirror based on download speed function ci_find_fast_ubuntu_mirror() { local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"} local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"} { # choose mirrors that are up-to-date by checking the Last-Modified header for { # randomly choose up to 10 mirrors using http:// protocol # (https isn't supported in docker containers that don't have ca-certificates installed) curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 # also consider Azure's Ubuntu mirror echo http://azure.archive.ubuntu.com/ubuntu/ } | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ |sort -g -r |head -1| awk '{ print $2 }' } function ci_pick_ubuntu_mirror() { echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror) if [ -z "$UBUNTU_MIRROR" ]; then # fallback to no mirror UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/" UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/" else UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}" fi echo "Picked '$UBUNTU_MIRROR'." # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables # that can be used by docker builds export UBUNTU_MIRROR export UBUNTU_SECURITY_MIRROR # make environment variables available for later GitHub Actions steps if [ -n "$GITHUB_ENV" ]; then echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV fi } if [ -z "$1" ]; then echo "usage: $0 [ci_tool_function_name]" echo "Available ci tool functions:" ci_list_functions exit 1 fi ci_function_name="ci_$1" shift if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then eval "$ci_function_name" "$@" else echo "Invalid ci tool function" echo "Available ci tool functions:" ci_list_functions exit 1 fi