kubernetes/hack/make-rules/test.sh

#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
source "${KUBE_ROOT}/hack/lib/init.sh"

kube::golang::setup_env
kube::util::require-jq

# start the cache mutation detector by default so that cache mutators will be found
KUBE_CACHE_MUTATION_DETECTOR="${KUBE_CACHE_MUTATION_DETECTOR:-true}"
export KUBE_CACHE_MUTATION_DETECTOR

# panic the server on watch decode errors since they are considered coder mistakes
KUBE_PANIC_WATCH_DECODE_ERROR="${KUBE_PANIC_WATCH_DECODE_ERROR:-true}"
export KUBE_PANIC_WATCH_DECODE_ERROR

kube::test::find_go_packages() {
  (
    cd "${KUBE_ROOT}"

    # Get a list of all the modules in this workspace.
    local -a workspace_module_patterns
    kube::util::read-array workspace_module_patterns < <(go list -m -json | jq -r '.Path + "/..."')

    # Get a list of all packages which have test files, but filter out ones
    # that we don't want to run by default (i.e. are not unit-tests).
    go list -find \
        -f '{{if or (gt (len .TestGoFiles) 0) (gt (len .XTestGoFiles) 0)}}{{.ImportPath}}{{end}}' \
        "${workspace_module_patterns[@]}" \
        | grep -vE \
            -e '^k8s.io/kubernetes/third_party(/.*)?$' \
            -e '^k8s.io/kubernetes/cmd/kubeadm/test(/.*)?$' \
            -e '^k8s.io/kubernetes/test/e2e$' \
            -e '^k8s.io/kubernetes/test/e2e_dra$' \
            -e '^k8s.io/kubernetes/test/e2e_node(/.*)?$' \
            -e '^k8s.io/kubernetes/test/e2e_kubeadm(/.*)?$' \
            -e '^k8s.io/.*/test/integration(/.*)?$'
  )

  # Some of our modules are not in this workspace. We have to change directories
  # to find their tests and then add the module path as prefix to the
  # packages as indicator that we need to do the same when executing those tests.
  local module
  for module in $(find hack/tools -name go.mod | sed -e 's;/go.mod;;'); do
   (
     cd "${module}"
     go list -find \
             -f '{{if or (gt (len .TestGoFiles) 0) (gt (len .XTestGoFiles) 0)}}{{.ImportPath}}{{end}}' \
             ./... |
     sed -e "s;^;${module}/;"
   )
  done
}

set -x

# TODO: This timeout should really be lower, this is a *long* time to test one
# package, however pkg/api/testing in particular will fail with a lower timeout
# currently. We should attempt to lower this over time.
KUBE_TIMEOUT=${KUBE_TIMEOUT:--timeout=180s}
KUBE_COVER=${KUBE_COVER:-n} # set to 'y' to enable coverage collection
KUBE_COVERMODE=${KUBE_COVERMODE:-atomic}
# The directory to save test coverage reports to, if generating them. If unset,
# a semi-predictable temporary directory will be used.
KUBE_COVER_REPORT_DIR="${KUBE_COVER_REPORT_DIR:-}"
# use KUBE_RACE="" to disable the race detector
# this is defaulted to "-race" in make test as well
# NOTE: DO NOT ADD A COLON HERE. KUBE_RACE="" is meaningful!
KUBE_RACE=${KUBE_RACE-"-race"}
# Set to the goveralls binary path to report coverage results to Coveralls.io.
KUBE_GOVERALLS_BIN=${KUBE_GOVERALLS_BIN:-}
# once we have multiple group supports
# Create a junit-style XML test report in this directory if set.
KUBE_JUNIT_REPORT_DIR=${KUBE_JUNIT_REPORT_DIR:-}
# If KUBE_JUNIT_REPORT_DIR is unset, and ARTIFACTS is set, then have them match.
if [[ -z "${KUBE_JUNIT_REPORT_DIR:-}" && -n "${ARTIFACTS:-}" ]]; then
    export KUBE_JUNIT_REPORT_DIR="${ARTIFACTS}"
fi
# Set to 'y' to keep the verbose stdout from tests when KUBE_JUNIT_REPORT_DIR is
# set.
KUBE_KEEP_VERBOSE_TEST_OUTPUT=${KUBE_KEEP_VERBOSE_TEST_OUTPUT:-n}
# Set to 'false' to disable reduction of the JUnit file to only the top level tests.
KUBE_PRUNE_JUNIT_TESTS=${KUBE_PRUNE_JUNIT_TESTS:-true}

set +x

kube::test::usage() {
  kube::log::usage_from_stdin <<EOF
usage: $0 [OPTIONS] [TARGETS]

OPTIONS:
  -p <number>   : number of parallel workers, must be >= 1
EOF
}

isnum() {
  [[ "$1" =~ ^[0-9]+$ ]]
}

PARALLEL="${PARALLEL:--1}"
while getopts "hp:i:" opt ; do
  case ${opt} in
    h)
      kube::test::usage
      exit 0
      ;;
    p)
      PARALLEL="${OPTARG}"
      if ! isnum "${PARALLEL}" || [[ "${PARALLEL}" -le 0 ]]; then
        kube::log::usage "'$0': argument to -p must be numeric and greater than 0"
        kube::test::usage
        exit 1
      fi
      ;;
    i)
      kube::log::usage "'$0': use GOFLAGS='-count <num-iterations>'"
      kube::test::usage
      exit 1
      ;;
    :)
      kube::log::usage "Option -${OPTARG} <value>"
      kube::test::usage
      exit 1
      ;;
    ?)
      kube::test::usage
      exit 1
      ;;
  esac
done
shift $((OPTIND - 1))

# Use eval to preserve embedded quoted strings.
#
# KUBE_TEST_ARGS contains arguments for `go test` (like -short)
# and may end with `-args <arguments for test binary>`, so it
# has to be passed to `go test` at the end of the invocation.
testargs=()
eval "testargs=(${KUBE_TEST_ARGS:-})"

# gotestsum --format value
# "standard-quiet" let's some stderr log messages through, "pkgname-and-test-fails" is similar and doesn't (https://github.com/kubernetes/kubernetes/issues/130934#issuecomment-2739957840).
gotestsum_format=pkgname-and-test-fails
if [[ -n "${FULL_LOG:-}" ]] ; then
  gotestsum_format=standard-verbose
fi

goflags=()

# Filter out arguments that start with "-" and move them to goflags.
testcases=()
for arg; do
  if [[ "${arg}" == -* ]]; then
    goflags+=("${arg}")
  else
    testcases+=("${arg}")
  fi
done
if [[ ${#testcases[@]} -eq 0 ]]; then
  # If the user passed no targets in, we want ~everything.
  # In addition also test some specific upstream packages.
  kube::util::read-array testcases < <(
    kube::test::find_go_packages
    grep -v -e '^#' -e '^$' "${KUBE_ROOT}/hack/dependency-unit-tests.conf"
  )
else
  # If the user passed targets, we should normalize them.
  # This can be slow for large numbers of inputs.
  #
  # Targets have to be part of the Kubernetes source code, i.e.
  # WHAT=hack/tools/golangci-lint/sigs.k8s.io/logtools
  # is not supported.
  kube::log::status "Normalizing Go targets"
  kube::util::read-array testcases < <(kube::golang::normalize_go_targets "${testcases[@]}")
fi
set -- "${testcases[@]+${testcases[@]}}"

if [[ -n "${KUBE_RACE}" ]] ; then
  goflags+=("${KUBE_RACE}")
fi

if [[ "${PARALLEL}" -gt 0 ]]; then
  goflags+=(-p "${PARALLEL}")
fi

junitFilenamePrefix() {
  if [[ -z "${KUBE_JUNIT_REPORT_DIR}" ]]; then
    echo ""
    return
  fi
  mkdir -p "${KUBE_JUNIT_REPORT_DIR}"
  echo -n "${KUBE_JUNIT_REPORT_DIR}/junit_$(echo -n "${1}" | tr /- _)$(kube::util::sortable_date)"
}

installTools() {
  if ! command -v gotestsum >/dev/null 2>&1; then
    kube::log::status "gotestsum not found; installing from ./hack/tools"
    GOTOOLCHAIN="$(kube::golang::hack_tools_gotoolchain)" go -C "${KUBE_ROOT}/hack/tools" install gotest.tools/gotestsum
  fi

  if ! command -v prune-junit-xml >/dev/null 2>&1; then
    kube::log::status "prune-junit-xml not found; installing from ./cmd"
    go -C "${KUBE_ROOT}/cmd/prune-junit-xml" install .
  fi
}

# filterTests distinguishes between tests that run in the main workspace
# (empty directory prefix, like k8s.io/kubernetes/pkg/kubelet) and those
# which run only inside specific directories (non-empty directory prefix,
# like hack/tools/golangci-lint/sigs.k8s.io/logtools).
#
# It prints matching tests without the directory prefix.
filterTests() {
  local prefix="$1"
  shift

  for test in "$@"; do
    if [[ -z "${prefix}" ]]; then
      # Filter out tests with special setup requirements.
      if ! [[ "${test}" =~ ^(hack/tools/*|vendor)/ ]]; then
        echo "${test}"
      fi
    else
      # Filter out tests not in the right sub-directory.
      if [[ "${test}" =~ ^${prefix} ]]; then
        echo "${test#"${prefix}"}"
      fi
    fi
  done
}

runTests() {
  installTools

  # Enable coverage data collection?
  local cover_msg
  local COMBINED_COVER_PROFILE
  local cover_flags=()

  if [[ ${KUBE_COVER} =~ ^[yY]$ ]]; then
    cover_msg="with code coverage"
    if [[ -z "${KUBE_COVER_REPORT_DIR}" ]]; then
      cover_report_dir="/tmp/k8s_coverage/$(kube::util::sortable_date)"
    else
      cover_report_dir="${KUBE_COVER_REPORT_DIR}"
    fi
    kube::log::status "Saving coverage output in '${cover_report_dir}'"
    mkdir -p "${@+${@/#/${cover_report_dir}/}}"
    COMBINED_COVER_PROFILE="${cover_report_dir}/combined-coverage.out"
    cover_flags=(-cover -covermode="${KUBE_COVERMODE}" -coverprofile="${COMBINED_COVER_PROFILE}")
  else
    cover_msg="without code coverage"
  fi

  kube::log::status "Running tests ${cover_msg} ${KUBE_RACE:+"and with ${KUBE_RACE}"}"
  rc=0
  local junitfiles=()
  # The different prefixes must be disjoint, i.e. one prefix cannot be contained in another.
  # We cannot test packages in hack/tools because it would be ambiguous whether
  # hack/tools/foo/bar is "foo/bar" in hack/tools or "bar" in "hack/tools/foo".
  for prefix in "" "vendor/" $(find hack/tools/*/* -name go.mod | sed -e 's;/go.mod;/;'); do
    unset testcases
    kube::util::read-array testcases < <(filterTests "${prefix}" "$@")

    if [[ ${#testcases[@]} -eq 0 ]]; then
      continue
    fi

    local junit_filename_prefix
    junit_filename_prefix=$(junitFilenamePrefix "${prefix}")

    local jsonfile=""
    local junitfile=""
    if [[ -n "${junit_filename_prefix}" ]]; then
      junitfile="${junit_filename_prefix}.xml"
      junitfiles+=( "${junitfile}" )
      # Keep the raw JSON output in addition to the JUnit file?
      if [[ ${KUBE_KEEP_VERBOSE_TEST_OUTPUT} =~ ^[yY]$ ]]; then
        jsonfile="${junit_filename_prefix}.stdout"
      fi
    fi

    if ! (
           case "${prefix}" in
             "")
               # Collect coverage only for the main workspace. Every "go test"
               # invocation overwrites the shared profile file, and
               # "go tool cover" below cannot resolve packages that are
               # outside the workspace.
               goflags+=("${cover_flags[@]:+${cover_flags[@]}}")
               ;;
             "vendor/")
               # Upstream tests are not vendored, so we have to download without modifying the vendor directory.
               # Preserves existing GOFLAGS if set.
               # The final value gets injected and logged below via env.
               GOFLAGS="${GOFLAGS:+${GOFLAGS} }-mod=readonly" ;;
             hack/tools/*)
               GOTOOLCHAIN="$(kube::golang::hack_tools_gotoolchain)"
               # hack/tools is not part of the workspace, must change the directory.
               cd "${prefix}"
               ;;
             *)
               # Some other non-workspace test, change the directory.
               cd "${prefix}" ;;
           esac
           pwd
           kube::log::run env GOTOOLCHAIN="${GOTOOLCHAIN:-}" GOFLAGS="${GOFLAGS:-}" gotestsum --format="${gotestsum_format}" \
            --jsonfile="${jsonfile}" \
            --junitfile="${junitfile}" \
            --raw-command \
            -- \
            go test -json \
            "${goflags[@]:+${goflags[@]}}" \
            "${KUBE_TIMEOUT}" \
            "${testcases[@]}" \
            "${testargs[@]:+${testargs[@]}}"
      ); then
      rc=1
    fi
  done

  for junitfile in "${junitfiles[@]}"; do
    prune-junit-xml -prune-tests="${KUBE_PRUNE_JUNIT_TESTS}" "${junitfile}"
  done

  if [[ ${KUBE_COVER} =~ ^[yY]$ ]]; then
    coverage_html_file="${cover_report_dir}/combined-coverage.html"
    go tool cover -html="${COMBINED_COVER_PROFILE}" -o="${coverage_html_file}"
    kube::log::status "Combined coverage report: ${coverage_html_file}"
  fi

  return "${rc}"
}

reportCoverageToCoveralls() {
  if [[ ${KUBE_COVER} =~ ^[yY]$ ]] && [[ -x "${KUBE_GOVERALLS_BIN}" ]]; then
    kube::log::status "Reporting coverage results to Coveralls for service ${CI_NAME:-}"
    ${KUBE_GOVERALLS_BIN} -coverprofile="${COMBINED_COVER_PROFILE}" \
    ${CI_NAME:+"-service=${CI_NAME}"} \
    ${COVERALLS_REPO_TOKEN:+"-repotoken=${COVERALLS_REPO_TOKEN}"} \
      || true
  fi
}

checkFDs() {
  # several unittests panic when httptest cannot open more sockets
  # due to the low default files limit on OS X.  Warn about low limit.
  local fileslimit
  fileslimit="$(ulimit -n)"
  if [[ "${fileslimit}" != "unlimited" && "${fileslimit}" -lt 1000 ]]; then
    echo "WARNING: ulimit -n (files) should be at least 1000, is ${fileslimit}, may cause test failure";
  fi
}

checkFDs

runTests "$@"

# We might run the tests for multiple versions, but we want to report only
# one of them to coveralls. Here we report coverage from the last run.
reportCoverageToCoveralls