Merge pull request #3525 from native-api/test_improvements

Test improvements

Makefile

@@ -62,7 +62,8 @@ $(TEST_PYTHON_BUILD_DOCKER_TARGETS): $(TEST_PYTHON_BUILD_DOCKER_PREFIX)-% : $(TE
 		$${CI+-e CI="$${CI}"} \
 		$(INTERACTIVE) \
 		$(DOCKER_IMAGE):$(DOCKER_TAG) \
-		bats $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} plugins/python-build/test/$${BATS_FILE_FILTER}
+		bats $${CI:+-F "/code/test/libexec/bats-format-tap-suite"} \
+		$${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} plugins/python-build/test/$${BATS_FILE_FILTER}
 
 .PHONY: $(TEST_BINARY_DOCKER_PREFIX)
 $(TEST_BINARY_DOCKER_PREFIX): $(TEST_BINARY_DOCKER_TARGETS)
@@ -84,7 +85,8 @@ $(TEST_BINARY_DOCKER_TARGETS): $(TEST_BINARY_DOCKER_PREFIX)-% : $(TEST_BATS_IMAG
 		$${CI+-e CI="$${CI}"} \
 		$(INTERACTIVE) \
 		$(DOCKER_IMAGE):$(DOCKER_TAG) \
-		bats $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} plugins/pyenv-binary/test/$${BATS_FILE_FILTER}
+		bats $${CI:+-F "/code/test/libexec/bats-format-tap-suite"} \
+		$${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} plugins/pyenv-binary/test/$${BATS_FILE_FILTER}
 
 # Build all images needed for bats under docker
 .PHONY: $(TEST_BATS_IMAGE_PREFIX)
@@ -120,10 +122,12 @@ test-unit: bats
 	PATH="./bats/bin:$$PATH" test/run
 	
 test-python-build: bats
-	cd plugins/python-build && $(PWD)/bats/bin/bats $${CI:+--tap} $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} test/$${BATS_FILE_FILTER}
+	cd plugins/python-build && $(PWD)/bats/bin/bats $${CI:+-F "$(PWD)/test/libexec/bats-format-tap-suite"} \
+		$${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} test/$${BATS_FILE_FILTER}
 
 test-binary: bats
-	cd plugins/pyenv-binary && $(PWD)/bats/bin/bats $${CI:+--tap} $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} test/$${BATS_FILE_FILTER}
+	cd plugins/pyenv-binary && $(PWD)/bats/bin/bats $${CI:+-F "$(PWD)/test/libexec/bats-format-tap-suite"} \
+		$${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} test/$${BATS_FILE_FILTER}
 
 .SECONDARY: bats-$(TEST_BATS_VERSION)
 bats-$(TEST_BATS_VERSION):

plugins/python-build/test/build.bats

@@ -1276,6 +1276,7 @@ OUT
   mkdir -p "$INSTALL_ROOT/bin"
   touch "$INSTALL_ROOT/bin/package"
   chmod -w "$INSTALL_ROOT/bin/package"
+  skip_if_nonposix_security -w "$INSTALL_ROOT/bin/package"
 
   install_fixture definitions/without-checksum
   assert_success
@@ -1288,6 +1289,7 @@ OUT
   export TMPDIR="${BATS_TEST_TMPDIR}/build"
   mkdir -p "$TMPDIR"
   chmod -w "$TMPDIR"
+  skip_if_nonposix_security -w "$TMPDIR"
 
   touch "${BATS_TEST_TMPDIR}/build-definition"
   run python-build "${BATS_TEST_TMPDIR}/build-definition" "$INSTALL_ROOT"
@@ -1298,6 +1300,7 @@ OUT
   export TMPDIR="${BATS_TEST_TMPDIR}/build"
   mkdir -p "$TMPDIR"
   chmod -x "$TMPDIR"
+  skip_if_nonposix_security -x "$TMPDIR"
 
   touch "${BATS_TEST_TMPDIR}/build-definition"
   run python-build "${BATS_TEST_TMPDIR}/build-definition" "$INSTALL_ROOT"

plugins/python-build/test/test_helper.bash

@@ -162,3 +162,7 @@ path_without() {
   echo "$path"
 }
 
+skip_if_nonposix_security() {
+  true "${1:?}" "${2:?}"
+  test "$@" && skip "UNIX permission bits are being overridden" || true
+}

test/libexec/bats-format-tap-suite

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -e
+
+# Wraps Bats TAP formatter, adding file names to the output
+
+_BATS_FORMATTER_LIB_RELPATH="lib/bats-core/formatter.bash"
+source "$BATS_ROOT/$_BATS_FORMATTER_LIB_RELPATH"
+
+
+# `bats-format-tap` calls this fn at the end
+# We must stub it to be able to source and alter it
+# https://stackoverflow.com/questions/1203583/how-do-i-rename-a-bash-function
+eval orig_"$(declare -f bats_parse_internal_extended_tap)"
+bats_parse_internal_extended_tap() { true; }
+
+
+# stub sourcing lib from `bats-format-tap`
+# as that would've reset the stubbed fn
+FAKE_BATS_ROOT=
+_trap_rm_fakeroot() { rm -rf "$FAKE_BATS_ROOT"; }
+REAL_BATS_ROOT="$BATS_ROOT"
+FAKE_BATS_ROOT=$(mktemp -d)
+trap _trap_rm_fakeroot ERR EXIT
+mkdir -p "$(dirname "$FAKE_BATS_ROOT/$_BATS_FORMATTER_LIB_RELPATH")"
+touch "$FAKE_BATS_ROOT/$_BATS_FORMATTER_LIB_RELPATH"
+BATS_ROOT="$FAKE_BATS_ROOT"
+
+source "$REAL_BATS_ROOT/libexec/bats-core/bats-format-tap"
+
+_trap_rm_fakeroot
+trap - ERR EXIT
+BATS_ROOT="$REAL_BATS_ROOT"
+unset FAKE_BATS_ROOT REAL_BATS_ROOT _trap_rm_fakeroot
+
+
+# Finally, alter `bats-format-tap`'s logic and invoke it
+
+# print as "Anything" line
+bats_tap_stream_suite() { # <file name>
+  # bats only passes --base-name to specific built-in formatters
+  # so we don't have info where the test root is
+  printf '%s\n' "${1##*/}"
+}
+
+orig_bats_parse_internal_extended_tap

test/rehash.bats

@@ -13,6 +13,8 @@ load test_helper
 @test "non-writable shims directory" {
   mkdir -p "${PYENV_ROOT}/shims"
   chmod -w "${PYENV_ROOT}/shims"
+  skip_if_nonposix_security -w "${PYENV_ROOT}/shims"
+
   run pyenv-rehash
   assert_failure "pyenv: cannot rehash: ${PYENV_ROOT}/shims isn't writable"
 }

test/run

@@ -6,4 +6,4 @@ if [ -n "$PYENV_NATIVE_EXT" ]; then
   make -C src
 fi
 
-exec bats ${CI:+--tap} ${BATS_TEST_FILTER:+--filter "${BATS_TEST_FILTER}"} test/${BATS_FILE_FILTER}
+exec bats ${CI:+-F "$PWD/test/libexec/bats-format-tap-suite"} ${BATS_TEST_FILTER:+--filter "${BATS_TEST_FILTER}"} test/${BATS_FILE_FILTER}

test/shims-linked-from-elsewhere.bats

@@ -1,39 +0,0 @@
-#!/usr/bin/env bats
-
-load test_helper
-
-@test "shims linked from elsewhere are chained, other programs at the same paths are unaffected (integration)" {
-  bats_require_minimum_version 1.5.0
-  progname=shimmed_program
-  called_progname=another_program
-  create_alt_executable_in_version "custom" "$progname" <<!
-echo "called from \$0"
-pyenv-exec "$called_progname"
-!
-  pyenv-rehash
-  PATH="$(path_without "$progname" "$called_progname")"
-  for disguised_shim_path in "$BATS_TEST_TMPDIR/"{weird-location,even/weirder/location}; do
-    mkdir -p "$disguised_shim_path"
-    ln -s "${PYENV_ROOT}/shims/$progname" "$disguised_shim_path/$progname"
-    PATH="$PATH:$disguised_shim_path"
-  done
-  create_executable "$disguised_shim_path" "$called_progname" <<!
-echo "convoluted call success!"
-!
-  real_path="$BATS_TEST_TMPDIR/real-location"
-  mkdir -p "$real_path"
-  ln -s "${PYENV_ROOT}/versions/custom/bin/$progname" "$real_path/$progname"
-  PATH="$PATH:$real_path"
-  
-  run "$progname"
-  assert_success
-  assert_output <<!
-called from $real_path/$progname
-convoluted call success!
-!
-
-  rm "$real_path/$progname"
-  run -127 "$progname"
-  assert_failure
-  assert_line 0 "pyenv: shimmed_program: command not found"
-}

test/shims.bats

@@ -27,3 +27,39 @@ load test_helper
   assert_line "irb"
   assert_line "python"
 }
+
+@test "shims linked from elsewhere are chained, other programs at the same paths are unaffected (integration)" {
+  bats_require_minimum_version 1.5.0
+  progname=shimmed_program
+  called_progname=another_program
+  create_alt_executable_in_version "custom" "$progname" <<!
+echo "called from \$0"
+pyenv-exec "$called_progname"
+!
+  pyenv-rehash
+  PATH="$(path_without "$progname" "$called_progname")"
+  for disguised_shim_path in "$BATS_TEST_TMPDIR/"{weird-location,even/weirder/location}; do
+    mkdir -p "$disguised_shim_path"
+    ln -s "${PYENV_ROOT}/shims/$progname" "$disguised_shim_path/$progname"
+    PATH="$PATH:$disguised_shim_path"
+  done
+  create_executable "$disguised_shim_path" "$called_progname" <<!
+echo "convoluted call success!"
+!
+  real_path="$BATS_TEST_TMPDIR/real-location"
+  mkdir -p "$real_path"
+  ln -s "${PYENV_ROOT}/versions/custom/bin/$progname" "$real_path/$progname"
+  PATH="$PATH:$real_path"
+  
+  run "$progname"
+  assert_success
+  assert_output <<!
+called from $real_path/$progname
+convoluted call success!
+!
+
+  rm "$real_path/$progname"
+  run -127 "$progname"
+  assert_failure
+  assert_line 0 "pyenv: shimmed_program: command not found"
+}

test/test_helper.bash

@@ -223,3 +223,8 @@ create_hook() {
     cat > "${PYENV_HOOK_PATH}/$1/$2"
   fi
 }
+
+skip_if_nonposix_security() {
+  true "${1:?}" "${2:?}"
+  test "$@" && skip "UNIX permission bits are being overridden" || true
+}