Merge pull request #691 from tlhackque/master

Prefer API V2 when both offered in a single directory, improve test error reports

Author: timkimber

getssl

@@ -266,6 +266,8 @@
 # 2021-07-22 Only pass +noidnout param to dig/drill(#682)(2.38)
 # 2021-07-25 Fix copy_file_to_location failures with ssh when suffix applied to file lacking an extension (tlhackque)(#686)
 # 2021-07-27 Support ftps://, FTPS_OPTIONS, remove default --insecure parameter to ftpes. Report caller(s) of error_exit in debug and test modes (tlhackque)(#687)(2.39)
+# 2021-07-30 Prefer API V2 when both offered (tlhackque) (#690) (2.40)
+# 2021-07-30 Run tests with -d to catch intermittent failures, Use fork's repo for upgrade tests. (tlhackque) (#692) (2.41)
 # ----------------------------------------------------------------------------------------
 
 case :$SHELLOPTS: in
@@ -274,7 +276,7 @@ esac
 
 PROGNAME=${0##*/}
 PROGDIR="$(cd "$(dirname "$0")" || exit; pwd -P;)"
-VERSION="2.39"
+VERSION="2.41"
 
 # defaults
 ACCOUNT_KEY_LENGTH=4096
@@ -284,7 +286,11 @@ CA="https://acme-staging-v02.api.letsencrypt.org/directory"
 CHALLENGE_CHECK_TYPE="http"
 CHECK_REMOTE_WAIT=0
 CHECK_REMOTE="true"
-CODE_LOCATION="https://raw.githubusercontent.com/srvrco/getssl/master/getssl"
+if [[ -n "${GITHUB_REPOSITORY}" ]] ; then
+  CODE_LOCATION="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/master/getssl"
+else
+  CODE_LOCATION="https://raw.githubusercontent.com/srvrco/getssl/master/getssl"
+fi
 CSR_SUBJECT="/"
 CURL_USERAGENT="${PROGNAME}/${VERSION}"
 DEACTIVATE_AUTH="false"
@@ -330,10 +336,12 @@ DNS_WAIT_RETRY_ADD="false"      # Try the dns_add_command again if the DNS recor
 # Private variables
 _CHECK_ALL=0
 _CREATE_CONFIG=0
+_CURL_VERSION=""
 _FORCE_RENEW=0
 _KEEP_VERSIONS=""
 _MUTE=0
 _NOTIFY_VALID=0
+_NOMETER=""
 _QUIET=0
 _RECREATE_CSR=0
 _REDIRECT_OUTPUT="1>/dev/null 2>&1"
@@ -562,15 +570,15 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
         check_result=$($DNS_CHECK_FUNC $DNS_CHECK_OPTIONS TXT "${rr}" "@${ns}" \
                       | grep -i "^${rr}" \
                       | grep 'IN\WTXT'|awk -F'"' '{ print $2}')
-        debug "check_result=$check_result"
+        debug "check_result=\"$check_result\""
         if [[ -z "$check_result" ]]; then
           # shellcheck disable=SC2086
           debug "$DNS_CHECK_FUNC" $DNS_CHECK_OPTIONS ANY "${rr}" "@${ns}"
           # shellcheck disable=SC2086
           check_result=$($DNS_CHECK_FUNC $DNS_CHECK_OPTIONS ANY "${rr}" "@${ns}" \
                       | grep -i "^${rr}" \
                       | grep 'IN\WTXT'|awk -F'"' '{ print $2}')
-          debug "check_result=$check_result"
+          debug "check_result=\"$check_result\""
         fi
       elif [[ "$DNS_CHECK_FUNC" == "host" ]]; then
         check_result=$($DNS_CHECK_FUNC -t TXT "${rr}" "${ns}" \
@@ -583,8 +591,8 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
                       | grep 'text ='|awk -F'"' '{ print $2}')
         fi
       fi
-      debug "expecting  $auth_key"
-      debug "${ns} gave ... $check_result"
+      debug "expecting  \"$auth_key\""
+      debug "${ns} gave ... \"$check_result\""
 
       if [[ "$check_result" == *"$auth_key"* ]]; then
         check_dns="success"
@@ -606,7 +614,7 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
           debug "dns check failed - removing existing value"
           del_dns_rr "${d}" "${auth_key}"
 
-          error_exit "checking ${rr} gave $check_result not $auth_key"
+          error_exit "checking \"${rr}\" gave \"$check_result\" not \"$auth_key\""
         fi
       fi
     done
@@ -758,7 +766,7 @@ check_getssl_upgrade() { # check if a more recent version of code is available a
   if [ "$TEMP_UPGRADE_FILE" == "" ]; then
     error_exit "mktemp failed"
   fi
-  curl --user-agent "$CURL_USERAGENT" --silent "$CODE_LOCATION" --output "$TEMP_UPGRADE_FILE"
+  curl ${_NOMETER} --user-agent "$CURL_USERAGENT" --silent "$CODE_LOCATION" --output "$TEMP_UPGRADE_FILE"
   errcode=$?
   if [[ $errcode -eq 60 ]]; then
     error_exit "curl needs updating, your version does not support SNI (multiple SSL domains on a single IP)"
@@ -941,7 +949,7 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required.
       fromfile=$(basename "$from")
       debug "davs user=$davsuser - pass=$davspass - host=$davshost port=$davsport dir=$davsdirn file=$davsfile"
       debug "from dir=$fromdir  file=$fromfile"
-      curl -u "${davsuser}:${davspass}" -T "${fromdir}/${fromfile}" "https://${davshost}:${davsport}${davsdirn}${davsfile}"
+      curl ${_NOMETER} -u "${davsuser}:${davspass}" -T "${fromdir}/${fromfile}" "https://${davshost}:${davsport}${davsdirn}${davsfile}"
     elif [[ "${to:0:6}" == "ftpes:" ]] || [[ "${to:0:5}" == "ftps:" ]] ; then
       debug "using ftp to copy the file from $from"
       ftpuser=$(echo "$to"| awk -F: '{print $2}')
@@ -956,10 +964,10 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required.
       debug "from dir=$fromdir  file=$fromfile"
       if [[ "${to:0:5}" == "ftps:" ]] ; then
         # shellcheck disable=SC2086
-        curl $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}:990/"
+        curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}:990/"
       else
         # shellcheck disable=SC2086
-        curl $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/"
+        curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${ftpdirn}/"
       fi
     else
       if ! mkdir -p "$(dirname "$to")" ; then
@@ -1377,7 +1385,7 @@ for d in "${alldomains[@]}"; do
         # ACME only allows port 80 (http), but redirects may use https.  --insecure is used in case
         # those certificates are being renewed.  Let's Encrypt does the same.  In this case, we verify
         # that the correct data is returned, so this is safe.
-        if [[ ! "$(curl --user-agent "$CURL_USERAGENT" --insecure --silent --location "$wellknown_url")" == "$keyauthorization" ]]; then
+        if [[ ! "$(curl ${_NOMETER} --user-agent "$CURL_USERAGENT" --insecure --silent --location "$wellknown_url")" == "$keyauthorization" ]]; then
           error_exit "for some reason could not reach $wellknown_url - please check it manually"
         fi
       fi
@@ -1621,7 +1629,7 @@ get_certificate() { # get certificate for csr, if all domains validated.
     CertData=$(awk ' $1 ~ "^Location" {print $2}' "$CURL_HEADER" |tr -d '\r')
     if [[ "$CertData" ]] ; then
       echo -----BEGIN CERTIFICATE----- > "$gc_certfile"
-      curl --user-agent "$CURL_USERAGENT" --silent "$CertData" | openssl base64 -e  >> "$gc_certfile"
+      curl ${_NOMETER} --user-agent "$CURL_USERAGENT" --silent "$CertData" | openssl base64 -e  >> "$gc_certfile"
       echo -----END CERTIFICATE-----  >> "$gc_certfile"
       info "Certificate saved in $CERT_FILE"
     fi
@@ -1641,7 +1649,7 @@ get_certificate() { # get certificate for csr, if all domains validated.
                 | sed 's/>//g')
     if [[ "$IssuerData" ]] ; then
       echo -----BEGIN CERTIFICATE----- > "$gc_cafile"
-      curl --user-agent "$CURL_USERAGENT" --silent "$IssuerData" | openssl base64 -e  >> "$gc_cafile"
+      curl ${_NOMETER} --user-agent "$CURL_USERAGENT" --silent "$IssuerData" | openssl base64 -e  >> "$gc_cafile"
       echo -----END CERTIFICATE-----  >> "$gc_cafile"
       info "The intermediate CA cert is in $gc_cafile"
     fi
@@ -1699,7 +1707,7 @@ get_certificate() { # get certificate for csr, if all domains validated.
       cp "$gc_certfile" "$gc_fullchain"
       while [[ -n "$issuer_url" ]]; do
         debug Fetching certificate issuer from "$issuer_url"
-        issuer_cert=$(curl --user-agent "$CURL_USERAGENT" --silent "$issuer_url" | openssl x509 -inform der -outform pem)
+        issuer_cert=$(curl ${_NOMETER} --user-agent "$CURL_USERAGENT" --silent "$issuer_url" | openssl x509 -inform der -outform pem)
         debug Fetched issuer certificate "$(echo "$issuer_cert" | openssl x509 -inform pem -noout -text | awk 'BEGIN {FS="Subject: "} NF==2 {print $2; exit}')"
         echo "$issuer_cert" >> "$gc_fullchain"
 
@@ -1714,7 +1722,7 @@ get_certificate() { # get certificate for csr, if all domains validated.
 get_cr() { # get curl response
   url="$1"
   debug url "$url"
-  response=$(curl --user-agent "$CURL_USERAGENT" --silent "$url")
+  response=$(curl ${_NOMETER} --user-agent "$CURL_USERAGENT" --silent "$url")
   ret=$?
   debug response "${response//[$'\t\r\n']}"
   code=$(json_get "$response" status)
@@ -1797,7 +1805,7 @@ get_signing_params() { # get signing parameters from key
 }
 
 graceful_exit() { # normal exit function.
-  exit_code=$1
+  exit_code="${1-0}"
   clean_up
   # shellcheck disable=SC2086
   exit $exit_code
@@ -2055,7 +2063,7 @@ obtain_ca_resource_locations()
   for suffix in "" "/directory" "/dir";
   do
     # Obtain CA resource locations
-    ca_all_loc=$(curl --user-agent "$CURL_USERAGENT" "${CA}${suffix}" 2>/dev/null)
+    ca_all_loc=$(curl ${_NOMETER} --user-agent "$CURL_USERAGENT" "${CA}${suffix}" 2>/dev/null)
     debug "ca_all_loc from ${CA}${suffix} gives $ca_all_loc"
     # APIv1
     URL_new_reg=$(echo "$ca_all_loc" | grep "new-reg" | awk -F'"' '{print $4}')
@@ -2072,10 +2080,11 @@ obtain_ca_resource_locations()
     fi
   done
 
-  if [[ -n "$URL_new_reg" ]]; then
-    API=1
-  elif [[ -n "$URL_newAccount" ]]; then
+  # If a directory offers both versions, select V2.
+  if [[ -n "$URL_newAccount" ]]; then
     API=2
+  elif [[ -n "$URL_new_reg" ]]; then
+    API=1
   else
     error_exit "unknown API version"
   fi
@@ -2236,9 +2245,9 @@ send_signed_request() { # Sends a request to the ACME server, signed with your p
   CURL_HEADER="$TEMP_DIR/curl.header"
   dp="$TEMP_DIR/curl.dump"
 
-  CURL="curl "
+  CURL="curl ${_NOMETER} "
   # shellcheck disable=SC2072
-  if [[ "$($CURL -V | head -1 | cut -d' ' -f2 )" > "7.33" ]]; then
+  if [[ ! "${_CURL_VERSION}" < "7.33" ]]; then
     CURL="$CURL --http1.1 "
   fi
 
@@ -2314,7 +2323,7 @@ send_signed_request() { # Sends a request to the ACME server, signed with your p
       fi
 
       if [[ $errcode -gt 0 || ( "$response" == "" && $url != *"revoke"* ) ]]; then
-        error_exit "ERROR curl \"$url\" failed with $errcode and returned $response"
+        error_exit "ERROR curl \"$url\" failed with $errcode and returned \"$response\""
       fi
 
       responseHeaders=$(cat "$CURL_HEADER")
@@ -2414,13 +2423,14 @@ signal_exit() { # Handle trapped signals
 }
 
 traceback() { # Print function traceback
-    local i d=1 lbl=" called"
-    echo "Traceback" >&2
-    for ((i=$((${#FUNCNAME[@]}-1)); i>0; i--)); do
-      if [[ ${i} -eq 1 ]] ; then lbl=" called traceback" ; fi
-      printf "%*s%s() line %d%s\n" "$d" '' "${FUNCNAME[$i]}" "${BASH_LINENO[$((i-1))]}" "$lbl" >&2
-      ((d++))
-    done
+  local i d=1 lbl=" called"
+  debug "Traceback"
+  for ((i=$((${#FUNCNAME[@]}-1)); i>0; i--)); do
+    if [[ ${i} -eq 1 ]] ; then lbl=" called traceback" ; fi
+    debug "$(printf "%*s%s() line %d%s\n" "$d" '' "${FUNCNAME[$i]}" "${BASH_LINENO[$((i-1))]}" "$lbl")"
+    ((d++))
+  done
+  return 0
 }
 
 urlbase64() { # urlbase64: base64 encoded string with '+' replaced with '-' and '/' replaced with '_'
@@ -2625,6 +2635,12 @@ write_openssl_conf() { # write out a minimal openssl conf
 trap "signal_exit TERM" TERM HUP
 trap "signal_exit INT"  INT
 
+# When running tests, use debug mode to capture intermittent faults
+# Test harness will Save output in a temporary file, which is displayed if an error occurs
+if [[ ${_RUNNING_TEST} -eq 1 ]] ; then
+  _USE_DEBUG=1
+fi
+
 # Parse command-line
 while [[ -n ${1+defined} ]]; do
   case $1 in
@@ -2720,6 +2736,17 @@ requires sed
 requires sort
 requires mktemp
 
+# Make sure cURL doesn't display a progress meter (if it's new enough)
+# --silent also does this, but suppresses warnings and informational messages too.
+# TODO: see where --silent can be removed (if _NOMETER defaults to --silent for old versions?)
+#       This would help with debugging transfer errors.
+
+_CURL_VERSION="$(curl -V | head -1 | cut -d' ' -f2 )"
+# shellcheck disable=SC2072
+if [[ ! "${_CURL_VERSION}" < "7.67" ]]; then
+    _NOMETER="--no-progress-meter"
+fi
+
 # Check if upgrades are available (unless they have specified -U to ignore Upgrade checks)
 if [[ $_UPGRADE_CHECK -eq 1 ]]; then
   check_getssl_upgrade

test/1-simple-http01-dig.bats

@@ -7,6 +7,7 @@ load '/getssl/test/test_helper.bash'
 
 # This is run for every test
 setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
     export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
     if [ -f /usr/bin/host ]; then
         mv /usr/bin/host /usr/bin/host.getssl.bak
@@ -18,6 +19,7 @@ setup() {
 
 
 teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
     if [ -f /usr/bin/host.getssl.bak ]; then
         mv /usr/bin/host.getssl.bak /usr/bin/host
     fi

test/1-simple-http01-nslookup.bats

@@ -7,6 +7,7 @@ load '/getssl/test/test_helper.bash'
 
 # This is run for every test
 setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
     export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
     if [ -f /usr/bin/dig ]; then
         mv /usr/bin/dig /usr/bin/dig.getssl.bak
@@ -18,6 +19,7 @@ setup() {
 
 
 teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
     if [ -f /usr/bin/dig.getssl.bak ]; then
         mv /usr/bin/dig.getssl.bak /usr/bin/dig
     fi

test/1-simple-http01-two-acl.bats

@@ -6,7 +6,12 @@ load '/getssl/test/test_helper.bash'
 
 
 # This is run for every test
+teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
+}
+
 setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
     export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
 }
 

test/1-simple-http01.bats

@@ -6,7 +6,12 @@ load '/getssl/test/test_helper.bash'
 
 
 # This is run for every test
+teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
+}
+
 setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
     export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
 }
 

test/10-mixed-case.bats

@@ -6,7 +6,12 @@ load '/getssl/test/test_helper.bash'
 
 
 # This is run for every test
+teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
+}
+
 setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
     if [ -z "$STAGING" ]; then
         export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
     fi

test/11-test--install.bats

@@ -6,7 +6,12 @@ load '/getssl/test/test_helper.bash'
 
 
 # This is run for every test
+teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
+}
+
 setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
     export CURL_CA_BUNDLE=/root/pebble-ca-bundle.crt
 }
 

test/11-test-no-domain-storage.bats

@@ -5,6 +5,13 @@ load '/bats-assert/load.bash'
 load '/getssl/test/test_helper.bash'
 
 
+setup() {
+    [ ! -f $BATS_TMPDIR/failed.skip ] || skip "skipping tests after first failure"
+}
+teardown() {
+    [ -n "$BATS_TEST_COMPLETED" ] || touch $BATS_TMPDIR/failed.skip
+}
+
 @test "Check that if domain storage isn't set getssl doesn't try to delete /tmp" {
     if [ -n "$STAGING" ]; then
         skip "Using staging server, skipping internal test"