Merge branch 'master' into fix-domain-template-url

Author: timkimber

.github/workflows/run-all-tests.yml

@@ -21,8 +21,16 @@ jobs:
       - uses: actions/checkout@v1
       - name: Build the docker-compose stack
         run: docker-compose up -d --build
-      - name: Run test suite on centos6
+      - name: Run test suite on CentOS6
         run: test/run-test.sh centos6
+  test-centos7:
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v1
+        - name: Build the docker-compose stack
+          run: docker-compose up -d --build
+        - name: Run test suite on CentOS7
+          run: test/run-test.sh centos7
   test-centos7-duckdns:
     runs-on: ubuntu-latest
     steps:

dns_scripts/dns_add_challtestsrv

@@ -4,4 +4,4 @@
 fulldomain="${1}"
 token="${2}"
 
-curl -X POST -d "{\"host\":\"_acme-challenge.${fulldomain}.\", \"value\": \"${token}\"}" http://10.30.50.3:8055/set-txt
+curl --silent -X POST -d "{\"host\":\"_acme-challenge.${fulldomain}.\", \"value\": \"${token}\"}" http://10.30.50.3:8055/set-txt

getssl

@@ -214,7 +214,10 @@
 # 2020-02-13 Fix bug with copying to all locations when creating RSA and ECDSA certs (2.20)
 # 2020-02-22 Change sign_string to use openssl asn1parse (better fix for #424)
 # 2020-02-23 Add dig to config check for systems without drill (ubuntu)
-# 2020-03-23 Fix staging server URL in domain template
+# 2020-03-11 Use dig +trace to find primary name server and improve dig parsing of CNAME
+# 2020-03-12 Fix bug with DNS validation and multiple domains (#524)
+# 2020-03-24 Find primary ns using all dns utils (dig, host, nslookup)
+# 2020-03-23 Fix staging server URL in domain template (2.21)
 # ----------------------------------------------------------------------------------------
 
 PROGNAME=${0##*/}
@@ -234,7 +237,7 @@ CSR_SUBJECT="/"
 CURL_USERAGENT="${PROGNAME}/${VERSION}"
 DEACTIVATE_AUTH="false"
 DEFAULT_REVOKE_CA="https://acme-v02.api.letsencrypt.org"
-DNS_EXTRA_WAIT=""
+DNS_EXTRA_WAIT=60
 DNS_WAIT=10
 DOMAIN_KEY_LENGTH=4096
 DUAL_RSA_ECDSA="false"
@@ -750,13 +753,37 @@ create_order() {
   OrderLink=$(echo "$responseHeaders" | grep -i location | awk '{print $2}'| tr -d '\r\n ')
   debug "Order link $OrderLink"
   FinalizeLink=$(json_get "$response" "finalize")
-  dn=0
-  for d in $alldomains; do
-    # get authorizations link
-    AuthLink[$dn]=$(json_get "$response" "identifiers" "value" "$d" "authorizations" "x")
-    debug "authorizations link for $d - ${AuthLink[$dn]}"
-    ((dn++))
-  done
+
+  if [[ $API -eq 1 ]]; then
+    dn=0
+    for d in $alldomains; do
+      # get authorizations link
+      AuthLink[$dn]=$(json_get "$response" "identifiers" "value" "$d" "authorizations" "x")
+      debug "authorizations link for $d - ${AuthLink[$dn]}"
+      ((dn++))
+    done
+  else
+    # Authorization links are unsorted, so fetch the authorization link, find the domain, save response in the correct array position
+    AuthLinks=$(json_get "$response" "authorizations")
+    AuthLinkResponse=()
+    AuthLinkResponseHeader=()
+    for l in $AuthLinks; do
+      debug "Requesting authorizations link for $l"
+      send_signed_request "$l" ""
+      # Get domain from response
+      authdomain=$(json_get "$response" "identifier" "value")
+      # find array position (This is O(n2) but that doubt we'll see performance issues)
+      dn=0
+      for d in $alldomains; do
+        if [ "$d" == "$authdomain" ]; then
+          debug "Saving authorization response for $authdomain for domain alldomains[$dn]"
+          AuthLinkResponse[$dn]=$response
+          AuthLinkResponseHeader[$dn]=$responseHeaders
+        fi
+        ((dn++))
+      done
+    done
+  fi
 }
 
 date_epoc() { # convert the date into epoch time
@@ -801,6 +828,29 @@ error_exit() { # give error message on error exit
   exit 1
 }
 
+find_dns_utils() {
+    HAS_NSLOOKUP=false
+    HAS_DIG_OR_DRILL=""
+    HAS_HOST=false
+    if [[ -n "$(command -v nslookup)" ]]; then
+        debug "HAS NSLOOKUP=true"
+        HAS_NSLOOKUP=true
+    fi
+
+    if [[ -n "$(command -v drill)" ]]; then
+        debug "HAS DIG_OR_DRILL=drill"
+        HAS_DIG_OR_DRILL="drill"
+    elif [[ -n "$(command -v dig)" ]]; then
+        debug "HAS DIG_OR_DRILL=dig"
+        HAS_DIG_OR_DRILL="dig"
+    fi
+
+    if [[ -n "$(command -v host)" ]]; then
+        debug "HAS HOST=true"
+        HAS_HOST=true
+    fi
+}
+
 fulfill_challenges() {
 dn=0
 for d in $alldomains; do
@@ -823,7 +873,9 @@ for d in $alldomains; do
       error_exit "new-authz error: $response"
     fi
   else
-    send_signed_request "${AuthLink[$dn]}" ""
+    response=${AuthLinkResponse[$dn]}
+    responseHeaders=${AuthLinkResponseHeader[$dn]}
+    response_status=$(json_get "$response" status)
   fi
 
   if [[ $response_status == "valid" ]]; then
@@ -841,16 +893,14 @@ for d in $alldomains; do
     if [[ $VALIDATE_VIA_DNS == "true" ]]; then # set up the correct DNS token for verification
       if [[ $API -eq 1 ]]; then
         # get the dns component of the ACME response
-        # get the token from the dns component
+        # get the token and uri from the dns component
         token=$(json_get "$response" "token" "dns-01")
-        # get the uri from the dns component
         uri=$(json_get "$response" "uri" "dns-01")
         debug uri "$uri"
       else # APIv2
         debug "authlink response = $response"
-        # get the token from the http-01 component
+        # get the token and uri from the dns-01 component
         token=$(json_get "$response" "challenges" "type" "dns-01" "token")
-        # get the uri from the http component
         uri=$(json_get "$response" "challenges" "type" "dns-01" "url")
         debug uri "$uri"
       fi
@@ -901,7 +951,6 @@ for d in $alldomains; do
         uri=$(json_get "$response" "uri" "http-01")
         debug uri "$uri"
       else # APIv2
-        send_signed_request "${AuthLink[$dn]}" ""
         debug "authlink response = $response"
         # get the token from the http-01 component
         token=$(json_get "$response" "challenges" "type" "http-01" "token")
@@ -998,8 +1047,9 @@ if [[ $VALIDATE_VIA_DNS == "true" ]]; then
                            | grep ^_acme -A2\
                            | grep '"'|awk -F'"' '{ print $2}')
           elif [[ "$DNS_CHECK_FUNC" == "drill" ]] || [[ "$DNS_CHECK_FUNC" == "dig" ]]; then
+            debug "$DNS_CHECK_FUNC" TXT "_acme-challenge.${d}" "@${ns}"
             check_result=$($DNS_CHECK_FUNC TXT "_acme-challenge.${d}" "@${ns}" \
-                           | grep 'IN TXT'|awk -F'"' '{ print $2}')
+                           | grep 'IN\WTXT'|awk -F'"' '{ print $2}')
           elif [[ "$DNS_CHECK_FUNC" == "host" ]]; then
             check_result=$($DNS_CHECK_FUNC -t TXT "_acme-challenge.${d}" "${ns}" \
                            | grep 'descriptive text'|awk -F'"' '{ print $2}')
@@ -1054,10 +1104,11 @@ fi
 }
 
 get_auth_dns() { # get the authoritative dns server for a domain (sets primary_ns )
-  gad_d="$1" # domain name
+  orig_gad_d="$1" # domain name
   gad_s="$PUBLIC_DNS_SERVER" # start with PUBLIC_DNS_SERVER
 
   if [[ "$os" == "cygwin" ]]; then
+    gad_d="$orig_gad_d"
     all_auth_dns_servers=$(nslookup -type=soa "${d}" ${PUBLIC_DNS_SERVER} 2>/dev/null \
                           | grep "primary name server" \
                           | awk '{print $NF}')
@@ -1068,85 +1119,125 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n
     return
   fi
 
-  if [[ "$DNS_CHECK_FUNC" == "drill" ]] || [[ "$DNS_CHECK_FUNC" == "dig" ]]; then
-    if [[ -z "$gad_s" ]]; then #checking for CNAMEs
-      res=$($DNS_CHECK_FUNC CNAME "$gad_d"| grep "^$gad_d")
-    else
-      res=$($DNS_CHECK_FUNC CNAME "$gad_d" "@$gad_s"| grep "^$gad_d")
-    fi
-    if [[ -n "$res" ]]; then # domain is a CNAME so get main domain
-      gad_d=$(echo "$res"| awk '{print $5}' |sed 's/\.$//g')
-    fi
-    if [[ -z "$gad_s" ]]; then #checking for CNAMEs
-      res=$($DNS_CHECK_FUNC NS "$gad_d"| grep "^$gad_d")
+  if [[ -n "$HAS_DIG_OR_DRILL" ]]; then
+    gad_d="$orig_gad_d"
+    debug Using "$HAS_DIG_OR_DRILL SOA +trace +nocomments $gad_d @$gad_s" to find primary nameserver
+    # Use SOA +trace to find the name server
+    if [[ -z "$gad_s" ]]; then
+        res=$($HAS_DIG_OR_DRILL SOA +trace +nocomments "$gad_d" 2>/dev/null | grep "IN\WNS\W" | tail -1)
     else
-      res=$($DNS_CHECK_FUNC NS "$gad_d" "@$gad_s"| grep "^$gad_d")
+        res=$($HAS_DIG_OR_DRILL SOA +trace +nocomments "$gad_d" "@$gad_s" 2>/dev/null | grep "IN\WNS\W" | tail -1)
     fi
+
+    # fallback to existing code
     if [[ -z "$res" ]]; then
-      error_exit "couldn't find primary DNS server - please set AUTH_DNS_SERVER in config"
-    else
-      all_auth_dns_servers=$(echo "$res" | awk '$4 ~ "NS" {print $5}' | sed 's/\.$//g'|tr '\n' ' ')
+      debug Checking for CNAME using "$HAS_DIG_OR_DRILL CNAME $gad_d @$gad_s"
+      if [[ -z "$gad_s" ]]; then #checking for CNAMEs (need grep as dig 9.11 sometimes returns everything not just CNAME entries)
+        res=$($HAS_DIG_OR_DRILL CNAME "$gad_d"| grep "^$gad_d" | grep CNAME)
+      else
+        res=$($HAS_DIG_OR_DRILL CNAME "$gad_d" "@$gad_s"| grep "^$gad_d" | grep CNAME)
+      fi
+      if [[ -n "$res" ]]; then # domain is a CNAME so get main domain
+        gad_d=$(echo "$res"| awk '{print $5}' |sed 's/\.$//g')
+        debug Domain is a CNAME, actual domain is "$gad_d"
+      fi
+      # If gad_d is an A record then this returns the SOA for the root domain, e.g. without the www
+      #   dig NS ubuntu.getssl.text
+      #   > getssl.test.      IN   SOA ns1.duckdns.org
+      # If gad_d is a CNAME record then this returns the NS for the domain pointed to by $gad_d
+      #   dig NS www.getssl.text
+      #   > www.getssl.test.  IN CNAME getssl.test
+      #   > getssl.test.      IN    NS ns1.duckdns.org
+      debug Using "$HAS_DIG_OR_DRILL NS $gad_d @$gad_s" to find primary nameserver
+      if [[ -z "$gad_s" ]]; then
+        res=$($HAS_DIG_OR_DRILL NS "$gad_d"| grep -E "IN\W(NS|SOA)\W" | tail -1)
+      else
+        res=$($HAS_DIG_OR_DRILL NS "$gad_d" "@$gad_s"| grep -E "IN\W(NS|SOA)\W" | tail -1)
+      fi
     fi
-    if [[ $CHECK_ALL_AUTH_DNS == "true" ]]; then
-      primary_ns="$all_auth_dns_servers"
-    else
-      primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
+    if [[ -n "$res" ]]; then
+      all_auth_dns_servers=$(echo "$res" | awk '$4 ~ "NS" {print $5}' | sed 's/\.$//g'|tr '\n' ' ')
+      if [[ $CHECK_ALL_AUTH_DNS == "true" ]]; then
+        primary_ns="$all_auth_dns_servers"
+      else
+        primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
+      fi
+      return
     fi
-    return
   fi
 
-  if [[ "$DNS_CHECK_FUNC" == "host" ]]; then
+  if [[ "$HAS_HOST" == true ]]; then
+    gad_d="$orig_gad_d"
+    debug Using "host -t NS" to find primary name server for "$gad_d"
     if [[ -z "$gad_s" ]]; then
-      res=$($DNS_CHECK_FUNC -t NS "$gad_d"| grep "name server")
+      res=$(host -t NS "$gad_d"| grep "name server")
     else
-      res=$($DNS_CHECK_FUNC -t NS "$gad_d" "$gad_s"| grep "name server")
+      res=$(host -t NS "$gad_d" "$gad_s"| grep "name server")
     fi
-    if [[ -z "$res" ]]; then
-      error_exit "couldn't find primary DNS server - please set AUTH_DNS_SERVER in config"
-    else
+    if [[ -n "$res" ]]; then
       all_auth_dns_servers=$(echo "$res" | awk '{print $4}' | sed 's/\.$//g'|tr '\n' ' ')
+      if [[ $CHECK_ALL_AUTH_DNS == "true" ]]; then
+        primary_ns="$all_auth_dns_servers"
+      else
+        primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
+      fi
+      return
     fi
-    if [[ $CHECK_ALL_AUTH_DNS == "true" ]]; then
-      primary_ns="$all_auth_dns_servers"
-    else
-      primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
-    fi
-    return
   fi
 
-  res=$(nslookup -debug -type=soa -type=ns "$gad_d" ${gad_s})
+  if [[ "$HAS_NSLOOKUP" == true ]]; then
+    gad_d="$orig_gad_d"
+    debug Using "nslookup -debug -type=soa -type=ns $gad_d $gad_s" to find primary name server
+    res=$(nslookup -debug -type=soa -type=ns "$gad_d" ${gad_s})
+
+    if [[ "$(echo "$res" | grep -c "Non-authoritative")" -gt 0 ]]; then
+      # this is a Non-authoritative server, need to check for an authoritative one.
+      gad_s=$(echo "$res" | awk '$2 ~ "nameserver" {print $4; exit }' |sed 's/\.$//g')
+      if [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
+        # if domain name doesn't exist, then find auth servers for next level up
+        gad_s=$(echo "$res" | awk '$1 ~ "origin" {print $3; exit }')
+        gad_d=$(echo "$res" | awk '$1 ~ "->" {print $2; exit}')
+        # handle scenario where awk returns nothing
+        if [[ -z "$gad_d" ]]; then
+          gad_d="$orig_gad_d"
+        fi
+      fi
 
-  if [[ "$(echo "$res" | grep -c "Non-authoritative")" -gt 0 ]]; then
-    # this is a Non-authoritative server, need to check for an authoritative one.
-    gad_s=$(echo "$res" | awk '$2 ~ "nameserver" {print $4; exit }' |sed 's/\.$//g')
-    if [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
-      # if domain name doesn't exist, then find auth servers for next level up
-      gad_s=$(echo "$res" | awk '$1 ~ "origin" {print $3; exit }')
-      gad_d=$(echo "$res" | awk '$1 ~ "->" {print $2; exit}')
+      # shellcheck disable=SC2086
+      res=$(nslookup -debug -type=soa -type=ns "$gad_d" ${gad_s})
     fi
-  fi
 
-  if [[ -z "$gad_s" ]]; then
-    res=$(nslookup -debug -type=soa -type=ns "$gad_d")
-  else
-    res=$(nslookup -debug -type=soa -type=ns "$gad_d" "${gad_s}")
-  fi
+    if [[ "$(echo "$res" | grep -c "canonical name")" -gt 0 ]]; then
+      gad_d=$(echo "$res" | awk ' $2 ~ "canonical" {print $5; exit }' |sed 's/\.$//g')
+    elif [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
+      gad_s=$(echo "$res" | awk ' $1 ~ "origin" {print $3; exit }')
+      gad_d=$(echo "$res"| awk '$1 ~ "->" {print $2; exit}')
+      # handle scenario where awk returns nothing
+      if [[ -z "$gad_d" ]]; then
+        gad_d="$orig_gad_d"
+      fi
+    fi
 
-  if [[ "$(echo "$res" | grep -c "canonical name")" -gt 0 ]]; then
-    gad_d=$(echo "$res" | awk ' $2 ~ "canonical" {print $5; exit }' |sed 's/\.$//g')
-  elif [[ "$(echo "$res" | grep -c "an't find")" -gt 0 ]]; then
-    gad_s=$(echo "$res" | awk ' $1 ~ "origin" {print $3; exit }')
-    gad_d=$(echo "$res"| awk '$1 ~ "->" {print $2; exit}')
-  fi
+    # shellcheck disable=SC2086
+    # not quoting gad_s fixes the nslookup: couldn't get address for '': not found warning (#332)
+    all_auth_dns_servers=$(nslookup -debug -type=soa -type=ns "$gad_d" $gad_s \
+                          | awk '$1 ~ "nameserver" {print $3}' \
+                          | sed 's/\.$//g'| tr '\n' ' ')
 
-  all_auth_dns_servers=$(nslookup -type=soa -type=ns "$gad_d" "$gad_s" \
-                        | awk ' $2 ~ "nameserver" {print $4}' \
-                        | sed 's/\.$//g'| tr '\n' ' ')
-  if [[ $CHECK_ALL_AUTH_DNS == "true" ]]; then
-    primary_ns="$all_auth_dns_servers"
-  else
-    primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
+    if [[ -n "$all_auth_dns_servers" ]]; then
+      if [[ $CHECK_ALL_AUTH_DNS == "true" ]]; then
+        primary_ns="$all_auth_dns_servers"
+      else
+        primary_ns=$(echo "$all_auth_dns_servers" | awk '{print $1}')
+      fi
+      return
+    fi
   fi
+
+  # nslookup on alpine/ubuntu containers doesn't support -debug, print a warning in this case
+  # This means getssl cannot check that the DNS record has been updated on the primary name server
+  info "Warning: Couldn't find primary DNS server - please set PUBLIC_DNS_SERVER or AUTH_DNS_SERVER in config"
+  info "This means getssl cannot check the DNS entry has been updated"
 }
 
 get_certificate() { # get certificate for csr, if all domains validated.
@@ -2248,6 +2339,9 @@ set_server_type
 # check config for typical errors.
 check_config
 
+# check what dns utils are installed
+find_dns_utils
+
 if [[ -e "$DOMAIN_DIR/FORCE_RENEWAL" ]]; then
   rm -f "$DOMAIN_DIR/FORCE_RENEWAL" || error_exit "problem deleting file $DOMAIN_DIR/FORCE_RENEWAL"
   _FORCE_RENEW=1