Wait for cert-manager to be ready

- cert-manager now uses a webhook validator so needs to be in a
ready state before resources can be applied via kubectl
- updates instructions on switching from staging to prod issuer

Signed-off-by: Alex Ellis <alexellis2@gmail.com>

Author: alexellis

README.md

@@ -122,18 +122,32 @@ In order to enable TLS, edit the following configuration:
 If you need to test in Staging and then go to Production without resetting the cluster:
 * Use `issuer_type: "staging"`
 * Run ofc-bootstrap with the instructions bellow
-* Once you want to switch to Production run
+* Once you want to switch to the Production issuer
+
+Flush out the staging certificates and orders
+
+```sh
+kubectl delete certificates --all  -n openfaas
+kubectl delete secret -n openfaas -l="certmanager.k8s.io/certificate-name"
+kubectl delete order -n openfaas --all
 ```
+
+Now update the staging references to "prod":
+
+```sh
 sed -i '' s/letsencrypt-staging/letsencrypt-prod/g ./tmp/generated-ingress-ingress-wildcard.yaml
-kubectl apply -f ./tmp/generated-ingress-ingress-wildcard.yaml
 sed -i '' s/letsencrypt-staging/letsencrypt-prod/g ./tmp/generated-ingress-ingress.yaml
-kubectl apply -f ./tmp/generated-ingress-ingress.yaml
 sed -i '' s/letsencrypt-staging/letsencrypt-prod/g ./tmp/generated-tls-auth-domain-cert.yml
-kubectl apply -f ./tmp/generated-tls-auth-domain-cert.yml
 sed -i '' s/letsencrypt-staging/letsencrypt-prod/g ./tmp/generated-tls-wildcard-domain-cert.yml
-kubectl apply -f ./tmp/generated-tls-wildcard-domain-cert.yml
+```
 
-kubectl delete certificates --all  -n openfaas
+Now create the new ingress and certificates:
+
+```sh
+kubectl apply -f ./tmp/generated-ingress-ingress-wildcard.yaml
+kubectl apply -f ./tmp/generated-ingress-ingress.yaml
+kubectl apply -f ./tmp/generated-tls-auth-domain-cert.yml
+kubectl apply -f ./tmp/generated-tls-wildcard-domain-cert.yml
 ```
 
 ### Run the Bootstrapper

init.yaml

@@ -135,7 +135,8 @@ enable_oauth: false
 ### your OpenFaaS Cloud.
 tls: true
 tls_config:
-  email: "user@domain"
+#  email: "user@domain"
+
 
   # issuer_type: "prod"
   issuer_type: "staging"

main.go

@@ -194,6 +194,15 @@ func process(plan types.Plan) error {
 			log.Println(ofErr)
 		}
 
+		for i := 0; i < retries; i++ {
+			log.Printf("Is cert-manager ready? %d/%d\n", i+1, retries)
+			ready := certManagerReady()
+			if ready {
+				break
+			}
+			time.Sleep(time.Second * 2)
+		}
+
 		ingressErr := ingress.Apply(plan)
 		if ingressErr != nil {
 			log.Println(ingressErr)
@@ -334,8 +343,7 @@ func installIngressController() error {
 		return err
 	}
 
-	log.Println(res.Stdout)
-	log.Println(res.Stderr)
+	log.Println(res.ExitCode, res.Stdout, res.Stderr)
 
 	return nil
 }
@@ -374,8 +382,7 @@ func installOpenfaas() error {
 		return err
 	}
 
-	log.Println(res.Stdout)
-	log.Println(res.Stderr)
+	log.Println(res.ExitCode, res.Stdout, res.Stderr)
 
 	return nil
 }
@@ -414,8 +421,7 @@ func installCertmanager() error {
 		return err
 	}
 
-	log.Println(res.Stdout)
-	log.Println(res.Stderr)
+	log.Println(res.ExitCode, res.Stdout, res.Stderr)
 
 	return nil
 }
@@ -434,8 +440,7 @@ func createNamespaces() error {
 		return err
 	}
 
-	log.Println(res.Stdout)
-	log.Println(res.Stderr)
+	log.Println(res.ExitCode, res.Stdout, res.Stderr)
 
 	return nil
 }
@@ -491,6 +496,17 @@ func exportSealedSecretPubCert() string {
 	return res.Stdout
 }
 
+func certManagerReady() bool {
+	task := execute.ExecTask{
+		Command: "./scripts/get-cert-manager.sh",
+		Shell:   true,
+	}
+
+	res, err := task.Execute()
+	fmt.Println("cert-manager", res.ExitCode, res.Stdout, res.Stderr, err)
+	return res.Stdout == "True"
+}
+
 func tillerReady() bool {
 
 	task := execute.ExecTask{

pkg/ingress/ingress.go

@@ -18,6 +18,8 @@ type IngressTemplate struct {
 	DNSService string
 }
 
+// Apply templates and applies any ingress records required
+// for the OpenFaaS Cloud ingress configuration
 func Apply(plan types.Plan) error {
 
 	err := apply("ingress-wildcard.yml", "ingress-wildcard", IngressTemplate{
@@ -90,7 +92,7 @@ func apply(source string, name string, ingress IngressTemplate) error {
 		return execErr
 	}
 
-	log.Println(execRes.Stdout)
+	log.Println(execRes.ExitCode, execRes.Stdout, execRes.Stderr)
 
 	return nil
 }

pkg/tls/tls.go

@@ -97,7 +97,7 @@ func applyTemplate(tempFilePath string) error {
 		return execErr
 	}
 
-	log.Println(execRes.Stdout)
+	log.Println(execRes.ExitCode, execRes.Stdout, execRes.Stderr)
 
 	return nil
 }

scripts/create-namespaces.sh

@@ -1,3 +1,8 @@
 #!/bin/bash
 
 kubectl apply -f https://raw.githubusercontent.com/openfaas/faas-netes/master/namespaces.yml
+
+kubectl create namespace cert-manager
+kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+
+kubectl get namespaces

scripts/get-cert-manager.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# cert-manager is ready for CRD objects when this condition is "True"
+CERT_READY=$(kubectl get cert/cert-manager-webhook-webhook-tls -n  cert-manager -o jsonpath="{.status.conditions[0].status}")
+WEBHOOK_READY=$(kubectl get deploy/cert-manager-webhook -n cert-manager -o jsonpath="{.status.conditions[0].status}")
+
+if [ "$CERT_READY" = "True" ]
+then
+    if [ "$WEBHOOK_READY" = "True" ]
+    then
+        echo -n True
+    fi
+fi

scripts/install-cert-manager.sh

@@ -3,9 +3,6 @@
 kubectl apply \
     -f https://raw.githubusercontent.com/jetstack/cert-manager/release-0.6/deploy/manifests/00-crds.yaml
 
-kubectl create namespace cert-manager
-kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
-
 helm install \
     --name cert-manager \
     --namespace cert-manager \