WIP: Recover properly when bastion goes down

pkg/terraform/providers/tarmak/data_source_bastion_instance.go

@@ -19,10 +19,6 @@ func dataSourceBastionInstance() *schema.Resource {
 				Type:     schema.TypeString,
 				Required: true,
 			},
-			"instance_id": {
-				Type:     schema.TypeString,
-				Optional: true,
-			},
 			"username": {
 				Type:     schema.TypeString,
 				Required: true,

pkg/terraform/providers/tarmak/resource_vault_cluster.go

@@ -18,6 +18,7 @@ func resourceTarmakVaultCluster() *schema.Resource {
 		Create: resourceTarmakVaultClusterCreate,
 		Read:   resourceTarmakVaultClusterRead,
 		Delete: resourceTarmakVaultClusterDelete,
+		Update: resourceTarmakVaultClusterCreate,
 
 		Schema: map[string]*schema.Schema{
 			"internal_fqdns": {
@@ -43,6 +44,10 @@ func resourceTarmakVaultCluster() *schema.Resource {
 				Required: true,
 				ForceNew: true,
 			},
+			"bastion_status": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
 			"status": {
 				Type:     schema.TypeString,
 				Computed: true,
@@ -52,12 +57,10 @@ func resourceTarmakVaultCluster() *schema.Resource {
 }
 
 func resourceTarmakVaultClusterCreate(d *schema.ResourceData, meta interface{}) (err error) {
+
 	client := meta.(*rpc.Client)
 
 	vaultInternalFQDNs := []string{}
-
-	//return fmt.Errorf("DEBUG: %#v", d.Get("internal_fqdns").([]interface{})[0])
-
 	for _, internalFQDN := range d.Get("internal_fqdns").([]interface{}) {
 		vaultInternalFQDNs = append(vaultInternalFQDNs, internalFQDN.(string))
 	}
@@ -94,6 +97,7 @@ func resourceTarmakVaultClusterCreate(d *schema.ResourceData, meta interface{})
 }
 
 func resourceTarmakVaultClusterRead(d *schema.ResourceData, meta interface{}) (err error) {
+
 	client := meta.(*rpc.Client)
 
 	vaultInternalFQDNs := []string{}

pkg/terraform/providers/tarmak/resource_vault_instance_role.go

@@ -2,7 +2,6 @@
 package tarmak
 
 import (
-	"fmt"
 	"log"
 	"net/rpc"
 
@@ -16,6 +15,7 @@ func resourceTarmakVaultInstanceRole() *schema.Resource {
 		Create: resourceTarmakVaultInstanceRoleCreate,
 		Read:   resourceTarmakVaultInstanceRoleRead,
 		Delete: resourceTarmakVaultInstanceRoleDelete,
+		Update: resourceTarmakVaultInstanceRoleCreate,
 
 		Schema: map[string]*schema.Schema{
 			"role_name": {
@@ -41,6 +41,10 @@ func resourceTarmakVaultInstanceRole() *schema.Resource {
 				Required: true,
 				ForceNew: true,
 			},
+			"vault_status": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
 			"init_token": {
 				Type:     schema.TypeString,
 				Computed: true,
@@ -52,6 +56,13 @@ func resourceTarmakVaultInstanceRole() *schema.Resource {
 func resourceTarmakVaultInstanceRoleCreate(d *schema.ResourceData, meta interface{}) (err error) {
 	client := meta.(*rpc.Client)
 
+	vaultStatus := d.Get("vault_status").(string)
+	if vaultStatus != tarmakRPC.VaultStatusReady {
+		log.Print("vault is not ready")
+		d.SetId("")
+		return nil
+	}
+
 	roleName := d.Get("role_name").(string)
 	clusterName := d.Get("vault_cluster_name").(string)
 	vaultInternalFQDNs := []string{}
@@ -72,12 +83,15 @@ func resourceTarmakVaultInstanceRoleCreate(d *schema.ResourceData, meta interfac
 	var reply tarmakRPC.VaultInstanceRoleReply
 	err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply)
 	if err != nil {
+		log.Printf("call to %s failed: %s", tarmakRPC.VaultInstanceRole, err)
 		d.SetId("")
-		return fmt.Errorf("call to %s failed: %s", tarmakRPC.VaultInstanceRole, err)
+		return nil
 	}
 
 	if err = d.Set("init_token", reply.InitToken); err != nil {
-		return fmt.Errorf("failed to set init token: %s", err)
+		log.Printf("failed to set init token: %s", err)
+		d.SetId("")
+		return
 	}
 
 	d.SetId(reply.InitToken)
@@ -88,6 +102,13 @@ func resourceTarmakVaultInstanceRoleCreate(d *schema.ResourceData, meta interfac
 func resourceTarmakVaultInstanceRoleRead(d *schema.ResourceData, meta interface{}) (err error) {
 	client := meta.(*rpc.Client)
 
+	vaultStatus := d.Get("vault_status").(string)
+	if vaultStatus != tarmakRPC.VaultStatusReady {
+		log.Printf("vault is not ready")
+		d.SetId("")
+		return nil
+	}
+
 	roleName := d.Get("role_name").(string)
 	clusterName := d.Get("vault_cluster_name").(string)
 	vaultInternalFQDNs := []string{}
@@ -108,6 +129,7 @@ func resourceTarmakVaultInstanceRoleRead(d *schema.ResourceData, meta interface{
 	var reply tarmakRPC.VaultInstanceRoleReply
 	err = client.Call(tarmakRPC.VaultInstanceRole, args, &reply)
 	if err != nil {
+		log.Printf("call to %s failed: %s", tarmakRPC.VaultInstanceRole, err)
 		d.SetId("")
 		return nil
 	}

pkg/terraform/providers/tarmak/rpc/bastion_instance_status.go

@@ -8,6 +8,13 @@ import (
 	cluster "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1"
 )
 
+const (
+	bastionVerifyTimeoutSeconds = 180
+	BastionStatusUnknown        = "unknown"
+	BastionStatusReady          = "ready"
+	BastionStatusDown           = "down"
+)
+
 var (
 	BastionInstanceStatusCall = fmt.Sprintf("%s.BastionInstanceStatus", RPCName)
 )
@@ -25,23 +32,53 @@ func (r *tarmakRPC) BastionInstanceStatus(args *BastionInstanceStatusArgs, resul
 	r.tarmak.Log().Debug("received rpc bastion status")
 
 	if r.cluster.GetState() == cluster.StateDestroy {
-		result.Status = "unknown"
+		result.Status = BastionStatusUnknown
 		return nil
 	}
 
-	var err error
-	for i := 1; i <= Retries; i++ {
-		if err = r.cluster.Environment().VerifyBastionAvailable(); err != nil {
-			r.tarmak.Log().Error(err)
-			time.Sleep(time.Second)
-		} else {
-			break
+	// check if bastion instance exists
+	instances, err := r.cluster.Environment().Provider().ListHosts(r.cluster.Environment().Hub())
+	if err != nil {
+		r.tarmak.Log().Debug("failed to list instances in hub: %s", err)
+		result.Status = BastionStatusUnknown
+		return nil
+	}
+	bastionExists := false
+	for _, instance := range instances {
+		for _, role := range instance.Roles() {
+			if role == cluster.InstancePoolTypeBastion {
+				bastionExists = true
+			}
 		}
 	}
-	if err != nil {
-		return fmt.Errorf("bastion instance is not ready: %s", err)
+	if !bastionExists {
+		r.tarmak.Log().Debug("bastion instance does not exist")
+		result.Status = BastionStatusDown
+		return nil
+	}
+
+	// verify bastion responsiveness
+	verifyChannel := make(chan bool)
+	go func() {
+		for {
+			if err := r.cluster.Environment().VerifyBastionAvailable(); err != nil {
+				r.tarmak.Log().Error(err)
+				time.Sleep(time.Second)
+				continue
+			}
+			verifyChannel <- true
+			return
+		}
+	}()
+
+	select {
+	case <-verifyChannel:
+	case <-time.After(bastionVerifyTimeoutSeconds * time.Second):
+		r.tarmak.Log().Debug("failed to verify bastion instance")
+		result.Status = BastionStatusDown
+		return nil
 	}
 
-	result.Status = "ready"
+	result.Status = BastionStatusReady
 	return nil
 }

pkg/terraform/providers/tarmak/rpc/vault_cluster_status.go

@@ -10,6 +10,11 @@ import (
 	cluster "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1"
 )
 
+const (
+	VaultStatusUnknown = "unknown"
+	VaultStatusReady   = "ready"
+)
+
 var (
 	VaultClusterStatusCall     = fmt.Sprintf("%s.VaultClusterStatus", RPCName)
 	VaultClusterInitStatusCall = fmt.Sprintf("%s.VaultClusterInitStatus", RPCName)
@@ -30,7 +35,7 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 	r.tarmak.Log().Debug("received rpc vault cluster status")
 
 	if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
-		result.Status = "unknown"
+		result.Status = VaultStatusUnknown
 		return nil
 	}
 
@@ -40,14 +45,16 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 	if err != nil {
 		err = fmt.Errorf("failed to initialise vault cluster: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 
 	vaultTunnel, err := vault.TunnelFromFQDNs(args.VaultInternalFQDNs, args.VaultCA)
 	if err != nil {
 		err = fmt.Errorf("failed to create vault tunnel: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 	defer vaultTunnel.Stop()
 
@@ -57,7 +64,8 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 	if err != nil {
 		err = fmt.Errorf("failed to retrieve vault root token: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 
 	vaultClient.SetToken(vaultRootToken)
@@ -68,18 +76,19 @@ func (r *tarmakRPC) VaultClusterStatus(args *VaultClusterStatusArgs, result *Vau
 	if err := k.Ensure(); err != nil {
 		err = fmt.Errorf("vault cluster is not ready: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 
-	result.Status = "ready"
+	result.Status = VaultStatusReady
 	return nil
 }
 
 func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result *VaultClusterStatusReply) error {
 	r.tarmak.Log().Debug("received rpc vault cluster status")
 
 	if r.tarmak.Cluster().GetState() == cluster.StateDestroy {
-		result.Status = "unknown"
+		result.Status = VaultStatusUnknown
 		return nil
 	}
 
@@ -89,7 +98,8 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
 	if err != nil {
 		err = fmt.Errorf("failed to create vault tunnel: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 	defer vaultTunnel.Stop()
 
@@ -99,7 +109,8 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
 	if err != nil {
 		err = fmt.Errorf("failed to retrieve vault root token: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 
 	vaultClient.SetToken(vaultRootToken)
@@ -117,14 +128,16 @@ func (r *tarmakRPC) VaultClusterInitStatus(args *VaultClusterStatusArgs, result
 	if err != nil {
 		err = fmt.Errorf("failed to retrieve init status: %s", err)
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 	if !up {
 		err = fmt.Errorf("failed to initialised vault cluster")
 		r.tarmak.Log().Error(err)
-		return err
+		result.Status = VaultStatusUnknown
+		return nil
 	}
 
-	result.Status = "ready"
+	result.Status = VaultStatusReady
 	return nil
 }

terraform/amazon/modules/bastion/bastion.tf

@@ -24,8 +24,14 @@ resource "aws_security_group" "bastion" {
   }
 }
 
+data "tarmak_bastion_instance" "bastion" {
+  hostname    = "bastion"
+  username    = "centos"
+
+  depends_on = ["aws_instance.bastion"]
+}
+
 resource "aws_instance" "bastion" {
-  count                  = 1
   ami                    = "${var.bastion_ami}"
   instance_type          = "${var.bastion_instance_type}"
   subnet_id              = "${var.public_subnet_ids[0]}"

terraform/amazon/modules/bastion/outputs.tf

@@ -1,24 +1,11 @@
-output "bastion_instance_id" {
-  value = "${element(concat(aws_instance.bastion.*.id, list("")), 0)}"
-}
-
-
-output "bastion_fqdn" {
-  value = "${aws_route53_record.bastion.fqdn}"
-}
-
-output "bastion_private_ip" {
-  value = "${aws_eip.bastion.public_ip}"
-}
-
-output "bastion_ip" {
-  value = "${aws_eip.bastion.public_ip}"
+output "bastion_status" {
+  value = "${data.tarmak_bastion_instance.bastion.status}"
 }
 
 output "bastion_security_group_id" {
   value = "${element(concat(aws_security_group.bastion.*.id, list("")), 0)}"
 }
 
-output "remote_admin_security_group_id" {
-  value = "${aws_security_group.remote_admin.id}"
+output "bastion_instance_id" {
+  value = "${element(concat(aws_instance.bastion.*.id, list("")), 0)}"
 }

terraform/amazon/modules/kubernetes/inputs.tf

@@ -47,10 +47,6 @@ variable "internal_fqdns" {
   type = "list"
 }
 
-variable "vault_kms_key_id" {}
-
-variable "vault_unseal_key_name" {}
-
 # template variables
 variable "availability_zones" {
   type = "list"
@@ -76,4 +72,6 @@ variable "public_zone_id" {}
 
 variable "vault_security_group_id" {}
 
-variable "bastion_security_group_id" {}
+variable "bastion_security_group_id" {}
+
+variable "vault_status" {}