Merge pull request #50 from gruntwork-io/yori-ubuntu18

Add ubuntu18.04 to examples and test

Author: yorinasub17

examples/couchbase-ami/couchbase.json

@@ -24,6 +24,25 @@
       "most_recent": true
     },
     "ssh_username": "ubuntu"
+  },{
+    "name": "ubuntu-18-ami",
+    "ami_name": "{{user `base_ami_name`}}-ubuntu-18-example-{{isotime | clean_ami_name}}",
+    "ami_description": "An Ubuntu 18.04 AMI that has Couchbase installed.",
+    "instance_type": "t2.micro",
+    "region": "{{user `aws_region`}}",
+    "type": "amazon-ebs",
+    "source_ami_filter": {
+      "filters": {
+        "virtualization-type": "hvm",
+        "architecture": "x86_64",
+        "name": "ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*",
+        "block-device-mapping.volume-type": "gp2",
+        "root-device-type": "ebs"
+      },
+      "owners": ["099720109477"],
+      "most_recent": true
+    },
+    "ssh_username": "ubuntu"
   },{
     "name": "amazon-linux-ami",
     "ami_name": "{{user `base_ami_name`}}-amazon-linux-example-{{isotime | clean_ami_name}}",
@@ -53,6 +72,16 @@
     "volumes": {
       "/sys/fs/cgroup": "/sys/fs/cgroup"
     }
+  },{
+    "name": "ubuntu-18-docker",
+    "type": "docker",
+    "image": "gruntwork/ubuntu-test:18.04",
+    "commit": true,
+    "run_command": ["-d", "-i", "-t", "{{.Image}}", "/sbin/init"],
+    "privileged": true,
+    "volumes": {
+      "/sys/fs/cgroup": "/sys/fs/cgroup"
+    }
   },{
     "name": "amazon-linux-docker",
     "type": "docker",
@@ -75,10 +104,10 @@
     "type": "shell",
     "pause_before": "5s",
     "inline": [
-      "DEBIAN_FRONTEND=noninteractive apt-get update",
-      "apt-get install -y git"
+      "apt-get update",
+      "DEBIAN_FRONTEND=noninteractive apt-get install -y git"
     ],
-    "only": ["ubuntu-docker"]
+    "only": ["ubuntu-docker", "ubuntu-18-docker"]
   },{
     "type": "shell",
     "pause_before": "30s",
@@ -91,11 +120,11 @@
     "type": "shell",
     "pause_before": "30s",
     "inline": [
-      "DEBIAN_FRONTEND=noninteractive sudo apt-get update",
-      "sudo apt-get install -y jq curl git python-pip",
+      "sudo apt-get update",
+      "sudo DEBIAN_FRONTEND=noninteractive apt-get install -y jq curl git python-pip",
       "sudo -H pip install --upgrade awscli"
     ],
-    "only": ["ubuntu-ami"]
+    "only": ["ubuntu-ami", "ubuntu-18-ami"]
   },{
     "type": "shell",
     "inline": [
@@ -126,6 +155,11 @@
     "repository": "gruntwork/couchbase-ubuntu-test",
     "tag": "latest",
     "only": ["ubuntu-docker"]
+  },{
+    "type": "docker-tag",
+    "repository": "gruntwork/couchbase-ubuntu-18-test",
+    "tag": "latest",
+    "only": ["ubuntu-18-docker"]
   },{
     "type": "docker-tag",
     "repository": "gruntwork/couchbase-amazon-linux-test",

modules/couchbase-cluster/main.tf

@@ -162,6 +162,13 @@ resource "aws_iam_instance_profile" "instance_profile" {
   lifecycle {
     create_before_destroy = true
   }
+
+  # IAM objects take time to propagate. This leads to subtle eventual consistency bugs where the ASG cannot be created
+  # because the IAM instance profile does not exist. We add a 15 second wait here to give the IAM instance profile a
+  # chance to propagate within AWS.
+  provisioner "local-exec" {
+    command = "echo 'Sleeping for 15 seconds to wait for IAM instance profile to be created'; sleep 15"
+  }
 }
 
 resource "aws_iam_role" "instance_role" {
@@ -176,6 +183,13 @@ resource "aws_iam_role" "instance_role" {
   lifecycle {
     create_before_destroy = true
   }
+
+  # IAM objects take time to propagate. This leads to subtle eventual consistency bugs where the ASG cannot be created
+  # because the IAM role does not exist. We add a 15 second wait here to give the IAM role a chance to propagate within
+  # AWS.
+  provisioner "local-exec" {
+    command = "echo 'Sleeping for 15 seconds to wait for IAM role to be created'; sleep 15"
+  }
 }
 
 data "aws_iam_policy_document" "instance_role" {

modules/install-couchbase-server/disable-thp

@@ -9,7 +9,7 @@
 # Short-Description: Disable THP
 # Description:       disables Transparent Huge Pages (THP) on boot
 ### END INIT INFO
-          
+
 case $1 in
 start)
   if [ -d /sys/kernel/mm/transparent_hugepage ]; then
@@ -22,4 +22,4 @@ start)
     return 0
   fi
 ;;
-esac
+esac

modules/install-couchbase-server/install-couchbase-server

@@ -45,7 +45,7 @@ function print_usage {
   echo
   echo "Usage: install-couchbase-server [options]"
   echo
-  echo "This script can be used to install Couchbase Server and its dependencies. This script has been tested with Ubuntu 16.04 and Amazon Linux 2."
+  echo "This script can be used to install Couchbase Server and its dependencies. This script has been tested with Ubuntu 16.04, 18.04, and Amazon Linux 2."
   echo
   echo "Options:"
   echo
@@ -65,29 +65,40 @@ function install_couchbase_on_ubuntu {
   local readonly version="$2"
   local readonly checksum="$3"
   local readonly checksum_type="$4"
+  local readonly ubuntu_version="$(lsb_release -r -s)"
+  local readonly dpkg_name="couchbase-server.deb"
 
-  log_info "Installing Couchbase $version ($edition edition) on Ubuntu"
+  log_info "Installing Couchbase $version ($edition edition) on Ubuntu $ubuntu_version"
 
-  local readonly filepath="couchbase-server-${edition}_${version}-ubuntu16.04_amd64.deb"
+  local readonly filepath="couchbase-server-${edition}_${version}-ubuntu${ubuntu_version}_amd64.deb"
   local readonly url="https://packages.couchbase.com/releases/$version/$filepath"
 
+  # NOTE: Couchbase only started publishing Ubuntu 18.04 deb packages for Enterprise edition version 6.0.2, which means
+  # that the vast majority of Couchbase server builds do not have official deb packages for 18.04. That said, the 16.04
+  # package has been verified to work, so we will fallback to using that if the 18.04 build is not available.
+  local readonly fallback_filepath="couchbase-server-${edition}_${version}-ubuntu16.04_amd64.deb"
+  local readonly fallback_url="https://packages.couchbase.com/releases/$version/$fallback_filepath"
+
   log_info "Installing Couchbase dependencies"
   sudo DEBIAN_FRONTEND=noninteractive apt-get update
   sudo apt-get install -y python-httplib2
 
   log_info "Downloading Couchbase from $url to $filepath"
-  curl --location --silent --fail --show-error -O "$url"
+  if [[ "$(curl --location --silent --fail --show-error -o "$dpkg_name" -w "%{http_code}" "$url")" == "404" ]]; then
+    log_warn "Failed to get Couchbase from $url. Trying $fallback_url."
+    curl --location --silent --fail --show-error -o "$dpkg_name" "$fallback_url"
+  fi
 
-  os_validate_checksum "$filepath" "$checksum" "$checksum_type"
+  os_validate_checksum "$dpkg_name" "$checksum" "$checksum_type"
 
   # Install Couchbase, but configure it to NOT start on boot. This allows the run-couchbase-server script to configure
   # Couchbase, including what ports to use, and THEN boot it up.
-  log_info "Installing Couchbase from $filepath"
-  sudo INSTALL_DONT_START_SERVER=1 dpkg -i "$filepath"
+  log_info "Installing Couchbase from $dpkg_name"
+  sudo INSTALL_DONT_START_SERVER=1 dpkg -i "$dpkg_name"
   sudo systemctl disable couchbase-server
 
-  log_info "Cleaning up $filepath"
-  rm -f "$filepath"
+  log_info "Cleaning up $dpkg_name"
+  rm -f "$dpkg_name"
 }
 
 function install_couchbase_on_amazon_linux {
@@ -136,7 +147,7 @@ function disable_transparent_huge_pages {
   elif os_is_amazon_linux; then
     sudo chkconfig disable-thp on
   else
-    log_error "This script only supports Ubuntu 16.04 and Amazon Linux 2."
+    log_error "This script only supports Ubuntu 16.04, 18.04, and Amazon Linux 2."
     exit 1
   fi
 }
@@ -248,7 +259,7 @@ function install {
 
   log_info "Starting Couchbase install..."
 
-  if os_is_ubuntu "16.04"; then
+  if os_is_ubuntu "16.04" || os_is_ubuntu "18.04"; then
     install_couchbase_on_ubuntu "$edition" "$version" "$checksum" "$checksum_type"
   elif os_is_amazon_linux "2"; then
     install_couchbase_on_amazon_linux "$edition" "$version" "$checksum" "$checksum_type"

modules/install-sync-gateway/install-sync-gateway

@@ -48,7 +48,7 @@ function print_usage {
   echo
   echo "Usage: install-sync-gateway [options]"
   echo
-  echo "This script can be used to install Couchbase Sync Gateway and its dependencies. This script has been tested with Ubuntu 16.04 and Amazon Linux 2."
+  echo "This script can be used to install Couchbase Sync Gateway and its dependencies. This script has been tested with Ubuntu 16.04, 18.04, and Amazon Linux 2."
   echo
   echo "Options:"
   echo
@@ -257,12 +257,12 @@ function install {
 
   log_info "Starting Sync Gateway install..."
 
-  if os_is_ubuntu "16.04"; then
+  if os_is_ubuntu "16.04" || os_is_ubuntu "18.04"; then
     install_sync_gateway_on_ubuntu "$edition" "$version" "$checksum" "$checksum_type"
   elif os_is_amazon_linux "2"; then
     install_sync_gateway_on_amazon_linux "$edition" "$version" "$checksum" "$checksum_type"
   else
-    log_error "This script only supports Ubuntu 16.04 and Amazon Linux 2."
+    log_error "This script only supports Ubuntu 16.04, 18.04, and Amazon Linux 2."
     exit 1
   fi
 

test/couchbase_multi_cluster_test.go

@@ -1,24 +1,30 @@
 package test
 
 import (
-	"testing"
-	"path/filepath"
 	"fmt"
-	"github.com/gruntwork-io/terratest/modules/test-structure"
-	"github.com/gruntwork-io/terratest/modules/terraform"
-	"github.com/gruntwork-io/terratest/modules/random"
+	"path/filepath"
+	"testing"
+
 	"github.com/gruntwork-io/terratest/modules/aws"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/gruntwork-io/terratest/modules/test-structure"
 )
 
 const dataNodeClusterVarName = "couchbase_data_node_cluster_name"
 const indexQuerySearchClusterVarName = "couchbase_index_query_search_node_cluster_name"
 const syncGatewayClusterVarName = "sync_gateway_cluster_name"
 
-func TestIntegrationCouchbaseEnterpriseMultiClusterUbuntu(t *testing.T) {
+func TestIntegrationCouchbaseEnterpriseMultiClusterUbuntu16(t *testing.T) {
 	t.Parallel()
 	testCouchbaseMultiCluster(t, "ubuntu", "enterprise")
 }
 
+func TestIntegrationCouchbaseEnterpriseMultiClusterUbuntu18(t *testing.T) {
+	t.Parallel()
+	testCouchbaseMultiCluster(t, "ubuntu-18", "enterprise")
+}
+
 func TestIntegrationCouchbaseEnterpriseMultiClusterAmazonLinux(t *testing.T) {
 	t.Parallel()
 	testCouchbaseMultiCluster(t, "amazon-linux", "enterprise")
@@ -65,11 +71,11 @@ func testCouchbaseMultiCluster(t *testing.T, osName string, edition string) {
 
 		terraformOptions := &terraform.Options{
 			TerraformDir: couchbaseMultiClusterDir,
-			Vars: map[string]interface{} {
+			Vars: map[string]interface{}{
 				"ami_id":                       amiId,
 				dataNodeClusterVarName:         formatCouchbaseClusterName("data", uniqueId),
 				indexQuerySearchClusterVarName: formatCouchbaseClusterName("search", uniqueId),
-				syncGatewayClusterVarName: 		formatCouchbaseClusterName("sync", uniqueId),
+				syncGatewayClusterVarName:      formatCouchbaseClusterName("sync", uniqueId),
 			},
 			EnvVars: map[string]string{
 				AWS_DEFAULT_REGION_ENV_VAR: awsRegion,
@@ -96,4 +102,3 @@ func testCouchbaseMultiCluster(t *testing.T, osName string, edition string) {
 		checkSyncGatewayWorking(t, syncGatewayUrl)
 	})
 }
-

test/couchbase_multi_datacenter_replication_test.go

@@ -26,11 +26,16 @@ const savedUniqueIdReplica = "UniqueIdReplica"
 const providersFile = "providers.tf"
 const providersFileBackup = "providers.tf.bak"
 
-func TestIntegrationCouchbaseEnterpriseMultiDataCenterReplicationUbuntu(t *testing.T) {
+func TestIntegrationCouchbaseEnterpriseMultiDataCenterReplicationUbuntu16(t *testing.T) {
 	t.Parallel()
 	testCouchbaseMultiDataCenterReplication(t, "ubuntu", "enterprise")
 }
 
+func TestIntegrationCouchbaseEnterpriseMultiDataCenterReplicationUbuntu18(t *testing.T) {
+	t.Parallel()
+	testCouchbaseMultiDataCenterReplication(t, "ubuntu-18", "enterprise")
+}
+
 func TestIntegrationCouchbaseEnterpriseMultiDataCenterReplicationAmazonLinux(t *testing.T) {
 	t.Parallel()
 	testCouchbaseMultiDataCenterReplication(t, "amazon-linux", "enterprise")

test/couchbase_single_cluster_dns_tls_test.go

@@ -16,11 +16,16 @@ const domainNameForTest = "gruntwork.in"
 // filter them down to the real public hosted zone for domainNameForTest.
 var domainNameTags = map[string]string{"original": "true"}
 
-func TestIntegrationCouchbaseCommunitySingleClusterDnsTlsUbuntu(t *testing.T) {
+func TestIntegrationCouchbaseCommunitySingleClusterDnsTlsUbuntu16(t *testing.T) {
 	t.Parallel()
 	testCouchbaseSingleClusterDnsTls(t, "ubuntu", "community")
 }
 
+func TestIntegrationCouchbaseCommunitySingleClusterDnsTlsUbuntu18(t *testing.T) {
+	t.Parallel()
+	testCouchbaseSingleClusterDnsTls(t, "ubuntu-18", "community")
+}
+
 func testCouchbaseSingleClusterDnsTls(t *testing.T, osName string, edition string) {
 	examplesFolder := test_structure.CopyTerraformFolderToTemp(t, "../", "examples")
 	couchbaseAmiDir := filepath.Join(examplesFolder, "couchbase-ami")

test/couchbase_single_cluster_test.go

@@ -11,21 +11,31 @@ import (
 
 const couchbaseClusterVarName = "cluster_name"
 
-func TestIntegrationCouchbaseCommunitySingleClusterUbuntu(t *testing.T) {
+func TestIntegrationCouchbaseCommunitySingleClusterUbuntu16(t *testing.T) {
 	t.Parallel()
 	testCouchbaseSingleCluster(t, "ubuntu", "community")
 }
 
+func TestIntegrationCouchbaseCommunitySingleClusterUbuntu18(t *testing.T) {
+	t.Parallel()
+	testCouchbaseSingleCluster(t, "ubuntu-18", "community")
+}
+
 func TestIntegrationCouchbaseCommunitySingleClusterAmazonLinux(t *testing.T) {
 	t.Parallel()
 	testCouchbaseSingleCluster(t, "amazon-linux", "community")
 }
 
-func TestIntegrationCouchbaseEnterpriseSingleClusterUbuntu(t *testing.T) {
+func TestIntegrationCouchbaseEnterpriseSingleClusterUbuntu16(t *testing.T) {
 	t.Parallel()
 	testCouchbaseSingleCluster(t, "ubuntu", "enterprise")
 }
 
+func TestIntegrationCouchbaseEnterpriseSingleClusterUbuntu18(t *testing.T) {
+	t.Parallel()
+	testCouchbaseSingleCluster(t, "ubuntu-18", "enterprise")
+}
+
 func TestIntegrationCouchbaseEnterpriseSingleClusterAmazonLinux(t *testing.T) {
 	t.Parallel()
 	testCouchbaseSingleCluster(t, "amazon-linux", "enterprise")