Merge pull request #790 from guidonguido/gng/websockify

Author: kingmakerbot

deploy/crownlabs/values.yaml

@@ -75,6 +75,7 @@ instance-operator:
       novncImage: crownlabs/novnc
       filebrowserImage: filebrowser/filebrowser
       filebrowserImageTag: latest
+      instmetricsServerEndpoint: crownlabs-instmetrics.crownlabs-production:9090
     containerVmSnapshots:
       kanikoImage: gcr.io/kaniko-project/executor
       exportImage: "crownlabs/img-exporter"

operators/cmd/instance-operator/main.go

@@ -86,6 +86,7 @@ func main() {
 	flag.StringVar(&containerEnvOpts.ContentDownloaderImg, "container-env-content-downloader-img", "latest", "The image name for the init-container to download and unarchive initial content to the instance volume.")
 	flag.StringVar(&containerEnvOpts.ContentUploaderImg, "container-env-content-uploader-img", "latest", "The image name for the job to compress and upload instance content from a persistent instance.")
 	flag.StringVar(&containerEnvOpts.MyDriveImgAndTag, "container-env-mydrive-img-and-tag", "filebrowser/filebrowser:latest", "The image name and tag for the filebrowser image (sidecar for gui-based file manager)")
+	flag.StringVar(&containerEnvOpts.InstMetricsEndpoint, "container-env-instmetrics-server-endpoint", "instmetrics:9090", "The endpoint of the InstMetrics gRPC server")
 
 	flag.StringVar(&instSnapOpts.VMRegistry, "vm-registry", "", "The registry where VMs should be uploaded")
 	flag.StringVar(&instSnapOpts.RegistrySecretName, "vm-registry-secret", "", "The name of the secret for the VM registry")

operators/deploy/instance-operator/templates/deployment.yaml

@@ -45,6 +45,7 @@ spec:
             - "--container-env-content-downloader-img={{ .Values.configurations.containerEnvironmentOptions.contentDownloaderImage }}"
             - "--container-env-content-uploader-img={{ .Values.configurations.containerEnvironmentOptions.contentUploaderImage }}"
             - "--container-env-mydrive-img-and-tag={{ .Values.configurations.containerEnvironmentOptions.mydriveImageAndTag }}"
+            - "--container-env-instmetrics-server-endpoint={{ .Values.configurations.containerEnvironmentOptions.instmetricsServerEndpoint }}"
             - "--vm-registry={{ .Values.configurations.privateContainerRegistry.url }}"
             - "--vm-registry-secret={{ .Values.configurations.privateContainerRegistry.secretName }}"
             - "--container-export-img={{ .Values.configurations.containerVmSnapshots.exportImage }}:{{ include "instance-operator.containerExportImageTag" . }}"

operators/deploy/instance-operator/values.yaml

@@ -19,6 +19,7 @@ configurations:
     contentDownloaderImage: crownlabs/content-downloader
     contentUploaderImage: crownlabs/content-uploader
     mydriveImageAndTag: filebrowser/filebrowser:latest
+    instmetricsServerEndpoint: crownlabs-instmetrics.crownlabs-production:9090
   containerVmSnapshots:
     kanikoImage: gcr.io/kaniko-project/executor:latest
     exportImage: "crownlabs/img-exporter"

operators/pkg/forge/containers.go

@@ -53,10 +53,23 @@ const (
 	CrownLabsUserID = int64(1010)
 	// SubmissionJobMaxRetries -> max number of retries for submission jobs.
 	SubmissionJobMaxRetries = 10
+	// AppCPULimitsEnvName -> name of the env variable containing AppContainer CPU limits.
+	AppCPULimitsEnvName = "APP_CPU_LIMITS"
+	// AppMEMLimitsEnvName -> name of the env variable containing AppContainer memory limits.
+	AppMEMLimitsEnvName = "APP_MEM_LIMITS"
+	// PodNameEnvName -> name of the env variable containing the Pod Name.
+	PodNameEnvName = "POD_NAME"
 
 	containersTerminationGracePeriod = 10
 )
 
+var (
+	// DefaultDivisor -> "0".
+	DefaultDivisor = *resource.NewQuantity(0, "")
+	// MilliDivisor -> "1m".
+	MilliDivisor = *resource.NewMilliQuantity(1, resource.DecimalSI)
+)
+
 // ContainerEnvOpts contains images name and tag for container environment.
 type ContainerEnvOpts struct {
 	ImagesTag            string
@@ -65,6 +78,7 @@ type ContainerEnvOpts struct {
 	MyDriveImgAndTag     string
 	ContentDownloaderImg string
 	ContentUploaderImg   string
+	InstMetricsEndpoint  string
 }
 
 // PVCSpec forges a ReadWriteOnce PersistentVolumeClaimSpec
@@ -166,7 +180,7 @@ func ContainersSpec(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir
 	volumeMountPath := MyDriveMountPath(environment)
 	switch environment.EnvironmentType {
 	case clv1alpha2.ClassContainer:
-		containers = append(containers, WebsockifyContainer(opts, environment), XVncContainer(opts), AppContainer(instance, environment, volumeMountPath))
+		containers = append(containers, WebsockifyContainer(opts, environment, instance), XVncContainer(opts), AppContainer(instance, environment, volumeMountPath))
 	case clv1alpha2.ClassStandalone:
 		containers = append(containers, StandaloneContainer(instance, environment, volumeMountPath))
 	default:
@@ -179,14 +193,22 @@ func ContainersSpec(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir
 
 // WebsockifyContainer forges the sidecar container to proxy requests from websocket
 // to the VNC server.
-func WebsockifyContainer(opts *ContainerEnvOpts, environment *clv1alpha2.Environment) corev1.Container {
+func WebsockifyContainer(opts *ContainerEnvOpts, environment *clv1alpha2.Environment, instance *clv1alpha2.Instance) corev1.Container {
 	websockifyContainer := GenericContainer(WebsockifyName, fmt.Sprintf("%s:%s", opts.WebsockifyImg, opts.ImagesTag))
 	SetContainerResources(&websockifyContainer, 0.01, 0.1, 30, 100)
+	AddEnvVariableFromFieldToContainer(&websockifyContainer, PodNameEnvName, "metadata.name")
+	AddEnvVariableFromResourcesToContainer(&websockifyContainer, AppCPULimitsEnvName, environment.Name, corev1.ResourceLimitsCPU, MilliDivisor)
+	AddEnvVariableFromResourcesToContainer(&websockifyContainer, AppMEMLimitsEnvName, environment.Name, corev1.ResourceLimitsMemory, DefaultDivisor)
 	AddTCPPortToContainer(&websockifyContainer, GUIPortName, GUIPortNumber)
 	AddTCPPortToContainer(&websockifyContainer, MetricsPortName, MetricsPortNumber)
 	AddContainerArg(&websockifyContainer, "http-addr", fmt.Sprintf(":%d", GUIPortNumber))
+	AddContainerArg(&websockifyContainer, "base-path", IngressGUICleanPath(instance))
 	AddContainerArg(&websockifyContainer, "metrics-addr", fmt.Sprintf(":%d", MetricsPortNumber))
 	AddContainerArg(&websockifyContainer, "show-controls", fmt.Sprint(environment.Mode == clv1alpha2.ModeStandard))
+	AddContainerArg(&websockifyContainer, "instmetrics-server-endpoint", opts.InstMetricsEndpoint)
+	AddContainerArg(&websockifyContainer, "pod-name", fmt.Sprintf("$(%s)", PodNameEnvName))
+	AddContainerArg(&websockifyContainer, "cpu-limit", fmt.Sprintf("$(%s)", AppCPULimitsEnvName))
+	AddContainerArg(&websockifyContainer, "memory-limit", fmt.Sprintf("$(%s)", AppMEMLimitsEnvName))
 	SetContainerReadinessTCPProbe(&websockifyContainer, GUIPortName)
 	return websockifyContainer
 }
@@ -236,8 +258,8 @@ func StandaloneContainer(instance *clv1alpha2.Instance, environment *clv1alpha2.
 func AppContainer(instance *clv1alpha2.Instance, environment *clv1alpha2.Environment, volumeMountPath string) corev1.Container {
 	appContainer := GenericContainer(environment.Name, environment.Image)
 	SetContainerResourcesFromEnvironment(&appContainer, environment)
-	AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_REQUESTS", corev1.ResourceRequestsCPU)
-	AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_LIMITS", corev1.ResourceLimitsCPU)
+	AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_REQUESTS", appContainer.Name, corev1.ResourceRequestsCPU, DefaultDivisor)
+	AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_LIMITS", appContainer.Name, corev1.ResourceLimitsCPU, DefaultDivisor)
 	if NeedsContainerVolume(instance, environment) {
 		AddContainerVolumeMount(&appContainer, MyDriveName, volumeMountPath)
 	}
@@ -316,14 +338,27 @@ func AddEnvVariableToContainer(c *corev1.Container, name, value string) {
 	})
 }
 
-// AddEnvVariableFromResourcesToContainer appends an environment variable to the given container's env with a resource-referenced value.
-func AddEnvVariableFromResourcesToContainer(c *corev1.Container, name string, resName corev1.ResourceName) {
+// AddEnvVariableFromFieldToContainer appends an environment variable to the given container's env with a generic field-referenced value.
+func AddEnvVariableFromFieldToContainer(c *corev1.Container, name, value string) {
 	c.Env = append(c.Env, corev1.EnvVar{
 		Name: name,
+		ValueFrom: &corev1.EnvVarSource{
+			FieldRef: &corev1.ObjectFieldSelector{
+				FieldPath: value,
+			},
+		},
+	})
+}
+
+// AddEnvVariableFromResourcesToContainer appends an environment variable to the given container's env with a resource-referenced value from the source container.
+func AddEnvVariableFromResourcesToContainer(c *corev1.Container, envVarName, srcContainerName string, resName corev1.ResourceName, divisor resource.Quantity) {
+	c.Env = append(c.Env, corev1.EnvVar{
+		Name: envVarName,
 		ValueFrom: &corev1.EnvVarSource{
 			ResourceFieldRef: &corev1.ResourceFieldSelector{
-				ContainerName: c.Name,
+				ContainerName: srcContainerName,
 				Resource:      resName.String(),
+				Divisor:       divisor,
 			},
 		},
 	})

operators/pkg/forge/containers_test.go

@@ -276,7 +276,7 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 				EnvironmentType: clv1alpha2.ClassContainer,
 				ExpectedOutput: func(i *clv1alpha2.Instance, e *clv1alpha2.Environment) []corev1.Container {
 					return []corev1.Container{
-						forge.WebsockifyContainer(&opts, &environment),
+						forge.WebsockifyContainer(&opts, &environment, i),
 						forge.XVncContainer(&opts),
 						forge.MyDriveContainer(i, &opts, myDriveMountPath),
 						forge.AppContainer(i, e, myDriveMountPath),
@@ -289,7 +289,7 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 				EnvironmentType: clv1alpha2.ClassContainer,
 				ExpectedOutput: func(i *clv1alpha2.Instance, e *clv1alpha2.Environment) []corev1.Container {
 					return []corev1.Container{
-						forge.WebsockifyContainer(&opts, &environment),
+						forge.WebsockifyContainer(&opts, &environment, i),
 						forge.XVncContainer(&opts),
 						forge.AppContainer(i, e, myDriveMountPath),
 					}
@@ -301,7 +301,7 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 				EnvironmentType: clv1alpha2.ClassContainer,
 				ExpectedOutput: func(i *clv1alpha2.Instance, e *clv1alpha2.Environment) []corev1.Container {
 					return []corev1.Container{
-						forge.WebsockifyContainer(&opts, &environment),
+						forge.WebsockifyContainer(&opts, &environment, i),
 						forge.XVncContainer(&opts),
 						forge.AppContainer(i, e, myDriveMountPath),
 					}
@@ -360,8 +360,8 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 			expected.Name = envName
 			forge.AddEnvVariableToContainer(&expected, "CROWNLABS_BASE_PATH", forge.IngressGUICleanPath(&instance))
 			forge.AddEnvVariableToContainer(&expected, "CROWNLABS_LISTEN_PORT", "6080")
-			forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", corev1.ResourceRequestsCPU)
-			forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", corev1.ResourceLimitsCPU)
+			forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", expected.Name, corev1.ResourceRequestsCPU, forge.DefaultDivisor)
+			forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", expected.Name, corev1.ResourceLimitsCPU, forge.DefaultDivisor)
 			Expect(actual.Env).To(ConsistOf(expected.Env))
 		})
 	})
@@ -370,7 +370,7 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 		var actual, expected corev1.Container
 		JustBeforeEach(func() {
 			expected = corev1.Container{}
-			actual = forge.WebsockifyContainer(&opts, &environment)
+			actual = forge.WebsockifyContainer(&opts, &environment, &instance)
 		})
 
 		It("Should set the correct container name and image", func() {
@@ -387,13 +387,17 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 			forge.AddTCPPortToContainer(&expected, "metrics", 9090)
 			Expect(actual.Ports).To(Equal(expected.Ports))
 		})
-		It("Should set no environment variables", func() {
-			Expect(actual.Env).To(BeEmpty())
-		})
 		It("Should set the readiness probe", func() {
 			forge.SetContainerReadinessTCPProbe(&expected, "gui")
 			Expect(actual.ReadinessProbe).To(Equal(expected.ReadinessProbe))
 		})
+		It("Should set the env varibles", func() {
+			expected.Name = forge.WebsockifyName
+			forge.AddEnvVariableFromFieldToContainer(&expected, forge.PodNameEnvName, "metadata.name")
+			forge.AddEnvVariableFromResourcesToContainer(&expected, forge.AppCPULimitsEnvName, environment.Name, corev1.ResourceLimitsCPU, forge.MilliDivisor)
+			forge.AddEnvVariableFromResourcesToContainer(&expected, forge.AppMEMLimitsEnvName, environment.Name, corev1.ResourceLimitsMemory, forge.DefaultDivisor)
+			Expect(actual.Env).To(ConsistOf(expected.Env))
+		})
 
 		When("the environment mode is Standard", func() {
 			BeforeEach(func() {
@@ -403,8 +407,13 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 			It("Should set the correct arguments", func() {
 				Expect(actual.Args).To(ConsistOf([]string{
 					fmt.Sprintf("--http-addr=:%d", forge.GUIPortNumber),
+					fmt.Sprintf("--base-path=%s", forge.IngressGUICleanPath(&instance)),
 					fmt.Sprintf("--metrics-addr=:%d", forge.MetricsPortNumber),
 					"--show-controls=true",
+					fmt.Sprintf("--instmetrics-server-endpoint=%s", opts.InstMetricsEndpoint),
+					fmt.Sprintf("--pod-name=$(%s)", forge.PodNameEnvName),
+					fmt.Sprintf("--cpu-limit=$(%s)", forge.AppCPULimitsEnvName),
+					fmt.Sprintf("--memory-limit=$(%s)", forge.AppMEMLimitsEnvName),
 				}))
 			})
 		})
@@ -417,8 +426,13 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 			It("Should set the correct arguments", func() {
 				Expect(actual.Args).To(ConsistOf([]string{
 					fmt.Sprintf("--http-addr=:%d", forge.GUIPortNumber),
+					fmt.Sprintf("--base-path=%s", forge.IngressGUICleanPath(&instance)),
 					fmt.Sprintf("--metrics-addr=:%d", forge.MetricsPortNumber),
 					"--show-controls=false",
+					fmt.Sprintf("--instmetrics-server-endpoint=%s", opts.InstMetricsEndpoint),
+					fmt.Sprintf("--pod-name=$(%s)", forge.PodNameEnvName),
+					fmt.Sprintf("--cpu-limit=$(%s)", forge.AppCPULimitsEnvName),
+					fmt.Sprintf("--memory-limit=$(%s)", forge.AppMEMLimitsEnvName),
 				}))
 			})
 		})
@@ -516,8 +530,8 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 			})
 			It("Should set the env varibles", func() {
 				expected.Name = envName
-				forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", corev1.ResourceRequestsCPU)
-				forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", corev1.ResourceLimitsCPU)
+				forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", expected.Name, corev1.ResourceRequestsCPU, forge.DefaultDivisor)
+				forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", expected.Name, corev1.ResourceLimitsCPU, forge.DefaultDivisor)
 				Expect(actual.Env).To(ConsistOf(expected.Env))
 			})
 		})
@@ -779,7 +793,7 @@ var _ = Describe("Containers and Deployment spec forging", func() {
 	Describe("The forge.AddEnvVariableFromResourcesToContainer", func() {
 		JustBeforeEach(func() {
 			container.Name = envName
-			forge.AddEnvVariableFromResourcesToContainer(&container, envVarName, corev1.ResourceRequestsCPU)
+			forge.AddEnvVariableFromResourcesToContainer(&container, envVarName, container.Name, corev1.ResourceRequestsCPU, forge.DefaultDivisor)
 		})
 		It("Should add a single env entry with the specified parameters", func() {
 			Expect(container.Env).To(ConsistOf(corev1.EnvVar{

operators/pkg/forge/ingresses.go

@@ -31,8 +31,8 @@ const (
 	IngressGUINameSuffix = "gui"
 	// IngressMyDriveNameSuffix -> the suffix added to the name of the ingress targeting the environment "MyDrive".
 	IngressMyDriveNameSuffix = "mydrive"
-	// IngressStandaloneSuffix -> the suffix added to the path of the ingress targeting the standalone application.
-	IngressStandaloneSuffix = "app"
+	// IngressAppSuffix -> the suffix added to the path of the ingress targeting standalone and container environments.
+	IngressAppSuffix = "app"
 
 	// IngressDefaultCertificateName -> the name of the secret containing the crownlabs certificate.
 	IngressDefaultCertificateName = "crownlabs-ingress-secret"
@@ -81,8 +81,6 @@ func IngressGUIAnnotations(environment *clv1alpha2.Environment, annotations map[
 	}
 	if environment.EnvironmentType == clv1alpha2.ClassStandalone && environment.RewriteURL {
 		annotations["nginx.ingress.kubernetes.io/rewrite-target"] = StandaloneRewriteEndpoint
-	} else if environment.EnvironmentType == clv1alpha2.ClassContainer {
-		annotations["nginx.ingress.kubernetes.io/rewrite-target"] = WebsockifyRewriteEndpoint
 	}
 	annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = "3600"
 	annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = "3600"
@@ -142,26 +140,28 @@ func IngressGUIPath(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir
 	switch environment.EnvironmentType {
 	case clv1alpha2.ClassStandalone:
 		if environment.RewriteURL {
-			return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressStandaloneSuffix+"(/|$)(.*)"), "/")
+			return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix+"(/|$)(.*)"), "/")
 		}
-		return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressStandaloneSuffix), "/")
-	case clv1alpha2.ClassContainer, clv1alpha2.ClassCloudVM, clv1alpha2.ClassVM:
+		return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix), "/")
+	case clv1alpha2.ClassContainer:
+		return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix), "/")
+	case clv1alpha2.ClassCloudVM, clv1alpha2.ClassVM:
 		return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressVNCGUIPathSuffix), "/")
 	}
 	return ""
 }
 
 // IngressGUICleanPath returns the path of the ingress targeting the environment GUI vnc or Standalone, without the url-rewrite's regex.
 func IngressGUICleanPath(instance *clv1alpha2.Instance) string {
-	return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressStandaloneSuffix), "/")
+	return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix), "/")
 }
 
 // IngressGuiStatusURL returns the path of the ingress targeting the environment.
 func IngressGuiStatusURL(host string, environment *clv1alpha2.Environment, instance *clv1alpha2.Instance) string {
 	switch environment.EnvironmentType {
-	case clv1alpha2.ClassStandalone:
-		return fmt.Sprintf("https://%v%v/%v/%v/", host, IngressInstancePrefix, instance.UID, IngressStandaloneSuffix)
-	case clv1alpha2.ClassContainer, clv1alpha2.ClassVM, clv1alpha2.ClassCloudVM:
+	case clv1alpha2.ClassStandalone, clv1alpha2.ClassContainer:
+		return fmt.Sprintf("https://%v%v/%v/%v/", host, IngressInstancePrefix, instance.UID, IngressAppSuffix)
+	case clv1alpha2.ClassVM, clv1alpha2.ClassCloudVM:
 		return fmt.Sprintf("https://%v%v/%v/", host, IngressInstancePrefix, instance.UID)
 	}
 	return ""
@@ -171,7 +171,7 @@ func IngressGuiStatusURL(host string, environment *clv1alpha2.Environment, insta
 func IngressGUIName(environment *clv1alpha2.Environment) string {
 	switch environment.EnvironmentType {
 	case clv1alpha2.ClassStandalone:
-		return IngressStandaloneSuffix
+		return IngressAppSuffix
 	case clv1alpha2.ClassContainer, clv1alpha2.ClassVM, clv1alpha2.ClassCloudVM:
 		return IngressGUINameSuffix
 	}