Merge pull request #91 from Calvin-Huang/fix/gcp

Fix GCP provider

Author: claffin

cloudproxy/check.py

@@ -51,7 +51,22 @@ def fetch_ip(ip_address):
 
 def check_alive(ip_address):
     try:
-        result = requests.get("http://ipecho.net/plain", proxies={'http': "http://" + ip_address + ":8899"}, timeout=10)
+        if settings.config["no_auth"]:
+            proxies = {
+                "http": "http://" + ip_address + ":8899",
+                "https": "http://" + ip_address + ":8899",
+            }
+        else:
+            auth = (
+                settings.config["auth"]["username"] + ":" + settings.config["auth"]["password"]
+            )
+
+            proxies = {
+                "http": "http://" + auth + "@" + ip_address + ":8899",
+                "https": "http://" + auth + "@" + ip_address + ":8899",
+            }
+    
+        result = requests.get("http://ipecho.net/plain", proxies=proxies, timeout=10)
         if result.status_code in (200, 407):
             return True
         else:

cloudproxy/providers/gcp/functions.py

@@ -9,28 +9,66 @@
 from cloudproxy.providers.config import set_auth
 from cloudproxy.providers.settings import config
 
-gcp = config["providers"]["gcp"]
-if gcp["enabled"] == 'True':
+gcp = None
+compute = None
+
+def get_client(instance_config=None):
+    """
+    Initialize and return a GCP client based on the provided configuration.
+    
+    Args:
+        instance_config: The specific instance configuration
+    
+    Returns:
+        tuple: (config, gcp_client)
+    """
+
+    global gcp, compute
+    if gcp is not None and compute is not None:
+        return gcp, compute
+
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    gcp = config["providers"]["gcp"]
     try:
-        credentials = service_account.Credentials.from_service_account_info(
-            json.loads(gcp["secrets"]["service_account_key"])
-        )
+        if 'sa_json' in instance_config["secrets"]:
+            credentials = service_account.Credentials.from_service_account_file(
+                instance_config["secrets"]["sa_json"]
+            )
+        else:
+            credentials = service_account.Credentials.from_service_account_info(
+                json.loads(instance_config["secrets"]["service_account_key"])
+            )
         compute = googleapiclient.discovery.build('compute', 'v1', credentials=credentials)
+
+        return gcp, compute
     except TypeError:
         logger.error("GCP -> Invalid service account key")
 
 
-def create_proxy():
+def create_proxy(instance_config=None):
+    """
+    Create a GCP proxy instance.
+    
+    Args:
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    gcp, compute = get_client(instance_config)
+
     image_response = compute.images().getFromFamily(
-        project=gcp["image_project"], 
-        family=gcp["image_family"]
+        project=instance_config["image_project"], 
+        family=instance_config["image_family"]
     ).execute()
     source_disk_image = image_response['selfLink']
 
     body = {
         'name': 'cloudproxy-' + str(uuid.uuid4()),
         'machineType': 
-            f"zones/{gcp['zone']}/machineTypes/{gcp['size']}",
+            f"zones/{instance_config['zone']}/machineTypes/{instance_config['size']}",
         'tags': {
             'items': [
                 'cloudproxy'
@@ -67,48 +105,93 @@ def create_proxy():
     }
 
     return compute.instances().insert(
-        project=gcp["project"],
-        zone=gcp["zone"],
+        project=instance_config["project"],
+        zone=instance_config["zone"],
         body=body
     ).execute()
 
-def delete_proxy(name):
+def delete_proxy(name, instance_config=None):
+    """
+    Delete a GCP proxy instance.
+    
+    Args:
+        name: Name of the instance to delete
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    gcp, compute = get_client(instance_config)
+
     try:
         return compute.instances().delete(
-            project=gcp["project"],
-            zone=gcp["zone"],
+            project=instance_config["project"],
+            zone=instance_config["zone"],
             instance=name
         ).execute()
     except googleapiclient.errors.HttpError:
         logger.info(f"GCP --> HTTP Error when trying to delete proxy {name}. Probably has already been deleted.")
         return None
 
-def stop_proxy(name):
+def stop_proxy(name, instance_config=None):
+    """
+    Stop a GCP proxy instance.
+    
+    Args:
+        name: Name of the instance to stop
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
     try:
         return compute.instances().stop(
-            project=gcp["project"],
-            zone=gcp["zone"],
+            project=instance_config["project"],
+            zone=instance_config["zone"],
             instance=name
         ).execute()
     except googleapiclient.errors.HttpError:
         logger.info(f"GCP --> HTTP Error when trying to stop proxy {name}. Probably has already been deleted.")
         return None
 
-def start_proxy(name):
+def start_proxy(name, instance_config=None):
+    """
+    Start a GCP proxy instance.
+    
+    Args:
+        name: Name of the instance to start
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    gcp, compute = get_client(instance_config)
+
     try:
         return compute.instances().start(
-            project=gcp["project"],
-            zone=gcp["zone"],
+            project=instance_config["project"],
+            zone=instance_config["zone"],
             instance=name
         ).execute()
     except googleapiclient.errors.HttpError:
         logger.info(f"GCP --> HTTP Error when trying to start proxy {name}. Probably has already been deleted.")
         return None
 
-def list_instances():
+def list_instances(instance_config=None):
+    """
+    List all GCP proxy instances.
+    
+    Args:
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    gcp, compute = get_client(instance_config)
+
     result = compute.instances().list(
-        project=gcp["project"], 
-        zone=gcp["zone"], 
+        project=instance_config["project"], 
+        zone=instance_config["zone"], 
         filter='labels.cloudproxy eq cloudproxy'
     ).execute()
     return result['items'] if 'items' in result else []

cloudproxy/providers/gcp/main.py

@@ -13,30 +13,49 @@
 )
 from cloudproxy.providers.settings import delete_queue, restart_queue, config
 
-def gcp_deployment(min_scaling):
-    total_instances = len(list_instances())
+def gcp_deployment(min_scaling, instance_config=None):
+    """
+    Deploy GCP instances based on min_scaling requirements.
+    
+    Args:
+        min_scaling: The minimum number of instances to maintain
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    total_instances = len(list_instances(instance_config))
     if min_scaling < total_instances:
         logger.info("Overprovisioned: GCP destroying.....")
         for instance in itertools.islice(
-            list_instances(), 0, (total_instances - min_scaling)
+            list_instances(instance_config), 0, (total_instances - min_scaling)
         ):
             access_configs = instance['networkInterfaces'][0]['accessConfigs'][0]
             msg = f"{instance['name']} {access_configs['natIP']}"
-            delete_proxy(instance['name'])
+            delete_proxy(instance['name'], instance_config)
             logger.info("Destroyed: GCP -> " + msg)
     if min_scaling - total_instances < 1:
         logger.info("Minimum GCP instances met")
     else:
         total_deploy = min_scaling - total_instances
         logger.info("Deploying: " + str(total_deploy) + " GCP instances")
         for _ in range(total_deploy):
-            create_proxy()
+            create_proxy(instance_config)
             logger.info("Deployed")
-    return len(list_instances())
+    return len(list_instances(instance_config))
+
+def gcp_check_alive(instance_config=None):
+    """
+    Check if any GCP instances are alive.
+    
+    Args:
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
 
-def gcp_check_alive():
     ip_ready = []
-    for instance in list_instances():
+    for instance in list_instances(instance_config):
         try:
             elapsed = datetime.datetime.now(
                 datetime.timezone.utc
@@ -50,7 +69,7 @@ def gcp_check_alive():
 
             elif instance['status'] == "TERMINATED":
                 logger.info("Waking up: GCP -> Instance " + instance['name'])
-                started = start_proxy(instance['name'])
+                started = start_proxy(instance['name'], instance_config)
                 if not started:
                     logger.info("Could not wake up, trying again later.")
 
@@ -83,27 +102,57 @@ def gcp_check_alive():
             logger.info("Pending: GCP -> Allocating IP")
     return ip_ready
 
-def gcp_check_delete():
-    for instance in list_instances():
+def gcp_check_delete(instance_config=None):
+    """
+    Check if any GCP instances need to be deleted.
+    
+    Args:
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    for instance in list_instances(instance_config):
         access_configs = instance['networkInterfaces'][0]['accessConfigs'][0]
         if 'natIP' in  access_configs and access_configs['natIP'] in delete_queue: 
             msg = f"{instance['name']}, {access_configs['natIP']}"
             delete_proxy(instance['name'])
             logger.info("Destroyed: not wanted -> " + msg)
             delete_queue.remove(access_configs['natIP'])
 
-def gcp_check_stop():
-    for instance in list_instances():
+def gcp_check_stop(instance_config=None):
+    """
+    Check if any GCP instances need to be stopped.
+    
+    Args:
+        instance_config: The specific instance configuration
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    for instance in list_instances(instance_config):
         access_configs = instance['networkInterfaces'][0]['accessConfigs'][0]
         if 'natIP' in  access_configs and access_configs['natIP'] in restart_queue:
             msg = f"{instance['name']}, {access_configs['natIP']}"
-            stop_proxy(instance['name'])
+            stop_proxy(instance['name'], instance_config)
             logger.info("Stopped: getting new IP -> " + msg)
             restart_queue.remove(access_configs['natIP'])
 
-def gcp_start():
-    gcp_check_delete()
-    gcp_check_stop()
-    gcp_deployment(config["providers"]["gcp"]["scaling"]["min_scaling"])
-    ip_ready = gcp_check_alive()
+def gcp_start(instance_config=None):
+    """
+    Start the GCP provider lifecycle.
+    
+    Args:
+        instance_config: The specific instance configuration
+    
+    Returns:
+        list: List of ready IP addresses
+    """
+    if instance_config is None:
+        instance_config = config["providers"]["gcp"]["instances"]["default"]
+
+    gcp_check_delete(instance_config)
+    gcp_check_stop(instance_config)
+    gcp_deployment(instance_config["scaling"]["min_scaling"], instance_config)
+    ip_ready = gcp_check_alive(instance_config)
     return ip_ready

cloudproxy/providers/settings.py

@@ -148,14 +148,15 @@
 config["providers"]["gcp"]["instances"]["default"]["secrets"]["service_account_key"] = os.environ.get(
     "GCP_SERVICE_ACCOUNT_KEY"
 )
+config["providers"]["gcp"]["instances"]["default"]["secrets"]["sa_json"] = os.environ.get("GCP_SA_JSON")
 config["providers"]["gcp"]["instances"]["default"]["scaling"]["min_scaling"] = int(
     os.environ.get("GCP_MIN_SCALING", 2)
 )
 config["providers"]["gcp"]["instances"]["default"]["scaling"]["max_scaling"] = int(
     os.environ.get("GCP_MAX_SCALING", 2)
 )
 config["providers"]["gcp"]["instances"]["default"]["size"] = os.environ.get("GCP_SIZE", "f1-micro")
-config["providers"]["gcp"]["instances"]["default"]["zone"] = os.environ.get("GCP_REGION", "us-central1-a")
+config["providers"]["gcp"]["instances"]["default"]["zone"] = os.environ.get("GCP_ZONE", "us-central1-a")
 config["providers"]["gcp"]["instances"]["default"]["image_project"] = os.environ.get("GCP_IMAGE_PROJECT", "ubuntu-os-cloud")
 config["providers"]["gcp"]["instances"]["default"]["image_family"] = os.environ.get("GCP_IMAGE_FAMILY", "ubuntu-minimal-2004-lts")
 config["providers"]["gcp"]["instances"]["default"]["display_name"] = os.environ.get("GCP_DISPLAY_NAME", "GCP")

docs/gcp.md

@@ -21,7 +21,9 @@ Now you have your credentials, you can use GCP as a proxy provider. Set up the e
 #### Required:
 ``GCP_ENABLED`` - to enable GCP as a provider, set as True. Default value: False
 
-``GCP_SA_JSON`` - the path to the service account JSON key file. For Docker, mount the file to the container and provide the path.
+``GCP_SA_JSON`` or `GCP_SERVICE_ACCOUNT_KEY` - the path to the service account JSON key file. For Docker, mount the file to the container and provide the path. If both `GCP_SA_JSON` and `GCP_SERVICE_ACCOUNT_KEY` are set, `GCP_SA_JSON` will override the other.
+
+``GCP_SERVICE_ACCOUNT_KEY`` or (`GCP_SA_JSON`) - service acount JSON key content. If both `GCP_SA_JSON` and `GCP_SERVICE_ACCOUNT_KEY` are set, `GCP_SA_JSON` will override the other.
 
 ``GCP_ZONE`` - the GCP zone where the instances will be created. Default value: us-central1-a