Use existing build output (#94)

* specify stage for make

* correct default pipeline input

* remove limit param validation for now

* increase timeout and memory on validation Lambda

* set release pattern regex

* fix premature exit on build errors causing container to stop

* capture error output to disk as ndjson

* consolidate error handling within process_allele function; remove sqs

* remove failed alleles queue

* script proceeds if some alleles fail

* validate build for single release version

* fix restore target

* fix input paths in state machine

* use existing build output if available

* wait for backup document to finish

* refactor input array for cleaner output

* add use_existing_build to pipeline params

* evaluate use_existing_build after file validation

* refactor environment validation target

* add skip_load to pipeline params

* update README

Author: chrisammon3000

Makefile

@@ -38,6 +38,11 @@ export PIPELINE_STATE_PATH := config/IMGTHLA-repository-state.json
 export PIPELINE_PARAMS_PATH := config/pipeline-input.json
 export FUNCTIONS_PATH := ${APP_NAME}/pipeline/functions
 
+# Required environment variables
+REQUIRED_VARS := STAGE APP_NAME AWS_ACCOUNT AWS_REGION AWS_PROFILE SUBSCRIBE_EMAILS \
+                 GITHUB_REPOSITORY_OWNER GITHUB_REPOSITORY_NAME GITHUB_PERSONAL_ACCESS_TOKEN \
+                 HOST_DOMAIN SUBDOMAIN ADMIN_EMAIL NEO4J_AMI_ID APOC_VERSION GDS_VERSION
+
 # print colors
 define blue
 	@tput setaf 4
@@ -170,51 +175,8 @@ else
 endif
 
 env.validate: check.dependencies
-ifndef STAGE
-$(error STAGE is not set. Please add STAGE to the environment variables.)
-endif
-ifndef APP_NAME
-$(error APP_NAME is not set. Please add APP_NAME to the environment variables.)
-endif
-ifndef AWS_ACCOUNT
-	$(error AWS_ACCOUNT is not set. Please add AWS_ACCOUNT to the environment variables.)
-endif
-ifndef AWS_REGION
-	$(error AWS_REGION is not set. Please add AWS_REGION to the environment variables.)
-endif
-ifndef AWS_PROFILE
-	$(error AWS_PROFILE is not set. Please select an AWS profile to use.)
-endif
-ifndef SUBSCRIBE_EMAILS
-$(error SUBSCRIBE_EMAILS is not set. Please add SUBSCRIBE_EMAILS to the environment variables.)
-endif
-ifndef GITHUB_REPOSITORY_OWNER
-$(error GITHUB_REPOSITORY_OWNER is not set. Please add GITHUB_REPOSITORY_OWNER to the environment variables.)
-endif
-ifndef GITHUB_REPOSITORY_NAME
-$(error GITHUB_REPOSITORY_NAME is not set. Please add GITHUB_REPOSITORY_NAME to the environment variables.)
-endif
-ifndef GITHUB_PERSONAL_ACCESS_TOKEN
-	$(error GITHUB_PERSONAL_ACCESS_TOKEN is not set. Please add GITHUB_PERSONAL_ACCESS_TOKEN to the environment variables.)
-endif
-ifndef HOST_DOMAIN
-	$(error HOST_DOMAIN is not set. Please add HOST_DOMAIN to the environment variables.)
-endif
-ifndef SUBDOMAIN
-$(error SUBDOMAIN is not set. Please add SUBDOMAIN to the environment variables.)
-endif
-ifndef ADMIN_EMAIL
-	$(error ADMIN_EMAIL is not set. Please add ADMIN_EMAIL to the environment variables.)
-endif
-ifndef NEO4J_AMI_ID
-$(error NEO4J_AMI_ID is not set. Please add NEO4J_AMI_ID to the environment variables.)
-endif
-ifndef APOC_VERSION
-$(error APOC_VERSION is not set. Please add APOC_VERSION to the environment variables.)
-endif
-ifndef GDS_VERSION
-$(error GDS_VERSION is not set. Please add GDS_VERSION to the environment variables.)
-endif
+	$(foreach var,$(REQUIRED_VARS),\
+		$(if $(value $(var)),,$(error $(var) is not set. Please add $(var) to the environment variables.)))
 ifndef CREATE_VPC
 	$(info 'CREATE_VPC' is not set. Defaulting to 'false')
 	$(eval export CREATE_VPC := false)
@@ -264,7 +226,9 @@ database.load.run: # args: align, kir, limit, releases
 	[ "$$kir" ] && kir="$$kir" || kir=false && \
 	[ "$$limit" ] && limit="$$limit" || limit="" && \
 	[ "$$releases" ] && releases="$$releases" || releases="" && \
-	payload="{ \"align\": $$align, \"kir\": $$kir, \"limit\": \"$$limit\", \"releases\": \"$$releases\", \"mem_profile\": false }" && \
+	[ "$$use_existing_build" ] && use_existing_build="$$use_existing_build" || use_existing_build=false && \
+	[ "$$skip_load" ] && skip_load="$$skip_load" || skip_load=false && \
+	payload="{\"align\":$$align,\"kir\":$$kir,\"limit\":\"$$limit\",\"releases\":\"$$releases\",\"mem_profile\":false,\"use_existing_build\":$$use_existing_build,\"skip_load\":$$skip_load}"&&\
 	echo "$$payload" | jq -r && \
 	echo "$$payload" | jq > payload.json
 	@echo "Run pipeline with this payload? [y/N] \c " && read ans && [ $${ans:-N} = y ]
@@ -370,7 +334,7 @@ database.delete:
 pipeline.delete:
 	$(MAKE) -C ${APP_NAME}/pipeline/ service.delete
 
-pipeline.functions.delete:
+pipeline.service.delete:
 	$(MAKE) -C ${APP_NAME}/pipeline/ service.functions.delete
 
 pipeline.jobs.delete:

README.md

@@ -265,8 +265,14 @@ STAGE=<stage> make deploy
 # Deploy config files and scripts to S3
 STAGE=<stage> make config.deploy
 
-# Run the StepFunctions State Machine to load Neo4j
-STAGE=<stage> make database.load.run releases=<version> align=<boolean> kir=<boolean> limit=<int>
+# Run the Step Functions State Machine to load Neo4j
+STAGE=<stage> make database.load.run \
+    releases=<version> \
+    align=<boolean> \
+    kir=<boolean> \
+    limit=<int> \
+    use_existing_build=<boolean> \
+    skip_load=<boolean>
 
 # Retrieve Neo4j credentials after deployment
 STAGE=<stage> make database.get.credentials
@@ -281,7 +287,7 @@ STAGE=<stage> make get.logs
 STAGE=<stage> make get.data
 
 # Delete all CloudFormation based services and data, default is data=false
-STAGE=<stage> make delete data=<true/false>
+STAGE=<stage> make delete data=<boolean>
 
 # Delete a specific layer
 STAGE=<stage> make pipeline.delete
@@ -344,23 +350,33 @@ Base input parameters (excluding the `releases` value) are passed to the Step Fu
 ```json
 // pipeline-input.json
 {
-  "align": "False",
-  "kir": "False",
-  "mem_profile": "False",
-  "limit": ""
+  "align": false,
+  "kir": false,
+  "mem_profile": false,
+  "limit": "", // Optional, defaults to false
+  "use_existing_build": false, // Optional, defaults to false
+  "skip_load": false // Optional, defaults to false
 }
 
 ```
-| Variable    | Example Value | Type   | Description                                                        |
-| ----------- | ------------- | ------ | ------------------------------------------------------------------ |
-| LIMIT       | 1000          | string | Number of alleles to build. Leave blank ("") to build all alleles. |
-| ALIGN       | False         | string | Include or exclude alignments in the build                         |
-| KIR         | False         | string | Include or exclude KIR data alignments in the build                |
-| MEM_PROFILE | False         | string | Enable memory profiling (for catching memory leaks during build)   |
+| Variable           | Example Value | Type   | Description                                                                    |
+| ------------------ | ------------- | ------ | ------------------------------------------------------------------------------ |
+| LIMIT              | 1000          | string | Number of alleles to build. Leave blank ("") to build all alleles.             |
+| ALIGN              | false         | string | Include or exclude alignments in the build                                     |
+| KIR                | false         | string | Include or exclude KIR data alignments in the build                            |
+| MEM_PROFILE        | false         | string | Enable memory profiling (for catching memory leaks during build)               |
+| USE_EXISTING_BUILD | false         | string | Use existing build files in S3 (if available) instead of building from scratch |
+| SKIP_LOAD          | false         | string | Skip loading the database after building                                       |
 
 The data pipeline can also be invoked from the command line:
 ```bash
-STAGE=<stage> make database.load.run releases=<version> align=<boolean> kir=<boolean> limit=<int>
+STAGE=<stage> make database.load.run \
+    releases=<version> \
+    align=<boolean> \
+    kir=<boolean> \
+    limit=<int> \
+    use_existing_build=<boolean> \
+    skip_load=<boolean>
 ```
 
 #### IMGT/HLA Release Versions State
@@ -393,14 +409,20 @@ STAGE=<stage> make database.load.run releases="<version>"
 # Example for single version
 STAGE=<stage> make database.load.run releases="3510"
 
-# Example for multiple versions
-STAGE=<stage> make database.load.run releases="3490,3500,3510"
+# Example for multiple versions where 3510 has already been built
+STAGE=<stage> make database.load.run \
+    releases="3490,3500,3510" \
+    use_existing_build=true
 
 # Example with limit
 STAGE=<stage> make database.load.run releases="3510" limit="1000"
 
 # Example with all arguments included
-STAGE=<stage> make database.load.run releases="3510" limit="" align="False" kir="False"
+STAGE=<stage> make database.load.run releases="3510" limit="" align=false kir=false
+
+# Example of how to build all releases and skip loading
+STAGE=dev make database.load.run releases="300,310,320,330,340,350,360,370,380,390,3100,3110,3120,3130,3140,3150,3160,3170,3180,3190,3200,3210,3220,3230,3240,3250,3260,3270,3280,3290,3300,3310,3320,3330,3340,3350,3360,3370,3380,3390,3400,3410,3420,3430,3440,3450,3460,3470,3480,3490,3500,3510,3520,3530" skip_load=true
+```
 
 These commands build an event payload to send to the `invoke-gfe-db-pipeline` Lambda.
 ```json
@@ -422,9 +444,9 @@ The Lambda function returns the following object which can be viewed in CloudWat
   "message": "Pipeline triggered",
   "input": [
     {
-      "ALIGN": "False",
-      "KIR": "False",
-      "MEM_PROFILE": "False",
+      "ALIGN": false,
+      "KIR": false,
+      "MEM_PROFILE": false,
       "LIMIT": "",
       "RELEASES": "3510"
     },
@@ -436,7 +458,7 @@ The Lambda function returns the following object which can be viewed in CloudWat
 ### Clean Up
 To tear down resources run the command. You will need to manually delete the data in the S3 bucket first to avoid an error in CloudFormation.
 ```bash
-STAGE=<stage> make delete data=<true/false>
+STAGE=<stage> make delete data=<boolean>
 ```
 Use the following commands to tear down individual services. Make sure to [backup](#backup--restore) your data first.
 ```bash

gfe-db/database/Makefile

@@ -118,10 +118,10 @@ service.restore: #from_date=<YYYY/MM/DD/HH>
 
 service.restore.pre-validate:
 	@echo "Validating restore pre-conditions"
-	@echo "Checking if backup exists"
-	@backup_object=$$(aws s3 ls --recursive s3://${DATA_BUCKET_NAME}/backups/neo4j/$$from_date/ | cut -d' ' -f7 | cut -d'/' -f3-6) && \
+	@echo "Checking if backup exists for \`s3://${DATA_BUCKET_NAME}/backups/neo4j/$$from_date/\`"
+	@backup_object=$$(aws s3 ls --recursive s3://${DATA_BUCKET_NAME}/backups/neo4j/$$from_date/ | cut -d ' ' -f 6-) && \
 	[ "$$backup_object" != "" ] || (echo "ERROR: Backup does not exist" && exit 1) && \
-	echo "Found backup for $$backup_object"
+	echo "Found backup target for $$backup_object"
 
 delete: ##=> Delete resources
 	@echo "$$(gdate -u +'%Y-%m-%d %H:%M:%S.%3N') - Deleting ${SERVICE} service" 2>&1 | tee -a $$CFN_LOG_PATH

gfe-db/pipeline/functions/invoke_backup_script/app.py

@@ -42,10 +42,16 @@ def lambda_handler(event, context):
             raise Exception("Failed to send command")
         else:
             logger.info(f"Neo4j backup invoked on instance {neo4j_database_instance_id}")
+            
+            # wait for the command to complete successfully
+            waiter = ssm.get_waiter('command_executed')
+            waiter.wait(
+                CommandId=response['Command']['CommandId'],
+                InstanceId=neo4j_database_instance_id
+            )
+
+            logger.info(f"Neo4j backup completed on instance {neo4j_database_instance_id}")
 
-            # TODO poll SSM until command is complete
-            # try: poll; except Failed Command: raise Exception
-    
     except Exception as err:
         logger.error(err)
         raise err

gfe-db/pipeline/functions/invoke_pipeline/app.py

@@ -29,7 +29,7 @@
 s3 = session.client('s3')
 sfn = session.client('stepfunctions')
 
-release_pattern = r"^\d{3}0$"
+release_pattern = r"^\d{2,3}0$"
 
 def lambda_handler(event, context):
     """Checks for new IMGT/HLA releases and triggers the update
@@ -41,8 +41,15 @@ def lambda_handler(event, context):
         if "releases" in event:
 
             # align, kir, mem_profile are booleans
-            if not all([ isinstance(event[arg], bool) for arg in [ 'align', 'kir', 'mem_profile' ] ]):
-                raise ValueError('align, kir, and mem_profile must be boolean values')
+            execution_input_bool_keys = [ 
+                'align', 
+                'kir', 
+                'mem_profile' ,
+                'use_existing_build',
+                'skip_load'
+            ]
+            if not all([ isinstance(event[arg], bool) for arg in execution_input_bool_keys if arg in event ]):
+                raise ValueError(f'{", ".join(execution_input_bool_keys)} must be boolean values')
 
             # conform booleans to the current argument format
             event = { arg: str(val) for arg, val in event.items() }
@@ -304,7 +311,7 @@ def parse_state(state_path, params_path):
     import os
     from pathlib import Path
 
-    path = Path(__file__).parent / "bad-event.json"
+    path = Path(__file__).parent / "event-use-existing-true.json"
     with open(path, "r") as f:
         event = json.load(f)
 

gfe-db/pipeline/functions/invoke_pipeline/event-use-existing-false-no-key.json

@@ -0,0 +1,8 @@
+{
+    "align": false,
+    "kir": false,
+    "limit": "",
+    "releases": "310",
+    "mem_profile": false,
+    "skip_load": true
+}

gfe-db/pipeline/functions/invoke_pipeline/event-use-existing-false.json

@@ -0,0 +1,8 @@
+{
+    "align": false,
+    "kir": false,
+    "limit": "",
+    "releases": "310",
+    "mem_profile": false,
+    "use_existing_build": false
+}

gfe-db/pipeline/functions/invoke_pipeline/event-use-existing-true.json

@@ -0,0 +1,9 @@
+{
+    "align": false,
+    "kir": false,
+    "limit": "",
+    "releases": "310",
+    "mem_profile": false,
+    "use_existing_build": true,
+    "skip_load": false
+}