From c27cfa497ae6fd9f8bcade58cf896649e4d61586 Mon Sep 17 00:00:00 2001 From: Kartikeya Pophali Date: Thu, 25 Sep 2025 16:13:10 +0530 Subject: [PATCH 1/3] Add EC2 deployment configuration for Celery workers and Flower, and update Docker Compose for Flower service --- .env.example | 20 ++++++++ .github/workflows/cd-production.yml | 75 +++++++++++++++++++++++++++-- .github/workflows/cd-staging.yml | 73 +++++++++++++++++++++++++++- docker-compose.dev.yml | 25 ++++++++++ 4 files changed, 188 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index b25b0f24..0a43d976 100644 --- a/.env.example +++ b/.env.example @@ -74,7 +74,27 @@ CELERY_ENABLE_UTC=true # India Standard Time (UTC+05:30) CELERY_TIMEZONE=Asia/Kolkata +# Flower Configuration (Celery Monitoring) +FLOWER_PORT=5555 +FLOWER_BASIC_AUTH=admin:changethis +FLOWER_URL_PREFIX= +FLOWER_MAX_TASKS=10000 # Callback Timeouts (in seconds) CALLBACK_CONNECT_TIMEOUT = 3 CALLBACK_READ_TIMEOUT = 10 + + +# EC2 Deployment Configuration for Celery Workers +# Staging EC2 Instance +EC2_STAGING_HOST=staging-celery-host.example.com +EC2_STAGING_USER=ubuntu +EC2_STAGING_KEY_NAME=staging-ec2-key + +# Production EC2 Instance +EC2_PRODUCTION_HOST=production-celery-host.example.com +EC2_PRODUCTION_USER=ubuntu +EC2_PRODUCTION_KEY_NAME=production-ec2-key + +# Docker Registry for Celery Images +DOCKER_IMAGE_CELERY=celery-worker diff --git a/.github/workflows/cd-production.yml b/.github/workflows/cd-production.yml index 7bd0c70d..2993cc79 100644 --- a/.github/workflows/cd-production.yml +++ b/.github/workflows/cd-production.yml @@ -1,4 +1,4 @@ -name: Deploy AI Platform to ECS Production +name: Deploy AI Platform to Production on: push: @@ -29,7 +29,7 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Build and Push Docker Image + - name: Build and Push Backend Docker Image env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} REPOSITORY: ${{ github.event.repository.name }}-repo @@ -38,9 +38,78 @@ jobs: docker build -t $REGISTRY/$REPOSITORY:latest ./backend docker push $REGISTRY/$REPOSITORY:latest - - name: Deploy to ECS + - name: Build and Push Celery Docker Image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ github.event.repository.name }}-celery-repo + run: | + docker build -f ./backend/Dockerfile.celery -t $REGISTRY/$REPOSITORY:production ./backend + docker push $REGISTRY/$REPOSITORY:production + + - name: Deploy Backend to ECS run: | aws ecs update-service \ --cluster ${{ github.event.repository.name }}-cluster \ --service ${{ github.event.repository.name }}-service \ --force-new-deployment + + - name: Deploy Celery Workers to EC2 + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ github.event.repository.name }}-celery-repo + run: | + # Create deployment script + cat > deploy-celery.sh << 'EOF' + #!/bin/bash + + # Stop existing containers + docker stop celery-prod-high celery-prod-other || true + docker rm celery-prod-high celery-prod-other || true + + # Pull latest image + aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin $1 + docker pull $1/$2:production + + # Start high priority worker + docker run -d --name celery-prod-high --restart unless-stopped \ + --env-file /home/ubuntu/.env.production \ + $1/$2:production \ + uv run celery -A app.celery.celery_app worker --queues=high_priority --loglevel=info --concurrency=2 --hostname=prod-high@%h + + # Start other queues worker + docker run -d --name celery-prod-other --restart unless-stopped \ + --env-file /home/ubuntu/.env.production \ + $1/$2:production \ + uv run celery -A app.celery.celery_app worker --queues=default,low_priority,cron --loglevel=info --concurrency=2 --hostname=prod-other@%h + EOF + + # Copy script to EC2 and execute + echo "${{ secrets.EC2_PRODUCTION_PRIVATE_KEY }}" > production_key.pem + chmod 600 production_key.pem + scp -i production_key.pem -o StrictHostKeyChecking=no deploy-celery.sh ubuntu@${{ secrets.EC2_PRODUCTION_HOST }}:/tmp/ + ssh -i production_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_PRODUCTION_HOST }} "chmod +x /tmp/deploy-celery.sh && /tmp/deploy-celery.sh $REGISTRY $REPOSITORY" + + - name: Deploy Flower to EC2 + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ github.event.repository.name }}-celery-repo + run: | + # Create flower deployment script + cat > deploy-flower.sh << 'EOF' + #!/bin/bash + + # Stop existing flower container + docker stop flower-production || true + docker rm flower-production || true + + # Start flower using the same celery image + docker run -d --name flower-production --restart unless-stopped \ + --env-file /home/ubuntu/.env.production \ + -p 5555:5555 \ + $1/$2:production \ + uv run celery -A app.celery.celery_app flower --port=5555 + EOF + + # Copy script to EC2 and execute + scp -i production_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ubuntu@${{ secrets.EC2_PRODUCTION_HOST }}:/tmp/ + ssh -i production_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_PRODUCTION_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 8ec570c3..78636f67 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -31,7 +31,7 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 - - name: Build and Push Docker Image + - name: Build and Push Backend Docker Image env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} REPOSITORY: ${{ github.event.repository.name }}-staging-repo @@ -39,6 +39,75 @@ jobs: docker build -t $REGISTRY/$REPOSITORY:latest ./backend docker push $REGISTRY/$REPOSITORY:latest - - name: Deploy to ECS + - name: Build and Push Celery Docker Image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ github.event.repository.name }}-celery-repo + run: | + docker build -f ./backend/Dockerfile.celery -t $REGISTRY/$REPOSITORY:staging ./backend + docker push $REGISTRY/$REPOSITORY:staging + + - name: Deploy Backend to ECS run: | aws ecs update-service --cluster ${{ github.event.repository.name }}-staging-cluster --service ${{ github.event.repository.name }}-staging-service --force-new-deployment + + - name: Deploy Celery Workers to EC2 + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ github.event.repository.name }}-celery-repo + run: | + # Create deployment script + cat > deploy-celery.sh << 'EOF' + #!/bin/bash + + # Stop existing containers + docker stop celery-stg-high celery-stg-other || true + docker rm celery-stg-high celery-stg-other || true + + # Pull latest image + aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin $1 + docker pull $1/$2:staging + + # Start high priority worker + docker run -d --name celery-stg-high --restart unless-stopped \ + --env-file /home/ubuntu/.env.staging \ + $1/$2:staging \ + uv run celery -A app.celery.celery_app worker --queues=high_priority --loglevel=info --concurrency=1 --hostname=stg-high@%h + + # Start other queues worker + docker run -d --name celery-stg-other --restart unless-stopped \ + --env-file /home/ubuntu/.env.staging \ + $1/$2:staging \ + uv run celery -A app.celery.celery_app worker --queues=default,low_priority,cron --loglevel=info --concurrency=1 --hostname=stg-other@%h + EOF + + # Copy script to EC2 and execute + echo "${{ secrets.EC2_STAGING_PRIVATE_KEY }}" > staging_key.pem + chmod 600 staging_key.pem + scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-celery.sh ubuntu@${{ secrets.EC2_STAGING_HOST }}:/tmp/ + ssh -i staging_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_STAGING_HOST }} "chmod +x /tmp/deploy-celery.sh && /tmp/deploy-celery.sh $REGISTRY $REPOSITORY" + + - name: Deploy Flower to EC2 + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ github.event.repository.name }}-celery-repo + run: | + # Create flower deployment script + cat > deploy-flower.sh << 'EOF' + #!/bin/bash + + # Stop existing flower container + docker stop flower-staging || true + docker rm flower-staging || true + + # Start flower using the same celery image + docker run -d --name flower-staging --restart unless-stopped \ + --env-file /home/ubuntu/.env.staging \ + -p 5555:5555 \ + $1/$2:staging \ + uv run celery -A app.celery.celery_app flower --port=5555 + EOF + + # Copy script to EC2 and execute + scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ubuntu@${{ secrets.EC2_STAGING_HOST }}:/tmp/ + ssh -i staging_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_STAGING_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ecf7c27c..f8bd69ae 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -92,6 +92,31 @@ services: - backend command: ["uv", "run", "celery", "-A", "app.celery.celery_app", "worker", "--loglevel=info", "--concurrency=2"] + flower: + build: + context: ./backend + dockerfile: Dockerfile.celery + environment: + - ENVIRONMENT=development + - REDIS_HOST=redis + - RABBITMQ_HOST=rabbitmq + - RABBITMQ_USER=guest + - RABBITMQ_PASSWORD=guest + env_file: + - ./.env + ports: + - "5555:5555" + volumes: + - ./backend:/app # Mount for live code changes + - /app/.venv # Exclude .venv from volume mount + networks: + - app-network + depends_on: + - redis + - rabbitmq + - celery-worker + command: ["uv", "run", "celery", "-A", "app.celery.celery_app", "flower", "--port=5555"] + networks: app-network: driver: bridge From 37111e53ebef6094b826df5c541036a6cd95bf8f Mon Sep 17 00:00:00 2001 From: Kartikeya Pophali Date: Tue, 30 Sep 2025 11:12:19 +0530 Subject: [PATCH 2/3] Refactor deployment scripts for EC2: clean up whitespace and improve readability --- .env.example | 2 +- .github/workflows/cd-production.yml | 18 +++++++++--------- .github/workflows/cd-staging.yml | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 0a43d976..f514c635 100644 --- a/.env.example +++ b/.env.example @@ -91,7 +91,7 @@ EC2_STAGING_HOST=staging-celery-host.example.com EC2_STAGING_USER=ubuntu EC2_STAGING_KEY_NAME=staging-ec2-key -# Production EC2 Instance +# Production EC2 Instance EC2_PRODUCTION_HOST=production-celery-host.example.com EC2_PRODUCTION_USER=ubuntu EC2_PRODUCTION_KEY_NAME=production-ec2-key diff --git a/.github/workflows/cd-production.yml b/.github/workflows/cd-production.yml index 2993cc79..7d02d48b 100644 --- a/.github/workflows/cd-production.yml +++ b/.github/workflows/cd-production.yml @@ -61,28 +61,28 @@ jobs: # Create deployment script cat > deploy-celery.sh << 'EOF' #!/bin/bash - + # Stop existing containers docker stop celery-prod-high celery-prod-other || true docker rm celery-prod-high celery-prod-other || true - + # Pull latest image aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin $1 docker pull $1/$2:production - + # Start high priority worker docker run -d --name celery-prod-high --restart unless-stopped \ --env-file /home/ubuntu/.env.production \ $1/$2:production \ uv run celery -A app.celery.celery_app worker --queues=high_priority --loglevel=info --concurrency=2 --hostname=prod-high@%h - - # Start other queues worker + + # Start other queues worker docker run -d --name celery-prod-other --restart unless-stopped \ --env-file /home/ubuntu/.env.production \ $1/$2:production \ uv run celery -A app.celery.celery_app worker --queues=default,low_priority,cron --loglevel=info --concurrency=2 --hostname=prod-other@%h EOF - + # Copy script to EC2 and execute echo "${{ secrets.EC2_PRODUCTION_PRIVATE_KEY }}" > production_key.pem chmod 600 production_key.pem @@ -97,11 +97,11 @@ jobs: # Create flower deployment script cat > deploy-flower.sh << 'EOF' #!/bin/bash - + # Stop existing flower container docker stop flower-production || true docker rm flower-production || true - + # Start flower using the same celery image docker run -d --name flower-production --restart unless-stopped \ --env-file /home/ubuntu/.env.production \ @@ -109,7 +109,7 @@ jobs: $1/$2:production \ uv run celery -A app.celery.celery_app flower --port=5555 EOF - + # Copy script to EC2 and execute scp -i production_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ubuntu@${{ secrets.EC2_PRODUCTION_HOST }}:/tmp/ ssh -i production_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_PRODUCTION_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 78636f67..a5de9503 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -59,28 +59,28 @@ jobs: # Create deployment script cat > deploy-celery.sh << 'EOF' #!/bin/bash - + # Stop existing containers docker stop celery-stg-high celery-stg-other || true docker rm celery-stg-high celery-stg-other || true - + # Pull latest image aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin $1 docker pull $1/$2:staging - + # Start high priority worker docker run -d --name celery-stg-high --restart unless-stopped \ --env-file /home/ubuntu/.env.staging \ $1/$2:staging \ uv run celery -A app.celery.celery_app worker --queues=high_priority --loglevel=info --concurrency=1 --hostname=stg-high@%h - - # Start other queues worker + + # Start other queues worker docker run -d --name celery-stg-other --restart unless-stopped \ --env-file /home/ubuntu/.env.staging \ $1/$2:staging \ uv run celery -A app.celery.celery_app worker --queues=default,low_priority,cron --loglevel=info --concurrency=1 --hostname=stg-other@%h EOF - + # Copy script to EC2 and execute echo "${{ secrets.EC2_STAGING_PRIVATE_KEY }}" > staging_key.pem chmod 600 staging_key.pem @@ -95,11 +95,11 @@ jobs: # Create flower deployment script cat > deploy-flower.sh << 'EOF' #!/bin/bash - + # Stop existing flower container docker stop flower-staging || true docker rm flower-staging || true - + # Start flower using the same celery image docker run -d --name flower-staging --restart unless-stopped \ --env-file /home/ubuntu/.env.staging \ @@ -107,7 +107,7 @@ jobs: $1/$2:staging \ uv run celery -A app.celery.celery_app flower --port=5555 EOF - + # Copy script to EC2 and execute scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ubuntu@${{ secrets.EC2_STAGING_HOST }}:/tmp/ ssh -i staging_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_STAGING_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" From 14db057fb42203b8ec414e5278b0134324ae3490 Mon Sep 17 00:00:00 2001 From: Kartikeya Pophali Date: Wed, 1 Oct 2025 13:36:07 +0530 Subject: [PATCH 3/3] Refactor EC2 deployment scripts for Celery and Flower: streamline variable names and enhance error handling --- .env.example | 15 ++-------- .github/workflows/cd-production.yml | 43 ++++++++++++++++++++++------- .github/workflows/cd-staging.yml | 43 ++++++++++++++++++++++------- 3 files changed, 69 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index f514c635..77bb4221 100644 --- a/.env.example +++ b/.env.example @@ -86,15 +86,6 @@ CALLBACK_READ_TIMEOUT = 10 # EC2 Deployment Configuration for Celery Workers -# Staging EC2 Instance -EC2_STAGING_HOST=staging-celery-host.example.com -EC2_STAGING_USER=ubuntu -EC2_STAGING_KEY_NAME=staging-ec2-key - -# Production EC2 Instance -EC2_PRODUCTION_HOST=production-celery-host.example.com -EC2_PRODUCTION_USER=ubuntu -EC2_PRODUCTION_KEY_NAME=production-ec2-key - -# Docker Registry for Celery Images -DOCKER_IMAGE_CELERY=celery-worker +EC2_HOST=production-celery-host.example.com +EC2_USER=ubuntu +EC2_KEY=production-ec2-key diff --git a/.github/workflows/cd-production.yml b/.github/workflows/cd-production.yml index 7d02d48b..17634312 100644 --- a/.github/workflows/cd-production.yml +++ b/.github/workflows/cd-production.yml @@ -61,13 +61,14 @@ jobs: # Create deployment script cat > deploy-celery.sh << 'EOF' #!/bin/bash + set -e # Stop existing containers - docker stop celery-prod-high celery-prod-other || true - docker rm celery-prod-high celery-prod-other || true + docker stop celery-prod-high celery-prod-other 2>/dev/null || true + docker rm celery-prod-high celery-prod-other 2>/dev/null || true # Pull latest image - aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin $1 + aws ecr get-login-password --region $3 | docker login --username AWS --password-stdin $1 docker pull $1/$2:production # Start high priority worker @@ -76,18 +77,28 @@ jobs: $1/$2:production \ uv run celery -A app.celery.celery_app worker --queues=high_priority --loglevel=info --concurrency=2 --hostname=prod-high@%h + # Verify high priority worker started + sleep 2 + docker ps | grep celery-prod-high || { echo "High priority worker failed to start"; exit 1; } + # Start other queues worker docker run -d --name celery-prod-other --restart unless-stopped \ --env-file /home/ubuntu/.env.production \ $1/$2:production \ uv run celery -A app.celery.celery_app worker --queues=default,low_priority,cron --loglevel=info --concurrency=2 --hostname=prod-other@%h + + # Verify other queues worker started + sleep 2 + docker ps | grep celery-prod-other || { echo "Other queues worker failed to start"; exit 1; } EOF # Copy script to EC2 and execute - echo "${{ secrets.EC2_PRODUCTION_PRIVATE_KEY }}" > production_key.pem + echo "${{ secrets.EC2_KEY }}" > production_key.pem chmod 600 production_key.pem - scp -i production_key.pem -o StrictHostKeyChecking=no deploy-celery.sh ubuntu@${{ secrets.EC2_PRODUCTION_HOST }}:/tmp/ - ssh -i production_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_PRODUCTION_HOST }} "chmod +x /tmp/deploy-celery.sh && /tmp/deploy-celery.sh $REGISTRY $REPOSITORY" + trap 'rm -f production_key.pem deploy-celery.sh' EXIT + scp -i production_key.pem -o StrictHostKeyChecking=no deploy-celery.sh ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }}:/tmp/ + ssh -i production_key.pem -o StrictHostKeyChecking=no ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }} "chmod +x /tmp/deploy-celery.sh && /tmp/deploy-celery.sh $REGISTRY $REPOSITORY ap-south-1" + rm -f production_key.pem deploy-celery.sh - name: Deploy Flower to EC2 env: @@ -97,10 +108,11 @@ jobs: # Create flower deployment script cat > deploy-flower.sh << 'EOF' #!/bin/bash + set -e # Stop existing flower container - docker stop flower-production || true - docker rm flower-production || true + docker stop flower-production 2>/dev/null || true + docker rm flower-production 2>/dev/null || true # Start flower using the same celery image docker run -d --name flower-production --restart unless-stopped \ @@ -108,8 +120,19 @@ jobs: -p 5555:5555 \ $1/$2:production \ uv run celery -A app.celery.celery_app flower --port=5555 + + # Verify Flower started + sleep 2 + docker ps | grep flower-production || { echo "Flower failed to start"; exit 1; } + + # Health check + curl -f http://localhost:5555/healthcheck || { echo "Flower health check failed"; exit 1; } EOF # Copy script to EC2 and execute - scp -i production_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ubuntu@${{ secrets.EC2_PRODUCTION_HOST }}:/tmp/ - ssh -i production_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_PRODUCTION_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" + echo "${{ secrets.EC2_KEY }}" > production_key.pem + chmod 600 production_key.pem + trap 'rm -f production_key.pem deploy-flower.sh' EXIT + scp -i production_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }}:/tmp/ + ssh -i production_key.pem -o StrictHostKeyChecking=no ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" + rm -f production_key.pem deploy-flower.sh diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index a5de9503..f3e83b12 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -59,13 +59,14 @@ jobs: # Create deployment script cat > deploy-celery.sh << 'EOF' #!/bin/bash + set -e # Stop existing containers - docker stop celery-stg-high celery-stg-other || true - docker rm celery-stg-high celery-stg-other || true + docker stop celery-stg-high celery-stg-other 2>/dev/null || true + docker rm celery-stg-high celery-stg-other 2>/dev/null || true # Pull latest image - aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin $1 + aws ecr get-login-password --region $3 | docker login --username AWS --password-stdin $1 docker pull $1/$2:staging # Start high priority worker @@ -74,18 +75,28 @@ jobs: $1/$2:staging \ uv run celery -A app.celery.celery_app worker --queues=high_priority --loglevel=info --concurrency=1 --hostname=stg-high@%h + # Verify high priority worker started + sleep 2 + docker ps | grep celery-stg-high || { echo "High priority worker failed to start"; exit 1; } + # Start other queues worker docker run -d --name celery-stg-other --restart unless-stopped \ --env-file /home/ubuntu/.env.staging \ $1/$2:staging \ uv run celery -A app.celery.celery_app worker --queues=default,low_priority,cron --loglevel=info --concurrency=1 --hostname=stg-other@%h + + # Verify other queues worker started + sleep 2 + docker ps | grep celery-stg-other || { echo "Other queues worker failed to start"; exit 1; } EOF # Copy script to EC2 and execute - echo "${{ secrets.EC2_STAGING_PRIVATE_KEY }}" > staging_key.pem + echo "${{ secrets.EC2_KEY }}" > staging_key.pem chmod 600 staging_key.pem - scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-celery.sh ubuntu@${{ secrets.EC2_STAGING_HOST }}:/tmp/ - ssh -i staging_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_STAGING_HOST }} "chmod +x /tmp/deploy-celery.sh && /tmp/deploy-celery.sh $REGISTRY $REPOSITORY" + trap 'rm -f staging_key.pem deploy-celery.sh' EXIT + scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-celery.sh ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }}:/tmp/ + ssh -i staging_key.pem -o StrictHostKeyChecking=no ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }} "chmod +x /tmp/deploy-celery.sh && /tmp/deploy-celery.sh $REGISTRY $REPOSITORY ap-south-1" + rm -f staging_key.pem deploy-celery.sh - name: Deploy Flower to EC2 env: @@ -95,10 +106,11 @@ jobs: # Create flower deployment script cat > deploy-flower.sh << 'EOF' #!/bin/bash + set -e # Stop existing flower container - docker stop flower-staging || true - docker rm flower-staging || true + docker stop flower-staging 2>/dev/null || true + docker rm flower-staging 2>/dev/null || true # Start flower using the same celery image docker run -d --name flower-staging --restart unless-stopped \ @@ -106,8 +118,19 @@ jobs: -p 5555:5555 \ $1/$2:staging \ uv run celery -A app.celery.celery_app flower --port=5555 + + # Verify Flower started + sleep 2 + docker ps | grep flower-staging || { echo "Flower failed to start"; exit 1; } + + # Health check + curl -f http://localhost:5555/healthcheck || { echo "Flower health check failed"; exit 1; } EOF # Copy script to EC2 and execute - scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ubuntu@${{ secrets.EC2_STAGING_HOST }}:/tmp/ - ssh -i staging_key.pem -o StrictHostKeyChecking=no ubuntu@${{ secrets.EC2_STAGING_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" + echo "${{ secrets.EC2_KEY }}" > staging_key.pem + chmod 600 staging_key.pem + trap 'rm -f staging_key.pem deploy-flower.sh' EXIT + scp -i staging_key.pem -o StrictHostKeyChecking=no deploy-flower.sh ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }}:/tmp/ + ssh -i staging_key.pem -o StrictHostKeyChecking=no ${{ secrets.EC2_USER }}@${{ secrets.EC2_HOST }} "chmod +x /tmp/deploy-flower.sh && /tmp/deploy-flower.sh $REGISTRY $REPOSITORY" + rm -f staging_key.pem deploy-flower.sh