From 1daf39dfb5b4bd9b5edff7762f67699a4b24bcd9 Mon Sep 17 00:00:00 2001 From: adela Date: Thu, 9 Jan 2025 18:02:59 +0800 Subject: [PATCH] add export --- .github/workflows/bb-export.yml | 173 ++++++++++++++++++ .../hr_prod/20250109_select_employee.sql | 1 + .../databases/hr_prod/manifest.toml | 4 + 3 files changed, 178 insertions(+) create mode 100644 .github/workflows/bb-export.yml create mode 100644 export/projects/sample-project/databases/hr_prod/20250109_select_employee.sql create mode 100644 export/projects/sample-project/databases/hr_prod/manifest.toml diff --git a/.github/workflows/bb-export.yml b/.github/workflows/bb-export.yml new file mode 100644 index 0000000..2762272 --- /dev/null +++ b/.github/workflows/bb-export.yml @@ -0,0 +1,173 @@ +name: Bytebase Export SQL +on: + pull_request: + types: [closed] + branches: + - main + paths: + - 'export/**' + workflow_dispatch: + +jobs: + bytebase-export: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Login Bytebase + id: bytebase-login + uses: bytebase/login-action@0.0.2 + with: + bytebase-url: ${{ secrets.BYTEBASE_URL }} + service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }} + service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v42 + with: + files: | + export/**/*.sql + since_last_remote_commit: true + + - name: Process SQL files + id: process-sql + if: steps.changed-files.outputs.any_changed == 'true' + run: | + # Function to make API calls with error handling + call_api() { + local url="$1" + local method="$2" + local data="$3" + local description="$4" + + echo "Making $description API call..." + response=$(curl -s -w "\n%{http_code}" \ + --request "$method" "$url" \ + --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ + --header "Content-Type: application/json" \ + --data "$data") + + status_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | sed '$d') + + echo "Status code: $status_code" + if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then + echo "Failed $description. Status: $status_code" + echo "Response: $body" + return 1 + fi + + echo "$body" + return 0 + } + + # Process each SQL file + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do + echo "Processing $file" + + # Find and parse manifest.toml + DIR_PATH=$(dirname "$file") + MANIFEST_PATH="" + while [[ "$DIR_PATH" == export* ]]; do + if [[ -f "$DIR_PATH/manifest.toml" ]]; then + MANIFEST_PATH="$DIR_PATH/manifest.toml" + break + fi + DIR_PATH=$(dirname "$DIR_PATH") + done + + if [[ -z "$MANIFEST_PATH" ]]; then + echo "Error: No manifest.toml found for $file" + exit 1 + fi + + # Parse manifest.toml + PROJECT=$(python3 -c "import toml; print(toml.load('$MANIFEST_PATH')['project'])") + INSTANCE=$(python3 -c "import toml; print(toml.load('$MANIFEST_PATH')['instance'])") + DATABASE=$(python3 -c "import toml; print(toml.load('$MANIFEST_PATH')['database'])") + FORMAT=$(python3 -c "import toml; config=toml.load('$MANIFEST_PATH'); print(config.get('format', 'JSON'))") + + # Read SQL content and encode to base64 + SQL_CONTENT=$(base64 < "$file") + + # Generate UUID for step ID + STEP_ID=$(python3 -c "import uuid; print(str(uuid.uuid4()))") + + BASE_URL="${{ steps.bytebase-login.outputs.api_url }}" + + # 1. Create Sheet + sheet_data=$(call_api \ + "$BASE_URL/v1/projects/$PROJECT/sheets" \ + "POST" \ + "{\"title\":\"\",\"content\":\"$SQL_CONTENT\",\"type\":\"TYPE_SQL\",\"source\":\"SOURCE_BYTEBASE_ARTIFACT\",\"visibility\":\"VISIBILITY_PUBLIC\"}" \ + "Create Sheet") || exit 1 + + SHEET_NAME=$(echo "$sheet_data" | python3 -c "import sys, json; print(json.load(sys.stdin)['name'])") + + # 2. Create Plan + plan_data=$(call_api \ + "$BASE_URL/v1/projects/$PROJECT/plans" \ + "POST" \ + "{\"steps\":[{\"specs\":[{\"id\":\"$STEP_ID\",\"export_data_config\":{\"target\":\"/instances/$INSTANCE/databases/$DATABASE\",\"format\":\"$FORMAT\",\"sheet\":\"$SHEET_NAME\"}}]}],\"title\":\"Export data from $DATABASE\",\"description\":\"EXPORT\"}" \ + "Create Plan") || exit 1 + + PLAN_NAME=$(echo "$plan_data" | python3 -c "import sys, json; print(json.load(sys.stdin)['name'])") + + # 3. Create Issue + issue_data=$(call_api \ + "$BASE_URL/v1/projects/$PROJECT/issues" \ + "POST" \ + "{\"approvers\":[],\"approvalTemplates\":[],\"subscribers\":[],\"title\":\"Issue: Export data from instances/$INSTANCE/databases/$DATABASE\",\"description\":\"SQL request from GitHub\",\"type\":\"DATABASE_DATA_EXPORT\",\"assignee\":\"\",\"plan\":\"$PLAN_NAME\"}" \ + "Create Issue") || exit 1 + + # 4. Create Rollout + rollout_data=$(call_api \ + "$BASE_URL/v1/projects/$PROJECT/rollouts" \ + "POST" \ + "{\"plan\":\"$PLAN_NAME\"}" \ + "Create Rollout") || exit 1 + + # Extract issue link for PR comment + ISSUE_NUMBER=$(echo "$issue_data" | python3 -c "import sys, json; print(json.load(sys.stdin)['name'].split('/')[-1])") + ISSUE_LINK="${{ secrets.BYTEBASE_URL }}/projects/$PROJECT/issues/$ISSUE_NUMBER" + echo "ISSUE_LINK=$ISSUE_LINK" >> $GITHUB_ENV + + echo "Successfully processed $file" + done + + - name: Comment on PR + uses: actions/github-script@v7 + if: always() + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + with: + script: | + const changedFiles = process.env.CHANGED_FILES || ''; + let commentBody = `### SQL Export Summary\n\n`; + + commentBody += `✅ **PR Status:** Merged\n\n`; + + commentBody += `📝 **Processed SQL Files:**\n\n`; + if (changedFiles.trim()) { + commentBody += changedFiles.split(' ').map(f => `- ${f}`).join('\n'); + } else { + commentBody += `None`; + } + + commentBody += '\n\n**Status:** ${process.env.STATUS || 'Completed'}`; + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: commentBody + }); diff --git a/export/projects/sample-project/databases/hr_prod/20250109_select_employee.sql b/export/projects/sample-project/databases/hr_prod/20250109_select_employee.sql new file mode 100644 index 0000000..99ff170 --- /dev/null +++ b/export/projects/sample-project/databases/hr_prod/20250109_select_employee.sql @@ -0,0 +1 @@ +SELECT * FROM employee; \ No newline at end of file diff --git a/export/projects/sample-project/databases/hr_prod/manifest.toml b/export/projects/sample-project/databases/hr_prod/manifest.toml new file mode 100644 index 0000000..5c48167 --- /dev/null +++ b/export/projects/sample-project/databases/hr_prod/manifest.toml @@ -0,0 +1,4 @@ +project = "sample-project" +instance = "prod-sample-instance" +database = "hr_prod" +format = "CSV" # Optional, defaults to "JSON" if not specified \ No newline at end of file