From e93484863f07a6ce99639564acddf7e220a02cc2 Mon Sep 17 00:00:00 2001 From: adela Date: Thu, 16 Jan 2025 11:14:57 +0800 Subject: [PATCH] update --- .github/workflows/.bb-masking-1.yml | 189 ---------------------- .github/workflows/.bb-masking-2.yml | 172 -------------------- .github/workflows/.bb-masking-3.yml | 175 -------------------- .github/workflows/try-login.yml | 30 ---- masking/global-masking-rule-advanced.json | 29 ++++ masking/global-masking-rule.json | 13 +- 6 files changed, 31 insertions(+), 577 deletions(-) delete mode 100644 .github/workflows/.bb-masking-1.yml delete mode 100644 .github/workflows/.bb-masking-2.yml delete mode 100644 .github/workflows/.bb-masking-3.yml delete mode 100644 .github/workflows/try-login.yml create mode 100644 masking/global-masking-rule-advanced.json diff --git a/.github/workflows/.bb-masking-1.yml b/.github/workflows/.bb-masking-1.yml deleted file mode 100644 index 60853cb..0000000 --- a/.github/workflows/.bb-masking-1.yml +++ /dev/null @@ -1,189 +0,0 @@ -name: Bytebase Masking Policy Update -on: - pull_request: - types: [closed] - branches: - - main - workflow_dispatch: - -jobs: - bytebase-masking-1: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - pull-requests: write - issues: write - contents: read - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - - name: Login Bytebase - id: bytebase-login - uses: bytebase/login-action@0.0.2 - with: - bytebase-url: ${{ secrets.BYTEBASE_URL }} - service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }} - service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }} - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v42 - with: - files: | - masking/databases/**/**/column-masking.json - masking/projects/**/masking-exception.json - since_last_remote_commit: true - fetch_depth: 0 - include_all_old_new_renamed_files: true - - - name: Debug changed files in detail - run: | - echo "All changed files:" - echo "${{ steps.changed-files.outputs.all_changed_files }}" - echo "Added files:" - echo "${{ steps.changed-files.outputs.added_files }}" - echo "Modified files:" - echo "${{ steps.changed-files.outputs.modified_files }}" - echo "Contains column-masking.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'column-masking.json') }}" - echo "Contains masking-exception.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'masking-exception.json') }}" - echo "Raw output:" - echo "${{ toJSON(steps.changed-files.outputs) }}" - - - name: Apply column masking policy - id: apply-column-masking - if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/column-masking.json') }} - run: | - # Process all column-masking.json files - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "column-masking.json" | while read -r CHANGED_FILE; do - echo "Processing: $CHANGED_FILE" - INSTANCE_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/databases\/\([^/]*\)\/\([^/]*\).*/\1/p') - DATABASE_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/databases\/\([^/]*\)\/\([^/]*\).*/\2/p') - echo "INSTANCE_NAME=$INSTANCE_NAME" - echo "DATABASE_NAME=$DATABASE_NAME" - - response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/instances/${INSTANCE_NAME}/databases/${DATABASE_NAME}/policies/masking?allow_missing=true&update_mask=payload" \ - --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - --header "Content-Type: application/json" \ - --data @"$CHANGED_FILE") - - # Extract status code and response body - status_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - echo "Status code: $status_code" - echo "Response body: $body" - - # Append to outputs (with unique identifiers) - echo "status_code_${DATABASE_NAME}=${status_code}" >> $GITHUB_OUTPUT - echo "response_${DATABASE_NAME}<> $GITHUB_OUTPUT - echo "${body}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then - echo "Failed with status code: $status_code for database: $DATABASE_NAME" - exit 1 - fi - done - - - name: Apply masking exception policy - id: apply-masking-exception - if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/masking-exception.json') }} - run: | - # Process all masking-exception.json files - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "masking-exception.json" | while read -r CHANGED_FILE; do - echo "Processing: $CHANGED_FILE" - PROJECT_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/projects\/\([^/]*\).*/\1/p') - echo "PROJECT_NAME=$PROJECT_NAME" - - response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/projects/${PROJECT_NAME}/policies/masking_exception?allow_missing=true&update_mask=payload" \ - --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - --header "Content-Type: application/json" \ - --data @"$CHANGED_FILE") - - # Extract status code and response body - status_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - echo "Status code: $status_code" - echo "Response body: $body" - - # Append to outputs (with unique identifiers) - echo "status_code_${PROJECT_NAME}=${status_code}" >> $GITHUB_OUTPUT - echo "response_${PROJECT_NAME}<> $GITHUB_OUTPUT - echo "${body}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then - echo "Failed with status code: $status_code for project: $PROJECT_NAME" - exit 1 - fi - done - - - name: Comment on PR - uses: actions/github-script@v7 - env: - CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - with: - script: | - const changedFiles = process.env.CHANGED_FILES || ''; - let commentBody = `### Masking Policy Update Summary\n\n`; - - // Add status of merge - commentBody += `✅ **PR Status:** Merged\n\n`; - - // Add changed files section - commentBody += `📝 **Changed Files:**\n\n`; - if (changedFiles.trim()) { - commentBody += changedFiles.split(' ').map(f => `- ${f}`).join('\n'); - } else { - commentBody += `None`; - } - commentBody += '\n\n'; - - // Add API calls summary - commentBody += `🔄 **API Calls:**\n\n`; - let apiCallsFound = false; - - if (changedFiles.includes('column-masking.json')) { - const maskingStatuses = Object.keys(${{ toJSON(steps.apply-column-masking.outputs) }} || {}) - .filter(key => key.startsWith('status_code_')) - .map(key => ({ - name: key.replace('status_code_', ''), - status: ${{ toJSON(steps.apply-column-masking.outputs) }}[key] - })); - - maskingStatuses.forEach(({name, status}) => { - apiCallsFound = true; - const success = status >= 200 && status < 300; - commentBody += `- Column Masking (${name}): ${success ? '✅' : '❌'} ${status}\n`; - }); - } - - if (changedFiles.includes('masking-exception.json')) { - const exceptionStatuses = Object.keys(${{ toJSON(steps.apply-masking-exception.outputs) }} || {}) - .filter(key => key.startsWith('status_code_')) - .map(key => ({ - name: key.replace('status_code_', ''), - status: ${{ toJSON(steps.apply-masking-exception.outputs) }}[key] - })); - - exceptionStatuses.forEach(({name, status}) => { - apiCallsFound = true; - const success = status >= 200 && status < 300; - commentBody += `- Masking Exception (${name}): ${success ? '✅' : '❌'} ${status}\n`; - }); - } - - if (!apiCallsFound) { - commentBody += `None`; - } - - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: commentBody - }); diff --git a/.github/workflows/.bb-masking-2.yml b/.github/workflows/.bb-masking-2.yml deleted file mode 100644 index 116faf4..0000000 --- a/.github/workflows/.bb-masking-2.yml +++ /dev/null @@ -1,172 +0,0 @@ -name: Bytebase Masking Policy Update 2 -on: - pull_request: - types: [closed] - branches: - - main - workflow_dispatch: - -jobs: - bytebase-masking-2: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - pull-requests: write - issues: write - contents: read - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - - name: Login Bytebase - id: bytebase-login - uses: bytebase/login-action@0.0.2 - with: - bytebase-url: ${{ secrets.BYTEBASE_URL }} - service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }} - service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }} - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v42 - with: - files: | - masking/masking-algorithm.json - masking/semantic-type.json - since_last_remote_commit: true - fetch_depth: 0 - include_all_old_new_renamed_files: true - - - name: Debug changed files - run: | - echo "All changed and added files:" - echo "Modified files: ${{ steps.changed-files.outputs.modified_files }}" - echo "Added files: ${{ steps.changed-files.outputs.added_files }}" - echo "All changes: ${{ steps.changed-files.outputs.all_changed_files }}" - - - name: Debug changed files in detail - run: | - echo "All changed files:" - echo "${{ steps.changed-files.outputs.all_changed_files }}" - echo "Contains masking-algorithm.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'masking-algorithm.json') }}" - echo "Contains semantic-type.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'semantic-type.json') }}" - echo "Raw output:" - echo "${{ toJSON(steps.changed-files.outputs) }}" - - - name: Apply masking algorithm - id: apply-masking-algorithm - if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, 'masking-algorithm.json') }} - run: | - CHANGED_FILE="masking/masking-algorithm.json" - echo "Processing: $CHANGED_FILE" - - response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/settings/bb.workspace.masking-algorithm?allow_missing=true" \ - --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - --header "Content-Type: application/json" \ - --data @"$CHANGED_FILE") - - # Extract status code and response body - status_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - echo "status_code=${status_code}" >> $GITHUB_OUTPUT - echo "response_body<> $GITHUB_OUTPUT - echo "${body}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then - echo "Failed with status code: $status_code" - exit 1 - fi - - - name: Apply semantic type - id: apply-semantic-type - if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/semantic-type.json') }} - run: | - # Process all masking-exception.json files - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "semantic-type.json" | while read -r CHANGED_FILE; do - echo "Processing: $CHANGED_FILE" - - response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/settings/bb.workspace.semantic-types?allow_missing=true" \ - --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - --header "Content-Type: application/json" \ - --data @"$CHANGED_FILE") - - # Extract status code and response body - status_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - echo "Status code: $status_code" - echo "Response body: $body" - - # Append to outputs (with unique identifiers) - echo "${body}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then - echo "Failed with status code: $status_code" - exit 1 - fi - done - - - name: Comment on PR - uses: actions/github-script@v7 - env: - CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - with: - script: | - const changedFiles = process.env.CHANGED_FILES || ''; - let commentBody = `### Masking Policy Update 2 Summary\n\n`; - - // Add status of merge - commentBody += `✅ **PR Status:** Merged\n\n`; - - // Add changed files section - commentBody += `📝 **Changed Files:**\n\n`; - if (changedFiles.trim()) { - commentBody += changedFiles.split(' ').map(f => `- ${f}`).join('\n'); - } else { - commentBody += `None`; - } - commentBody += '\n\n'; - - // Add API calls summary - commentBody += `🔄 **API Calls:**\n\n`; - let apiCallsFound = false; - - if (changedFiles.includes('masking-algorithm.json')) { - const status = ${{ toJSON(steps.apply-masking-algorithm.outputs) }}.status_code; - if (status) { - apiCallsFound = true; - const success = status >= 200 && status < 300; - commentBody += `- Column Masking: ${success ? '✅' : '❌'} ${status}\n`; - } - } - - if (changedFiles.includes('semantic-type.json')) { - const exceptionStatuses = Object.keys(${{ toJSON(steps.apply-semantic-type.outputs) }} || {}) - .filter(key => key.startsWith('status_code_')) - .map(key => ({ - name: key.replace('status_code_', ''), - status: ${{ toJSON(steps.apply-semantic-type.outputs) }}[key] - })); - - exceptionStatuses.forEach(({name, status}) => { - apiCallsFound = true; - const success = status >= 200 && status < 300; - commentBody += `- Masking Exception (${name}): ${success ? '✅' : '❌'} ${status}\n`; - }); - } - - if (!apiCallsFound) { - commentBody += `None`; - } - - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: commentBody - }); diff --git a/.github/workflows/.bb-masking-3.yml b/.github/workflows/.bb-masking-3.yml deleted file mode 100644 index 0eb77e2..0000000 --- a/.github/workflows/.bb-masking-3.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Bytebase Masking Policy Update 3 -on: - pull_request: - types: [closed] - branches: - - main - workflow_dispatch: - -jobs: - bytebase-masking-3: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - pull-requests: write - issues: write - contents: read - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - - name: Login Bytebase - id: bytebase-login - uses: bytebase/login-action@0.0.2 - with: - bytebase-url: ${{ secrets.BYTEBASE_URL }} - service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }} - service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }} - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v42 - with: - files: | - masking/data-classification.json - masking/global-masking-rule.json - since_last_remote_commit: true - fetch_depth: 0 - include_all_old_new_renamed_files: true - - - name: Debug changed files - run: | - echo "All changed and added files:" - echo "Modified files: ${{ steps.changed-files.outputs.modified_files }}" - echo "Added files: ${{ steps.changed-files.outputs.added_files }}" - echo "All changes: ${{ steps.changed-files.outputs.all_changed_files }}" - - - name: Debug changed files in detail - run: | - echo "All changed files:" - echo "${{ steps.changed-files.outputs.all_changed_files }}" - echo "Contains data-classification.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'data-classification.json') }}" - echo "Contains global-masking-rule.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'global-masking-rule.json') }}" - echo "Raw output:" - echo "${{ toJSON(steps.changed-files.outputs) }}" - - - name: Apply data classification - id: apply-data-classification - if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, 'data-classification.json') }} - run: | - CHANGED_FILE="masking/data-classification.json" - echo "Processing: $CHANGED_FILE" - - response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/settings/bb.workspace.data-classification" \ - --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - --header "Content-Type: application/json" \ - --data @"$CHANGED_FILE") - - # Extract status code and response body - status_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - echo "status_code=${status_code}" >> $GITHUB_OUTPUT - echo "response_body<> $GITHUB_OUTPUT - echo "${body}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then - echo "Failed with status code: $status_code" - exit 1 - fi - - - name: Apply global masking rule - id: apply-global-masking-rule - if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, 'global-masking-rule.json') }} - run: | - # Process all global-masking-rule.json files - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "global-masking-rule.json" | while read -r CHANGED_FILE; do - echo "Processing: $CHANGED_FILE" - - response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/policies/masking_rule?allow_missing=true&update_mask=payload" \ - --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - --header "Content-Type: application/json" \ - --data @"$CHANGED_FILE") - - # Extract status code and response body - status_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - echo "Status code: $status_code" - echo "Response body: $body" - - # Append to outputs (with unique identifiers) - if [[ $status_code -ge 200 && $status_code -lt 300 ]]; then - echo "${body}" >> $GITHUB_OUTPUT - else - echo "Failed with status code: $status_code" - echo "Response body: ${body}" - if [[ $status_code -eq 403 ]]; then - echo "Access denied. Please check your permissions and API token." - fi - exit 1 - fi - done - - - name: Comment on PR - uses: actions/github-script@v7 - env: - CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - with: - script: | - const changedFiles = process.env.CHANGED_FILES || ''; - let commentBody = `### Masking Policy Update 3 Summary\n\n`; - - // Add status of merge - commentBody += `✅ **PR Status:** Merged\n\n`; - - // Add changed files section - commentBody += `📝 **Changed Files:**\n\n`; - if (changedFiles.trim()) { - commentBody += changedFiles.split(' ').map(f => `- ${f}`).join('\n'); - } else { - commentBody += `None`; - } - commentBody += '\n\n'; - - // Add API calls summary - commentBody += `🔄 **API Calls:**\n\n`; - let apiCallsFound = false; - - if (changedFiles.includes('masking-algorithm.json')) { - const status = ${{ toJSON(steps.apply-masking-algorithm.outputs) }}.status_code; - if (status) { - apiCallsFound = true; - const success = status >= 200 && status < 300; - commentBody += `- Column Masking: ${success ? '✅' : '❌'} ${status}\n`; - } - } - - if (changedFiles.includes('semantic-type.json')) { - const exceptionStatuses = Object.keys(${{ toJSON(steps.apply-semantic-type.outputs) }} || {}) - .filter(key => key.startsWith('status_code_')) - .map(key => ({ - name: key.replace('status_code_', ''), - status: ${{ toJSON(steps.apply-semantic-type.outputs) }}[key] - })); - - exceptionStatuses.forEach(({name, status}) => { - apiCallsFound = true; - const success = status >= 200 && status < 300; - commentBody += `- Masking Exception (${name}): ${success ? '✅' : '❌'} ${status}\n`; - }); - } - - if (!apiCallsFound) { - commentBody += `None`; - } - - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: commentBody - }); diff --git a/.github/workflows/try-login.yml b/.github/workflows/try-login.yml deleted file mode 100644 index 9d8b898..0000000 --- a/.github/workflows/try-login.yml +++ /dev/null @@ -1,30 +0,0 @@ -on: - push: - branches: - - main - workflow_dispatch: - pull_request: - -jobs: - bytebase-ci: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Login Bytebase - id: bytebase-login - uses: bytebase/login-action@main - with: - bytebase-url: ${{ secrets.BYTEBASE_URL }} - service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }} - service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }} - - - name: Debug API URL - run: | - echo "API URL Length: $(echo -n "${{ steps.bytebase-login.outputs.api_url }}" | wc -c)" - echo "Token Length: $(echo -n "${{ steps.bytebase-login.outputs.token }}" | wc -c)" - - - name: List projects - run: | - curl "${{steps.bytebase-login.outputs.api_url}}/projects" \ - -H "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \ - -H 'Content-Type: application/json; charset=utf-8' \ No newline at end of file diff --git a/masking/global-masking-rule-advanced.json b/masking/global-masking-rule-advanced.json new file mode 100644 index 0000000..4ff3dec --- /dev/null +++ b/masking/global-masking-rule-advanced.json @@ -0,0 +1,29 @@ +{ + "name": "policies/masking_rule", + "inheritFromParent": false, + "type": "MASKING_RULE", + "maskingRulePolicy": { + "rules": [ + { + "id": "76356d81-6231-4128-9be7-2c549fc505f5", + "condition": { + "expression": "classification_level in [\"2\", \"3\"]", + "title": "", + "description": "" + }, + "semanticType": "bb.default-partial" + }, + { + "id": "1ddc47c9-6ab6-4760-accd-947bc1a5f155", + "condition": { + "expression": "classification_level in [\"4\"]", + "title": "", + "description": "" + }, + "semanticType": "bb.default" + } + ] + }, + "enforce": true, + "resourceType": "WORKSPACE" +} diff --git a/masking/global-masking-rule.json b/masking/global-masking-rule.json index 4ff3dec..ec6533e 100644 --- a/masking/global-masking-rule.json +++ b/masking/global-masking-rule.json @@ -7,16 +7,7 @@ { "id": "76356d81-6231-4128-9be7-2c549fc505f5", "condition": { - "expression": "classification_level in [\"2\", \"3\"]", - "title": "", - "description": "" - }, - "semanticType": "bb.default-partial" - }, - { - "id": "1ddc47c9-6ab6-4760-accd-947bc1a5f155", - "condition": { - "expression": "classification_level in [\"4\"]", + "expression": "environment_id == \"prod\"", "title": "", "description": "" }, @@ -26,4 +17,4 @@ }, "enforce": true, "resourceType": "WORKSPACE" -} +} \ No newline at end of file