feat: support semantic type setting (#94)

ecmadao · web-flow · commit 87a42a590693 · 2025-02-11T18:18:29.000+08:00
* chore: use set for unordered collection

* chore: update doc

* fix: list all databases in the project

* chore: update

* fix: lint

* chore: add databases for instance

* chore: more error log

* chore: update

* chore: update

* feat: database resource

* fix: lint

* feat: support semantic type setting

* fix: lint

* chore: update
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.0.14
+1.0.15
diff --git a/api/setting.go b/api/setting.go
@@ -12,6 +12,8 @@ const (
 	SettingWorkspaceExternalApproval SettingName = "bb.workspace.approval.external"
 	// SettingDataClassification is the setting name for data classification.
 	SettingDataClassification SettingName = "bb.workspace.data-classification"
+	// SettingSemanticTypes is the setting name for semantic types.
+	SettingSemanticTypes SettingName = "bb.workspace.semantic-types"
 )
 
 // RiskLevel is the approval risk level.
diff --git a/client/cel.go b/client/cel.go
@@ -6,8 +6,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/hashicorp/terraform-plugin-log/tflog"
-
 	v1pb "github.com/bytebase/bytebase/proto/generated-go/v1"
 	"github.com/pkg/errors"
 	v1alpha1 "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
@@ -33,8 +31,6 @@ func (c *client) ParseExpression(ctx context.Context, expression string) (*v1alp
 		return nil, err
 	}
 
-	tflog.Debug(ctx, fmt.Sprintf("parse cel response:\n%v", string(body)))
-
 	var res v1pb.BatchParseResponse
 	if err := ProtojsonUnmarshaler.Unmarshal(body, &res); err != nil {
 		return nil, err
diff --git a/docs/data-sources/setting.md b/docs/data-sources/setting.md
@@ -22,6 +22,7 @@ The setting data source.
 ### Optional
 
 - `classification` (Block List, Max: 1) Classification for data masking. Require ENTERPRISE subscription. (see [below for nested schema](#nestedblock--classification))
+- `semantic_types` (Block Set) Semantic types for data masking. Require ENTERPRISE subscription. (see [below for nested schema](#nestedblock--semantic_types))
 - `workspace_profile` (Block List, Max: 1) (see [below for nested schema](#nestedblock--workspace_profile))
 
 ### Read-Only
@@ -63,6 +64,73 @@ Optional:
 
 
 
+<a id="nestedblock--semantic_types"></a>
+### Nested Schema for `semantic_types`
+
+Optional:
+
+- `algorithm` (Block List, Max: 1) The semantic type algorithm. Required. (see [below for nested schema](#nestedblock--semantic_types--algorithm))
+- `description` (String) The semantic type description. Optional.
+- `id` (String) The semantic type unique uuid.
+- `title` (String) The semantic type title. Required.
+
+<a id="nestedblock--semantic_types--algorithm"></a>
+### Nested Schema for `semantic_types.algorithm`
+
+Optional:
+
+- `full_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--full_mask))
+- `inner_outer_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--inner_outer_mask))
+- `md5_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--md5_mask))
+- `range_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--range_mask))
+
+<a id="nestedblock--semantic_types--algorithm--full_mask"></a>
+### Nested Schema for `semantic_types.algorithm.full_mask`
+
+Optional:
+
+- `substitution` (String) Substitution is the string used to replace the original value, the max length of the string is 16 bytes.
+
+
+<a id="nestedblock--semantic_types--algorithm--inner_outer_mask"></a>
+### Nested Schema for `semantic_types.algorithm.inner_outer_mask`
+
+Optional:
+
+- `prefix_len` (Number)
+- `substitution` (String)
+- `suffix_len` (Number)
+- `type` (String)
+
+
+<a id="nestedblock--semantic_types--algorithm--md5_mask"></a>
+### Nested Schema for `semantic_types.algorithm.md5_mask`
+
+Optional:
+
+- `salt` (String) Salt is the salt value to generate a different hash that with the word alone.
+
+
+<a id="nestedblock--semantic_types--algorithm--range_mask"></a>
+### Nested Schema for `semantic_types.algorithm.range_mask`
+
+Optional:
+
+- `slices` (Block List) (see [below for nested schema](#nestedblock--semantic_types--algorithm--range_mask--slices))
+
+<a id="nestedblock--semantic_types--algorithm--range_mask--slices"></a>
+### Nested Schema for `semantic_types.algorithm.range_mask.slices`
+
+Optional:
+
+- `end` (Number) End is the stop index of the original value, should be less than the length of the original value.
+- `start` (Number) Start is the start index of the original value, start from 0 and should be less than stop.
+- `substitution` (String) Substitution is the string used to replace the OriginalValue[start:end).
+
+
+
+
+
 <a id="nestedblock--workspace_profile"></a>
 ### Nested Schema for `workspace_profile`
 
diff --git a/docs/resources/setting.md b/docs/resources/setting.md
@@ -24,6 +24,7 @@ The setting resource.
 - `approval_flow` (Block List) Configure risk level and approval flow for different tasks. Require ENTERPRISE subscription. (see [below for nested schema](#nestedblock--approval_flow))
 - `classification` (Block List, Max: 1) Classification for data masking. Require ENTERPRISE subscription. (see [below for nested schema](#nestedblock--classification))
 - `external_approval_nodes` (Block List) Configure external nodes in the approval flow. Require ENTERPRISE subscription. (see [below for nested schema](#nestedblock--external_approval_nodes))
+- `semantic_types` (Block Set) Semantic types for data masking. Require ENTERPRISE subscription. (see [below for nested schema](#nestedblock--semantic_types))
 - `workspace_profile` (Block List, Max: 1) (see [below for nested schema](#nestedblock--workspace_profile))
 
 ### Read-Only
@@ -136,6 +137,73 @@ Required:
 
 
 
+<a id="nestedblock--semantic_types"></a>
+### Nested Schema for `semantic_types`
+
+Optional:
+
+- `algorithm` (Block List, Max: 1) The semantic type algorithm. Required. (see [below for nested schema](#nestedblock--semantic_types--algorithm))
+- `description` (String) The semantic type description. Optional.
+- `id` (String) The semantic type unique uuid.
+- `title` (String) The semantic type title. Required.
+
+<a id="nestedblock--semantic_types--algorithm"></a>
+### Nested Schema for `semantic_types.algorithm`
+
+Optional:
+
+- `full_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--full_mask))
+- `inner_outer_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--inner_outer_mask))
+- `md5_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--md5_mask))
+- `range_mask` (Block List, Max: 1) (see [below for nested schema](#nestedblock--semantic_types--algorithm--range_mask))
+
+<a id="nestedblock--semantic_types--algorithm--full_mask"></a>
+### Nested Schema for `semantic_types.algorithm.full_mask`
+
+Optional:
+
+- `substitution` (String) Substitution is the string used to replace the original value, the max length of the string is 16 bytes.
+
+
+<a id="nestedblock--semantic_types--algorithm--inner_outer_mask"></a>
+### Nested Schema for `semantic_types.algorithm.inner_outer_mask`
+
+Optional:
+
+- `prefix_len` (Number)
+- `substitution` (String)
+- `suffix_len` (Number)
+- `type` (String)
+
+
+<a id="nestedblock--semantic_types--algorithm--md5_mask"></a>
+### Nested Schema for `semantic_types.algorithm.md5_mask`
+
+Optional:
+
+- `salt` (String) Salt is the salt value to generate a different hash that with the word alone.
+
+
+<a id="nestedblock--semantic_types--algorithm--range_mask"></a>
+### Nested Schema for `semantic_types.algorithm.range_mask`
+
+Optional:
+
+- `slices` (Block List) (see [below for nested schema](#nestedblock--semantic_types--algorithm--range_mask--slices))
+
+<a id="nestedblock--semantic_types--algorithm--range_mask--slices"></a>
+### Nested Schema for `semantic_types.algorithm.range_mask.slices`
+
+Optional:
+
+- `end` (Number) End is the stop index of the original value, should be less than the length of the original value.
+- `start` (Number) Start is the start index of the original value, start from 0 and should be less than stop.
+- `substitution` (String) Substitution is the string used to replace the OriginalValue[start:end).
+
+
+
+
+
 <a id="nestedblock--workspace_profile"></a>
 ### Nested Schema for `workspace_profile`
 
diff --git a/examples/environments/main.tf b/examples/environments/main.tf
@@ -2,7 +2,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/groups/main.tf b/examples/groups/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/instances/main.tf b/examples/instances/main.tf
@@ -2,7 +2,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/policies/main.tf b/examples/policies/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/projects/main.tf b/examples/projects/main.tf
@@ -2,7 +2,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/settings/main.tf b/examples/settings/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
@@ -33,6 +33,10 @@ data "bytebase_setting" "classification" {
   name = "bb.workspace.data-classification"
 }
 
+data "bytebase_setting" "semantic_types" {
+  name = "bb.workspace.semantic-types"
+}
+
 output "approval_flow" {
   value = data.bytebase_setting.approval_flow
 }
@@ -48,3 +52,7 @@ output "workspace_profile" {
 output "classification" {
   value = data.bytebase_setting.classification
 }
+
+output "semantic_types" {
+  value = data.bytebase_setting.semantic_types
+}
diff --git a/examples/setup/data_masking.tf b/examples/setup/data_masking.tf
@@ -40,6 +40,50 @@ resource "bytebase_setting" "classification" {
   }
 }
 
+resource "bytebase_setting" "semantic_types" {
+  name = "bb.workspace.semantic-types"
+
+  semantic_types {
+    id    = "bb.default"
+    title = "Default"
+  }
+
+  semantic_types {
+    id    = "bb.default-partial"
+    title = "Default partial"
+  }
+
+  semantic_types {
+    id    = "9c84e2a6-02e5-4031-89c5-13342b568f7b"
+    title = "Full mask"
+    algorithm {
+      full_mask {
+        substitution = "***"
+      }
+    }
+  }
+
+  semantic_types {
+    id    = "bb14d0ae-5aff-4f65-9143-bced63f8b054"
+    title = "Range mask"
+    algorithm {
+      range_mask {
+        slices {
+          start        = 0
+          end          = 1
+          substitution = "*"
+        }
+
+        slices {
+          start        = 2
+          end          = 3
+          substitution = "*"
+        }
+      }
+    }
+  }
+}
+
 resource "bytebase_policy" "masking_exception_policy" {
   depends_on = [
     bytebase_project.sample_project,
diff --git a/examples/setup/database.tf b/examples/setup/database.tf
@@ -15,11 +15,11 @@ resource "bytebase_database" "database" {
         name = "salary"
         columns {
           name          = "amount"
-          semantic_type = "default"
+          semantic_type = "bb.default"
         }
         columns {
           name           = "emp_no"
-          semantic_type  = "default-partial"
+          semantic_type  = "bb.default-partial"
           classification = "1-1"
           labels = {
             tenant = "example"
diff --git a/examples/setup/main.tf b/examples/setup/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/setup/project.tf b/examples/setup/project.tf
@@ -11,13 +11,6 @@ resource "bytebase_project" "sample_project" {
   title       = "Sample project"
   key         = "SAMM"
 
-  dynamic "databases" {
-    for_each = bytebase_instance.prod.databases
-    content {
-      name = databases.value.name
-    }
-  }
-
   members {
     member = format("user:%s", bytebase_user.workspace_dba.email)
     role   = "roles/projectOwner"
diff --git a/examples/users/main.tf b/examples/users/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/examples/vcs/main.tf b/examples/vcs/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     bytebase = {
-      version = "1.0.14"
+      version = "1.0.15"
       # For local development, please use "terraform.local/bytebase/bytebase" instead
       source = "registry.terraform.io/bytebase/bytebase"
     }
diff --git a/provider/data_source_setting.go b/provider/data_source_setting.go
diff --git a/provider/resource_project.go b/provider/resource_project.go
diff --git a/provider/resource_setting.go b/provider/resource_setting.go

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,8 @@ const (`
`12`	`12`	`SettingWorkspaceExternalApproval SettingName = "bb.workspace.approval.external"`
`13`	`13`	`// SettingDataClassification is the setting name for data classification.`
`14`	`14`	`SettingDataClassification SettingName = "bb.workspace.data-classification"`
	`15`	`+ // SettingSemanticTypes is the setting name for semantic types.`
	`16`	`+ SettingSemanticTypes SettingName = "bb.workspace.semantic-types"`
`15`	`17`	`)`
`16`	`18`
`17`	`19`	`// RiskLevel is the approval risk level.`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`terraform {`
`3`	`3`	`required_providers {`
`4`	`4`	`bytebase = {`
`5`		`- version = "1.0.14"`
	`5`	`+ version = "1.0.15"`
`6`	`6`	`# For local development, please use "terraform.local/bytebase/bytebase" instead`
`7`	`7`	`source = "registry.terraform.io/bytebase/bytebase"`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`terraform {`
`2`	`2`	`required_providers {`
`3`	`3`	`bytebase = {`
`4`		`- version = "1.0.14"`
	`4`	`+ version = "1.0.15"`
`5`	`5`	`# For local development, please use "terraform.local/bytebase/bytebase" instead`
`6`	`6`	`source = "registry.terraform.io/bytebase/bytebase"`
`7`	`7`	`}`