Merge pull request #109 from cipherstash/rename-fields

Rename search term for consistency 
PR_BYPASS: required for customer demo. minor changes to the names of several functions and internal fields.

Author: tobyhede

DEVELOPMENT.md

@@ -318,12 +318,12 @@ Each type of encrypted index (`unique`, `match`, `ore`) has an associated type,
 These are transient runtime types, used internally by EQL functions and operators:
 
 - `eql_v2.blake3`
-- `eql_v2.unique_index`
-- `eql_v2.match`
+- `eql_v2.hmac_256`
+- `eql_v2.bloom_filter`
 - `eql_v2.ore_cllw_u64_8`
 - `eql_v2.ore_cllw_var_8`
-- `eql_v2.ore_64_8_v2`
-- `eql_v2.ore_64_8_v2_term`
+- `eql_v2.ore_block_u64_8_256`
+- `eql_v2.ore_block_u64_8_256_term`
 
 The data in the column is converted into these types, when any operations are being performed on that encrypted data.
 
@@ -348,7 +348,7 @@ For example, it is possible to have both `unique` and `ore` indexes defined.
 For equality (`=`, `<>`) operations, a `unique` index term is a text comparison and should be preferred over an `ore` index term.
 
 The index term types and functions are internal implementation details and should not be exposed as operators on the `eql_v2_encrypted` type.
-For example, `eql_v2_encrypted` should not have an operator with the `ore_64_8_v2` type.
+For example, `eql_v2_encrypted` should not have an operator with the `ore_block_u64_8_256` type.
 Users should never need to think about or interact with EQL internals.
 
 #### Working without operators

README.md

@@ -148,8 +148,8 @@ Data is stored in the PostgreSQL database as:
     "t": "users"
   },
   "k": "ct",
-  "m": null,
-  "o": null,
+  "bf": null,
+  "ob": null,
   "u": null,
   "v": 1
 }
@@ -231,7 +231,7 @@ In order to use the specialized functions, you must first configure the correspo
 
 ### Equality search
 
-Enable equality search on encrypted data using the `eql_v2.unique` function.
+Enable equality search on encrypted data using the `eql_v2.hmac_256` function.
 
 **Index configuration example:**
 
@@ -248,8 +248,8 @@ SELECT eql_v2.add_search_config(
 
 ```sql
 SELECT * FROM users
-WHERE eql_v2.unique(encrypted_email) = eql_v2.unique(
-  '{"v":1,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"},"q":"unique"}'
+WHERE eql_v2.hmac_256(encrypted_email) = eql_v2.hmac_256(
+  '{"v":1,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"},"q":"hmac_256"}'
 );
 ```
 
@@ -261,7 +261,7 @@ SELECT * FROM users WHERE email = 'test@example.com';
 
 ### Full-text search
 
-Enables basic full-text search on encrypted data using the `eql_v2.match` function.
+Enables basic full-text search on encrypted data using the `eql_v2.bloom_filter` function.
 
 **Index configuration example:**
 
@@ -279,7 +279,7 @@ SELECT eql_v2.add_search_config(
 
 ```sql
 SELECT * FROM users
-WHERE eql_v2.match(encrypted_email) @> eql_v2.match(
+WHERE eql_v2.bloom_filter(encrypted_email) @> eql_v2.bloom_filter(
   '{"v":1,"k":"pt","p":"test","i":{"t":"users","c":"encrypted_email"},"q":"match"}'
 );
 ```
@@ -292,7 +292,7 @@ SELECT * FROM users WHERE email LIKE '%test%';
 
 ### Range queries
 
-Enable range queries on encrypted data using the `eql_v2.ore_64_8_v2`, `eql_v2.ore_cllw_u64_8`, or `eql_v2.ore_cllw_var_8` functions. Supports:
+Enable range queries on encrypted data using the `eql_v2.ore_block_u64_8_256`, `eql_v2.ore_cllw_u64_8`, or `eql_v2.ore_cllw_var_8` functions. Supports:
 
 - `ORDER BY`
 - `WHERE`
@@ -301,7 +301,7 @@ Enable range queries on encrypted data using the `eql_v2.ore_64_8_v2`, `eql_v2.o
 
 ```sql
 SELECT * FROM users
-WHERE eql_v2.ore_64_8_v2(encrypted_date) < eql_v2.ore_64_8_v2(
+WHERE eql_v2.ore_block_u64_8_256(encrypted_date) < eql_v2.ore_block_u64_8_256(
   '{"v":1,"k":"pt","p":"2023-10-05","i":{"t":"users","c":"encrypted_date"},"q":"ore"}'
 );
 ```
@@ -316,7 +316,7 @@ SELECT * FROM users WHERE date < '2023-10-05';
 
 ```sql
 SELECT id FROM users
-ORDER BY eql_v2.ore_64_8_v2(encrypted_field) DESC;
+ORDER BY eql_v2.ore_block_u64_8_256(encrypted_field) DESC;
 ```
 
 Equivalent plaintext query:

SUPABASE.md

@@ -24,15 +24,15 @@ In EQL, PostgreSQL operators are an alias for a function, so the implementation
 
 ### Core Functions
 
-| Function                     | Description                                          | Example                                         |
-| ---------------------------- | ---------------------------------------------------- | ----------------------------------------------- |
-| `eql_v2.ciphertext(val)`     | Extract ciphertext from encrypted value              | `SELECT eql_v2.ciphertext(encrypted_field)`     |
-| `eql_v2.blake3(val)`         | Extract blake3 hash from encrypted value             | `SELECT eql_v2.blake3(encrypted_field)`         |
-| `eql_v2.unique(val)`         | Extract unique index from encrypted value            | `SELECT eql_v2.unique(encrypted_field)`         |
-| `eql_v2.match(val)`          | Extract match index from encrypted value             | `SELECT eql_v2.match(encrypted_field)`          |
-| `eql_v2.ore_64_8_v2(val)`    | Extract ORE index from encrypted value               | `SELECT eql_v2.ore_64_8_v2(encrypted_field)`    |
-| `eql_v2.ore_cllw_u64_8(val)` | Extract CLLW ORE index from encrypted value          | `SELECT eql_v2.ore_cllw_u64_8(encrypted_field)` |
-| `eql_v2.ore_cllw_var_8(val)` | Extract variable CLLW ORE index from encrypted value | `SELECT eql_v2.ore_cllw_var_8(encrypted_field)` |
+| Function                          | Description                                          | Exa     mple                                         |
+| --------------------------------- | --------------------------------------------------------- | ----------------------------------------------- |
+| `eql_v2.ciphertext(val)`          | Extract ciphertext from encrypted value              | `SELECT eql_v2.ciphertext     (encrypted_field)`     |
+| `eql_v2.blake3(val)`              | Extract blake3 hash from encrypted value             | `SELECT eql_v2.blake3(     encrypted_field)`         |
+| `eql_v2.hmac_256(val)`            | Extract hmac_256 index from encrypted value            | `SELECT eql_v2.hmac_256(encrypted_fie     ld)`         |
+| `eql_v2.bloom_filter(val)`        | Extract match index from encrypted value             | `SELECT eql_v2.bloom_filter(encrypted_field)`               |
+| `eql_v2.ore_block_u64_8_256(val)` | Extract ORE index from encrypted value               | `SELECT eql_v2.ore_block_u64_8_256(encrypted_field)`   |
+| `eql_v2.ore_cllw_u64_8(val)`      | Extract CLLW ORE index from encrypted value          | `SELECT eql_v2.ore_cllw_u64_8(encrypted_fie     ld)` |
+| `eql_v2.ore_cllw_var_8(val)`      | Extract variable CLLW ORE index from encrypted value | `SELECT eql_v2.ore_cllw_var_8(     encrypted_field)` |
 
 ### Aggregate Functions
 
@@ -90,19 +90,17 @@ The behaviour of EQL's encrypted `LIKE` operators is slightly different to the b
 In EQL, the `LIKE` operator can be used on `match` indexes.
 Case sensitivity is determined by the [index term configuration](./docs/reference/INDEX.md#options-for-match-indexes-opts) of `match` indexes.
 A `match` index term can be configured to enable case sensitive searches with token filters (for example, `downcase` and `upcase`).
-The data is encrypted based on the index term configuration.
-The `LIKE` operation is always the same, even if the data is tokenised differently.
-The different operators are kept to preserve the semantics of SQL statements in client applications.
+The data is encrypted based on the index term configurat     ion.
+The `LIKE` operation is always the same, even if the data is----- tokenised differently.
+The different operators are kept to preserve the semantics of SQL statements in client      applications.
 
-### `ORDER BY`
+### `ORDER      BY`
 
-Ordering requires wrapping the ordered column in the `eql_v2.order_by` function, like this:
+Ordering requires wrapping the ordered column in the `eql_v2.order_by` function, lik     e this:
 
 ```sql
 SELECT * FROM users ORDER BY eql_v2.order_by(encrypted_created_at) DESC
-```
-
-PostgreSQL uses operators when handling `ORDER BY` operations. The `eql_v2.order_by` function behaves in the same way as the comparison operators, using the appropriate index type (ore_64_8_v2, ore_cllw_u64_8, or ore_cllw_var_8) to determine the ordering.
+``     ` PostgreSQL uses operators when handling `ORDER BY` operations. The `eql_v2.order_by` function behaves in the same way as the comparison operators, using the appropriate index type (ore_block_u64_8_256     , ore_cllw_u64_8, or ore_cllw_var_8) to determine the      ordering.
 
 ### JSONB Support
 

docs/reference/INDEX.md

@@ -49,7 +49,7 @@ The default match index options are:
 ```json
   {
     "k": 6,
-    "m": 2048,
+    "bf": 2048,
     "include_original": true,
     "tokenizer": {
       "kind": "ngram",

docs/tutorials/GETTINGSTARTED.md

@@ -193,15 +193,15 @@ For ordering or comparison queries we add an `ore` index:
 
 ```sql
 SELECT cs_add_index_v2('users', 'email_encrypted', 'ore', 'text');
-CREATE INDEX ON users (cs_ore_64_8_v2(email_encrypted));
+CREATE INDEX ON users (ore_block_u64_8_256(email_encrypted));
 ```
 
 After adding these indexes, our `eql_v2_configuration` table will look like this:
 
 ```bash
 id         | 1
 state      | pending
-data       | {"v": 1, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "m": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}}
+data       | {"v": 1, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "bf": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}}
 ```
 
 The initial `state` will be set as pending.
@@ -218,7 +218,7 @@ The `cs_configured_v2` table will now have a state of `active`.
 ```bash
 id         | 1
 state      | active
-data       | {"v": 1, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "m": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}}
+data       | {"v": 1, "tables": {"users": {"email_encrypted": {"cast_as": "text", "indexes": {"ore": {}, "match": {"k": 6, "bf": 2048, "tokenizer": {"kind": "ngram", "token_length": 3}, "token_filters": [{"kind": "downcase"}], "include_original": true}, "unique": {"token_filters": [{"kind": "downcase"}]}}}}}}
 ```
 
 ### Encrypting existing plaintext data
@@ -325,9 +325,9 @@ It creates an EQL payload that looks similar to this and inserts this into the e
     "t": "users", // Table
     "c": "email_encrypted" // Encrypted column
   },
-  "m": [42], // The ciphertext used for free text queries i.e match index
+  "bf": [42], // The ciphertext used for free text queries i.e match index
   "u": "unique ciphertext", // The ciphertext used for unique queries. i.e unique index
-  "o": ["a", "b", "c"], // The ciphertext used for order or comparison queries. i.e ore index
+  "ob": ["a", "b", "c"], // The ciphertext used for order or comparison queries. i.e ore index
   "v": 1
 }
 ```
@@ -386,9 +386,9 @@ The json stored in the database looks similar to this:
     "t": "users", // Table
     "c": "email_encrypted" // Encrypted column
   },
-  "m": [42], // The ciphertext used for free text queries i.e match index
+  "bf": [42], // The ciphertext used for free text queries i.e match index
   "u": "unique ciphertext", // The ciphertext used for unique queries. i.e unique index
-  "o": ["a", "b", "c"], // The ciphertext used for order or comparison queries. i.e ore index
+  "ob": ["a", "b", "c"], // The ciphertext used for order or comparison queries. i.e ore index
   "v": 1
 }
 ```
@@ -509,7 +509,7 @@ Prerequsites:
 
 - An [ore index](#adding-indexes) is needed on the encrypted column to support this operation.
 
-EQL function to use: `cs_ore_64_8_v2(val JSONB)`.
+EQL function to use: `ore_block_u64_8_256(val JSONB)`.
 
 A plaintext query order by email looks like this:
 
@@ -520,7 +520,7 @@ SELECT * FROM users ORDER BY email ASC;
 The EQL equivalent of this query is:
 
 ```sql
-SELECT * FROM users ORDER BY cs_ore_64_8_v2(email_encrypted) ASC;
+SELECT * FROM users ORDER BY ore_block_u64_8_256(email_encrypted) ASC;
 ```
 
 This query returns:
@@ -538,7 +538,7 @@ Prerequsites:
 
 - A [unique index](#adding-indexes) is needed on the encrypted column to support this operation.
 
-EQL function to use: `cs_ore_64_8_v2(val JSONB)`.
+EQL function to use: `ore_block_u64_8_256(val JSONB)`.
 
 EQL query payload for a comparison query:
 
@@ -564,7 +564,7 @@ SELECT * FROM users WHERE email > 'gracehopper@test.com';
 The EQL equivalent of this query is:
 
 ```sql
-SELECT * FROM users WHERE cs_ore_64_8_v2(email_encrypted) > cs_ore_64_8_v2(
+SELECT * FROM users WHERE ore_block_u64_8_256(email_encrypted) > ore_block_u64_8_256(
   '{"v":1,"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"q":"ore"}'
   );
 ```

src/blake3/functions.sql

@@ -4,21 +4,22 @@
 
 -- extracts blake3 index from a jsonb value
 
+
 CREATE FUNCTION eql_v2.blake3(val jsonb)
   RETURNS eql_v2.blake3
   IMMUTABLE STRICT PARALLEL SAFE
 AS $$
 	BEGIN
 
-    IF NOT (val ? 'b') NULL THEN
-        RAISE 'Expected a blake3 index (b) value in json: %', val;
+    IF NOT (val ? 'b3') NULL THEN
+        RAISE 'Expected a blake3 index (b3) value in json: %', val;
     END IF;
 
-    IF val->>'b' IS NULL THEN
+    IF val->>'b3' IS NULL THEN
       RETURN NULL;
     END IF;
 
-    RETURN val->>'b';
+    RETURN val->>'b3';
   END;
 $$ LANGUAGE plpgsql;
 

src/bloom_filter/functions.sql

@@ -0,0 +1,28 @@
+-- REQUIRE: src/schema.sql
+
+
+-- extracts match index from an emcrypted column
+
+CREATE FUNCTION eql_v2.bloom_filter(val jsonb)
+  RETURNS eql_v2.bloom_filter
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    IF val ? 'bf' THEN
+      RETURN ARRAY(SELECT jsonb_array_elements(val->'bf'))::eql_v2.bloom_filter;
+    END IF;
+    RAISE 'Expected a match index (bf) value in json: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+
+-- extracts unique index from an encrypted column
+
+CREATE FUNCTION eql_v2.bloom_filter(val eql_v2_encrypted)
+  RETURNS eql_v2.bloom_filter
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+    RETURN (SELECT eql_v2.bloom_filter(val.data));
+  END;
+$$ LANGUAGE plpgsql;

src/match/functions_test.sql renamed to src/bloom_filter/functions_test.sql

@@ -4,11 +4,11 @@ DO $$
   BEGIN
     PERFORM assert_result(
         'Extract match index term from encrypted',
-        'SELECT eql_v2.match(''{"m": []}''::jsonb)');
+        'SELECT eql_v2.bloom_filter(''{"bf": []}''::jsonb)');
 
     PERFORM assert_exception(
         'Missing match index term in encrypted raises exception',
-        'SELECT eql_v2.match(''{}''::jsonb)');
+        'SELECT eql_v2.bloom_filter(''{}''::jsonb)');
 
   END;
 $$ LANGUAGE plpgsql;

src/bloom_filter/types.sql

@@ -0,0 +1,4 @@
+-- REQUIRE: src/schema.sql
+
+CREATE DOMAIN eql_v2.bloom_filter AS smallint[];
+

src/config/functions_private.sql

@@ -88,7 +88,7 @@ LANGUAGE sql STRICT PARALLEL SAFE
 BEGIN ATOMIC
   SELECT jsonb_build_object(
             'k', 6,
-            'm', 2048,
+            'bf', 2048,
             'include_original', true,
             'tokenizer', json_build_object('kind', 'ngram', 'token_length', 3),
             'token_filters', json_build_array(json_build_object('kind', 'downcase')));