Merge pull request #103 from cipherstash/orderby-function

Enhance functions for operator-free environments like Supabase

Author: tobyhede

.github/workflows/test-eql.yml

@@ -5,18 +5,17 @@ on:
       - main
     paths:
       - ".github/workflows/test-eql.yml"
-      - "src/*.sql"
-      - "sql/*.sql"
+      - "src/**/*.sql"
+      - "sql/**/*.sql"
       - "tests/**/*"
       - "tasks/**/*"
 
   pull_request:
-    branches:
-      - main
+    # run on all pull requests
     paths:
       - ".github/workflows/test-eql.yml"
-      - "src/*.sql"
-      - "sql/*.sql"
+      - "src/**/*.sql"
+      - "sql/**/*.sql"
       - "tests/**/*"
       - "tasks/**/*"
 

SUPABASE.md

@@ -0,0 +1,74 @@
+# Supabase
+
+
+## No operators, no problems
+
+Supabase [does not currently support](https://github.com/supabase/supautils/issues/72) custom operators.
+The EQL operator functions can be used in this situation.
+
+In EQL, PostgreSQL operators are an alias for a function, so the implementation and behaviour remains the same across operators and functions.
+
+| Operator | Function                                           | Example                                                           |
+| -------- | -------------------------------------------------- | ----------------------------------------------------------------- |
+| `=`      | `eql_v1.eq(eql_v1_encrypted, eql_v1_encrypted)`    | `SELECT * FROM users WHERE eql_v1.eq(encrypted_email, $1)`<br>    |
+| `<>`     | `eql_v1.neq(eql_v1_encrypted, eql_v1_encrypted)`   | `SELECT * FROM users WHERE eql_v1.neq(encrypted_email, $1)`<br>   |
+| `<`      | `eql_v1.lt(eql_v1_encrypted, eql_v1_encrypted)`    | `SELECT * FROM users WHERE eql_v1.lt(encrypted_email, $1)`<br>    |
+| `<=`     | `eql_v1.lte(eql_v1_encrypted, eql_v1_encrypted)`   | `SELECT * FROM users WHERE eql_v1.lte(encrypted_email, $1)`<br>   |
+| `>`      | `eql_v1.gt(eql_v1_encrypted, eql_v1_encrypted)`    | `SELECT * FROM users WHERE eql_v1.gt(encrypted_email, $1)`<br>    |
+| `>=`     | `eql_v1.gte(eql_v1_encrypted, eql_v1_encrypted)`   | `SELECT * FROM users WHERE eql_v1.gte(encrypted_email, $1)`<br>   |
+| `~~`     | `eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)`  | `SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)`<br>  |
+| `~~*`    | `eql_v1.ilike(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.ilike(encrypted_email, $1)`<br> |
+| `LIKE`   | `eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)`  | `SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)`<br>  |
+| `ILIKE`  | `eql_v1.ilike(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.ilike(encrypted_email, $1)`<br> |
+
+### Example SQL Statements
+
+#### Equality `=`
+
+
+**Operator**
+```sql
+SELECT * FROM users WHERE encrypted_email = $1
+```
+
+**Function**
+```sql
+SELECT * FROM users WHERE eql_v1.eq(encrypted_email, $1)
+```
+
+
+#### Like & ILIKE `~~, ~~*`
+
+
+**Operator**
+```sql
+SELECT * FROM users WHERE encrypted_email LIKE $1
+```
+
+**Function**
+```sql
+SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)
+```
+
+#### Case Sensitivity
+
+The EQL `eql_v1.like` and `eql_v1.ilike` functions are equivalent.
+
+The behaviour of EQL's encrypted `LIKE` operators is slightly different to the behaviour of PostgreSQL's `LIKE` operator.
+In EQL, the `LIKE` operator can be used on `match` indexes.
+Case sensitivity is determined by the [index term configuration](./docs/reference/INDEX.md#options-for-match-indexes-opts) of `match` indexes.
+A `match` index term can be configured to enable case sensitive searches with token filters (for example, `downcase` and `upcase`).
+The data is encrypted based on the index term configuration.
+The `LIKE` operation is always the same, even if the data is tokenised differently.
+The different operators are kept to preserve the semantics of SQL statements in client applications.
+
+### `ORDER BY`
+
+Ordering requires wrapping the ordered column in the `eql_v1.order_by` function, like this:
+
+```sql
+SELECT * FROM users ORDER BY eql_v1.order_by(encrypted_created_at) DESC
+```
+
+PostgreSQL uses operators when handling `ORDER BY` operations. The `eql_v1.order_by` function behaves in
+

src/encryptindex/functions_test.sql

@@ -8,7 +8,7 @@
 TRUNCATE TABLE eql_v1_configuration;
 
 -- Create a table with a plaintext column
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -63,7 +63,7 @@ $$ LANGUAGE plpgsql;
 TRUNCATE TABLE eql_v1_configuration;
 
 -- Create a table with multiple plaintext columns
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -119,7 +119,7 @@ $$ LANGUAGE plpgsql;
 -- The schema should be validated first.
 -- Users table does not exist, so should fail.
 -- -----------------------------------------------
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 TRUNCATE TABLE eql_v1_configuration;
 
 
@@ -148,7 +148,7 @@ $$ LANGUAGE plpgsql;
 --
 -- Schema validation is skipped
 -- -----------------------------------------------
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 TRUNCATE TABLE eql_v1_configuration;
 
 DO $$
@@ -194,7 +194,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES (
 );
 
 -- Create a table with plaintext and encrypted columns
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -244,7 +244,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES (
 );
 
 -- Create a table with plaintext and jsonb column
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -295,7 +295,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES (
 
 
 -- Create a table with multiple plaintext columns
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,

src/operators/order_by.sql

@@ -0,0 +1,72 @@
+-- REQUIRE: src/encrypted/types.sql
+-- REQUIRE: src/ore/types.sql
+-- REQUIRE: src/ore/functions.sql
+-- REQUIRE: src/ore/operators.sql
+-- REQUIRE: src/ore_cllw_u64_8/types.sql
+-- REQUIRE: src/ore_cllw_u64_8/functions.sql
+-- REQUIRE: src/ore_cllw_u64_8/operators.sql
+
+-- order_by function for ordering when operators are not available.
+--
+-- There are multiple index terms that provide equality comparisons
+--   - ore_cllw_u64_8
+--   - ore_cllw_var_8
+--   - ore_64_8_v1
+--
+-- We check these index terms in this order and use the first one that exists for both parameters
+--
+--
+
+-- DROP FUNCTION IF EXISTS eql_v1.order_by(a eql_v1_encrypted, b eql_v1_encrypted);
+
+CREATE FUNCTION eql_v1.order_by(a eql_v1_encrypted)
+  RETURNS eql_v1.ore_64_8_v1
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+    BEGIN
+      RETURN eql_v1.ore_64_8_v1(a);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_64_8_v1 index');
+    END;
+
+    RETURN false;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- TODO: make this work
+--       fails with jsonb format issue, which I think is due to the type casting
+--
+CREATE FUNCTION eql_v1.order_by_any(a anyelement)
+  RETURNS anyelement
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  DECLARE
+    e eql_v1_encrypted;
+    result ALIAS FOR $0;
+  BEGIN
+
+    e := a::eql_v1_encrypted;
+
+    BEGIN
+      result := eql_v1.ore_cllw_u64_8(e);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_cllw_u64_8 index');
+    END;
+
+    BEGIN
+      result := eql_v1.ore_cllw_var_8(e);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_cllw_u64_8 index');
+    END;
+
+    BEGIN
+      result := eql_v1.ore_64_8_v1(e);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_64_8_v1 index');
+    END;
+
+    RETURN result;
+  END;
+$$ LANGUAGE plpgsql;
+

src/operators/order_by_test.sql

@@ -0,0 +1,28 @@
+\set ON_ERROR_STOP on
+
+--
+-- ORE - ORDER BY ore_64_8_v1(eql_v1_encrypted)
+--
+DO $$
+DECLARE
+    e eql_v1_encrypted;
+    ore_term eql_v1_encrypted;
+  BEGIN
+      SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+      PERFORM assert_count(
+        'ORDER BY eql_v1.order_by(e) DESC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) DESC', ore_term),
+        41);
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.order_by(e) DESC returns correct record',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) DESC LIMIT 1', ore_term),
+        '41');
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.order_by(e) ASC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) ASC LIMIT 1', ore_term),
+        '1');
+  END;
+$$ LANGUAGE plpgsql;

src/operators/~~.sql

@@ -15,12 +15,23 @@
 
 -- DROP FUNCTION IF EXISTS eql_v1.match(a eql_v1_encrypted, b eql_v1_encrypted);
 
-CREATE FUNCTION eql_v1.match(a eql_v1_encrypted, b eql_v1_encrypted)
+CREATE FUNCTION eql_v1.like(a eql_v1_encrypted, b eql_v1_encrypted)
 RETURNS boolean AS $$
   SELECT eql_v1.match(a) @> eql_v1.match(b);
 $$ LANGUAGE SQL;
 
 
+--
+-- Case sensitivity depends on the index term configuration
+-- Function preserves the SQL semantics
+--
+CREATE FUNCTION eql_v1.ilike(a eql_v1_encrypted, b eql_v1_encrypted)
+RETURNS boolean AS $$
+  SELECT eql_v1.match(a) @> eql_v1.match(b);
+$$ LANGUAGE SQL;
+
+
+
 -- DROP OPERATOR BEFORE FUNCTION
 -- DROP OPERATOR IF EXISTS ~~ (eql_v1_encrypted, eql_v1_encrypted);
 -- DROP OPERATOR IF EXISTS ~~* (eql_v1_encrypted, eql_v1_encrypted);
@@ -31,7 +42,7 @@ CREATE FUNCTION eql_v1."~~"(a eql_v1_encrypted, b eql_v1_encrypted)
   RETURNS boolean
 AS $$
   BEGIN
-    RETURN eql_v1.match(a, b);
+    RETURN eql_v1.like(a, b);
   END;
 $$ LANGUAGE plpgsql;
 
@@ -65,7 +76,7 @@ CREATE FUNCTION eql_v1."~~"(a eql_v1_encrypted, b jsonb)
   RETURNS boolean
 AS $$
   BEGIN
-    RETURN eql_v1.match(a, b::eql_v1_encrypted);
+    RETURN eql_v1.like(a, b::eql_v1_encrypted);
   END;
 $$ LANGUAGE plpgsql;
 
@@ -100,7 +111,7 @@ CREATE FUNCTION eql_v1."~~"(a jsonb, b eql_v1_encrypted)
   RETURNS boolean
 AS $$
   BEGIN
-    RETURN eql_v1.match(a::eql_v1_encrypted, b);
+    RETURN eql_v1.like(a::eql_v1_encrypted, b);
   END;
 $$ LANGUAGE plpgsql;
 

src/operators/~~_test.sql

@@ -87,17 +87,17 @@ DECLARE
       e := create_encrypted_json(i, 'm');
 
       PERFORM assert_result(
-        format('eql_v1.match(eql_v1_encrypted, eql_v1_encrypted)', i),
-        format('SELECT e FROM encrypted WHERE eql_v1.match(e, %L);', e));
+        format('eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)', i),
+        format('SELECT e FROM encrypted WHERE eql_v1.like(e, %L);', e));
 
     end loop;
 
     -- Partial match
     e := create_encrypted_json('m')::jsonb || '{"m": [10, 11]}';
 
     PERFORM assert_result(
-        'eql_v1.match(eql_v1_encrypted, eql_v1_encrypted)',
-        format('SELECT e FROM encrypted WHERE eql_v1.match(e, %L);', e));
+        'eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)',
+        format('SELECT e FROM encrypted WHERE eql_v1.like(e, %L);', e));
 
   END;
 $$ LANGUAGE plpgsql;

src/ore/functions_test.sql

@@ -11,4 +11,31 @@ DO $$
         'SELECT eql_v1.ore_64_8_v1(''{}''::jsonb)');
 
   END;
+$$ LANGUAGE plpgsql;
+
+--
+-- ORE - ORDER BY ore_64_8_v1(eql_v1_encrypted)
+--
+DO $$
+DECLARE
+    e eql_v1_encrypted;
+    ore_term eql_v1_encrypted;
+  BEGIN
+      SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+      PERFORM assert_count(
+        'ORDER BY eql_v1.ore_64_8_v1(e) DESC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) DESC', ore_term),
+        41);
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.ore_64_8_v1(e) DESC returns correct record',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) DESC LIMIT 1', ore_term),
+        '41');
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.ore_64_8_v1(e) ASC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) ASC LIMIT 1', ore_term),
+        '1');
+  END;
 $$ LANGUAGE plpgsql;