Skip to content

Commit 6306871

Browse files
tobyhedetobyhede
authored
Merge pull request #116 from cipherstash/enable-index-on-hmac_256
Enable simpler indexing on encrypted records with hmac_256
2 parents 6d56789 + a09c575 commit 6306871

File tree

14 files changed

+393
-19
lines changed

14 files changed

+393
-19
lines changed

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,8 +103,8 @@ SELECT eql_v2.add_column('users', 'encrypted_email');
103103
After modifying configurations, activate them by running:
104104

105105
```sql
106-
SELECT eql_v2.encrypt();
107-
SELECT eql_v2.activate();
106+
SELECT eql_v2.migrate_config();
107+
SELECT eql_v2.activate_config();
108108
```
109109

110110
**Important:** These functions must be run after any modifications to the configuration.
@@ -219,8 +219,8 @@ SELECT eql_v2.add_search_config(
219219
After adding an index, you have to activate the configuration:
220220

221221
```sql
222-
SELECT eql_v2.encrypt();
223-
SELECT eql_v2.activate();
222+
SELECT eql_v2.migrate_config();
223+
SELECT eql_v2.activate_config();
224224
```
225225

226226
## Searching data with EQL

‎src/blake3/functions.sql‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@ CREATE FUNCTION eql_v2.blake3(val jsonb)
1010
IMMUTABLE STRICT PARALLEL SAFE
1111
AS $$
1212
BEGIN
13+
IF val IS NULL THEN
14+
RETURN NULL;
15+
END IF;
1316

14-
IF NOT (val ? 'b3') NULL THEN
17+
IF NOT (val ? 'b3') THEN
1518
RAISE 'Expected a blake3 index (b3) value in json: %', val;
1619
END IF;
1720

@@ -34,3 +37,24 @@ AS $$
3437
RETURN (SELECT eql_v2.blake3(val.data));
3538
END;
3639
$$ LANGUAGE plpgsql;
40+
41+
42+
CREATE FUNCTION eql_v2.has_blake3(val jsonb)
43+
RETURNS boolean
44+
IMMUTABLE STRICT PARALLEL SAFE
45+
AS $$
46+
BEGIN
47+
RETURN val ? 'b3';
48+
END;
49+
$$ LANGUAGE plpgsql;
50+
51+
52+
CREATE FUNCTION eql_v2.has_blake3(val eql_v2_encrypted)
53+
RETURNS boolean
54+
IMMUTABLE STRICT PARALLEL SAFE
55+
AS $$
56+
BEGIN
57+
RETURN eql_v2.has_blake3(val.data);
58+
END;
59+
$$ LANGUAGE plpgsql;
60+

‎src/bloom_filter/functions.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,14 @@ CREATE FUNCTION eql_v2.bloom_filter(val jsonb)
88
IMMUTABLE STRICT PARALLEL SAFE
99
AS $$
1010
BEGIN
11+
IF val IS NULL THEN
12+
RETURN NULL;
13+
END IF;
14+
1115
IF val ? 'bf' THEN
1216
RETURN ARRAY(SELECT jsonb_array_elements(val->'bf'))::eql_v2.bloom_filter;
1317
END IF;
18+
1419
RAISE 'Expected a match index (bf) value in json: %', val;
1520
END;
1621
$$ LANGUAGE plpgsql;
@@ -26,3 +31,23 @@ AS $$
2631
RETURN (SELECT eql_v2.bloom_filter(val.data));
2732
END;
2833
$$ LANGUAGE plpgsql;
34+
35+
36+
CREATE FUNCTION eql_v2.has_bloom_filter(val jsonb)
37+
RETURNS boolean
38+
IMMUTABLE STRICT PARALLEL SAFE
39+
AS $$
40+
BEGIN
41+
RETURN val ? 'bf';
42+
END;
43+
$$ LANGUAGE plpgsql;
44+
45+
46+
CREATE FUNCTION eql_v2.has_bloom_filter(val eql_v2_encrypted)
47+
RETURNS boolean
48+
IMMUTABLE STRICT PARALLEL SAFE
49+
AS $$
50+
BEGIN
51+
RETURN eql_v2.has_bloom_filter(val.data);
52+
END;
53+
$$ LANGUAGE plpgsql;

‎src/config/functions.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ $$ LANGUAGE plpgsql;
144144
-- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
145145
--
146146

147-
CREATE FUNCTION eql_v2.encrypt()
147+
CREATE FUNCTION eql_v2.migrate_config()
148148
RETURNS boolean
149149
AS $$
150150
BEGIN
@@ -168,7 +168,7 @@ $$ LANGUAGE plpgsql;
168168

169169

170170

171-
CREATE FUNCTION eql_v2.activate()
171+
CREATE FUNCTION eql_v2.activate_config()
172172
RETURNS boolean
173173
AS $$
174174
BEGIN

‎src/encryptindex/functions_test.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ CREATE TABLE users
155155
DO $$
156156
BEGIN
157157
PERFORM eql_v2.add_search_config('users', 'name', 'match');
158-
PERFORM eql_v2.encrypt();
158+
PERFORM eql_v2.migrate_config();
159159

160160
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
161161
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -205,7 +205,7 @@ CREATE TABLE users
205205
DO $$
206206
BEGIN
207207
PERFORM eql_v2.add_search_config('users', 'name', 'match');
208-
PERFORM eql_v2.encrypt();
208+
PERFORM eql_v2.migrate_config();
209209

210210
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
211211
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -256,8 +256,8 @@ DO $$
256256
BEGIN
257257
PERFORM eql_v2.add_search_config('users', 'name', 'match');
258258

259-
PERFORM eql_v2.encrypt(); -- need to encrypt first
260-
PERFORM eql_v2.activate();
259+
PERFORM eql_v2.migrate_config(); -- need to encrypt first
260+
PERFORM eql_v2.activate_config();
261261

262262
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
263263
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'inactive'));

‎src/hmac_256/functions.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ CREATE FUNCTION eql_v2.hmac_256(val jsonb)
88
IMMUTABLE STRICT PARALLEL SAFE
99
AS $$
1010
BEGIN
11+
IF val IS NULL THEN
12+
RETURN NULL;
13+
END IF;
14+
1115
IF val ? 'hm' THEN
1216
RETURN val->>'hm';
1317
END IF;
@@ -16,6 +20,27 @@ AS $$
1620
$$ LANGUAGE plpgsql;
1721

1822

23+
CREATE FUNCTION eql_v2.has_hmac_256(val jsonb)
24+
RETURNS boolean
25+
IMMUTABLE STRICT PARALLEL SAFE
26+
AS $$
27+
BEGIN
28+
RETURN val ? 'hm';
29+
END;
30+
$$ LANGUAGE plpgsql;
31+
32+
33+
CREATE FUNCTION eql_v2.has_hmac_256(val eql_v2_encrypted)
34+
RETURNS boolean
35+
IMMUTABLE STRICT PARALLEL SAFE
36+
AS $$
37+
BEGIN
38+
RETURN eql_v2.has_hmac_256(val.data);
39+
END;
40+
$$ LANGUAGE plpgsql;
41+
42+
43+
1944
-- extracts hmac_256 index from an encrypted column
2045

2146
CREATE FUNCTION eql_v2.hmac_256(val eql_v2_encrypted)

‎src/hmac_256/functions_test.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,15 @@ DO $$
1212

1313
END;
1414
$$ LANGUAGE plpgsql;
15+
16+
17+
DO $$
18+
DECLARE
19+
e eql_v2_encrypted;
20+
BEGIN
21+
e := create_encrypted_json(1, 'hm');
22+
23+
ASSERT eql_v2.has_hmac_256(e);
24+
END;
25+
$$ LANGUAGE plpgsql;
26+

‎src/operators/operator_class.sql‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,66 @@
1010
-- REQUIRE: src/operators/>.sql
1111

1212

13+
--
14+
-- Compare two eql_v2_encrypted values
15+
-- Uses `ore_block_u64_8_256` or `has_hmac_256` index terms for comparison if defined on ONE of the compared value
16+
--
17+
-- Important note: -- Index order of operations is reversed from equality operator.
18+
-- In equality operations, `has_hmac_256` is preferred as it reduces to a text comparison and is more efficient
19+
-- As compare is used for ordering, `ore_block_u64_8_256` provides more complete ordering and is checked first.
20+
-- THe assumption is that if you add ore you are adding it because you want to use it specifically for comparison.
21+
22+
-- Thusly, the logic for determining which index term to use:
23+
-- Use ORE if BOTH parameters have ore index
24+
-- Fallback to hmac if BOTH parameters have hmac index
25+
-- Fallback to ORE if ONE of the parameters has ore index (will compare against a NULL term for the other parameter)
26+
-- Fallback to hmac if ONE of the parameters has hmac index (will compare against a NULL term term for the other parameter)
27+
--
28+
-- As a general rule, columns should have the same index terms as they are encrypted with the same configuration.
29+
-- Index terms should only be different during an encryption config migration.
30+
-- eg, when adding an ore index to a column any existing values will NOT have the ore index until encryptindexed/migrated
31+
--
1332
CREATE FUNCTION eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted)
1433
RETURNS integer
1534
IMMUTABLE STRICT PARALLEL SAFE
35+
AS $$
36+
BEGIN
37+
38+
-- PERFORM eql_v2.log('eql_v2.has_hmac_256(a)', eql_v2.has_hmac_256(a)::text);
39+
-- PERFORM eql_v2.log('eql_v2.has_hmac_256(b)', eql_v2.has_hmac_256(b)::text);
40+
-- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
41+
-- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
42+
43+
44+
-- Use ORE if BOTH parameters have ore index
45+
IF eql_v2.has_ore_block_u64_8_256(a) AND eql_v2.has_ore_block_u64_8_256(b) THEN
46+
RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
47+
END IF;
48+
49+
-- Fallback to hmac if BOTH parameters have hmac index
50+
IF eql_v2.has_hmac_256(a) AND eql_v2.has_hmac_256(b) THEN
51+
RETURN eql_v2.compare_hmac(a, b);
52+
END IF;
53+
54+
-- Fallback to ORE if one of the parameters has ore index
55+
IF eql_v2.has_ore_block_u64_8_256(a) OR eql_v2.has_ore_block_u64_8_256(b) THEN
56+
RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
57+
END IF;
58+
59+
-- Fallback to hmac if ONE of the parameters has hmac index
60+
IF eql_v2.has_hmac_256(a) OR eql_v2.has_hmac_256(b) THEN
61+
RETURN eql_v2.compare_hmac(a, b);
62+
END IF;
63+
64+
RAISE 'Expected an hmac_256 (hm) or ore_block_u64_8_256 (ob) value in json: %', val;
65+
END;
66+
$$ LANGUAGE plpgsql;
67+
68+
--------------------
69+
70+
CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted)
71+
RETURNS integer
72+
IMMUTABLE STRICT PARALLEL SAFE
1673
AS $$
1774
DECLARE
1875
a_ore eql_v2.ore_block_u64_8_256;
@@ -38,6 +95,51 @@ AS $$
3895
END;
3996
$$ LANGUAGE plpgsql;
4097

98+
99+
--------------------
100+
101+
CREATE FUNCTION eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted)
102+
RETURNS integer
103+
IMMUTABLE STRICT PARALLEL SAFE
104+
AS $$
105+
DECLARE
106+
a_hmac eql_v2.hmac_256;
107+
b_hmac eql_v2.hmac_256;
108+
BEGIN
109+
110+
a_hmac = eql_v2.hmac_256(a);
111+
b_hmac = eql_v2.hmac_256(b);
112+
113+
IF a_hmac IS NULL AND b_hmac IS NULL THEN
114+
RETURN 0;
115+
END IF;
116+
117+
IF a_hmac IS NULL THEN
118+
RETURN -1;
119+
END IF;
120+
121+
IF b_hmac IS NULL THEN
122+
RETURN 1;
123+
END IF;
124+
125+
IF a_hmac = b_hmac THEN
126+
RETURN 0;
127+
END IF;
128+
129+
IF a_hmac < b_hmac THEN
130+
RETURN -1;
131+
END IF;
132+
133+
IF a_hmac > b_hmac THEN
134+
RETURN 1;
135+
END IF;
136+
137+
END;
138+
$$ LANGUAGE plpgsql;
139+
140+
141+
--------------------
142+
41143
CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree;
42144

43145
CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS
@@ -48,3 +150,28 @@ CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypte
48150
OPERATOR 5 >,
49151
FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted);
50152

153+
154+
--------------------
155+
156+
-- CREATE OPERATOR FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 USING btree;
157+
158+
-- CREATE OPERATOR CLASS eql_v2.encrypted_operator_ore_block_u64_8_256 FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 AS
159+
-- OPERATOR 1 <,
160+
-- OPERATOR 2 <=,
161+
-- OPERATOR 3 =,
162+
-- OPERATOR 4 >=,
163+
-- OPERATOR 5 >,
164+
-- FUNCTION 1 eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted);
165+
166+
-- --------------------
167+
168+
-- CREATE OPERATOR FAMILY eql_v2.encrypted_hmac_256_operator USING btree;
169+
170+
-- CREATE OPERATOR CLASS eql_v2.encrypted_hmac_256_operator FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_hmac_256_operator AS
171+
-- OPERATOR 1 <,
172+
-- OPERATOR 2 <=,
173+
-- OPERATOR 3 =,
174+
-- OPERATOR 4 >=,
175+
-- OPERATOR 5 >,
176+
-- FUNCTION 1 eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted);
177+

0 commit comments

Comments
 (0)