Skip to content

Commit 629d266

Browse files
utayutay
committed
Add support for SSE-C encryption on S3
1 parent 21e996f commit 629d266

File tree

4 files changed

+46
-3
lines changed

4 files changed

+46
-3
lines changed

‎extension/httpfs/create_secret_functions.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
8282
secret->secret_map["use_ssl"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
8383
} else if (lower_name == "kms_key_id") {
8484
secret->secret_map["kms_key_id"] = named_param.second.ToString();
85+
} else if (lower_name == "sse_c_key") {
86+
secret->secret_map["sse_c_key"] = named_param.second.ToString();
87+
} else if (lower_name == "sse_c_key_md5") {
88+
secret->secret_map["sse_c_key_md5"] = named_param.second.ToString();
8589
} else if (lower_name == "url_compatibility_mode") {
8690
if (named_param.second.type() != LogicalType::BOOLEAN) {
8791
throw InvalidInputException("Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'",
@@ -120,6 +124,16 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
120124
}
121125
}
122126

127+
bool has_sse_c_key = secret->secret_map.find("sse_c_key") != secret->secret_map.end();
128+
bool has_sse_c_key_md5 = secret->secret_map.find("sse_c_key_md5") != secret->secret_map.end();
129+
if (has_sse_c_key != has_sse_c_key_md5) {
130+
throw InvalidInputException("Both `sse_c_key` and `sse_c_key_md5` must be set together, or neither should be set");
131+
}
132+
bool has_kms_key_id = secret->secret_map.find("kms_key_id") != secret->secret_map.end();
133+
if (has_kms_key_id && has_sse_c_key) {
134+
throw InvalidInputException("Cannot set `kms_key_id` and `sse_c_key` at the same time");
135+
}
136+
123137
return std::move(secret);
124138
}
125139

@@ -190,6 +204,8 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
190204
function.named_parameters["url_style"] = LogicalType::VARCHAR;
191205
function.named_parameters["use_ssl"] = LogicalType::BOOLEAN;
192206
function.named_parameters["kms_key_id"] = LogicalType::VARCHAR;
207+
function.named_parameters["sse_c_key"] = LogicalType::VARCHAR;
208+
function.named_parameters["sse_c_key_md5"] = LogicalType::VARCHAR;
193209
function.named_parameters["url_compatibility_mode"] = LogicalType::BOOLEAN;
194210
function.named_parameters["requester_pays"] = LogicalType::BOOLEAN;
195211

‎extension/httpfs/httpfs_extension.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ static void LoadInternal(DatabaseInstance &instance) {
6868
config.AddExtensionOption("s3_url_style", "S3 URL style", LogicalType::VARCHAR, Value("vhost"));
6969
config.AddExtensionOption("s3_use_ssl", "S3 use SSL", LogicalType::BOOLEAN, Value(true));
7070
config.AddExtensionOption("s3_kms_key_id", "S3 KMS Key ID", LogicalType::VARCHAR);
71+
config.AddExtensionOption("s3_sse_c_key", "S3 SSE-C Key", LogicalType::VARCHAR);
72+
config.AddExtensionOption("s3_sse_c_key_md5", "S3 SSE-C Key MD5", LogicalType::VARCHAR);
7173
config.AddExtensionOption("s3_url_compatibility_mode", "Disable Globs and Query Parameters on S3 URLs",
7274
LogicalType::BOOLEAN, Value(false));
7375
config.AddExtensionOption("s3_requester_pays", "S3 use requester pays mode", LogicalType::BOOLEAN, Value(false));

‎extension/httpfs/include/s3fs.hpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ struct S3AuthParams {
2727
string session_token;
2828
string endpoint;
2929
string kms_key_id;
30+
string sse_c_key;
31+
string sse_c_key_md5;
3032
string url_style;
3133
bool use_ssl = true;
3234
bool s3_url_compatibility_mode = false;
@@ -44,9 +46,10 @@ struct AWSEnvironmentCredentialsProvider {
4446
static constexpr const char *DUCKDB_ENDPOINT_ENV_VAR = "DUCKDB_S3_ENDPOINT";
4547
static constexpr const char *DUCKDB_USE_SSL_ENV_VAR = "DUCKDB_S3_USE_SSL";
4648
static constexpr const char *DUCKDB_KMS_KEY_ID_ENV_VAR = "DUCKDB_S3_KMS_KEY_ID";
49+
static constexpr const char *DUCKDB_SSE_C_KEY_ENV_VAR = "DUCKDB_S3_SSE_C_KEY";
50+
static constexpr const char *DUCKDB_SSE_C_KEY_MD5_ENV_VAR = "DUCKDB_S3_SSE_C_KEY_MD5";
4751
static constexpr const char *DUCKDB_REQUESTER_PAYS_ENV_VAR = "DUCKDB_S3_REQUESTER_PAYS";
4852

49-
5053
explicit AWSEnvironmentCredentialsProvider(DBConfig &config) : config(config) {};
5154

5255
DBConfig &config;

‎extension/httpfs/s3fs.cpp

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
5151
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-kms-encryption.html#sse-request-headers-kms
5252
bool use_sse_kms = auth_params.kms_key_id.length() > 0 && (method == "POST" || method == "PUT") &&
5353
query.find("uploadId") == std::string::npos;
54+
bool use_sse_c = auth_params.sse_c_key.length() > 0 && auth_params.sse_c_key_md5.length() > 0;
5455

5556
res["x-amz-date"] = datetime_now;
5657
res["x-amz-content-sha256"] = payload_hash;
@@ -61,13 +62,18 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
6162
res["x-amz-server-side-encryption"] = "aws:kms";
6263
res["x-amz-server-side-encryption-aws-kms-key-id"] = auth_params.kms_key_id;
6364
}
65+
if (use_sse_c) {
66+
res["x-amz-server-side-encryption-customer-algorithm"] = "AES256";
67+
res["x-amz-server-side-encryption-customer-key"] = auth_params.sse_c_key;
68+
res["x-amz-server-side-encryption-customer-key-md5"] = auth_params.sse_c_key_md5;
69+
}
6470

6571
bool use_requester_pays = auth_params.requester_pays;
6672
if (use_requester_pays) {
6773
res["x-amz-request-payer"] = "requester";
6874
}
6975

70-
string signed_headers = "";
76+
string signed_headers = "";
7177
hash_bytes canonical_request_hash;
7278
hash_str canonical_request_hash_str;
7379
if (content_type.length() > 0) {
@@ -80,10 +86,13 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
8086
if (use_sse_kms) {
8187
signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id";
8288
}
89+
if (use_sse_c) {
90+
signed_headers += ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;x-amz-server-side-encryption-customer-key-md5";
91+
}
8392
if (use_requester_pays) {
8493
signed_headers += ";x-amz-request-payer";
8594
}
86-
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
95+
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
8796
if (content_type.length() > 0) {
8897
canonical_request += "\ncontent-type:" + content_type;
8998
}
@@ -95,6 +104,11 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
95104
canonical_request += "\nx-amz-server-side-encryption:aws:kms";
96105
canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id;
97106
}
107+
if (use_sse_c) {
108+
canonical_request += "\nx-amz-server-side-encryption-customer-algorithm:AES256";
109+
canonical_request += "\nx-amz-server-side-encryption-customer-key:" + auth_params.sse_c_key;
110+
canonical_request += "\nx-amz-server-side-encryption-customer-key-md5:" + auth_params.sse_c_key_md5;
111+
}
98112
if (use_requester_pays) {
99113
canonical_request += "\nx-amz-request-payer:requester";
100114
}
@@ -154,6 +168,8 @@ void AWSEnvironmentCredentialsProvider::SetAll() {
154168
this->SetExtensionOptionValue("s3_endpoint", DUCKDB_ENDPOINT_ENV_VAR);
155169
this->SetExtensionOptionValue("s3_use_ssl", DUCKDB_USE_SSL_ENV_VAR);
156170
this->SetExtensionOptionValue("s3_kms_key_id", DUCKDB_KMS_KEY_ID_ENV_VAR);
171+
this->SetExtensionOptionValue("s3_sse_c_key", DUCKDB_SSE_C_KEY_ENV_VAR);
172+
this->SetExtensionOptionValue("s3_sse_c_key_md5", DUCKDB_SSE_C_KEY_MD5_ENV_VAR);
157173
this->SetExtensionOptionValue("s3_requester_pays", DUCKDB_REQUESTER_PAYS_ENV_VAR);
158174
}
159175

@@ -167,6 +183,8 @@ S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() {
167183
params.session_token = SESSION_TOKEN_ENV_VAR;
168184
params.endpoint = DUCKDB_ENDPOINT_ENV_VAR;
169185
params.kms_key_id = DUCKDB_KMS_KEY_ID_ENV_VAR;
186+
params.sse_c_key = DUCKDB_SSE_C_KEY_ENV_VAR;
187+
params.sse_c_key_md5 = DUCKDB_SSE_C_KEY_MD5_ENV_VAR;
170188
params.use_ssl = DUCKDB_USE_SSL_ENV_VAR;
171189
params.requester_pays = DUCKDB_REQUESTER_PAYS_ENV_VAR;
172190

@@ -192,6 +210,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
192210
secret_reader.TryGetSecretKeyOrSetting("region", "s3_region", result.region);
193211
secret_reader.TryGetSecretKeyOrSetting("use_ssl", "s3_use_ssl", result.use_ssl);
194212
secret_reader.TryGetSecretKeyOrSetting("kms_key_id", "s3_kms_key_id", result.kms_key_id);
213+
secret_reader.TryGetSecretKeyOrSetting("sse_c_key", "s3_sse_c_key", result.sse_c_key);
214+
secret_reader.TryGetSecretKeyOrSetting("sse_c_key_md5", "s3_sse_c_key_md5", result.sse_c_key_md5);
195215
secret_reader.TryGetSecretKeyOrSetting("s3_url_compatibility_mode", "s3_url_compatibility_mode",
196216
result.s3_url_compatibility_mode);
197217
secret_reader.TryGetSecretKeyOrSetting("requester_pays", "s3_requester_pays",
@@ -233,6 +253,8 @@ unique_ptr<KeyValueSecret> CreateSecret(vector<string> &prefix_paths_p, string &
233253
return_value->secret_map["url_style"] = params.url_style;
234254
return_value->secret_map["use_ssl"] = params.use_ssl;
235255
return_value->secret_map["kms_key_id"] = params.kms_key_id;
256+
return_value->secret_map["sse_c_key"] = params.sse_c_key;
257+
return_value->secret_map["sse_c_key_md5"] = params.sse_c_key_md5;
236258
return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode;
237259
return_value->secret_map["requester_pays"] = params.requester_pays;
238260

0 commit comments

Comments
 (0)