Skip to content

Commit ef0f1b8

Browse files
carlopicarlopi
committed
Merge remote-tracking branch 'origin/main' into code_quality
2 parents 7906808 + da28219 commit ef0f1b8

File tree

12 files changed

+113
-56
lines changed

12 files changed

+113
-56
lines changed

‎.github/workflows/MainDistributionPipeline.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ jobs:
1919
extension_name: httpfs
2020
duckdb_version: v1.3-ossivalis
2121
ci_tools_version: main
22-
exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
2322

2423

2524
duckdb-stable-deploy:
@@ -32,7 +31,6 @@ jobs:
3231
duckdb_version: v1.3-ossivalis
3332
ci_tools_version: main
3433
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
35-
exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
3634

3735
code-quality:
3836
name: Code Quality

‎CMakeLists.txt

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,24 @@ add_extension_definitions()
99
include_directories(extension/httpfs/include
1010
${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
1111

12+
if (NOT EMSCRIPTEN)
13+
set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_client.cpp)
14+
add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
15+
else()
16+
set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
17+
endif()
18+
1219
build_static_extension(
1320
httpfs
1421
extension/httpfs/hffs.cpp
1522
extension/httpfs/s3fs.cpp
1623
extension/httpfs/httpfs.cpp
17-
extension/httpfs/httpfs_client.cpp
1824
extension/httpfs/http_state.cpp
1925
extension/httpfs/crypto.cpp
26+
extension/httpfs/hash_functions.cpp
2027
extension/httpfs/create_secret_functions.cpp
21-
extension/httpfs/httpfs_extension.cpp)
28+
extension/httpfs/httpfs_extension.cpp
29+
${EXTRA_SOURCES} )
2230

2331
set(PARAMETERS "-warnings")
2432
build_loadable_extension(
@@ -27,11 +35,12 @@ build_loadable_extension(
2735
extension/httpfs/hffs.cpp
2836
extension/httpfs/s3fs.cpp
2937
extension/httpfs/httpfs.cpp
30-
extension/httpfs/httpfs_client.cpp
3138
extension/httpfs/http_state.cpp
3239
extension/httpfs/crypto.cpp
40+
extension/httpfs/hash_functions.cpp
3341
extension/httpfs/create_secret_functions.cpp
34-
extension/httpfs/httpfs_extension.cpp)
42+
extension/httpfs/httpfs_extension.cpp
43+
${EXTRA_SOURCES} )
3544

3645
if(MINGW)
3746
set(OPENSSL_USE_STATIC_LIBS TRUE)

‎extension/httpfs/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
Documentation on S3 tests setup can be found [here](../../test/sql/copy/s3/README.md)
1+
Documentation on S3 tests setup can be found [in the duckdb/duckdb repository](https://github.com/duckdb/duckdb/blob/main/test/sql/copy/s3/README.md)

‎extension/httpfs/crypto.cpp

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "crypto.hpp"
2+
#include "hash_functions.hpp"
23
#include "mbedtls_wrapper.hpp"
34
#include <iostream>
45
#include "duckdb/common/common.hpp"
@@ -9,28 +10,6 @@
910

1011
namespace duckdb {
1112

12-
void sha256(const char *in, size_t in_len, hash_bytes &out) {
13-
duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
14-
}
15-
16-
void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) {
17-
duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out);
18-
}
19-
20-
void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
21-
hmac256(message, (char *)secret, sizeof(hash_bytes), out);
22-
}
23-
24-
void hex256(hash_bytes &in, hash_str &out) {
25-
const char *hex = "0123456789abcdef";
26-
unsigned char *pin = in;
27-
unsigned char *pout = out;
28-
for (; pin < in + sizeof(in); pout += 2, pin++) {
29-
pout[0] = hex[(*pin >> 4) & 0xF];
30-
pout[1] = hex[*pin & 0xF];
31-
}
32-
}
33-
3413
AESStateSSL::AESStateSSL(const std::string *key) : context(EVP_CIPHER_CTX_new()) {
3514
if (!(context)) {
3615
throw InternalException("AES GCM failed with initializing context");

‎extension/httpfs/hash_functions.cpp

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#include "mbedtls_wrapper.hpp"
2+
#include "hash_functions.hpp"
3+
4+
namespace duckdb {
5+
6+
void sha256(const char *in, size_t in_len, hash_bytes &out) {
7+
duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
8+
}
9+
10+
void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) {
11+
duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out);
12+
}
13+
14+
void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
15+
hmac256(message, (char *)secret, sizeof(hash_bytes), out);
16+
}
17+
18+
void hex256(hash_bytes &in, hash_str &out) {
19+
const char *hex = "0123456789abcdef";
20+
unsigned char *pin = in;
21+
unsigned char *pout = out;
22+
for (; pin < in + sizeof(in); pout += 2, pin++) {
23+
pout[0] = hex[(*pin >> 4) & 0xF];
24+
pout[1] = hex[*pin & 0xF];
25+
}
26+
}
27+
28+
} // namespace duckdb

‎extension/httpfs/httpfs.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,4 +728,9 @@ void HTTPFileHandle::StoreClient(unique_ptr<HTTPClient> client) {
728728
HTTPFileHandle::~HTTPFileHandle() {
729729
DUCKDB_LOG_FILE_SYSTEM_CLOSE((*this));
730730
};
731+
732+
string HTTPFSUtil::GetName() const {
733+
return "HTTPFS";
734+
}
735+
731736
} // namespace duckdb

‎extension/httpfs/httpfs_client.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,4 @@ unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text)
160160
return result;
161161
}
162162

163-
string HTTPFSUtil::GetName() const {
164-
return "HTTPFS";
165-
}
166-
167163
} // namespace duckdb

‎extension/httpfs/httpfs_client_wasm.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#include "httpfs_client.hpp"
2+
#include "http_state.hpp"
3+
4+
namespace duckdb {
5+
6+
unique_ptr<HTTPClient> HTTPFSUtil::InitializeClient(HTTPParams &http_params, const string &proto_host_port) {
7+
throw InternalException("HTTPFSUtil::InitializeClient is not expected to be called");
8+
}
9+
10+
unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text) {
11+
unordered_map<string, string> result;
12+
// TODO: HTTPFSUtil::ParseGetParameters is currently not implemented
13+
return result;
14+
}
15+
16+
} // namespace duckdb

‎extension/httpfs/httpfs_extension.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
#include "duckdb.hpp"
77
#include "s3fs.hpp"
88
#include "hffs.hpp"
9+
#ifdef OVERRIDE_ENCRYPTION_UTILS
910
#include "crypto.hpp"
11+
#endif // OVERRIDE_ENCRYPTION_UTILS
1012

1113
namespace duckdb {
1214

@@ -61,16 +63,23 @@ static void LoadInternal(DatabaseInstance &instance) {
6163
// HuggingFace options
6264
config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests",
6365
LogicalType::UBIGINT, Value::UBIGINT(0));
64-
config.http_util = make_shared_ptr<HTTPFSUtil>();
66+
67+
if (config.http_util && config.http_util->GetName() == "WasmHTTPUtils") {
68+
// Already handled, do not override
69+
} else {
70+
config.http_util = make_shared_ptr<HTTPFSUtil>();
71+
}
6572

6673
auto provider = make_uniq<AWSEnvironmentCredentialsProvider>(config);
6774
provider->SetAll();
6875

6976
CreateS3SecretFunctions::Register(instance);
7077
CreateBearerTokenFunctions::Register(instance);
7178

79+
#ifdef OVERRIDE_ENCRYPTION_UTILS
7280
// set pointer to OpenSSL encryption state
7381
config.encryption_util = make_shared_ptr<AESStateSSLFactory>();
82+
#endif // OVERRIDE_ENCRYPTION_UTILS
7483
}
7584
void HttpfsExtension::Load(DuckDB &db) {
7685
LoadInternal(*db.instance);

‎extension/httpfs/include/hash_functions.hpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#pragma once
2+
3+
#include "duckdb/common/helper.hpp"
4+
5+
namespace duckdb {
6+
7+
typedef unsigned char hash_bytes[32];
8+
typedef unsigned char hash_str[64];
9+
10+
void sha256(const char *in, size_t in_len, hash_bytes &out);
11+
12+
void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out);
13+
14+
void hmac256(std::string message, hash_bytes secret, hash_bytes &out);
15+
16+
void hex256(hash_bytes &in, hash_str &out);
17+
18+
} // namespace duckdb

0 commit comments

Comments
 (0)