diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 18faf1b..866dbc0 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -12,42 +12,11 @@ concurrency: cancel-in-progress: true jobs: - duckdb-main-build: - name: Build main extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main - with: - ci_tools_version: main - duckdb_version: main - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' - extension_name: ui - - duckdb-next-patch-build: - name: Build next patch extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 - with: - ci_tools_version: v1.2.1 - duckdb_version: v1.2-histrionicus - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' - extension_name: ui - duckdb-stable-build: name: Build stable extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 + uses: ./.github/workflows/_extension_distribution.yml with: ci_tools_version: v1.2.1 duckdb_version: v1.2.1 exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' extension_name: ui - - duckdb-stable-deploy: - if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} - name: Deploy stable extension binaries - needs: duckdb-stable-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.2.1 - secrets: inherit - with: - extension_name: ui - duckdb_version: v1.2.1 - ci_tools_version: v1.2.1 - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' - deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml new file mode 100644 index 0000000..3691502 --- /dev/null +++ b/.github/workflows/_extension_distribution.yml @@ -0,0 +1,738 @@ +# Reusable workflow for building DuckDB extensions using a standardized environment +# +# The workflow: +# - builds the extension using the CI workflow from the corresponding DuckDB version +# - uploads the extensions as gh actions artifacts in the following format: +# --extension- +# +# note: extensions are simply uploaded to GitHub actions, deploying the extensions is done a separate step. More info on +# this can be found in https://github.com/duckdb/extension-template + +name: Extension distribution +on: + workflow_call: + inputs: + # The name with which the extension will be built + extension_name: + required: true + type: string + # DuckDB version to build against, should in most cases be identical to + duckdb_version: + required: true + type: string + # The version of the https://github.com/duckdb/extension-ci-tools submodule of the extension. In most cases will be identical to `duckdb_version`. + # Passing this explicitly is required because of https://github.com/actions/toolkit/issues/1264 + ci_tools_version: + required: true + type: string + # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' + exclude_archs: + required: false + type: string + default: "" + # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times + artifact_postfix: + required: false + type: string + default: "" + # Override the default vcpkg repository + vcpkg_url: + required: false + type: string + default: "https://github.com/microsoft/vcpkg.git" + # Override the default vcpkg commit used by this version of DuckDB + vcpkg_commit: + required: false + type: string + default: "5e5d0e1cd7785623065e77eff011afdeec1a3574" + # Override the default script producing the matrices. Allows specifying custom matrices. + matrix_parse_script: + required: false + type: string + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" + # Enable building the DuckDB Shell + build_duckdb_shell: + required: false + type: boolean + default: true + # Supply an override repository to build, instead of using the current one + override_repository: + required: false + type: string + default: "" + # The git ref used for the override_repository + override_ref: + required: false + type: string + default: "" + # Override the repo for the CI tools (for testing CI tools itself) + override_ci_tools_repository: + required: false + type: string + default: "duckdb/extension-ci-tools" + # Pass extra toolchains + # available: (parser_tools, rust, fortran, omp, python3) + extra_toolchains: + required: false + type: string + default: "" + rust_logs: + required: false + type: boolean + default: false + # Optional tag the build extension should have -- this is easy to misuse, and subject to change, for internal use only + extension_tag: + required: false + type: string + default: "" + # Optional tag the referenced duckdb should have -- this is a easy to misuse, and subject to change, for internal use only + duckdb_tag: + required: false + type: string + default: "" + # If set tot true, skip tests + skip_tests: + required: false + type: boolean + default: false + # DEPRECATED: use extra_toolchains instead + enable_rust: + required: false + type: boolean + default: false + +jobs: + generate_matrix: + name: Generate matrix + runs-on: ubuntu-latest + outputs: + linux_matrix: ${{ steps.set-matrix-linux.outputs.linux_matrix }} + windows_matrix: ${{ steps.set-matrix-windows.outputs.windows_matrix }} + osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }} + wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }} + steps: + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} + + - id: parse-matrices + run: | + mkdir build + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + + - id: set-matrix-linux + run: | + linux_matrix="`cat build/linux_matrix.json`" + echo linux_matrix=$linux_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + - id: set-matrix-osx + run: | + osx_matrix="`cat build/osx_matrix.json`" + echo osx_matrix=$osx_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + - id: set-matrix-windows + run: | + windows_matrix="`cat build/windows_matrix.json`" + echo windows_matrix=$windows_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + - id: set-matrix-wasm + run: | + wasm_matrix="`cat build/wasm_matrix.json`" + echo wasm_matrix=$wasm_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + linux: + name: Linux + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.linux_matrix != '{}' && needs.generate_matrix.outputs.linux_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.linux_matrix)}} + env: + VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + GEN: ninja + BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + + steps: + - name: Free up some unused space + continue-on-error: true + run: | + docker images -a -q > package.list + if [ -s package.list ]; then + echo "To be deleted" + cat package.list + echo "---" + docker rmi $(cat package.list) + fi + rm package.list + + - uses: actions/checkout@v4 + name: Checkout override repository + if: ${{inputs.override_repository != ''}} + with: + repository: ${{ inputs.override_repository }} + ref: ${{ inputs.override_ref }} + fetch-depth: 0 + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout current repository + if: ${{inputs.override_repository == ''}} + with: + fetch-depth: 0 + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} + + - name: Checkout DuckDB to version + if: ${{inputs.duckdb_version != ''}} + run: | + DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version + + - name: Tag extension + if: ${{inputs.extension_tag != ''}} + run: | + git tag ${{ inputs.extension_tag }} + + - name: Tag DuckDB extension + if: ${{inputs.duckdb_tag != ''}} + run: | + DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag + + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} + fetch-depth: 0 + + - name: Build Docker image + shell: bash + run: | + docker build \ + --build-arg 'vcpkg_url=${{ inputs.vcpkg_url }}' \ + --build-arg 'vcpkg_commit=${{ inputs.vcpkg_commit }}' \ + --build-arg 'extra_toolchains=${{ inputs.enable_rust && format(';{0};rust;', inputs.extra_toolchains) || format(';{0};', inputs.extra_toolchains) }}' \ + -t duckdb/${{ matrix.duckdb_arch }} \ + ./extension-ci-tools/docker/${{ matrix.duckdb_arch }} + + - name: Create env file for docker + run: | + touch docker_env.txt + echo "VCPKG_TARGET_TRIPLET=${{ matrix.vcpkg_triplet }}" >> docker_env.txt + echo "BUILD_SHELL=${{ inputs.build_duckdb_shell && '1' || '0' }}" >> docker_env.txt + echo "OPENSSL_ROOT_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> docker_env.txt + echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> docker_env.txt + echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt + echo "DUCKDB_PLATFORM=${{ matrix.duckdb_arch }}" >> docker_env.txt + echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt + echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt + echo "TOOLCHAIN_FLAGS=${{ matrix.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt + + - name: Generate timestamp for Ccache entry + shell: cmake -P {0} + id: ccache_timestamp + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) + message("::set-output name=timestamp::${current_date}") + + - name: Create Ccache directory + run: | + mkdir ccache_dir + + - name: Load Ccache + uses: actions/cache@v4 + with: + path: ./ccache_dir + key: ccache-extension-distribution-${{ matrix.duckdb_arch }}-${{ steps.ccache_timestamp.outputs.timestamp }} + restore-keys: | + ccache-extension-distribution-${{ matrix.duckdb_arch }}- + + - name: Run configure (outside Docker) + shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 + run: | + make configure_ci + + - name: Run configure (inside Docker) + shell: bash + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make configure_ci + + - name: Build extension (inside Docker) + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release + + - name: Test extension (inside docker) + if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release + + - name: Test extension (outside docker) + if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 + run: | + make test_release + + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension + + - uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: test-ssl-${{ inputs.duckdb_version }}-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: build/release/extension/ui/test-ssl/test_ssl + + - name: Print Rust logs + if: ${{ inputs.rust_logs && (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;')) }} + run: | + for filename in build/release/rust/src/*/*build-*.log; do + echo Printing logs for file $filename + cat $filename; + echo Done printing logs for $filename + done + + macos: + name: MacOS + runs-on: macos-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.osx_matrix != '{}' && needs.generate_matrix.outputs.osx_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.osx_matrix)}} + env: + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} + OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }} + GEN: ninja + BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + + steps: + - uses: actions/checkout@v4 + name: Checkout override repository + if: ${{inputs.override_repository != ''}} + with: + repository: ${{ inputs.override_repository }} + ref: ${{ inputs.override_ref }} + fetch-depth: 0 + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout current repository + if: ${{inputs.override_repository == ''}} + with: + fetch-depth: 0 + submodules: 'true' + + - name: Install Ninja + run: | + brew install ninja autoconf make libtool automake autoconf-archive + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + continue-on-error: true + with: + key: extension-distribution-${{ matrix.duckdb_arch }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} + + - name: Checkout DuckDB to version + if: ${{inputs.duckdb_version != ''}} + run: | + DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version + + - name: Tag extension + if: ${{inputs.extension_tag != ''}} + run: | + git tag ${{ inputs.extension_tag }} + + - name: Tag DuckDB extension + if: ${{inputs.duckdb_tag != ''}} + run: | + DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} + vcpkgGitURL: ${{ inputs.vcpkg_url }} + + - name: Install Rust cross compile dependency + if: ${{ (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;')) && matrix.osx_build_arch == 'x86_64'}} + run: | + rustup target add x86_64-apple-darwin + + - name: 'Setup go' + if: ${{ (inputs.enable_go || contains(format(';{0};', inputs.extra_toolchains), ';go;'))}} + uses: actions/setup-go@v4 + with: + go-version: '1.23' + + - name: Install parser tools + if: ${{ contains(format(';{0};', inputs.extra_toolchains), ';parser_tools;')}} + run: | + brew install bison flex + + - name: install omp (x86) + if: ${{ contains(format(';{0};', inputs.extra_toolchains), ';omp;') && matrix.duckdb_arch == 'osx_amd64' }} + run: | + arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" + (echo; echo 'eval "$(/usr/local/bin/brew shellenv)"') >> /Users/runner/.bash_profile + eval "$(/usr/local/bin/brew shellenv)" + arch -x86_64 brew install libomp + echo "LDFLAGS=-L/usr/local/opt/libomp/lib" >> $GITHUB_ENV + echo "CFLAGS=-I/usr/local/opt/libomp/include" >> $GITHUB_ENV + echo "CPPFLAGS=-I/usr/local/opt/libomp/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I/usr/local/opt/libomp/include" >> $GITHUB_ENV + + - name: install omp (arm) + if: ${{ contains(format(';{0};', inputs.extra_toolchains), ';omp;') && matrix.duckdb_arch == 'osx_arm64' }} + run: | + brew install libomp + echo "LDFLAGS=-L/opt/homebrew/opt/libomp/lib" >> $GITHUB_ENV + echo "CFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV + echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV + echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV + + - name: Run configure + shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + run: | + make configure_ci + + - name: Build extension + shell: bash + run: | + make release + + - name: Test Extension + if: ${{ matrix.osx_build_arch == 'arm64' && inputs.skip_tests == false }} + shell: bash + run: | + make test_release + + - uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension + + - uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: test-ssl-${{ inputs.duckdb_version }}-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: build/release/extension/ui/test-ssl/test_ssl + + - name: Print Rust logs + if: ${{ inputs.rust_logs && (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;')) }} + run: | + for filename in build/release/rust/src/*/*build-*.log; do + echo Printing logs for file $filename + cat $filename; + echo Done printing logs for $filename + done + + windows: + name: Windows + runs-on: windows-2019 + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.windows_matrix != '{}' && needs.generate_matrix.outputs.windows_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.windows_matrix)}} + env: + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} + BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + CC: ${{ (matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw') && 'gcc' || '' }} + CXX: ${{ (matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw') && 'g++' || '' }} + + steps: + - name: Keep \n line endings + shell: bash + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@v4 + name: Checkout override repository + if: ${{inputs.override_repository != ''}} + with: + repository: ${{ inputs.override_repository }} + ref: ${{ inputs.override_ref }} + fetch-depth: 0 + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout current repository + if: ${{inputs.override_repository == ''}} + with: + fetch-depth: 0 + submodules: 'true' + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Setup Rust + if: (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;')) + uses: dtolnay/rust-toolchain@stable + + - name: 'Setup go' + if: ${{ (inputs.enable_go || contains(format(';{0};', inputs.extra_toolchains), ';go;'))}} + uses: actions/setup-go@v4 + with: + go-version: '1.23' + + - name: Install parser tools + if: ${{ contains(format(';{0};', inputs.extra_toolchains), ';parser_tools;')}} + run: | + choco install winflexbison3 + + - uses: r-lib/actions/setup-r@v2 + if: matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw' + with: + r-version: 'devel' + update-rtools: true + rtools-version: '42' # linker bug in 43 + + - name: setup rtools gcc for vcpkg + if: matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw' + run: | + cp C:/rtools42/x86_64-w64-mingw32.static.posix/bin/gcc.exe C:/rtools42/x86_64-w64-mingw32.static.posix/bin/x86_64-w64-mingw32-gcc.exe + cp C:/rtools42/x86_64-w64-mingw32.static.posix/bin/g++.exe C:/rtools42/x86_64-w64-mingw32.static.posix/bin/x86_64-w64-mingw32-g++.exe + cp C:/rtools42/x86_64-w64-mingw32.static.posix/bin/gfortran.exe C:/rtools42/x86_64-w64-mingw32.static.posix/bin/x86_64-w64-mingw32-gfortran.exe + + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} + + - name: Checkout DuckDB to version + if: ${{inputs.duckdb_version != ''}} + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + run: | + make set_duckdb_version + + - name: Tag extension + if: ${{inputs.extension_tag != ''}} + run: | + git tag ${{ inputs.extension_tag }} + + - name: Tag DuckDB extension + if: ${{inputs.duckdb_tag != ''}} + env: + DUCKDB_TAG: ${{ inputs.duckdb_tag }} + run: | + make set_duckdb_tag + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + continue-on-error: true + with: + key: ${{ github.job }}-${{ matrix.duckdb_arch }} + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} + vcpkgGitURL: ${{ inputs.vcpkg_url }} + + - name: Run configure + shell: bash + env: + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + DUCKDB_PLATFORM_RTOOLS: ${{ (matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw') && 1 || 0 }} + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + run: | + make configure_ci + + - name: Build extension + env: + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + DUCKDB_PLATFORM_RTOOLS: ${{ (matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw') && 1 || 0 }} + run: | + make release + + - name: Test extension + if: ${{ inputs.skip_tests == false }} + env: + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + DUCKDB_PLATFORM_RTOOLS: ${{ (matrix.duckdb_arch == 'windows_amd64_rtools' || matrix.duckdb_arch == 'windows_amd64_mingw') && 1 || 0 }} + run: | + make test_release + + - uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + build/release/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension + + - name: Find test_ssl + shell: bash + run: find . -name 'test_ssl.exe' + + - uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: test-ssl-${{ inputs.duckdb_version }}-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + # MinGW: build/release/extension/ui/test-ssl/test_ssl.exe + # Windows: build/release/extension/ui/test-ssl/Release/test_ssl.exe + path: build/release/extension/ui/test-ssl/**/test_ssl.exe + + + - name: Print Rust logs + if: ${{ inputs.rust_logs && (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;')) }} + shell: bash + run: | + for filename in build/release/rust/src/*/*build-*.log; do + echo Printing logs for file $filename + cat $filename; + echo Done printing logs for $filename + done + + wasm: + name: DuckDB-Wasm + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.wasm_matrix != '{}' && needs.generate_matrix.outputs.wasm_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.wasm_matrix)}} + env: + VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + + steps: + - uses: actions/checkout@v4 + name: Checkout override repository + if: ${{inputs.override_repository != ''}} + with: + repository: ${{ inputs.override_repository }} + ref: ${{ inputs.override_ref }} + fetch-depth: 0 + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout current repository + if: ${{inputs.override_repository == ''}} + with: + fetch-depth: 0 + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} + + - name: Checkout DuckDB to version + if: ${{inputs.duckdb_version != ''}} + run: | + DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version + + - name: Tag extension + if: ${{inputs.extension_tag != ''}} + run: | + git tag ${{ inputs.extension_tag }} + + - name: Tag DuckDB extension + if: ${{inputs.duckdb_tag != ''}} + run: | + DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag + + - uses: mymindstorm/setup-emsdk@v13 + with: + version: 3.1.71 + + - name: Setup Rust for cross compilation + if: ${{ (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;'))}} + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-emscripten + + - name: 'Setup go' + if: ${{ (inputs.enable_go || contains(format(';{0};', inputs.extra_toolchains), ';go;'))}} + uses: actions/setup-go@v4 + with: + go-version: '1.23' + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} + vcpkgGitURL: ${{ inputs.vcpkg_url }} + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + continue-on-error: true + with: + key: ${{ github.job }}-${{ matrix.duckdb_arch }} + + - name: Run configure + shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + run: | + make configure_ci + + - name: Build Wasm module + run: | + make ${{ matrix.duckdb_arch }} + + - uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm + + - name: Print Rust logs + if: ${{ inputs.rust_logs && (inputs.enable_rust || contains(format(';{0};', inputs.extra_toolchains), ';rust;')) }} + shell: bash + run: | + for filename in build/release/rust/src/*/*build-*.log; do + echo Printing logs for file $filename + cat $filename; + echo Done printing logs for $filename + done diff --git a/CMakeLists.txt b/CMakeLists.txt index d681500..4e1fc29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,3 +60,5 @@ install( EXPORT "${DUCKDB_EXPORT_SET}" LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") + +add_subdirectory(test-ssl) \ No newline at end of file diff --git a/test-ssl/CMakeLists.txt b/test-ssl/CMakeLists.txt new file mode 100644 index 0000000..610bca4 --- /dev/null +++ b/test-ssl/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.5...3.31.5) + +find_package(OpenSSL REQUIRED) + +project("Test ssl") +set(CMAKE_CXX_STANDARD 11) + +add_definitions(-DNO_DUCKDB_RE2 -DCMAKE_BUILD_TYPE=Debug) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") + +include_directories( + ${PROJECT_SOURCE_DIR}/../third_party/httplib + ${PROJECT_SOURCE_DIR}/../duckdb/src/include) + +add_executable("test_ssl" "test_ssl.cc") + +target_link_libraries("test_ssl" OpenSSL::SSL OpenSSL::Crypto) + +install(TARGETS "test_ssl") + +# cmake -S . -G Ninja -B build && cmake --build build \ No newline at end of file diff --git a/test-ssl/test_ssl.cc b/test-ssl/test_ssl.cc new file mode 100644 index 0000000..39a44bd --- /dev/null +++ b/test-ssl/test_ssl.cc @@ -0,0 +1,16 @@ +#define CPPHTTPLIB_OPENSSL_SUPPORT +#include "httplib.hpp" + +using namespace duckdb_httplib_openssl; + +int main() { + Client client("https://ui.duckdb.org"); + auto res = client.Get("/"); + if (res) { + std::cout << "Status: " << res->status << std::endl; + std::cout << "Body: " << res->body.substr(0, 42) << "... (" << res->body.size() << ")" << std::endl; + } else { + std::cout << "Error: " << res.error() << std::endl; + } + return 0; +} \ No newline at end of file diff --git a/third_party/httplib/httplib.hpp b/third_party/httplib/httplib.hpp index f12a80d..7b51bb5 100644 --- a/third_party/httplib/httplib.hpp +++ b/third_party/httplib/httplib.hpp @@ -248,8 +248,78 @@ using socket_t = int; #include #include #include + +#ifdef NO_DUCKDB_RE2 +namespace duckdb_re2 { +enum class RegexOptions : uint8_t { NONE, CASE_INSENSITIVE }; +class Regex { +public: + explicit Regex(const std::string &pattern, RegexOptions options = RegexOptions::NONE): re(pattern) {} + explicit Regex(const char *pattern, RegexOptions options = RegexOptions::NONE) : Regex(std::string(pattern)) { + } + // const duckdb_re2::RE2 &GetRegex() const { + // return *regex; + // } + std::regex re; +}; +struct GroupMatch { + std::string text; + uint32_t position; + + const std::string &str() const { // NOLINT + return text; + } + operator std::string() const { // NOLINT: allow implicit cast + return text; + } +}; + +struct Match { + + GroupMatch GetGroup(uint64_t index) { + return {str(index), static_cast(position(index))}; + } + + std::string str(uint64_t index) { // NOLINT + return m.str(index); + } + + uint64_t position(uint64_t index) { // NOLINT + return m.position(index); + } + + uint64_t length(uint64_t index) { // NOLINT + throw std::runtime_error("uint64_t length(uint64_t index) - NA"); + } + + GroupMatch operator[](uint64_t i) { + return GetGroup(i); + } + std::cmatch m; +}; + +bool RegexSearch(const std::string &input, Match &match, const Regex ®ex) { + throw std::runtime_error("bool RegexSearch(const std::string &input, Match &match, const Regex ®ex) - NA"); +} + +bool RegexMatch(const std::string &input, Match &match, const Regex ®ex) { + return std::regex_match(input.c_str(), match.m, regex.re); +} + +bool RegexMatch(const char *start, const char *end, Match &match, const Regex ®ex) { + throw std::runtime_error("bool RegexMatch(const char *start, const char *end, Match &match, const Regex ®ex) - NA"); +} + +bool RegexMatch(const std::string &input, const Regex ®ex) { + std::cmatch m; + return std::regex_match(input.c_str(), m, regex.re); +} +} + +#else #include "duckdb/common/re2_regex.hpp" #include "duckdb/common/random_engine.hpp" +#endif #ifdef CPPHTTPLIB_OPENSSL_SUPPORT #ifdef _WIN32 @@ -4643,6 +4713,9 @@ inline std::string to_lower(const char *beg, const char *end) { } inline std::string make_multipart_data_boundary() { +#ifdef NO_DUCKDB_RE2 + throw std::runtime_error("NA"); +#else static const char data[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; @@ -4653,6 +4726,7 @@ inline std::string make_multipart_data_boundary() { } return result; +#endif } inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { @@ -5109,6 +5183,7 @@ inline bool parse_www_authenticate(const Response &res, if (type == "Basic") { return false; } else if (type == "Digest") { +#ifndef NO_DUCKDB_RE2 s = s.substr(pos + 1); auto matches = duckdb_re2::RegexFindAll(s, re); for (auto &m : matches) { @@ -5122,6 +5197,9 @@ inline bool parse_www_authenticate(const Response &res, auth[key] = val; } return true; +#else + throw std::runtime_error("parse_www_authenticate- NA"); +#endif } } } @@ -8166,6 +8244,10 @@ inline SSL *ssl_new(socket_t sock, SSL_CTX *ctx, std::mutex &ctx_mutex, } if (ssl) { +#ifdef NO_DUCKDB_RE2 + SSL_set_msg_callback(ssl, SSL_trace); + SSL_set_msg_callback_arg(ssl, BIO_new_fp(stdout, 0)); +#endif set_nonblocking(sock, true); auto bio = BIO_new_socket(static_cast(sock), BIO_NOCLOSE); BIO_set_nbio(bio, 1); @@ -8678,6 +8760,7 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { verify_result_ = SSL_get_verify_result(ssl2); if (verify_result_ != X509_V_OK) { + std::cerr << "SSL_get_verify_result failed: " << verify_result_ << std::endl; error = Error::SSLServerVerification; return false; } @@ -8686,10 +8769,12 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { if (server_cert == nullptr) { error = Error::SSLServerVerification; + std::cerr << "SSL_get1_peer_certificate failed" << std::endl; return false; } if (!verify_host(server_cert)) { + std::cerr << "verify_host failed" << std::endl; X509_free(server_cert); error = Error::SSLServerVerification; return false;