From f714c7ff6349eda13e9af2fe2939a729f0bbac55 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 18 Jun 2025 13:52:03 +0200 Subject: [PATCH 01/63] wip --- .builders/scripts/build_wheels.py | 3 ++ .builders/scripts/utils.py | 63 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 926724815007f..a663ca97aaa6f 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -119,6 +119,9 @@ def main(): check_process(command_args, env=env_vars) + # Remove test files + + # Repair wheels check_process([ sys.executable, '-u', str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'), diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 0750f1ba8237b..3d2e6ccaab0d5 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import email +import os import re from pathlib import Path from zipfile import ZipFile @@ -32,3 +33,65 @@ def extract_metadata(wheel: Path) -> email.Message: raise RuntimeError(message) from None return email.message_from_string(metadata_file_contents) + +def remove_test_files(wheel: Path) -> None: + with ZipFile(str(wheel)) as zip_archive: + for path in zip_archive.namelist(): + if is_excluded_from_wheel(path): + zip_archive.remove(path) + +def is_excluded_from_wheel(path: str) -> bool: + ''' + These files are excluded from the wheel in the agent build: + https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb + In order to have more accurate results, this files are excluded when computing the size of the dependencies while + the wheels still include them. + ''' + excluded_test_paths = [ + os.path.normpath(path) + for path in [ + 'idlelib/idle_test', + 'bs4/tests', + 'Cryptodome/SelfTest', + 'gssapi/tests', + 'keystoneauth1/tests', + 'openstack/tests', + 'os_service_types/tests', + 'pbr/tests', + 'pkg_resources/tests', + 'psutil/tests', + 'securesystemslib/_vendor/ed25519/test_data', + 'setuptools/_distutils/tests', + 'setuptools/tests', + 'simplejson/tests', + 'stevedore/tests', + 'supervisor/tests', + 'test', # cm-client + 'vertica_python/tests', + 'websocket/tests', + ] + ] + + type_annot_libraries = [ + 'krb5', + 'Cryptodome', + 'ddtrace', + 'pyVmomi', + 'gssapi', + ] + rel_path = Path(path).as_posix() + + # Test folders + for test_folder in excluded_test_paths: + if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): + return True + + # Python type annotations + path_parts = Path(rel_path).parts + if path_parts: + dependency_name = path_parts[0] + if dependency_name in type_annot_libraries: + if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': + return True + + return False \ No newline at end of file From 38bc7c0e8f6aff9f1e8cfb80fd7060a38b68f404 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 1 Jul 2025 12:03:16 +0200 Subject: [PATCH 02/63] remove changes --- .builders/scripts/build_wheels.py | 1 - .builders/scripts/utils.py | 61 ------------------------------- 2 files changed, 62 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index a663ca97aaa6f..bdb90a00122cc 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -119,7 +119,6 @@ def main(): check_process(command_args, env=env_vars) - # Remove test files # Repair wheels diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 3d2e6ccaab0d5..c80b46bab0de5 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -34,64 +34,3 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) -def remove_test_files(wheel: Path) -> None: - with ZipFile(str(wheel)) as zip_archive: - for path in zip_archive.namelist(): - if is_excluded_from_wheel(path): - zip_archive.remove(path) - -def is_excluded_from_wheel(path: str) -> bool: - ''' - These files are excluded from the wheel in the agent build: - https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb - In order to have more accurate results, this files are excluded when computing the size of the dependencies while - the wheels still include them. - ''' - excluded_test_paths = [ - os.path.normpath(path) - for path in [ - 'idlelib/idle_test', - 'bs4/tests', - 'Cryptodome/SelfTest', - 'gssapi/tests', - 'keystoneauth1/tests', - 'openstack/tests', - 'os_service_types/tests', - 'pbr/tests', - 'pkg_resources/tests', - 'psutil/tests', - 'securesystemslib/_vendor/ed25519/test_data', - 'setuptools/_distutils/tests', - 'setuptools/tests', - 'simplejson/tests', - 'stevedore/tests', - 'supervisor/tests', - 'test', # cm-client - 'vertica_python/tests', - 'websocket/tests', - ] - ] - - type_annot_libraries = [ - 'krb5', - 'Cryptodome', - 'ddtrace', - 'pyVmomi', - 'gssapi', - ] - rel_path = Path(path).as_posix() - - # Test folders - for test_folder in excluded_test_paths: - if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): - return True - - # Python type annotations - path_parts = Path(rel_path).parts - if path_parts: - dependency_name = path_parts[0] - if dependency_name in type_annot_libraries: - if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': - return True - - return False \ No newline at end of file From 5e442f026f94ad37556fb2aac47dae7f46749ab9 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 1 Jul 2025 15:20:49 +0200 Subject: [PATCH 03/63] remove tests --- .builders/scripts/build_wheels.py | 20 ++++++++- .builders/scripts/utils.py | 69 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index bdb90a00122cc..c2a1c8b3c7cc4 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -4,11 +4,12 @@ import os import subprocess import sys +import zipfile from pathlib import Path from tempfile import TemporaryDirectory from dotenv import dotenv_values -from utils import extract_metadata, normalize_project_name +from utils import extract_metadata, normalize_project_name, remove_test_files INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' @@ -119,7 +120,22 @@ def main(): check_process(command_args, env=env_vars) - + # Remove test files + print(f"[INFO] Removing test files from wheels in {staged_wheel_dir}") + for wheel_file in staged_wheel_dir.glob("*.whl"): + print(f"[INFO] Processing wheel: {wheel_file.name}") + with zipfile.ZipFile(wheel_file, "r") as z: + before_files = z.namelist() + # print(f"[INFO] Files before removing test files: {before_files}") + remove_test_files(wheel_file) + with zipfile.ZipFile(wheel_file, "r") as z: + after_files = z.namelist() + # print(f"[INFO] Files after removing test files: {after_files}") + removed = set(before_files) - set(after_files) + if removed: + print(f"[INFO] Removed files: {sorted(removed)}") + else: + print(f"[INFO] No files were removed from: {wheel_file.name}") # Repair wheels check_process([ diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index c80b46bab0de5..8231650bb6be6 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -34,3 +34,72 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) +def remove_test_files(wheel: Path) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + temp_wheel = Path(tmpdir) / wheel.name + + # Copy files except those excluded + with ZipFile(wheel, "r") as zin, ZipFile(temp_wheel, "w") as zout: + for item in zin.infolist(): + if not is_excluded_from_wheel(item.filename): + zout.writestr(item, zin.read(item.filename)) + + shutil.move(str(temp_wheel), str(wheel)) + + + +def is_excluded_from_wheel(path: str) -> bool: + """ + These files are excluded from the wheel in the agent build: + https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb + In order to have more accurate results, this files are excluded when computing the size of the dependencies while + the wheels still include them. + """ + excluded_test_paths = [ + os.path.normpath(path) + for path in [ + "idlelib/idle_test", + "bs4/tests", + "Cryptodome/SelfTest", + "gssapi/tests", + "keystoneauth1/tests", + "openstack/tests", + "os_service_types/tests", + "pbr/tests", + "pkg_resources/tests", + "psutil/tests", + "securesystemslib/_vendor/ed25519/test_data", + "setuptools/_distutils/tests", + "setuptools/tests", + "simplejson/tests", + "stevedore/tests", + "supervisor/tests", + "test", # cm-client + "vertica_python/tests", + "websocket/tests", + ] + ] + + type_annot_libraries = [ + "krb5", + "Cryptodome", + "ddtrace", + "pyVmomi", + "gssapi", + ] + rel_path = Path(path).as_posix() + + # Test folders + for test_folder in excluded_test_paths: + if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): + return True + + # Python type annotations + path_parts = Path(rel_path).parts + if path_parts: + dependency_name = path_parts[0] + if dependency_name in type_annot_libraries: + if path.endswith(".pyi") or os.path.basename(path) == "py.typed": + return True + + return False From 5b4db44544c77608280a286939cd69be16f81675 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 2 Jul 2025 13:01:42 +0200 Subject: [PATCH 04/63] remove funcions --- .builders/scripts/build_wheels.py | 16 ------- .builders/scripts/utils.py | 70 +------------------------------ 2 files changed, 1 insertion(+), 85 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index c2a1c8b3c7cc4..09c9bcc0d3ad7 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -120,22 +120,6 @@ def main(): check_process(command_args, env=env_vars) - # Remove test files - print(f"[INFO] Removing test files from wheels in {staged_wheel_dir}") - for wheel_file in staged_wheel_dir.glob("*.whl"): - print(f"[INFO] Processing wheel: {wheel_file.name}") - with zipfile.ZipFile(wheel_file, "r") as z: - before_files = z.namelist() - # print(f"[INFO] Files before removing test files: {before_files}") - remove_test_files(wheel_file) - with zipfile.ZipFile(wheel_file, "r") as z: - after_files = z.namelist() - # print(f"[INFO] Files after removing test files: {after_files}") - removed = set(before_files) - set(after_files) - if removed: - print(f"[INFO] Removed files: {sorted(removed)}") - else: - print(f"[INFO] No files were removed from: {wheel_file.name}") # Repair wheels check_process([ diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 8231650bb6be6..fbcdcb590b67f 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -4,6 +4,7 @@ import os import re from pathlib import Path + from zipfile import ZipFile UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') @@ -34,72 +35,3 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) -def remove_test_files(wheel: Path) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - temp_wheel = Path(tmpdir) / wheel.name - - # Copy files except those excluded - with ZipFile(wheel, "r") as zin, ZipFile(temp_wheel, "w") as zout: - for item in zin.infolist(): - if not is_excluded_from_wheel(item.filename): - zout.writestr(item, zin.read(item.filename)) - - shutil.move(str(temp_wheel), str(wheel)) - - - -def is_excluded_from_wheel(path: str) -> bool: - """ - These files are excluded from the wheel in the agent build: - https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb - In order to have more accurate results, this files are excluded when computing the size of the dependencies while - the wheels still include them. - """ - excluded_test_paths = [ - os.path.normpath(path) - for path in [ - "idlelib/idle_test", - "bs4/tests", - "Cryptodome/SelfTest", - "gssapi/tests", - "keystoneauth1/tests", - "openstack/tests", - "os_service_types/tests", - "pbr/tests", - "pkg_resources/tests", - "psutil/tests", - "securesystemslib/_vendor/ed25519/test_data", - "setuptools/_distutils/tests", - "setuptools/tests", - "simplejson/tests", - "stevedore/tests", - "supervisor/tests", - "test", # cm-client - "vertica_python/tests", - "websocket/tests", - ] - ] - - type_annot_libraries = [ - "krb5", - "Cryptodome", - "ddtrace", - "pyVmomi", - "gssapi", - ] - rel_path = Path(path).as_posix() - - # Test folders - for test_folder in excluded_test_paths: - if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): - return True - - # Python type annotations - path_parts = Path(rel_path).parts - if path_parts: - dependency_name = path_parts[0] - if dependency_name in type_annot_libraries: - if path.endswith(".pyi") or os.path.basename(path) == "py.typed": - return True - - return False From fdb7e1e000d5be19c27f31bb98cbdf9597a8d20e Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 2 Jul 2025 15:30:33 +0200 Subject: [PATCH 05/63] patch --- .builders/build.py | 8 +++ .builders/patches/remove_tests.patch | 104 +++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 .builders/patches/remove_tests.patch diff --git a/.builders/build.py b/.builders/build.py index 328441670a76d..133b02f109832 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -108,6 +108,10 @@ def build_macos(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') + # Apply the remove_tests patch + test_patch = mount_dir / 'patches' / 'remove_tests.patch' + check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) + prefix_path = builder_root / 'prefix' env = { **os.environ, @@ -210,6 +214,10 @@ def build_image(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') + # Apply the remove_tests patch + test_patch = mount_dir / 'patches' / 'remove_tests.patch' + check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir / 'scripts') + # Create outputs on the host so they can be removed wheels_dir = mount_dir / 'wheels' wheels_dir.mkdir() diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch new file mode 100644 index 0000000000000..6b8a2792c334e --- /dev/null +++ b/.builders/patches/remove_tests.patch @@ -0,0 +1,104 @@ +diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py +index 09c9bcc0d3..983998011e 100644 +--- a/scripts/build_wheels.py +--- b/scripts/build_wheels.py +@@ -120,6 +120,10 @@ def main(): + + check_process(command_args, env=env_vars) + ++ # Remove test files ++ for wheel_file in staged_wheel_dir.glob("*.whl"): ++ remove_test_files(wheel_file) ++ + + # Repair wheels + check_process([ +diff --git a/scripts/utils.py b/scripts/utils.py +index fbcdcb590b..ce3fdbf7ec 100644 +--- a/scripts/utils.py +--- b/scripts/utils.py +@@ -4,7 +4,8 @@ import email + import os + import re + from pathlib import Path +- ++import shutil ++import tempfile + from zipfile import ZipFile + + UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') +@@ -35,3 +36,74 @@ def extract_metadata(wheel: Path) -> email.Message: + + return email.message_from_string(metadata_file_contents) + ++ ++def remove_test_files(wheel: Path) -> None: ++ with tempfile.TemporaryDirectory() as tmpdir: ++ temp_wheel = Path(tmpdir) / wheel.name ++ ++ # Copy files except those excluded ++ with ZipFile(wheel, "r") as zin, ZipFile(temp_wheel, "w") as zout: ++ for item in zin.infolist(): ++ if not is_excluded_from_wheel(item.filename): ++ zout.writestr(item, zin.read(item.filename)) ++ ++ shutil.move(str(temp_wheel), str(wheel)) ++ ++ ++ ++def is_excluded_from_wheel(path: str) -> bool: ++ """ ++ These files are excluded from the wheel in the agent build: ++ https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb ++ In order to have more accurate results, this files are excluded when computing the size of the dependencies while ++ the wheels still include them. ++ """ ++ excluded_test_paths = [ ++ os.path.normpath(path) ++ for path in [ ++ "idlelib/idle_test", ++ "bs4/tests", ++ "Cryptodome/SelfTest", ++ "gssapi/tests", ++ "keystoneauth1/tests", ++ "openstack/tests", ++ "os_service_types/tests", ++ "pbr/tests", ++ "pkg_resources/tests", ++ "psutil/tests", ++ "securesystemslib/_vendor/ed25519/test_data", ++ "setuptools/_distutils/tests", ++ "setuptools/tests", ++ "simplejson/tests", ++ "stevedore/tests", ++ "supervisor/tests", ++ "test", # cm-client ++ "vertica_python/tests", ++ "websocket/tests", ++ "win32com/test" ++ ] ++ ] ++ ++ type_annot_libraries = [ ++ "krb5", ++ "Cryptodome", ++ "ddtrace", ++ "pyVmomi", ++ "gssapi", ++ ] ++ rel_path = Path(path).as_posix() ++ ++ # Test folders ++ for test_folder in excluded_test_paths: ++ if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): ++ return True ++ ++ # Python type annotations ++ path_parts = Path(rel_path).parts ++ if path_parts: ++ dependency_name = path_parts[0] ++ if dependency_name in type_annot_libraries: ++ if path.endswith(".pyi") or os.path.basename(path) == "py.typed": ++ return True ++ ++ return False From 0672b2e6a86d334010900560c712cd0cab64d70e Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 2 Jul 2025 16:04:22 +0200 Subject: [PATCH 06/63] remove imports for patch --- .builders/scripts/build_wheels.py | 3 +-- .builders/scripts/utils.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 09c9bcc0d3ad7..0c76a93f4820f 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -4,12 +4,11 @@ import os import subprocess import sys -import zipfile from pathlib import Path from tempfile import TemporaryDirectory from dotenv import dotenv_values -from utils import extract_metadata, normalize_project_name, remove_test_files +from utils import extract_metadata, normalize_project_name INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index fbcdcb590b67f..63772a277df46 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -1,10 +1,8 @@ from __future__ import annotations import email -import os import re from pathlib import Path - from zipfile import ZipFile UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') From 2c63f2c21869364a1da05dd7a3b2724c83ab1963 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 2 Jul 2025 16:14:15 +0200 Subject: [PATCH 07/63] patch --- .builders/patches/remove_tests.patch | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 6b8a2792c334e..ee7daa1e1c693 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,8 +1,17 @@ diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py -index 09c9bcc0d3..983998011e 100644 +index 0c76a93f48..93278ebe1a 100644 --- a/scripts/build_wheels.py --- b/scripts/build_wheels.py -@@ -120,6 +120,10 @@ def main(): +@@ -8,7 +8,7 @@ from pathlib import Path + from tempfile import TemporaryDirectory + + from dotenv import dotenv_values +-from utils import extract_metadata, normalize_project_name ++from utils import extract_metadata, normalize_project_name, remove_test_files + + INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' + CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' +@@ -119,6 +119,10 @@ def main(): check_process(command_args, env=env_vars) @@ -14,20 +23,21 @@ index 09c9bcc0d3..983998011e 100644 # Repair wheels check_process([ diff --git a/scripts/utils.py b/scripts/utils.py -index fbcdcb590b..ce3fdbf7ec 100644 +index 63772a277d..1a140b0d7b 100644 --- a/scripts/utils.py --- b/scripts/utils.py -@@ -4,7 +4,8 @@ import email - import os +@@ -1,7 +1,10 @@ + from __future__ import annotations + + import email ++import os import re - from pathlib import Path -- +import shutil +import tempfile + from pathlib import Path from zipfile import ZipFile - UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') -@@ -35,3 +36,74 @@ def extract_metadata(wheel: Path) -> email.Message: +@@ -33,3 +36,74 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) From 9cd4abeb4feef05be3674a18eff5f353a3062df5 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 09:10:04 +0200 Subject: [PATCH 08/63] print in patch --- .builders/patches/remove_tests.patch | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index ee7daa1e1c693..a9fd6958f1bd4 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ -diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py +diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0c76a93f48..93278ebe1a 100644 --- a/scripts/build_wheels.py ---- b/scripts/build_wheels.py ++++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -23,7 +23,7 @@ index 0c76a93f48..93278ebe1a 100644 # Repair wheels check_process([ diff --git a/scripts/utils.py b/scripts/utils.py -index 63772a277d..1a140b0d7b 100644 +index 63772a277d..32dfdd3f72 100644 --- a/scripts/utils.py --- b/scripts/utils.py @@ -1,7 +1,10 @@ @@ -37,7 +37,7 @@ index 63772a277d..1a140b0d7b 100644 from pathlib import Path from zipfile import ZipFile -@@ -33,3 +36,74 @@ def extract_metadata(wheel: Path) -> email.Message: +@@ -33,3 +36,77 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) @@ -45,14 +45,17 @@ index 63772a277d..1a140b0d7b 100644 +def remove_test_files(wheel: Path) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + temp_wheel = Path(tmpdir) / wheel.name -+ ++ count = 0 + # Copy files except those excluded + with ZipFile(wheel, "r") as zin, ZipFile(temp_wheel, "w") as zout: + for item in zin.infolist(): + if not is_excluded_from_wheel(item.filename): + zout.writestr(item, zin.read(item.filename)) -+ ++ else: ++ count += 1 + shutil.move(str(temp_wheel), str(wheel)) ++ if count > 0: ++ print(f"Removed {count} test files from {wheel.name}") + + + From fbad066cdd3ec5a021334bcac035e4daf368d983 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 10:03:59 +0200 Subject: [PATCH 09/63] fix typo --- .builders/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/build.py b/.builders/build.py index 133b02f109832..de3b512c8270b 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -216,7 +216,7 @@ def build_image(): # Apply the remove_tests patch test_patch = mount_dir / 'patches' / 'remove_tests.patch' - check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir / 'scripts') + check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) # Create outputs on the host so they can be removed wheels_dir = mount_dir / 'wheels' From e11957c07805adcbf2fb832870ec806e87ccdf97 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 12:27:52 +0200 Subject: [PATCH 10/63] repack wheels --- .builders/patches/remove_tests.patch | 90 ++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index a9fd6958f1bd4..8b45fc84283f9 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0c76a93f48..93278ebe1a 100644 ---- a/scripts/build_wheels.py -+++ b/scripts/build_wheels.py +index 0c76a93f48..13d46e52e0 100644 +--- a/.builders/scripts/build_wheels.py ++++ b/.builders/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -11,22 +11,37 @@ index 0c76a93f48..93278ebe1a 100644 INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' -@@ -119,6 +119,10 @@ def main(): +@@ -107,6 +107,8 @@ def main(): + if constraints_file := env_vars.get('PIP_CONSTRAINT'): + env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) + ++ ++ + # Fetch or build wheels + command_args = [ + str(python_path), '-m', 'pip', 'wheel', +@@ -114,11 +116,16 @@ def main(): + '--wheel-dir', str(staged_wheel_dir), + '--extra-index-url', CUSTOM_EXTERNAL_INDEX, + ] ++ ++ + if args.use_built_index: + command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) check_process(command_args, env=env_vars) + # Remove test files + for wheel_file in staged_wheel_dir.glob("*.whl"): + remove_test_files(wheel_file) -+ # Repair wheels check_process([ -diff --git a/scripts/utils.py b/scripts/utils.py -index 63772a277d..32dfdd3f72 100644 ---- a/scripts/utils.py ---- b/scripts/utils.py -@@ -1,7 +1,10 @@ +diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py +index 63772a277d..54178e25eb 100644 +--- a/.builders/scripts/utils.py ++++ b/.builders/scripts/utils.py +@@ -1,10 +1,16 @@ from __future__ import annotations import email @@ -37,25 +52,50 @@ index 63772a277d..32dfdd3f72 100644 from pathlib import Path from zipfile import ZipFile -@@ -33,3 +36,77 @@ def extract_metadata(wheel: Path) -> email.Message: ++from wheel.cli.pack import pack ++from wheel.cli.unpack import unpack ++ + UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') + + +@@ -33,3 +39,96 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) + -+def remove_test_files(wheel: Path) -> None: -+ with tempfile.TemporaryDirectory() as tmpdir: -+ temp_wheel = Path(tmpdir) / wheel.name -+ count = 0 -+ # Copy files except those excluded -+ with ZipFile(wheel, "r") as zin, ZipFile(temp_wheel, "w") as zout: -+ for item in zin.infolist(): -+ if not is_excluded_from_wheel(item.filename): -+ zout.writestr(item, zin.read(item.filename)) -+ else: -+ count += 1 -+ shutil.move(str(temp_wheel), str(wheel)) -+ if count > 0: -+ print(f"Removed {count} test files from {wheel.name}") ++def remove_test_files(wheel_path: Path) -> None: ++ """ ++ Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. ++ """ ++ removed = False ++ with tempfile.TemporaryDirectory() as td: ++ td_path = Path(td) ++ ++ # Unpack the wheel into temp dir ++ unpack(wheel_path, dest=td_path) ++ unpacked_dir = next(td_path.iterdir()) ++ ++ # Remove excluded files/folders ++ for root, dirs, files in os.walk(td, topdown=False): ++ for d in list(dirs): ++ ++ full_dir = Path(root) / d ++ rel = full_dir.relative_to(td_path).as_posix() ++ if is_excluded_from_wheel(rel): ++ shutil.rmtree(full_dir) ++ removed = True ++ dirs.remove(d) ++ for f in files: ++ rel = Path(root).joinpath(f).relative_to(td_path).as_posix() ++ if is_excluded_from_wheel(rel): ++ os.remove(Path(root) / f) ++ removed = True ++ if removed: ++ print(f"Tests removed from {wheel_path.name}") ++ ++ # Repack to same directory, regenerating RECORD ++ pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) ++ + + + From cd33605e6b8aa5698da03b8fa93018ac29953e1b Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 12:54:10 +0200 Subject: [PATCH 11/63] fix --- .builders/patches/remove_tests.patch | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 8b45fc84283f9..da376051ba6ad 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0c76a93f48..13d46e52e0 100644 ---- a/.builders/scripts/build_wheels.py -+++ b/.builders/scripts/build_wheels.py +--- a/scripts/build_wheels.py ++++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -39,8 +39,8 @@ index 0c76a93f48..13d46e52e0 100644 check_process([ diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 63772a277d..54178e25eb 100644 ---- a/.builders/scripts/utils.py -+++ b/.builders/scripts/utils.py +--- a/scripts/utils.py ++++ b/scripts/utils.py @@ -1,10 +1,16 @@ from __future__ import annotations @@ -80,13 +80,13 @@ index 63772a277d..54178e25eb 100644 + for d in list(dirs): + + full_dir = Path(root) / d -+ rel = full_dir.relative_to(td_path).as_posix() ++ rel = full_dir.relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + shutil.rmtree(full_dir) + removed = True + dirs.remove(d) + for f in files: -+ rel = Path(root).joinpath(f).relative_to(td_path).as_posix() ++ rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + os.remove(Path(root) / f) + removed = True From 3681ff65caf3c01400a9d0d2b78a4469041da937 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 14:25:08 +0200 Subject: [PATCH 12/63] add wheel library --- .builders/images/runner_dependencies.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.builders/images/runner_dependencies.txt b/.builders/images/runner_dependencies.txt index 3c3b0f9b9b352..d15c3893dfbfb 100644 --- a/.builders/images/runner_dependencies.txt +++ b/.builders/images/runner_dependencies.txt @@ -3,3 +3,4 @@ urllib3==2.2.0 auditwheel==6.0.0; sys_platform == 'linux' delvewheel==1.5.2; sys_platform == 'win32' delocate==0.13.0; sys_platform == 'darwin' +wheel==0.45.1 From ecc2fd9a378da64004248b5b9df19698b6a11664 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 14:44:28 +0200 Subject: [PATCH 13/63] comment patch --- .builders/build.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builders/build.py b/.builders/build.py index de3b512c8270b..c8c8a478ecde7 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -108,9 +108,9 @@ def build_macos(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') - # Apply the remove_tests patch - test_patch = mount_dir / 'patches' / 'remove_tests.patch' - check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) + # # Apply the remove_tests patch + # test_patch = mount_dir / 'patches' / 'remove_tests.patch' + # check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) prefix_path = builder_root / 'prefix' env = { @@ -214,9 +214,9 @@ def build_image(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') - # Apply the remove_tests patch - test_patch = mount_dir / 'patches' / 'remove_tests.patch' - check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) + # # Apply the remove_tests patch + # test_patch = mount_dir / 'patches' / 'remove_tests.patch' + # check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) # Create outputs on the host so they can be removed wheels_dir = mount_dir / 'wheels' From 34136f986341f911731684e10101eca5987d4612 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 3 Jul 2025 16:56:17 +0200 Subject: [PATCH 14/63] after repair --- .builders/patches/remove_tests.patch | 29 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index da376051ba6ad..1446edcbb642b 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0c76a93f48..13d46e52e0 100644 ---- a/scripts/build_wheels.py -+++ b/scripts/build_wheels.py +index 0c76a93f48..61fd144e9c 100644 +--- a/.builders/scripts/build_wheels.py ++++ b/.builders/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -20,7 +20,7 @@ index 0c76a93f48..13d46e52e0 100644 # Fetch or build wheels command_args = [ str(python_path), '-m', 'pip', 'wheel', -@@ -114,11 +116,16 @@ def main(): +@@ -114,11 +116,14 @@ def main(): '--wheel-dir', str(staged_wheel_dir), '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] @@ -31,16 +31,25 @@ index 0c76a93f48..13d46e52e0 100644 check_process(command_args, env=env_vars) -+ # Remove test files -+ for wheel_file in staged_wheel_dir.glob("*.whl"): -+ remove_test_files(wheel_file) ++ # Repair wheels check_process([ +@@ -128,6 +133,10 @@ def main(): + '--external-dir', str(external_wheels_dir), + ]) + ++ # Remove test files ++ for wheel_file in staged_wheel_dir.glob("*.whl"): ++ remove_test_files(wheel_file) ++ + dependencies: dict[str, tuple[str, str]] = {} + for wheel_dir in wheels_dir.iterdir(): + for entry in wheel_dir.iterdir(): diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py -index 63772a277d..54178e25eb 100644 ---- a/scripts/utils.py -+++ b/scripts/utils.py +index 63772a277d..f763b08f24 100644 +--- a/.builders/scripts/utils.py ++++ b/.builders/scripts/utils.py @@ -1,10 +1,16 @@ from __future__ import annotations From b6e8713f0a89ad2c89c4211741b7ff7a3aa79198 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 4 Jul 2025 09:24:17 +0200 Subject: [PATCH 15/63] patch --- .builders/patches/remove_tests.patch | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 1446edcbb642b..519030391c64c 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0c76a93f48..61fd144e9c 100644 ---- a/.builders/scripts/build_wheels.py -+++ b/.builders/scripts/build_wheels.py +index 0c76a93f48..83fa5f8ca7 100644 +--- a/scripts/build_wheels.py ++++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -35,21 +35,18 @@ index 0c76a93f48..61fd144e9c 100644 # Repair wheels check_process([ -@@ -128,6 +133,10 @@ def main(): - '--external-dir', str(external_wheels_dir), - ]) - -+ # Remove test files -+ for wheel_file in staged_wheel_dir.glob("*.whl"): -+ remove_test_files(wheel_file) -+ +@@ -131,6 +136,7 @@ def main(): dependencies: dict[str, tuple[str, str]] = {} for wheel_dir in wheels_dir.iterdir(): for entry in wheel_dir.iterdir(): -diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py ++ remove_test_files(entry) + project_metadata = extract_metadata(entry) + project_name = normalize_project_name(project_metadata['Name']) + project_version = project_metadata['Version'] +diff --git a/scripts/utils.py b/scripts/utils.py index 63772a277d..f763b08f24 100644 ---- a/.builders/scripts/utils.py -+++ b/.builders/scripts/utils.py +--- a/scripts/utils.py +--- b/scripts/utils.py @@ -1,10 +1,16 @@ from __future__ import annotations From 40dfd38fefc62a77d5dcd29a99817204b84332f2 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 4 Jul 2025 09:25:38 +0200 Subject: [PATCH 16/63] uncomment patch call --- .builders/build.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builders/build.py b/.builders/build.py index c8c8a478ecde7..de3b512c8270b 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -108,9 +108,9 @@ def build_macos(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') - # # Apply the remove_tests patch - # test_patch = mount_dir / 'patches' / 'remove_tests.patch' - # check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) + # Apply the remove_tests patch + test_patch = mount_dir / 'patches' / 'remove_tests.patch' + check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) prefix_path = builder_root / 'prefix' env = { @@ -214,9 +214,9 @@ def build_image(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') - # # Apply the remove_tests patch - # test_patch = mount_dir / 'patches' / 'remove_tests.patch' - # check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) + # Apply the remove_tests patch + test_patch = mount_dir / 'patches' / 'remove_tests.patch' + check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) # Create outputs on the host so they can be removed wheels_dir = mount_dir / 'wheels' From 989dcd2a217c5f4ada07f656ffe28198c6d31bce Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 8 Jul 2025 14:36:27 +0200 Subject: [PATCH 17/63] skip unchanged --- .builders/patches/remove_tests.patch | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 519030391c64c..57fd27a134fc8 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ -diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py +diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py index 0c76a93f48..83fa5f8ca7 100644 --- a/scripts/build_wheels.py -+++ b/scripts/build_wheels.py +--- b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -44,10 +44,10 @@ index 0c76a93f48..83fa5f8ca7 100644 project_name = normalize_project_name(project_metadata['Name']) project_version = project_metadata['Version'] diff --git a/scripts/utils.py b/scripts/utils.py -index 63772a277d..f763b08f24 100644 +index 63772a277d..278cfee068 100644 --- a/scripts/utils.py --- b/scripts/utils.py -@@ -1,10 +1,16 @@ +@@ -1,10 +1,17 @@ from __future__ import annotations import email @@ -55,6 +55,7 @@ index 63772a277d..f763b08f24 100644 import re +import shutil +import tempfile ++import zipfile from pathlib import Path from zipfile import ZipFile @@ -64,7 +65,7 @@ index 63772a277d..f763b08f24 100644 UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') -@@ -33,3 +39,96 @@ def extract_metadata(wheel: Path) -> email.Message: +@@ -33,3 +40,103 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) @@ -73,6 +74,13 @@ index 63772a277d..f763b08f24 100644 + """ + Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. + """ ++ # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. ++ with zipfile.ZipFile(wheel_path, "r") as zf: ++ excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] ++ ++ if not excluded_members: ++ # Nothing to strip, so skip rewriting the wheel ++ return + removed = False + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) From 8508bb22fa5750eb2f6088d73c92ccf6db82d1c8 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 8 Jul 2025 14:44:11 +0200 Subject: [PATCH 18/63] skip unchanged --- .builders/patches/remove_tests.patch | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 57fd27a134fc8..bfe10992e3309 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ -diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py -index 0c76a93f48..83fa5f8ca7 100644 ---- a/scripts/build_wheels.py ---- b/scripts/build_wheels.py +diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py +index 0c76a93f48..0198d2241b 100644 +--- a/.builders/scripts/build_wheels.py ++++ b/.builders/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -20,7 +20,7 @@ index 0c76a93f48..83fa5f8ca7 100644 # Fetch or build wheels command_args = [ str(python_path), '-m', 'pip', 'wheel', -@@ -114,11 +116,14 @@ def main(): +@@ -114,11 +116,15 @@ def main(): '--wheel-dir', str(staged_wheel_dir), '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] @@ -31,22 +31,15 @@ index 0c76a93f48..83fa5f8ca7 100644 check_process(command_args, env=env_vars) -+ ++ for entry in staged_wheel_dir.iterdir(): ++ remove_test_files(entry) # Repair wheels check_process([ -@@ -131,6 +136,7 @@ def main(): - dependencies: dict[str, tuple[str, str]] = {} - for wheel_dir in wheels_dir.iterdir(): - for entry in wheel_dir.iterdir(): -+ remove_test_files(entry) - project_metadata = extract_metadata(entry) - project_name = normalize_project_name(project_metadata['Name']) - project_version = project_metadata['Version'] -diff --git a/scripts/utils.py b/scripts/utils.py +diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 63772a277d..278cfee068 100644 ---- a/scripts/utils.py ---- b/scripts/utils.py +--- a/.builders/scripts/utils.py ++++ b/.builders/scripts/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations From c8cdc85f5693511a9d91264d7ae52f2f24ff9e0d Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 8 Jul 2025 14:45:10 +0200 Subject: [PATCH 19/63] skip unchanged --- .builders/patches/remove_tests.patch | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index bfe10992e3309..897fa726189fa 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ -diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py +diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py index 0c76a93f48..0198d2241b 100644 ---- a/.builders/scripts/build_wheels.py -+++ b/.builders/scripts/build_wheels.py +--- a/scripts/build_wheels.py +--- b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -36,10 +36,10 @@ index 0c76a93f48..0198d2241b 100644 # Repair wheels check_process([ -diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py +diff --git a/scripts/utils.py b/scripts/utils.py index 63772a277d..278cfee068 100644 ---- a/.builders/scripts/utils.py -+++ b/.builders/scripts/utils.py +--- a/scripts/utils.py +--- b/scripts/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations From 623c0b4c1dca3762b755df613e229b4b65669854 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 8 Jul 2025 15:49:56 +0200 Subject: [PATCH 20/63] debug --- .builders/scripts/repair_wheels.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py index bae851cc2d895..391fad8c38d73 100644 --- a/.builders/scripts/repair_wheels.py +++ b/.builders/scripts/repair_wheels.py @@ -186,6 +186,21 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None: def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None: import subprocess + # If the wheel is ddtrace, print all files in that wheel + import zipfile + + def is_ddtrace_wheel(wheel): + # wheel can be a Path or str + name = str(wheel) + return 'ddtrace' in name + + for wheel in iter_wheels(source_dir): + if is_ddtrace_wheel(wheel): + print(f"All files in wheel: {wheel}") + with zipfile.ZipFile(wheel, 'r') as zf: + for file in zf.namelist(): + print(file) + exclusions = ['mqic.dll'] external_invalid_file_patterns = [ From 343a13a27f10392da2fe34fbc38b5cc402532414 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 9 Jul 2025 10:25:11 +0200 Subject: [PATCH 21/63] debug --- .builders/patches/remove_tests.patch | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 897fa726189fa..bb31ededa22ce 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ -diff --git a/scripts/build_wheels.py b/scripts/build_wheels.py +diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0c76a93f48..0198d2241b 100644 ---- a/scripts/build_wheels.py ---- b/scripts/build_wheels.py +--- a/.builders/scripts/build_wheels.py ++++ b/.builders/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -36,10 +36,10 @@ index 0c76a93f48..0198d2241b 100644 # Repair wheels check_process([ -diff --git a/scripts/utils.py b/scripts/utils.py -index 63772a277d..278cfee068 100644 ---- a/scripts/utils.py ---- b/scripts/utils.py +diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py +index 63772a277d..e1f680728f 100644 +--- a/.builders/scripts/utils.py ++++ b/.builders/scripts/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations @@ -58,7 +58,7 @@ index 63772a277d..278cfee068 100644 UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') -@@ -33,3 +40,103 @@ def extract_metadata(wheel: Path) -> email.Message: +@@ -33,3 +40,113 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) @@ -74,6 +74,13 @@ index 63772a277d..278cfee068 100644 + if not excluded_members: + # Nothing to strip, so skip rewriting the wheel + return ++ print(f"[FILES] Before removing tests: {wheel_path}") ++ if "ddtrace" in wheel_path.name: ++ print(f"All files in wheel: {wheel_path}") ++ with zipfile.ZipFile(wheel_path, 'r') as zf: ++ for file in zf.namelist(): ++ print(file) ++ + removed = False + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) @@ -103,7 +110,10 @@ index 63772a277d..278cfee068 100644 + # Repack to same directory, regenerating RECORD + pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) + -+ ++ print(f"[FILES] After removing tests: {wheel_path}") ++ with zipfile.ZipFile(wheel_path, 'r') as zf: ++ for file in zf.namelist(): ++ print(file) + + +def is_excluded_from_wheel(path: str) -> bool: From 925ed4b9099e9f1220b3d2bb3f221cb5451637a8 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 9 Jul 2025 10:28:19 +0200 Subject: [PATCH 22/63] debug --- .builders/patches/remove_tests.patch | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index bb31ededa22ce..dcc6b35869bc9 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0c76a93f48..0198d2241b 100644 ---- a/.builders/scripts/build_wheels.py -+++ b/.builders/scripts/build_wheels.py +--- a/scripts/build_wheels.py ++++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -38,8 +38,8 @@ index 0c76a93f48..0198d2241b 100644 check_process([ diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 63772a277d..e1f680728f 100644 ---- a/.builders/scripts/utils.py -+++ b/.builders/scripts/utils.py +--- a/scripts/utils.py +--- b/scripts/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations From a711b96990f6270700edd9df42bd9c528b313796 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 9 Jul 2025 15:15:13 +0200 Subject: [PATCH 23/63] after repair --- .builders/patches/remove_tests.patch | 59 +++++++--------------------- .builders/scripts/repair_wheels.py | 15 ------- .builders/scripts/utils.py | 1 - 3 files changed, 15 insertions(+), 60 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index dcc6b35869bc9..10205c8ba56ba 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0c76a93f48..0198d2241b 100644 ---- a/scripts/build_wheels.py -+++ b/scripts/build_wheels.py +index 0c76a93f48..7c6b86e553 100644 +--- a/.builders/scripts/build_wheels.py ++++ b/.builders/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -11,35 +11,18 @@ index 0c76a93f48..0198d2241b 100644 INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' -@@ -107,6 +107,8 @@ def main(): - if constraints_file := env_vars.get('PIP_CONSTRAINT'): - env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) - -+ -+ - # Fetch or build wheels - command_args = [ - str(python_path), '-m', 'pip', 'wheel', -@@ -114,11 +116,15 @@ def main(): - '--wheel-dir', str(staged_wheel_dir), - '--extra-index-url', CUSTOM_EXTERNAL_INDEX, - ] -+ -+ - if args.use_built_index: - command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) - - check_process(command_args, env=env_vars) - -+ for entry in staged_wheel_dir.iterdir(): +@@ -131,6 +131,7 @@ def main(): + dependencies: dict[str, tuple[str, str]] = {} + for wheel_dir in wheels_dir.iterdir(): + for entry in wheel_dir.iterdir(): + remove_test_files(entry) - - # Repair wheels - check_process([ + project_metadata = extract_metadata(entry) + project_name = normalize_project_name(project_metadata['Name']) + project_version = project_metadata['Version'] diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py -index 63772a277d..e1f680728f 100644 ---- a/scripts/utils.py ---- b/scripts/utils.py +index 63772a277d..310a04c798 100644 +--- a/.builders/scripts/utils.py ++++ b/.builders/scripts/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations @@ -58,7 +41,7 @@ index 63772a277d..e1f680728f 100644 UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') -@@ -33,3 +40,113 @@ def extract_metadata(wheel: Path) -> email.Message: +@@ -33,3 +40,101 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) @@ -74,13 +57,7 @@ index 63772a277d..e1f680728f 100644 + if not excluded_members: + # Nothing to strip, so skip rewriting the wheel + return -+ print(f"[FILES] Before removing tests: {wheel_path}") -+ if "ddtrace" in wheel_path.name: -+ print(f"All files in wheel: {wheel_path}") -+ with zipfile.ZipFile(wheel_path, 'r') as zf: -+ for file in zf.namelist(): -+ print(file) -+ ++ + removed = False + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) @@ -92,7 +69,6 @@ index 63772a277d..e1f680728f 100644 + # Remove excluded files/folders + for root, dirs, files in os.walk(td, topdown=False): + for d in list(dirs): -+ + full_dir = Path(root) / d + rel = full_dir.relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): @@ -110,11 +86,6 @@ index 63772a277d..e1f680728f 100644 + # Repack to same directory, regenerating RECORD + pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) + -+ print(f"[FILES] After removing tests: {wheel_path}") -+ with zipfile.ZipFile(wheel_path, 'r') as zf: -+ for file in zf.namelist(): -+ print(file) -+ + +def is_excluded_from_wheel(path: str) -> bool: + """ diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py index 391fad8c38d73..bae851cc2d895 100644 --- a/.builders/scripts/repair_wheels.py +++ b/.builders/scripts/repair_wheels.py @@ -186,21 +186,6 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None: def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None: import subprocess - # If the wheel is ddtrace, print all files in that wheel - import zipfile - - def is_ddtrace_wheel(wheel): - # wheel can be a Path or str - name = str(wheel) - return 'ddtrace' in name - - for wheel in iter_wheels(source_dir): - if is_ddtrace_wheel(wheel): - print(f"All files in wheel: {wheel}") - with zipfile.ZipFile(wheel, 'r') as zf: - for file in zf.namelist(): - print(file) - exclusions = ['mqic.dll'] external_invalid_file_patterns = [ diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 63772a277df46..0750f1ba8237b 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -32,4 +32,3 @@ def extract_metadata(wheel: Path) -> email.Message: raise RuntimeError(message) from None return email.message_from_string(metadata_file_contents) - From e8891c43d51018b74705e8fa3b97d2c1d580f89d Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 9 Jul 2025 15:15:49 +0200 Subject: [PATCH 24/63] after repair --- .builders/patches/remove_tests.patch | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 10205c8ba56ba..7148ccac2bc45 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0c76a93f48..7c6b86e553 100644 ---- a/.builders/scripts/build_wheels.py -+++ b/.builders/scripts/build_wheels.py +--- a/scripts/build_wheels.py ++++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -21,8 +21,8 @@ index 0c76a93f48..7c6b86e553 100644 project_version = project_metadata['Version'] diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 63772a277d..310a04c798 100644 ---- a/.builders/scripts/utils.py -+++ b/.builders/scripts/utils.py +--- a/scripts/utils.py ++++ b/scripts/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations From 4e9bdc8f238b683e2f232c504fff1cdbcd23fd57 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 9 Jul 2025 16:12:27 +0200 Subject: [PATCH 25/63] fix --- .builders/scripts/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 0750f1ba8237b..63772a277df46 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -32,3 +32,4 @@ def extract_metadata(wheel: Path) -> email.Message: raise RuntimeError(message) from None return email.message_from_string(metadata_file_contents) + From fd2b5071f126ce953506da2c2365659b5f328125 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 11 Jul 2025 11:11:37 +0200 Subject: [PATCH 26/63] classify wheels --- .builders/scripts/build_wheels.py | 142 ++++++++++++++++----------- .builders/scripts/classify_wheels.py | 74 ++++++++++++++ .builders/scripts/repair_wheels.py | 124 +++++++---------------- .builders/scripts/utils.py | 8 +- 4 files changed, 205 insertions(+), 143 deletions(-) create mode 100644 .builders/scripts/classify_wheels.py diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0c76a93f4820f..0716dcb325d21 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -8,37 +8,37 @@ from tempfile import TemporaryDirectory from dotenv import dotenv_values -from utils import extract_metadata, normalize_project_name +from utils import extract_metadata, iter_wheels, normalize_project_name -INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' -CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' -CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built' +INDEX_BASE_URL = "https://agent-int-packages.datadoghq.com" +CUSTOM_EXTERNAL_INDEX = f"{INDEX_BASE_URL}/external" +CUSTOM_BUILT_INDEX = f"{INDEX_BASE_URL}/built" -if sys.platform == 'win32': - PY3_PATH = Path('C:\\py3\\Scripts\\python.exe') - PY2_PATH = Path('C:\\py2\\Scripts\\python.exe') - MOUNT_DIR = Path('C:\\mnt') - ENV_FILE = Path('C:\\.env') +if sys.platform == "win32": + PY3_PATH = Path("C:\\py3\\Scripts\\python.exe") + PY2_PATH = Path("C:\\py2\\Scripts\\python.exe") + MOUNT_DIR = Path("C:\\mnt") + ENV_FILE = Path("C:\\.env") def join_command_args(args: list[str]) -> str: return subprocess.list2cmdline(args) def path_to_uri(path: str) -> str: - return f'file:///{os.path.abspath(path).replace(" ", "%20").replace(os.sep, "/")}' + return f"file:///{os.path.abspath(path).replace(' ', '%20').replace(os.sep, '/')}" else: import shlex - PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python')) - PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python')) - MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home')) - ENV_FILE = Path(os.environ.get('DD_ENV_FILE', '/.env')) + PY3_PATH = Path(os.environ.get("DD_PY3_BUILDENV_PATH", "/py3/bin/python")) + PY2_PATH = Path(os.environ.get("DD_PY2_BUILDENV_PATH", "/py2/bin/python")) + MOUNT_DIR = Path(os.environ.get("DD_MOUNT_DIR", "/home")) + ENV_FILE = Path(os.environ.get("DD_ENV_FILE", "/.env")) def join_command_args(args: list[str]) -> str: return shlex.join(args) def path_to_uri(path: str) -> str: - return f'file://{os.path.abspath(path).replace(" ", "%20")}' + return f"file://{os.path.abspath(path).replace(' ', '%20')}" def abort(message, *, code=1): @@ -47,7 +47,7 @@ def abort(message, *, code=1): def check_process(*args, **kwargs) -> subprocess.CompletedProcess: - print(f'Running: {args[0] if isinstance(args[0], str) else join_command_args(args[0])}', file=sys.stderr) + print(f"Running: {args[0] if isinstance(args[0], str) else join_command_args(args[0])}", file=sys.stderr) process = subprocess.run(*args, **kwargs) if process.returncode: sys.exit(process.returncode) @@ -56,44 +56,47 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: def main(): - parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False) - parser.add_argument('--python', required=True) - parser.add_argument('--use-built-index', action='store_true', default=False) + parser = argparse.ArgumentParser(prog="wheel-builder", allow_abbrev=False) + parser.add_argument("--python", required=True) + parser.add_argument("--use-built-index", action="store_true", default=False) args = parser.parse_args() python_version = args.python - if python_version == '3': + if python_version == "3": python_path = PY3_PATH - elif python_version == '2': + elif python_version == "2": python_path = PY2_PATH else: - abort(f'Invalid python version: {python_version}') + abort(f"Invalid python version: {python_version}") - wheels_dir = MOUNT_DIR / 'wheels' - built_wheels_dir = wheels_dir / 'built' - external_wheels_dir = wheels_dir / 'external' + wheels_dir = MOUNT_DIR / "wheels" + built_wheels_dir = wheels_dir / "built" + external_wheels_dir = wheels_dir / "external" # Install build dependencies - check_process([str(python_path), '-m', 'pip', 'install', '-r', str(MOUNT_DIR / 'build_dependencies.txt')]) + check_process([str(python_path), "-m", "pip", "install", "-r", str(MOUNT_DIR / "build_dependencies.txt")]) with TemporaryDirectory() as d: staged_wheel_dir = Path(d).resolve() + staged_built_wheels_dir = staged_wheel_dir / "built" + staged_external_wheels_dir = staged_wheel_dir / "external" + env_vars = dict(os.environ) - env_vars['PATH'] = f'{python_path.parent}{os.pathsep}{env_vars["PATH"]}' - env_vars['PIP_WHEEL_DIR'] = str(staged_wheel_dir) - env_vars['DD_BUILD_PYTHON_VERSION'] = python_version - env_vars['DD_MOUNT_DIR'] = str(MOUNT_DIR) - env_vars['DD_ENV_FILE'] = str(ENV_FILE) + env_vars["PATH"] = f"{python_path.parent}{os.pathsep}{env_vars['PATH']}" + env_vars["PIP_WHEEL_DIR"] = str(staged_wheel_dir) + env_vars["DD_BUILD_PYTHON_VERSION"] = python_version + env_vars["DD_MOUNT_DIR"] = str(MOUNT_DIR) + env_vars["DD_ENV_FILE"] = str(ENV_FILE) # Off is on, see: https://github.com/pypa/pip/issues/5735 - env_vars['PIP_NO_BUILD_ISOLATION'] = '0' + env_vars["PIP_NO_BUILD_ISOLATION"] = "0" # Spaces are used to separate multiple values which means paths themselves cannot contain spaces, see: # https://github.com/pypa/pip/issues/10114#issuecomment-1880125475 - env_vars['PIP_FIND_LINKS'] = path_to_uri(staged_wheel_dir) + env_vars["PIP_FIND_LINKS"] = path_to_uri(staged_wheel_dir) # Perform builder-specific logic if required - if build_command := os.environ.get('DD_BUILD_COMMAND'): + if build_command := os.environ.get("DD_BUILD_COMMAND"): check_process(build_command, env=env_vars, shell=True) # Load environment variables @@ -104,43 +107,72 @@ def main(): else: env_vars[key] = value - if constraints_file := env_vars.get('PIP_CONSTRAINT'): - env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) + if constraints_file := env_vars.get("PIP_CONSTRAINT"): + env_vars["PIP_CONSTRAINT"] = path_to_uri(constraints_file) # Fetch or build wheels command_args = [ - str(python_path), '-m', 'pip', 'wheel', - '-r', str(MOUNT_DIR / 'requirements.in'), - '--wheel-dir', str(staged_wheel_dir), - '--extra-index-url', CUSTOM_EXTERNAL_INDEX, + str(python_path), + "-m", + "pip", + "wheel", + "-r", + str(MOUNT_DIR / "requirements.in"), + "--wheel-dir", + str(staged_wheel_dir), + "--extra-index-url", + CUSTOM_EXTERNAL_INDEX, ] if args.use_built_index: - command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) + command_args.extend(["--extra-index-url", CUSTOM_BUILT_INDEX]) check_process(command_args, env=env_vars) + # Classify wheels + check_process( + [ + sys.executable, + "-u", + str(MOUNT_DIR / "scripts" / "classify_wheels.py"), + "--source-dir", + str(staged_wheel_dir), + "--built-dir", + str(staged_built_wheels_dir), + "--external-dir", + str(staged_external_wheels_dir), + ] + ) # Repair wheels - check_process([ - sys.executable, '-u', str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'), - '--source-dir', str(staged_wheel_dir), - '--built-dir', str(built_wheels_dir), - '--external-dir', str(external_wheels_dir), - ]) + check_process( + [ + sys.executable, + "-u", + str(MOUNT_DIR / "scripts" / "repair_wheels.py"), + "--source-built-dir", + str(staged_built_wheels_dir), + "--source-external-dir", + str(staged_external_wheels_dir), + "--built-dir", + str(built_wheels_dir), + "--external-dir", + str(external_wheels_dir), + ] + ) dependencies: dict[str, tuple[str, str]] = {} for wheel_dir in wheels_dir.iterdir(): - for entry in wheel_dir.iterdir(): - project_metadata = extract_metadata(entry) - project_name = normalize_project_name(project_metadata['Name']) - project_version = project_metadata['Version'] + for wheel in iter_wheels(wheel_dir): + project_metadata = extract_metadata(wheel) + project_name = normalize_project_name(project_metadata["Name"]) + project_version = project_metadata["Version"] dependencies[project_name] = project_version - final_requirements = MOUNT_DIR / 'frozen.txt' - with final_requirements.open('w', encoding='utf-8') as f: + final_requirements = MOUNT_DIR / "frozen.txt" + with final_requirements.open("w", encoding="utf-8") as f: for project_name, project_version in sorted(dependencies.items()): - f.write(f'{project_name}=={project_version}\n') + f.write(f"{project_name}=={project_version}\n") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/.builders/scripts/classify_wheels.py b/.builders/scripts/classify_wheels.py new file mode 100644 index 0000000000000..826b391de055c --- /dev/null +++ b/.builders/scripts/classify_wheels.py @@ -0,0 +1,74 @@ +import argparse +import shutil +import time +from functools import cache +from hashlib import sha256 +from pathlib import Path + +import urllib3 +from utils import extract_metadata, iter_wheels, normalize_project_name + + +@cache +def get_wheel_hashes(project) -> dict[str, str]: + retry_wait = 2 + while True: + try: + response = urllib3.request( + 'GET', + f'https://pypi.org/simple/{project}', + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + ) + except urllib3.exceptions.HTTPError as e: + err_msg = f'Failed to fetch hashes for `{project}`: {e}' + else: + if response.status == 200: + break + + err_msg = f'Failed to fetch hashes for `{project}`, status code: {response.status}' + + print(err_msg) + print(f'Retrying in {retry_wait} seconds') + time.sleep(retry_wait) + retry_wait *= 2 + continue + + data = response.json() + return { + file['filename']: file['hashes']['sha256'] + for file in data['files'] + if file['filename'].endswith('.whl') and 'sha256' in file['hashes'] + } + + +def wheel_was_built(wheel: Path) -> bool: + project_metadata = extract_metadata(wheel) + project_name = normalize_project_name(project_metadata['Name']) + wheel_hashes = get_wheel_hashes(project_name) + if wheel.name not in wheel_hashes: + return True + + file_hash = sha256(wheel.read_bytes()).hexdigest() + return file_hash != wheel_hashes[wheel.name] + + +def classify_wheels(source_dir: str, built_dir: str, external_dir: str) -> None: + for wheel in iter_wheels(source_dir): + if wheel_was_built(wheel): + shutil.move(wheel, built_dir) + else: + shutil.move(wheel, external_dir) + + +def main(): + parser = argparse.ArgumentParser("Classifies wheels into built and external depending on the hash of the wheel") + parser.add_argument('--source-dir', required=True) + parser.add_argument('--built-dir', required=True) + parser.add_argument('--external-dir', required=True) + args = parser.parse_args() + + classify_wheels(args.source_dir, args.built_dir, args.external_dir) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py index bae851cc2d895..564b430e61bd6 100644 --- a/.builders/scripts/repair_wheels.py +++ b/.builders/scripts/repair_wheels.py @@ -5,70 +5,17 @@ import re import shutil import sys -import time from fnmatch import fnmatch -from functools import cache -from hashlib import sha256 from pathlib import Path -from typing import Iterator, NamedTuple +from typing import NamedTuple from zipfile import ZipFile -import urllib3 -from utils import extract_metadata, normalize_project_name +from utils import iter_wheels # Packages for which we're skipping the openssl-3 build check OPENSSL_PACKAGE_BYPASS = ["psycopg"] -@cache -def get_wheel_hashes(project) -> dict[str, str]: - retry_wait = 2 - while True: - try: - response = urllib3.request( - 'GET', - f'https://pypi.org/simple/{project}', - headers={"Accept": "application/vnd.pypi.simple.v1+json"}, - ) - except urllib3.exceptions.HTTPError as e: - err_msg = f'Failed to fetch hashes for `{project}`: {e}' - else: - if response.status == 200: - break - - err_msg = f'Failed to fetch hashes for `{project}`, status code: {response.status}' - - print(err_msg) - print(f'Retrying in {retry_wait} seconds') - time.sleep(retry_wait) - retry_wait *= 2 - continue - - data = response.json() - return { - file['filename']: file['hashes']['sha256'] - for file in data['files'] - if file['filename'].endswith('.whl') and 'sha256' in file['hashes'] - } - - -def iter_wheels(source_dir: str) -> Iterator[Path]: - for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): - if entry.suffix == '.whl' and entry.is_file(): - yield entry - - -def wheel_was_built(wheel: Path) -> bool: - project_metadata = extract_metadata(wheel) - project_name = normalize_project_name(project_metadata['Name']) - wheel_hashes = get_wheel_hashes(project_name) - if wheel.name not in wheel_hashes: - return True - - file_hash = sha256(wheel.read_bytes()).hexdigest() - return file_hash != wheel_hashes[wheel.name] - - def find_patterns_in_wheel(wheel: Path, patterns: list[str]) -> list[str]: """Returns all found files inside `wheel` that match the given glob-style pattern""" with ZipFile(wheel) as zf: @@ -122,7 +69,7 @@ def check_unacceptable_files( sys.exit(1) -def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None: +def repair_linux(source_built_dir: str, source_external_dir: str, built_dir: str, external_dir: str) -> None: from auditwheel.patcher import Patchelf from auditwheel.policy import WheelPolicies from auditwheel.repair import repair_wheel @@ -148,20 +95,20 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None: policy['lib_whitelist'].remove('libz.so.1') del policy['symbol_versions']['ZLIB'] - for wheel in iter_wheels(source_dir): + for wheel in iter_wheels(source_external_dir): print(f'--> {wheel.name}') + print('Using existing wheel') - if not wheel_was_built(wheel): - print('Using existing wheel') - - check_unacceptable_files( - wheel, - bypass_prefixes=OPENSSL_PACKAGE_BYPASS, - invalid_file_patterns=external_invalid_file_patterns, - ) - shutil.move(wheel, external_dir) - continue + check_unacceptable_files( + wheel, + bypass_prefixes=OPENSSL_PACKAGE_BYPASS, + invalid_file_patterns=external_invalid_file_patterns, + ) + shutil.move(wheel, external_dir) + continue + for wheel in iter_wheels(source_built_dir): + print(f'--> {wheel.name}') try: repair_wheel( policies, @@ -183,7 +130,7 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None: print('Repaired wheel') -def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None: +def repair_windows(source_built_dir: str, source_external_dir: str, built_dir: str, external_dir: str) -> None: import subprocess exclusions = ['mqic.dll'] @@ -194,20 +141,20 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None: '*.libs/libcrypto-3*.dll', ] - for wheel in iter_wheels(source_dir): + for wheel in iter_wheels(source_external_dir): print(f'--> {wheel.name}') + print('Using existing wheel') - if not wheel_was_built(wheel): - print('Using existing wheel') - - check_unacceptable_files( - wheel, - bypass_prefixes=OPENSSL_PACKAGE_BYPASS, - invalid_file_patterns=external_invalid_file_patterns, - ) - shutil.move(wheel, external_dir) - continue + check_unacceptable_files( + wheel, + bypass_prefixes=OPENSSL_PACKAGE_BYPASS, + invalid_file_patterns=external_invalid_file_patterns, + ) + shutil.move(wheel, external_dir) + continue + for wheel in iter_wheels(source_built_dir): + print(f'--> {wheel.name}') # Platform independent wheels: move and rename to make platform specific wheel_name = WheelName.parse(wheel.name) if wheel_name.platform_tag == 'any': @@ -226,7 +173,7 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None: sys.exit(process.returncode) -def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None: +def repair_darwin(source_built_dir: str, source_external_dir: str, built_dir: str, external_dir: str) -> None: from delocate import delocate_wheel exclusions = [re.compile(s) for s in [ # pymqi @@ -262,14 +209,14 @@ def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None: def copy_filt_func(libname): return not any(excl.search(libname) for excl in exclusions) - for wheel in iter_wheels(source_dir): + for wheel in iter_wheels(source_external_dir): print(f'--> {wheel.name}') - if not wheel_was_built(wheel): - print('Using existing wheel') - - shutil.move(wheel, external_dir) - continue + print('Using existing wheel') + shutil.move(wheel, external_dir) + continue + for wheel in iter_wheels(source_built_dir): + print(f'--> {wheel.name}') # Platform independent wheels: move and rename to make platform specific wheel_name = WheelName.parse(wheel.name) if wheel_name.platform_tag == 'any': @@ -310,10 +257,13 @@ def main(): argparser.add_argument('--external-dir', required=True) args = argparser.parse_args() + source_built_dir = Path(args.source_dir) / 'built' + source_external_dir = Path(args.source_dir) / 'external' + print(f'Repairing wheels in: {args.source_dir}') print(f'Outputting built wheels to: {args.built_dir}') print(f'Outputting external wheels to: {args.external_dir}') - REPAIR_FUNCTIONS[sys.platform](args.source_dir, args.built_dir, args.external_dir) + REPAIR_FUNCTIONS[sys.platform](source_built_dir, source_external_dir, args.built_dir, args.external_dir) if __name__ == '__main__': diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 63772a277df46..b6358edd92191 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -4,10 +4,10 @@ import re from pathlib import Path from zipfile import ZipFile +from typing import Iterator UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') - def normalize_project_name(name: str) -> str: # https://peps.python.org/pep-0503/#normalized-names return UNNORMALIZED_PROJECT_NAME_CHARS.sub('-', name).lower() @@ -33,3 +33,9 @@ def extract_metadata(wheel: Path) -> email.Message: return email.message_from_string(metadata_file_contents) + + +def iter_wheels(source_dir: str) -> Iterator[Path]: + for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): + if entry.suffix == '.whl' and entry.is_file(): + yield entry \ No newline at end of file From 7b9041fcc4fb8788eb73cc8c24c329197cca47fd Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 11 Jul 2025 14:09:30 +0200 Subject: [PATCH 27/63] fix --- .builders/scripts/repair_wheels.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py index 564b430e61bd6..8c243d09e8bfc 100644 --- a/.builders/scripts/repair_wheels.py +++ b/.builders/scripts/repair_wheels.py @@ -252,18 +252,16 @@ def main(): argparser = argparse.ArgumentParser( description='Repair wheels found in a directory with the platform-specific tool' ) - argparser.add_argument('--source-dir', required=True) + argparser.add_argument('--source-built-dir', required=True) + argparser.add_argument('--source-external-dir', required=True) argparser.add_argument('--built-dir', required=True) argparser.add_argument('--external-dir', required=True) args = argparser.parse_args() - source_built_dir = Path(args.source_dir) / 'built' - source_external_dir = Path(args.source_dir) / 'external' - - print(f'Repairing wheels in: {args.source_dir}') + print(f'Repairing wheels in: {args.source_built_dir}') print(f'Outputting built wheels to: {args.built_dir}') print(f'Outputting external wheels to: {args.external_dir}') - REPAIR_FUNCTIONS[sys.platform](source_built_dir, source_external_dir, args.built_dir, args.external_dir) + REPAIR_FUNCTIONS[sys.platform](args.source_built_dir, args.source_external_dir, args.built_dir, args.external_dir) if __name__ == '__main__': From 242dc247a72374e22f39741bcd4b1df570e8be9b Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Mon, 14 Jul 2025 09:59:11 +0200 Subject: [PATCH 28/63] classify wheels --- .builders/scripts/build_wheels.py | 4 ++++ .builders/scripts/utils.py | 29 ++++++++++++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0716dcb325d21..0ae4035590305 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -81,6 +81,10 @@ def main(): staged_built_wheels_dir = staged_wheel_dir / "built" staged_external_wheels_dir = staged_wheel_dir / "external" + # Create the directories + staged_built_wheels_dir.mkdir(parents=True, exist_ok=True) + staged_external_wheels_dir.mkdir(parents=True, exist_ok=True) + env_vars = dict(os.environ) env_vars["PATH"] = f"{python_path.parent}{os.pathsep}{env_vars['PATH']}" env_vars["PIP_WHEEL_DIR"] = str(staged_wheel_dir) diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index b6358edd92191..112359127171c 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -1,34 +1,41 @@ from __future__ import annotations import email +import os import re +import shutil +import tempfile from pathlib import Path -from zipfile import ZipFile from typing import Iterator +from zipfile import ZipFile + +from wheel.cli.pack import pack +from wheel.cli.unpack import unpack + +UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r"[-_.]+") -UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') def normalize_project_name(name: str) -> str: # https://peps.python.org/pep-0503/#normalized-names - return UNNORMALIZED_PROJECT_NAME_CHARS.sub('-', name).lower() + return UNNORMALIZED_PROJECT_NAME_CHARS.sub("-", name).lower() def extract_metadata(wheel: Path) -> email.Message: with ZipFile(str(wheel)) as zip_archive: for path in zip_archive.namelist(): - root = path.split('/', 1)[0] - if root.endswith('.dist-info'): + root = path.split("/", 1)[0] + if root.endswith(".dist-info"): dist_info_dir = root break else: - message = f'Could not find the `.dist-info` directory in wheel: {wheel.name}' + message = f"Could not find the `.dist-info` directory in wheel: {wheel.name}" raise RuntimeError(message) try: - with zip_archive.open(f'{dist_info_dir}/METADATA') as zip_file: - metadata_file_contents = zip_file.read().decode('utf-8') + with zip_archive.open(f"{dist_info_dir}/METADATA") as zip_file: + metadata_file_contents = zip_file.read().decode("utf-8") except KeyError: - message = f'Could not find a `METADATA` file in the `{dist_info_dir}` directory' + message = f"Could not find a `METADATA` file in the `{dist_info_dir}` directory" raise RuntimeError(message) from None return email.message_from_string(metadata_file_contents) @@ -37,5 +44,5 @@ def extract_metadata(wheel: Path) -> email.Message: def iter_wheels(source_dir: str) -> Iterator[Path]: for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): - if entry.suffix == '.whl' and entry.is_file(): - yield entry \ No newline at end of file + if entry.suffix == ".whl" and entry.is_file(): + yield entry From a395b57363417bb65e8a57efadf3271b70106f63 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Mon, 14 Jul 2025 10:20:31 +0200 Subject: [PATCH 29/63] remove import --- .builders/scripts/utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 112359127171c..35d8ca138cf22 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -1,17 +1,11 @@ from __future__ import annotations import email -import os import re -import shutil -import tempfile from pathlib import Path from typing import Iterator from zipfile import ZipFile -from wheel.cli.pack import pack -from wheel.cli.unpack import unpack - UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r"[-_.]+") From 3d268c20d386ceeb1d67de6f9ceb8f2136aa9f54 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Mon, 14 Jul 2025 14:14:09 +0200 Subject: [PATCH 30/63] patch --- .builders/patches/remove_tests.patch | 110 +++++++++++++++++++-------- 1 file changed, 77 insertions(+), 33 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 7148ccac2bc45..d2055854a1836 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,29 +1,68 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0c76a93f48..7c6b86e553 100644 ---- a/scripts/build_wheels.py -+++ b/scripts/build_wheels.py +index 0ae4035590..6d38e1f197 100644 +--- a/.builders/scripts/build_wheels.py ++++ b/.builders/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory from dotenv import dotenv_values --from utils import extract_metadata, normalize_project_name -+from utils import extract_metadata, normalize_project_name, remove_test_files +-from utils import extract_metadata, iter_wheels, normalize_project_name ++from utils import extract_metadata, iter_wheels, normalize_project_name, remove_test_files + + INDEX_BASE_URL = "https://agent-int-packages.datadoghq.com" + CUSTOM_EXTERNAL_INDEX = f"{INDEX_BASE_URL}/external" +@@ -54,7 +54,12 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: + + return process + +- ++def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: ++ project_metadata = extract_metadata(wheel) ++ project_name = normalize_project_name(project_metadata["Name"]) ++ project_version = project_metadata["Version"] ++ dependencies[project_name] = project_version ++ + def main(): + parser = argparse.ArgumentParser(prog="wheel-builder", allow_abbrev=False) + parser.add_argument("--python", required=True) +@@ -165,17 +170,21 @@ def main(): + ) - INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' - CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' -@@ -131,6 +131,7 @@ def main(): dependencies: dict[str, tuple[str, str]] = {} - for wheel_dir in wheels_dir.iterdir(): - for entry in wheel_dir.iterdir(): -+ remove_test_files(entry) - project_metadata = extract_metadata(entry) - project_name = normalize_project_name(project_metadata['Name']) - project_version = project_metadata['Version'] +- for wheel_dir in wheels_dir.iterdir(): +- for wheel in iter_wheels(wheel_dir): +- project_metadata = extract_metadata(wheel) +- project_name = normalize_project_name(project_metadata["Name"]) +- project_version = project_metadata["Version"] +- dependencies[project_name] = project_version +- +- final_requirements = MOUNT_DIR / "frozen.txt" +- with final_requirements.open("w", encoding="utf-8") as f: +- for project_name, project_version in sorted(dependencies.items()): +- f.write(f"{project_name}=={project_version}\n") ++ # Handle wheels currently in the external directory and move them to the built directory if they were modified ++ for wheel in iter_wheels(external_wheels_dir): ++ was_modified = remove_test_files(wheel) ++ if was_modified: ++ # A modified wheel is no longer external → move it to built directory ++ new_path = built_wheels_dir / wheel.name ++ wheel.rename(new_path) ++ wheel = new_path ++ ++ add_dependency(dependencies, wheel) ++ ++ # Handle wheels already in the built directory ++ for wheel in iter_wheels(built_wheels_dir): ++ remove_test_files(wheel) ++ add_dependency(dependencies, wheel) + + + if __name__ == "__main__": diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py -index 63772a277d..310a04c798 100644 ---- a/scripts/utils.py -+++ b/scripts/utils.py -@@ -1,10 +1,17 @@ +index 35d8ca138c..431fd6ddbb 100644 +--- a/.builders/scripts/utils.py ++++ b/.builders/scripts/utils.py +@@ -1,11 +1,17 @@ from __future__ import annotations import email @@ -31,34 +70,33 @@ index 63772a277d..310a04c798 100644 import re +import shutil +import tempfile -+import zipfile from pathlib import Path + from typing import Iterator from zipfile import ZipFile +from wheel.cli.pack import pack +from wheel.cli.unpack import unpack + - UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') - + UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r"[-_.]+") -@@ -33,3 +40,101 @@ def extract_metadata(wheel: Path) -> email.Message: - - return email.message_from_string(metadata_file_contents) +@@ -40,3 +46,105 @@ def iter_wheels(source_dir: str) -> Iterator[Path]: + for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): + if entry.suffix == ".whl" and entry.is_file(): + yield entry + +def remove_test_files(wheel_path: Path) -> None: + """ + Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. + """ + # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. -+ with zipfile.ZipFile(wheel_path, "r") as zf: ++ with ZipFile(wheel_path, "r") as zf: + excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] + + if not excluded_members: + # Nothing to strip, so skip rewriting the wheel -+ return -+ -+ removed = False ++ return False ++ + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + @@ -73,19 +111,19 @@ index 63772a277d..310a04c798 100644 + rel = full_dir.relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + shutil.rmtree(full_dir) -+ removed = True + dirs.remove(d) + for f in files: + rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + os.remove(Path(root) / f) -+ removed = True -+ if removed: -+ print(f"Tests removed from {wheel_path.name}") ++ ++ print(f"Tests removed from {wheel_path.name}") + + # Repack to same directory, regenerating RECORD + pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) + ++ return True ++ + +def is_excluded_from_wheel(path: str) -> bool: + """ @@ -102,12 +140,17 @@ index 63772a277d..310a04c798 100644 + "Cryptodome/SelfTest", + "gssapi/tests", + "keystoneauth1/tests", ++ "lazy_loader/tests", + "openstack/tests", + "os_service_types/tests", + "pbr/tests", + "pkg_resources/tests", ++ "pip/_vendor/colorama/tests", + "psutil/tests", ++ "requests_unixsocket/tests", + "securesystemslib/_vendor/ed25519/test_data", ++ "setuptools/_distutils/compilers/C/tests", ++ "setuptools/_vendor/packaging/tests", + "setuptools/_distutils/tests", + "setuptools/tests", + "simplejson/tests", @@ -116,7 +159,7 @@ index 63772a277d..310a04c798 100644 + "test", # cm-client + "vertica_python/tests", + "websocket/tests", -+ "win32com/test" ++ "win32com/test", + ] + ] + @@ -143,3 +186,4 @@ index 63772a277d..310a04c798 100644 + return True + + return False +\ No newline at end of file From 2649bfaee4ce9411d190ba235cb4a774eadcaaf0 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Mon, 14 Jul 2025 14:26:43 +0200 Subject: [PATCH 31/63] patch fix --- .builders/patches/remove_tests.patch | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index d2055854a1836..68846a012235e 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,7 +1,7 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0ae4035590..6d38e1f197 100644 ---- a/.builders/scripts/build_wheels.py -+++ b/.builders/scripts/build_wheels.py +--- a/scripts/build_wheels.py ++++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path from tempfile import TemporaryDirectory @@ -60,8 +60,8 @@ index 0ae4035590..6d38e1f197 100644 if __name__ == "__main__": diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 35d8ca138c..431fd6ddbb 100644 ---- a/.builders/scripts/utils.py -+++ b/.builders/scripts/utils.py +--- a/scripts/utils.py ++++ b/scripts/utils.py @@ -1,11 +1,17 @@ from __future__ import annotations From bb898e06e9eec70e845e3baae85f063d1aed9ca5 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Mon, 14 Jul 2025 14:47:23 +0200 Subject: [PATCH 32/63] fix --- .builders/patches/remove_tests.patch | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 68846a012235e..6c4a6fef151c2 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -1,5 +1,5 @@ diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0ae4035590..6d38e1f197 100644 +index 0ae4035590..d664994402 100644 --- a/scripts/build_wheels.py +++ b/scripts/build_wheels.py @@ -8,7 +8,7 @@ from pathlib import Path @@ -25,7 +25,7 @@ index 0ae4035590..6d38e1f197 100644 def main(): parser = argparse.ArgumentParser(prog="wheel-builder", allow_abbrev=False) parser.add_argument("--python", required=True) -@@ -165,17 +170,21 @@ def main(): +@@ -165,12 +170,21 @@ def main(): ) dependencies: dict[str, tuple[str, str]] = {} @@ -35,11 +35,6 @@ index 0ae4035590..6d38e1f197 100644 - project_name = normalize_project_name(project_metadata["Name"]) - project_version = project_metadata["Version"] - dependencies[project_name] = project_version -- -- final_requirements = MOUNT_DIR / "frozen.txt" -- with final_requirements.open("w", encoding="utf-8") as f: -- for project_name, project_version in sorted(dependencies.items()): -- f.write(f"{project_name}=={project_version}\n") + # Handle wheels currently in the external directory and move them to the built directory if they were modified + for wheel in iter_wheels(external_wheels_dir): + was_modified = remove_test_files(wheel) @@ -56,8 +51,8 @@ index 0ae4035590..6d38e1f197 100644 + remove_test_files(wheel) + add_dependency(dependencies, wheel) - - if __name__ == "__main__": + final_requirements = MOUNT_DIR / "frozen.txt" + with final_requirements.open("w", encoding="utf-8") as f: diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 35d8ca138c..431fd6ddbb 100644 --- a/scripts/utils.py From f53a55c803e1728721062c34480f76ec0a8b9f7c Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 18 Jul 2025 10:53:31 +0200 Subject: [PATCH 33/63] remove quotes --- .builders/scripts/build_wheels.py | 124 +++++++++++++++--------------- .builders/scripts/utils.py | 18 ++--- 2 files changed, 71 insertions(+), 71 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0ae4035590305..e56ca529ee05f 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -10,35 +10,35 @@ from dotenv import dotenv_values from utils import extract_metadata, iter_wheels, normalize_project_name -INDEX_BASE_URL = "https://agent-int-packages.datadoghq.com" -CUSTOM_EXTERNAL_INDEX = f"{INDEX_BASE_URL}/external" -CUSTOM_BUILT_INDEX = f"{INDEX_BASE_URL}/built" +INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' +CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' +CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built' -if sys.platform == "win32": - PY3_PATH = Path("C:\\py3\\Scripts\\python.exe") - PY2_PATH = Path("C:\\py2\\Scripts\\python.exe") - MOUNT_DIR = Path("C:\\mnt") - ENV_FILE = Path("C:\\.env") +if sys.platform == 'win32': + PY3_PATH = Path('C:\\py3\\Scripts\\python.exe') + PY2_PATH = Path('C:\\py2\\Scripts\\python.exe') + MOUNT_DIR = Path('C:\\mnt') + ENV_FILE = Path('C:\\.env') def join_command_args(args: list[str]) -> str: return subprocess.list2cmdline(args) def path_to_uri(path: str) -> str: - return f"file:///{os.path.abspath(path).replace(' ', '%20').replace(os.sep, '/')}" + return f'file:///{os.path.abspath(path).replace(' ', '%20').replace(os.sep, '/')}' else: import shlex - PY3_PATH = Path(os.environ.get("DD_PY3_BUILDENV_PATH", "/py3/bin/python")) - PY2_PATH = Path(os.environ.get("DD_PY2_BUILDENV_PATH", "/py2/bin/python")) - MOUNT_DIR = Path(os.environ.get("DD_MOUNT_DIR", "/home")) - ENV_FILE = Path(os.environ.get("DD_ENV_FILE", "/.env")) + PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python')) + PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python')) + MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home')) + ENV_FILE = Path(os.environ.get('DD_ENV_FILE', '/.env')) def join_command_args(args: list[str]) -> str: return shlex.join(args) def path_to_uri(path: str) -> str: - return f"file://{os.path.abspath(path).replace(' ', '%20')}" + return f'file://{os.path.abspath(path).replace(" ", "%20")}' def abort(message, *, code=1): @@ -47,7 +47,7 @@ def abort(message, *, code=1): def check_process(*args, **kwargs) -> subprocess.CompletedProcess: - print(f"Running: {args[0] if isinstance(args[0], str) else join_command_args(args[0])}", file=sys.stderr) + print(f'Running: {args[0] if isinstance(args[0], str) else join_command_args(args[0])}', file=sys.stderr) process = subprocess.run(*args, **kwargs) if process.returncode: sys.exit(process.returncode) @@ -56,51 +56,51 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: def main(): - parser = argparse.ArgumentParser(prog="wheel-builder", allow_abbrev=False) - parser.add_argument("--python", required=True) - parser.add_argument("--use-built-index", action="store_true", default=False) + parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False) + parser.add_argument('--python', required=True) + parser.add_argument('--use-built-index', action='store_true', default=False) args = parser.parse_args() python_version = args.python - if python_version == "3": + if python_version == '3': python_path = PY3_PATH - elif python_version == "2": + elif python_version == '2': python_path = PY2_PATH else: - abort(f"Invalid python version: {python_version}") + abort(f'Invalid python version: {python_version}') - wheels_dir = MOUNT_DIR / "wheels" - built_wheels_dir = wheels_dir / "built" - external_wheels_dir = wheels_dir / "external" + wheels_dir = MOUNT_DIR / 'wheels' + built_wheels_dir = wheels_dir / 'built' + external_wheels_dir = wheels_dir / 'external' # Install build dependencies - check_process([str(python_path), "-m", "pip", "install", "-r", str(MOUNT_DIR / "build_dependencies.txt")]) + check_process([str(python_path), '-m', 'pip', 'install', '-r', str(MOUNT_DIR / 'build_dependencies.txt')]) with TemporaryDirectory() as d: staged_wheel_dir = Path(d).resolve() - staged_built_wheels_dir = staged_wheel_dir / "built" - staged_external_wheels_dir = staged_wheel_dir / "external" + staged_built_wheels_dir = staged_wheel_dir / 'built' + staged_external_wheels_dir = staged_wheel_dir / 'external' # Create the directories staged_built_wheels_dir.mkdir(parents=True, exist_ok=True) staged_external_wheels_dir.mkdir(parents=True, exist_ok=True) env_vars = dict(os.environ) - env_vars["PATH"] = f"{python_path.parent}{os.pathsep}{env_vars['PATH']}" - env_vars["PIP_WHEEL_DIR"] = str(staged_wheel_dir) - env_vars["DD_BUILD_PYTHON_VERSION"] = python_version - env_vars["DD_MOUNT_DIR"] = str(MOUNT_DIR) - env_vars["DD_ENV_FILE"] = str(ENV_FILE) + env_vars['PATH'] = f'{python_path.parent}{os.pathsep}{env_vars["PATH"]}' + env_vars['PIP_WHEEL_DIR'] = str(staged_wheel_dir) + env_vars['DD_BUILD_PYTHON_VERSION'] = python_version + env_vars['DD_MOUNT_DIR'] = str(MOUNT_DIR) + env_vars['DD_ENV_FILE'] = str(ENV_FILE) # Off is on, see: https://github.com/pypa/pip/issues/5735 - env_vars["PIP_NO_BUILD_ISOLATION"] = "0" + env_vars['PIP_NO_BUILD_ISOLATION'] = '0' # Spaces are used to separate multiple values which means paths themselves cannot contain spaces, see: # https://github.com/pypa/pip/issues/10114#issuecomment-1880125475 - env_vars["PIP_FIND_LINKS"] = path_to_uri(staged_wheel_dir) + env_vars['PIP_FIND_LINKS'] = path_to_uri(staged_wheel_dir) # Perform builder-specific logic if required - if build_command := os.environ.get("DD_BUILD_COMMAND"): + if build_command := os.environ.get('DD_BUILD_COMMAND'): check_process(build_command, env=env_vars, shell=True) # Load environment variables @@ -111,24 +111,24 @@ def main(): else: env_vars[key] = value - if constraints_file := env_vars.get("PIP_CONSTRAINT"): - env_vars["PIP_CONSTRAINT"] = path_to_uri(constraints_file) + if constraints_file := env_vars.get('PIP_CONSTRAINT'): + env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) # Fetch or build wheels command_args = [ str(python_path), - "-m", - "pip", - "wheel", - "-r", - str(MOUNT_DIR / "requirements.in"), - "--wheel-dir", + '-m', + 'pip', + 'wheel', + '-r', + str(MOUNT_DIR / 'requirements.in'), + '--wheel-dir', str(staged_wheel_dir), - "--extra-index-url", + '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] if args.use_built_index: - command_args.extend(["--extra-index-url", CUSTOM_BUILT_INDEX]) + command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) check_process(command_args, env=env_vars) @@ -136,13 +136,13 @@ def main(): check_process( [ sys.executable, - "-u", - str(MOUNT_DIR / "scripts" / "classify_wheels.py"), - "--source-dir", + '-u', + str(MOUNT_DIR / 'scripts' / 'classify_wheels.py'), + '--source-dir', str(staged_wheel_dir), - "--built-dir", + '--built-dir', str(staged_built_wheels_dir), - "--external-dir", + '--external-dir', str(staged_external_wheels_dir), ] ) @@ -151,15 +151,15 @@ def main(): check_process( [ sys.executable, - "-u", - str(MOUNT_DIR / "scripts" / "repair_wheels.py"), - "--source-built-dir", + '-u', + str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'), + '--source-built-dir', str(staged_built_wheels_dir), - "--source-external-dir", + '--source-external-dir', str(staged_external_wheels_dir), - "--built-dir", + '--built-dir', str(built_wheels_dir), - "--external-dir", + '--external-dir', str(external_wheels_dir), ] ) @@ -168,15 +168,15 @@ def main(): for wheel_dir in wheels_dir.iterdir(): for wheel in iter_wheels(wheel_dir): project_metadata = extract_metadata(wheel) - project_name = normalize_project_name(project_metadata["Name"]) - project_version = project_metadata["Version"] + project_name = normalize_project_name(project_metadata['Name']) + project_version = project_metadata['Version'] dependencies[project_name] = project_version - final_requirements = MOUNT_DIR / "frozen.txt" - with final_requirements.open("w", encoding="utf-8") as f: + final_requirements = MOUNT_DIR / 'frozen.txt' + with final_requirements.open('w', encoding='utf-8') as f: for project_name, project_version in sorted(dependencies.items()): - f.write(f"{project_name}=={project_version}\n") + f.write(f'{project_name}=={project_version}\n') -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 35d8ca138cf22..0208da31719ab 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -6,30 +6,30 @@ from typing import Iterator from zipfile import ZipFile -UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r"[-_.]+") +UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') def normalize_project_name(name: str) -> str: # https://peps.python.org/pep-0503/#normalized-names - return UNNORMALIZED_PROJECT_NAME_CHARS.sub("-", name).lower() + return UNNORMALIZED_PROJECT_NAME_CHARS.sub('-', name).lower() def extract_metadata(wheel: Path) -> email.Message: with ZipFile(str(wheel)) as zip_archive: for path in zip_archive.namelist(): - root = path.split("/", 1)[0] - if root.endswith(".dist-info"): + root = path.split('/', 1)[0] + if root.endswith('.dist-info'): dist_info_dir = root break else: - message = f"Could not find the `.dist-info` directory in wheel: {wheel.name}" + message = f'Could not find the `.dist-info` directory in wheel: {wheel.name}' raise RuntimeError(message) try: - with zip_archive.open(f"{dist_info_dir}/METADATA") as zip_file: - metadata_file_contents = zip_file.read().decode("utf-8") + with zip_archive.open(f'{dist_info_dir}/METADATA') as zip_file: + metadata_file_contents = zip_file.read().decode('utf-8') except KeyError: - message = f"Could not find a `METADATA` file in the `{dist_info_dir}` directory" + message = f'Could not find a `METADATA` file in the `{dist_info_dir}` directory' raise RuntimeError(message) from None return email.message_from_string(metadata_file_contents) @@ -38,5 +38,5 @@ def extract_metadata(wheel: Path) -> email.Message: def iter_wheels(source_dir: str) -> Iterator[Path]: for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): - if entry.suffix == ".whl" and entry.is_file(): + if entry.suffix == '.whl' and entry.is_file(): yield entry From 7e1fb3b1efb06c906d60de2d525d6bd458557be6 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 18 Jul 2025 10:57:30 +0200 Subject: [PATCH 34/63] replace quote --- .builders/scripts/build_wheels.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index e56ca529ee05f..bc47e46e21ca2 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -24,7 +24,7 @@ def join_command_args(args: list[str]) -> str: return subprocess.list2cmdline(args) def path_to_uri(path: str) -> str: - return f'file:///{os.path.abspath(path).replace(' ', '%20').replace(os.sep, '/')}' + return f'file:///{os.path.abspath(path).replace(" ", "%20").replace(os.sep, "/")}' else: import shlex @@ -116,16 +116,10 @@ def main(): # Fetch or build wheels command_args = [ - str(python_path), - '-m', - 'pip', - 'wheel', - '-r', - str(MOUNT_DIR / 'requirements.in'), - '--wheel-dir', - str(staged_wheel_dir), - '--extra-index-url', - CUSTOM_EXTERNAL_INDEX, + str(python_path), '-m', 'pip', 'wheel', + '-r', str(MOUNT_DIR / 'requirements.in'), + '--wheel-dir', str(staged_wheel_dir), + '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] if args.use_built_index: command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) From fb3cc7135dae9e5ac6f0f8ecf46da0f5252f8ce4 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 09:16:30 +0200 Subject: [PATCH 35/63] fix quotes in patch --- .builders/patches/remove_tests.patch | 98 ++++++++++++++-------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch index 6c4a6fef151c2..8f5b508465f8d 100644 --- a/.builders/patches/remove_tests.patch +++ b/.builders/patches/remove_tests.patch @@ -9,8 +9,8 @@ index 0ae4035590..d664994402 100644 -from utils import extract_metadata, iter_wheels, normalize_project_name +from utils import extract_metadata, iter_wheels, normalize_project_name, remove_test_files - INDEX_BASE_URL = "https://agent-int-packages.datadoghq.com" - CUSTOM_EXTERNAL_INDEX = f"{INDEX_BASE_URL}/external" + INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' + CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' @@ -54,7 +54,12 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: return process @@ -18,13 +18,13 @@ index 0ae4035590..d664994402 100644 - +def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: + project_metadata = extract_metadata(wheel) -+ project_name = normalize_project_name(project_metadata["Name"]) -+ project_version = project_metadata["Version"] ++ project_name = normalize_project_name(project_metadata['Name']) ++ project_version = project_metadata['Version'] + dependencies[project_name] = project_version + def main(): - parser = argparse.ArgumentParser(prog="wheel-builder", allow_abbrev=False) - parser.add_argument("--python", required=True) + parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False) + parser.add_argument('--python', required=True) @@ -165,12 +170,21 @@ def main(): ) @@ -32,8 +32,8 @@ index 0ae4035590..d664994402 100644 - for wheel_dir in wheels_dir.iterdir(): - for wheel in iter_wheels(wheel_dir): - project_metadata = extract_metadata(wheel) -- project_name = normalize_project_name(project_metadata["Name"]) -- project_version = project_metadata["Version"] +- project_name = normalize_project_name(project_metadata['Name']) +- project_version = project_metadata['Version'] - dependencies[project_name] = project_version + # Handle wheels currently in the external directory and move them to the built directory if they were modified + for wheel in iter_wheels(external_wheels_dir): @@ -51,8 +51,8 @@ index 0ae4035590..d664994402 100644 + remove_test_files(wheel) + add_dependency(dependencies, wheel) - final_requirements = MOUNT_DIR / "frozen.txt" - with final_requirements.open("w", encoding="utf-8") as f: + final_requirements = MOUNT_DIR / 'frozen.txt' + with final_requirements.open('w', encoding='utf-8') as f: diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 35d8ca138c..431fd6ddbb 100644 --- a/scripts/utils.py @@ -72,20 +72,20 @@ index 35d8ca138c..431fd6ddbb 100644 +from wheel.cli.pack import pack +from wheel.cli.unpack import unpack + - UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r"[-_.]+") + UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') @@ -40,3 +46,105 @@ def iter_wheels(source_dir: str) -> Iterator[Path]: for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): - if entry.suffix == ".whl" and entry.is_file(): + if entry.suffix == '.whl' and entry.is_file(): yield entry + +def remove_test_files(wheel_path: Path) -> None: -+ """ ++ ''' + Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. -+ """ ++ ''' + # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. -+ with ZipFile(wheel_path, "r") as zf: ++ with ZipFile(wheel_path, 'r') as zf: + excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] + + if not excluded_members: @@ -112,7 +112,7 @@ index 35d8ca138c..431fd6ddbb 100644 + if is_excluded_from_wheel(rel): + os.remove(Path(root) / f) + -+ print(f"Tests removed from {wheel_path.name}") ++ print(f'Tests removed from {wheel_path.name}') + + # Repack to same directory, regenerating RECORD + pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) @@ -121,49 +121,49 @@ index 35d8ca138c..431fd6ddbb 100644 + + +def is_excluded_from_wheel(path: str) -> bool: -+ """ ++ ''' + These files are excluded from the wheel in the agent build: + https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb + In order to have more accurate results, this files are excluded when computing the size of the dependencies while + the wheels still include them. -+ """ ++ ''' + excluded_test_paths = [ + os.path.normpath(path) + for path in [ -+ "idlelib/idle_test", -+ "bs4/tests", -+ "Cryptodome/SelfTest", -+ "gssapi/tests", -+ "keystoneauth1/tests", -+ "lazy_loader/tests", -+ "openstack/tests", -+ "os_service_types/tests", -+ "pbr/tests", -+ "pkg_resources/tests", -+ "pip/_vendor/colorama/tests", -+ "psutil/tests", -+ "requests_unixsocket/tests", -+ "securesystemslib/_vendor/ed25519/test_data", -+ "setuptools/_distutils/compilers/C/tests", -+ "setuptools/_vendor/packaging/tests", -+ "setuptools/_distutils/tests", -+ "setuptools/tests", -+ "simplejson/tests", -+ "stevedore/tests", -+ "supervisor/tests", -+ "test", # cm-client -+ "vertica_python/tests", -+ "websocket/tests", -+ "win32com/test", ++ 'idlelib/idle_test', ++ 'bs4/tests', ++ 'Cryptodome/SelfTest', ++ 'gssapi/tests', ++ 'keystoneauth1/tests', ++ 'lazy_loader/tests', ++ 'openstack/tests', ++ 'os_service_types/tests', ++ 'pbr/tests', ++ 'pkg_resources/tests', ++ 'pip/_vendor/colorama/tests', ++ 'psutil/tests', ++ 'requests_unixsocket/tests', ++ 'securesystemslib/_vendor/ed25519/test_data', ++ 'setuptools/_distutils/compilers/C/tests', ++ 'setuptools/_vendor/packaging/tests', ++ 'setuptools/_distutils/tests', ++ 'setuptools/tests', ++ 'simplejson/tests', ++ 'stevedore/tests', ++ 'supervisor/tests', ++ 'test', # cm-client ++ 'vertica_python/tests', ++ 'websocket/tests', ++ 'win32com/test', + ] + ] + + type_annot_libraries = [ -+ "krb5", -+ "Cryptodome", -+ "ddtrace", -+ "pyVmomi", -+ "gssapi", ++ 'krb5', ++ 'Cryptodome', ++ 'ddtrace', ++ 'pyVmomi', ++ 'gssapi', + ] + rel_path = Path(path).as_posix() + @@ -177,7 +177,7 @@ index 35d8ca138c..431fd6ddbb 100644 + if path_parts: + dependency_name = path_parts[0] + if dependency_name in type_annot_libraries: -+ if path.endswith(".pyi") or os.path.basename(path) == "py.typed": ++ if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': + return True + + return False From e626c0c731571f547dd3740f48b28846f397625b Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 11:06:02 +0200 Subject: [PATCH 36/63] remove patch --- .builders/patches/remove_tests.patch | 184 --------------------------- .builders/scripts/build_wheels.py | 96 +++++++++++--- .builders/scripts/classify_wheels.py | 74 ----------- .builders/scripts/utils.py | 108 ++++++++++++++++ 4 files changed, 184 insertions(+), 278 deletions(-) delete mode 100644 .builders/patches/remove_tests.patch delete mode 100644 .builders/scripts/classify_wheels.py diff --git a/.builders/patches/remove_tests.patch b/.builders/patches/remove_tests.patch deleted file mode 100644 index 8f5b508465f8d..0000000000000 --- a/.builders/patches/remove_tests.patch +++ /dev/null @@ -1,184 +0,0 @@ -diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py -index 0ae4035590..d664994402 100644 ---- a/scripts/build_wheels.py -+++ b/scripts/build_wheels.py -@@ -8,7 +8,7 @@ from pathlib import Path - from tempfile import TemporaryDirectory - - from dotenv import dotenv_values --from utils import extract_metadata, iter_wheels, normalize_project_name -+from utils import extract_metadata, iter_wheels, normalize_project_name, remove_test_files - - INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' - CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' -@@ -54,7 +54,12 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: - - return process - -- -+def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: -+ project_metadata = extract_metadata(wheel) -+ project_name = normalize_project_name(project_metadata['Name']) -+ project_version = project_metadata['Version'] -+ dependencies[project_name] = project_version -+ - def main(): - parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False) - parser.add_argument('--python', required=True) -@@ -165,12 +170,21 @@ def main(): - ) - - dependencies: dict[str, tuple[str, str]] = {} -- for wheel_dir in wheels_dir.iterdir(): -- for wheel in iter_wheels(wheel_dir): -- project_metadata = extract_metadata(wheel) -- project_name = normalize_project_name(project_metadata['Name']) -- project_version = project_metadata['Version'] -- dependencies[project_name] = project_version -+ # Handle wheels currently in the external directory and move them to the built directory if they were modified -+ for wheel in iter_wheels(external_wheels_dir): -+ was_modified = remove_test_files(wheel) -+ if was_modified: -+ # A modified wheel is no longer external → move it to built directory -+ new_path = built_wheels_dir / wheel.name -+ wheel.rename(new_path) -+ wheel = new_path -+ -+ add_dependency(dependencies, wheel) -+ -+ # Handle wheels already in the built directory -+ for wheel in iter_wheels(built_wheels_dir): -+ remove_test_files(wheel) -+ add_dependency(dependencies, wheel) - - final_requirements = MOUNT_DIR / 'frozen.txt' - with final_requirements.open('w', encoding='utf-8') as f: -diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py -index 35d8ca138c..431fd6ddbb 100644 ---- a/scripts/utils.py -+++ b/scripts/utils.py -@@ -1,11 +1,17 @@ - from __future__ import annotations - - import email -+import os - import re -+import shutil -+import tempfile - from pathlib import Path - from typing import Iterator - from zipfile import ZipFile - -+from wheel.cli.pack import pack -+from wheel.cli.unpack import unpack -+ - UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') - - -@@ -40,3 +46,105 @@ def iter_wheels(source_dir: str) -> Iterator[Path]: - for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): - if entry.suffix == '.whl' and entry.is_file(): - yield entry -+ -+def remove_test_files(wheel_path: Path) -> None: -+ ''' -+ Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. -+ ''' -+ # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. -+ with ZipFile(wheel_path, 'r') as zf: -+ excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] -+ -+ if not excluded_members: -+ # Nothing to strip, so skip rewriting the wheel -+ return False -+ -+ with tempfile.TemporaryDirectory() as td: -+ td_path = Path(td) -+ -+ # Unpack the wheel into temp dir -+ unpack(wheel_path, dest=td_path) -+ unpacked_dir = next(td_path.iterdir()) -+ -+ # Remove excluded files/folders -+ for root, dirs, files in os.walk(td, topdown=False): -+ for d in list(dirs): -+ full_dir = Path(root) / d -+ rel = full_dir.relative_to(unpacked_dir).as_posix() -+ if is_excluded_from_wheel(rel): -+ shutil.rmtree(full_dir) -+ dirs.remove(d) -+ for f in files: -+ rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() -+ if is_excluded_from_wheel(rel): -+ os.remove(Path(root) / f) -+ -+ print(f'Tests removed from {wheel_path.name}') -+ -+ # Repack to same directory, regenerating RECORD -+ pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) -+ -+ return True -+ -+ -+def is_excluded_from_wheel(path: str) -> bool: -+ ''' -+ These files are excluded from the wheel in the agent build: -+ https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb -+ In order to have more accurate results, this files are excluded when computing the size of the dependencies while -+ the wheels still include them. -+ ''' -+ excluded_test_paths = [ -+ os.path.normpath(path) -+ for path in [ -+ 'idlelib/idle_test', -+ 'bs4/tests', -+ 'Cryptodome/SelfTest', -+ 'gssapi/tests', -+ 'keystoneauth1/tests', -+ 'lazy_loader/tests', -+ 'openstack/tests', -+ 'os_service_types/tests', -+ 'pbr/tests', -+ 'pkg_resources/tests', -+ 'pip/_vendor/colorama/tests', -+ 'psutil/tests', -+ 'requests_unixsocket/tests', -+ 'securesystemslib/_vendor/ed25519/test_data', -+ 'setuptools/_distutils/compilers/C/tests', -+ 'setuptools/_vendor/packaging/tests', -+ 'setuptools/_distutils/tests', -+ 'setuptools/tests', -+ 'simplejson/tests', -+ 'stevedore/tests', -+ 'supervisor/tests', -+ 'test', # cm-client -+ 'vertica_python/tests', -+ 'websocket/tests', -+ 'win32com/test', -+ ] -+ ] -+ -+ type_annot_libraries = [ -+ 'krb5', -+ 'Cryptodome', -+ 'ddtrace', -+ 'pyVmomi', -+ 'gssapi', -+ ] -+ rel_path = Path(path).as_posix() -+ -+ # Test folders -+ for test_folder in excluded_test_paths: -+ if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): -+ return True -+ -+ # Python type annotations -+ path_parts = Path(rel_path).parts -+ if path_parts: -+ dependency_name = path_parts[0] -+ if dependency_name in type_annot_libraries: -+ if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': -+ return True -+ -+ return False -\ No newline at end of file diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index bc47e46e21ca2..97c8d1e454ef7 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -2,13 +2,18 @@ import argparse import os +import shutil import subprocess import sys +import time +from functools import cache +from hashlib import sha256 from pathlib import Path from tempfile import TemporaryDirectory +import urllib3 from dotenv import dotenv_values -from utils import extract_metadata, iter_wheels, normalize_project_name +from utils import extract_metadata, iter_wheels, normalize_project_name, remove_test_files INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' @@ -54,6 +59,56 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: return process +@cache +def get_wheel_hashes(project) -> dict[str, str]: + retry_wait = 2 + while True: + try: + response = urllib3.request( + 'GET', + f'https://pypi.org/simple/{project}', + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + ) + except urllib3.exceptions.HTTPError as e: + err_msg = f'Failed to fetch hashes for `{project}`: {e}' + else: + if response.status == 200: + break + + err_msg = f'Failed to fetch hashes for `{project}`, status code: {response.status}' + + print(err_msg) + print(f'Retrying in {retry_wait} seconds') + time.sleep(retry_wait) + retry_wait *= 2 + continue + + data = response.json() + return { + file['filename']: file['hashes']['sha256'] + for file in data['files'] + if file['filename'].endswith('.whl') and 'sha256' in file['hashes'] + } + + +def wheel_was_built(wheel: Path) -> bool: + project_metadata = extract_metadata(wheel) + project_name = normalize_project_name(project_metadata['Name']) + wheel_hashes = get_wheel_hashes(project_name) + if wheel.name not in wheel_hashes: + return True + + file_hash = sha256(wheel.read_bytes()).hexdigest() + return file_hash != wheel_hashes[wheel.name] + + + + +def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: + project_metadata = extract_metadata(wheel) + project_name = normalize_project_name(project_metadata['Name']) + project_version = project_metadata['Version'] + dependencies[project_name] = project_version def main(): parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False) @@ -127,19 +182,11 @@ def main(): check_process(command_args, env=env_vars) # Classify wheels - check_process( - [ - sys.executable, - '-u', - str(MOUNT_DIR / 'scripts' / 'classify_wheels.py'), - '--source-dir', - str(staged_wheel_dir), - '--built-dir', - str(staged_built_wheels_dir), - '--external-dir', - str(staged_external_wheels_dir), - ] - ) + for wheel in iter_wheels(staged_wheel_dir): + if wheel_was_built(wheel): + shutil.move(wheel, staged_built_wheels_dir) + else: + shutil.move(wheel, staged_external_wheels_dir) # Repair wheels check_process( @@ -159,12 +206,21 @@ def main(): ) dependencies: dict[str, tuple[str, str]] = {} - for wheel_dir in wheels_dir.iterdir(): - for wheel in iter_wheels(wheel_dir): - project_metadata = extract_metadata(wheel) - project_name = normalize_project_name(project_metadata['Name']) - project_version = project_metadata['Version'] - dependencies[project_name] = project_version + # Handle wheels currently in the external directory and move them to the built directory if they were modified + for wheel in iter_wheels(external_wheels_dir): + was_modified = remove_test_files(wheel) + if was_modified: + # A modified wheel is no longer external → move it to built directory + new_path = built_wheels_dir / wheel.name + wheel.rename(new_path) + wheel = new_path + + add_dependency(dependencies, wheel) + + # Handle wheels already in the built directory + for wheel in iter_wheels(built_wheels_dir): + remove_test_files(wheel) + add_dependency(dependencies, wheel) final_requirements = MOUNT_DIR / 'frozen.txt' with final_requirements.open('w', encoding='utf-8') as f: diff --git a/.builders/scripts/classify_wheels.py b/.builders/scripts/classify_wheels.py deleted file mode 100644 index 826b391de055c..0000000000000 --- a/.builders/scripts/classify_wheels.py +++ /dev/null @@ -1,74 +0,0 @@ -import argparse -import shutil -import time -from functools import cache -from hashlib import sha256 -from pathlib import Path - -import urllib3 -from utils import extract_metadata, iter_wheels, normalize_project_name - - -@cache -def get_wheel_hashes(project) -> dict[str, str]: - retry_wait = 2 - while True: - try: - response = urllib3.request( - 'GET', - f'https://pypi.org/simple/{project}', - headers={"Accept": "application/vnd.pypi.simple.v1+json"}, - ) - except urllib3.exceptions.HTTPError as e: - err_msg = f'Failed to fetch hashes for `{project}`: {e}' - else: - if response.status == 200: - break - - err_msg = f'Failed to fetch hashes for `{project}`, status code: {response.status}' - - print(err_msg) - print(f'Retrying in {retry_wait} seconds') - time.sleep(retry_wait) - retry_wait *= 2 - continue - - data = response.json() - return { - file['filename']: file['hashes']['sha256'] - for file in data['files'] - if file['filename'].endswith('.whl') and 'sha256' in file['hashes'] - } - - -def wheel_was_built(wheel: Path) -> bool: - project_metadata = extract_metadata(wheel) - project_name = normalize_project_name(project_metadata['Name']) - wheel_hashes = get_wheel_hashes(project_name) - if wheel.name not in wheel_hashes: - return True - - file_hash = sha256(wheel.read_bytes()).hexdigest() - return file_hash != wheel_hashes[wheel.name] - - -def classify_wheels(source_dir: str, built_dir: str, external_dir: str) -> None: - for wheel in iter_wheels(source_dir): - if wheel_was_built(wheel): - shutil.move(wheel, built_dir) - else: - shutil.move(wheel, external_dir) - - -def main(): - parser = argparse.ArgumentParser("Classifies wheels into built and external depending on the hash of the wheel") - parser.add_argument('--source-dir', required=True) - parser.add_argument('--built-dir', required=True) - parser.add_argument('--external-dir', required=True) - args = parser.parse_args() - - classify_wheels(args.source_dir, args.built_dir, args.external_dir) - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index 0208da31719ab..d0fc988d365ee 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -1,11 +1,17 @@ from __future__ import annotations import email +import os import re +import shutil +import tempfile from pathlib import Path from typing import Iterator from zipfile import ZipFile +from wheel.cli.pack import pack +from wheel.cli.unpack import unpack + UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') @@ -40,3 +46,105 @@ def iter_wheels(source_dir: str) -> Iterator[Path]: for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()): if entry.suffix == '.whl' and entry.is_file(): yield entry + +def remove_test_files(wheel_path: Path) -> None: + ''' + Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. + ''' + # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. + with ZipFile(wheel_path, 'r') as zf: + excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] + + if not excluded_members: + # Nothing to strip, so skip rewriting the wheel + return False + + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + + # Unpack the wheel into temp dir + unpack(wheel_path, dest=td_path) + unpacked_dir = next(td_path.iterdir()) + + # Remove excluded files/folders + for root, dirs, files in os.walk(td, topdown=False): + for d in list(dirs): + full_dir = Path(root) / d + rel = full_dir.relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + shutil.rmtree(full_dir) + dirs.remove(d) + for f in files: + rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + os.remove(Path(root) / f) + + print(f'Tests removed from {wheel_path.name}') + + # Repack to same directory, regenerating RECORD + pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) + + return True + + +def is_excluded_from_wheel(path: str) -> bool: + ''' + These files are excluded from the wheel in the agent build: + https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb + In order to have more accurate results, this files are excluded when computing the size of the dependencies while + the wheels still include them. + ''' + excluded_test_paths = [ + os.path.normpath(path) + for path in [ + 'idlelib/idle_test', + 'bs4/tests', + 'Cryptodome/SelfTest', + 'gssapi/tests', + 'keystoneauth1/tests', + 'lazy_loader/tests', + 'openstack/tests', + 'os_service_types/tests', + 'pbr/tests', + 'pkg_resources/tests', + 'pip/_vendor/colorama/tests', + 'psutil/tests', + 'requests_unixsocket/tests', + 'securesystemslib/_vendor/ed25519/test_data', + 'setuptools/_distutils/compilers/C/tests', + 'setuptools/_vendor/packaging/tests', + 'setuptools/_distutils/tests', + 'setuptools/tests', + 'simplejson/tests', + 'stevedore/tests', + 'supervisor/tests', + 'test', # cm-client + 'vertica_python/tests', + 'websocket/tests', + 'win32com/test', + ] + ] + + type_annot_libraries = [ + 'krb5', + 'Cryptodome', + 'ddtrace', + 'pyVmomi', + 'gssapi', + ] + rel_path = Path(path).as_posix() + + # Test folders + for test_folder in excluded_test_paths: + if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): + return True + + # Python type annotations + path_parts = Path(rel_path).parts + if path_parts: + dependency_name = path_parts[0] + if dependency_name in type_annot_libraries: + if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': + return True + + return False \ No newline at end of file From 9461354ef4cb31466d04d10de9153838be169359 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 11:07:26 +0200 Subject: [PATCH 37/63] remove patch --- .builders/build.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.builders/build.py b/.builders/build.py index de3b512c8270b..328441670a76d 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -108,10 +108,6 @@ def build_macos(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') - # Apply the remove_tests patch - test_patch = mount_dir / 'patches' / 'remove_tests.patch' - check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) - prefix_path = builder_root / 'prefix' env = { **os.environ, @@ -214,10 +210,6 @@ def build_image(): shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') shutil.copytree(HERE / 'patches', mount_dir / 'patches') - # Apply the remove_tests patch - test_patch = mount_dir / 'patches' / 'remove_tests.patch' - check_process(['patch', '--batch', '-p1', '-i', str(test_patch)], cwd=mount_dir) - # Create outputs on the host so they can be removed wheels_dir = mount_dir / 'wheels' wheels_dir.mkdir() From a7303bd0a129cf8d059b128ca7ff7f4f7a639809 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 14:38:40 +0200 Subject: [PATCH 38/63] simplify utils and .toml --- .builders/scripts/build_wheels.py | 120 ++++++++++++++++++++- .builders/scripts/files_to_remove.toml | 35 ++++++ .builders/scripts/utils.py | 141 ------------------------- 3 files changed, 150 insertions(+), 146 deletions(-) create mode 100644 .builders/scripts/files_to_remove.toml diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 97c8d1e454ef7..df326455c5559 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -1,7 +1,9 @@ from __future__ import annotations import argparse +import email import os +import re import shutil import subprocess import sys @@ -10,14 +12,19 @@ from hashlib import sha256 from pathlib import Path from tempfile import TemporaryDirectory +from zipfile import ZipFile +import toml import urllib3 from dotenv import dotenv_values -from utils import extract_metadata, iter_wheels, normalize_project_name, remove_test_files +from utils import iter_wheels +from wheel.cli.pack import pack +from wheel.cli.unpack import unpack INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built' +UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') if sys.platform == 'win32': PY3_PATH = Path('C:\\py3\\Scripts\\python.exe') @@ -59,6 +66,33 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess: return process + +def extract_metadata(wheel: Path) -> email.Message: + with ZipFile(str(wheel)) as zip_archive: + for path in zip_archive.namelist(): + root = path.split('/', 1)[0] + if root.endswith('.dist-info'): + dist_info_dir = root + break + else: + message = f'Could not find the `.dist-info` directory in wheel: {wheel.name}' + raise RuntimeError(message) + + try: + with zip_archive.open(f'{dist_info_dir}/METADATA') as zip_file: + metadata_file_contents = zip_file.read().decode('utf-8') + except KeyError: + message = f'Could not find a `METADATA` file in the `{dist_info_dir}` directory' + raise RuntimeError(message) from None + + return email.message_from_string(metadata_file_contents) + + +def normalize_project_name(name: str) -> str: + # https://peps.python.org/pep-0503/#normalized-names + return UNNORMALIZED_PROJECT_NAME_CHARS.sub('-', name).lower() + + @cache def get_wheel_hashes(project) -> dict[str, str]: retry_wait = 2 @@ -102,6 +136,75 @@ def wheel_was_built(wheel: Path) -> bool: return file_hash != wheel_hashes[wheel.name] +def remove_test_files(wheel_path: Path) -> None: + ''' + Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. + ''' + # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. + with ZipFile(wheel_path, 'r') as zf: + excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] + + if not excluded_members: + # Nothing to strip, so skip rewriting the wheel + return False + + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + + # Unpack the wheel into temp dir + unpack(wheel_path, dest=td_path) + unpacked_dir = next(td_path.iterdir()) + + # Remove excluded files/folders + for root, dirs, files in os.walk(td, topdown=False): + for d in list(dirs): + full_dir = Path(root) / d + rel = full_dir.relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + shutil.rmtree(full_dir) + dirs.remove(d) + for f in files: + rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() + if is_excluded_from_wheel(rel): + os.remove(Path(root) / f) + + print(f'Tests removed from {wheel_path.name}') + + # Repack to same directory, regenerating RECORD + pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) + + return True + + +def is_excluded_from_wheel(path: str) -> bool: + ''' + These files are excluded from the wheel in the agent build: + https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb + In order to have more accurate results, this files are excluded when computing the size of the dependencies while + the wheels still include them. + ''' + files_to_remove_path = Path(__file__).parent / "files_to_remove.toml" + with open(files_to_remove_path, "r", encoding="utf-8") as f: + config = toml.load(f) + excluded_test_paths = [os.path.normpath(path) for path in config.get("excluded_test_paths", [])] + + type_annot_libraries = config.get("type_annot_libraries", []) + rel_path = Path(path).as_posix() + + # Test folders + for test_folder in excluded_test_paths: + if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): + return True + + # Python type annotations + path_parts = Path(rel_path).parts + if path_parts: + dependency_name = path_parts[0] + if dependency_name in type_annot_libraries: + if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': + return True + + return False def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: @@ -110,6 +213,7 @@ def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: project_version = project_metadata['Version'] dependencies[project_name] = project_version + def main(): parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False) parser.add_argument('--python', required=True) @@ -171,10 +275,16 @@ def main(): # Fetch or build wheels command_args = [ - str(python_path), '-m', 'pip', 'wheel', - '-r', str(MOUNT_DIR / 'requirements.in'), - '--wheel-dir', str(staged_wheel_dir), - '--extra-index-url', CUSTOM_EXTERNAL_INDEX, + str(python_path), + '-m', + 'pip', + 'wheel', + '-r', + str(MOUNT_DIR / 'requirements.in'), + '--wheel-dir', + str(staged_wheel_dir), + '--extra-index-url', + CUSTOM_EXTERNAL_INDEX, ] if args.use_built_index: command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) diff --git a/.builders/scripts/files_to_remove.toml b/.builders/scripts/files_to_remove.toml new file mode 100644 index 0000000000000..125085bc5cc5e --- /dev/null +++ b/.builders/scripts/files_to_remove.toml @@ -0,0 +1,35 @@ +excluded_test_paths = [ + "idlelib/idle_test", + "bs4/tests", + "Cryptodome/SelfTest", + "gssapi/tests", + "keystoneauth1/tests", + "lazy_loader/tests", + "openstack/tests", + "os_service_types/tests", + "pbr/tests", + "pkg_resources/tests", + "pip/_vendor/colorama/tests", + "psutil/tests", + "requests_unixsocket/tests", + "securesystemslib/_vendor/ed25519/test_data", + "setuptools/_distutils/compilers/C/tests", + "setuptools/_vendor/packaging/tests", + "setuptools/_distutils/tests", + "setuptools/tests", + "simplejson/tests", + "stevedore/tests", + "supervisor/tests", + "test", + "vertica_python/tests", + "websocket/tests", + "win32com/test", +] + +type_annot_libraries = [ + 'krb5', + 'Cryptodome', + 'ddtrace', + 'pyVmomi', + 'gssapi', +] \ No newline at end of file diff --git a/.builders/scripts/utils.py b/.builders/scripts/utils.py index d0fc988d365ee..35e7436e69b21 100644 --- a/.builders/scripts/utils.py +++ b/.builders/scripts/utils.py @@ -1,45 +1,5 @@ -from __future__ import annotations - -import email -import os -import re -import shutil -import tempfile from pathlib import Path from typing import Iterator -from zipfile import ZipFile - -from wheel.cli.pack import pack -from wheel.cli.unpack import unpack - -UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') - - -def normalize_project_name(name: str) -> str: - # https://peps.python.org/pep-0503/#normalized-names - return UNNORMALIZED_PROJECT_NAME_CHARS.sub('-', name).lower() - - -def extract_metadata(wheel: Path) -> email.Message: - with ZipFile(str(wheel)) as zip_archive: - for path in zip_archive.namelist(): - root = path.split('/', 1)[0] - if root.endswith('.dist-info'): - dist_info_dir = root - break - else: - message = f'Could not find the `.dist-info` directory in wheel: {wheel.name}' - raise RuntimeError(message) - - try: - with zip_archive.open(f'{dist_info_dir}/METADATA') as zip_file: - metadata_file_contents = zip_file.read().decode('utf-8') - except KeyError: - message = f'Could not find a `METADATA` file in the `{dist_info_dir}` directory' - raise RuntimeError(message) from None - - return email.message_from_string(metadata_file_contents) - def iter_wheels(source_dir: str) -> Iterator[Path]: @@ -47,104 +7,3 @@ def iter_wheels(source_dir: str) -> Iterator[Path]: if entry.suffix == '.whl' and entry.is_file(): yield entry -def remove_test_files(wheel_path: Path) -> None: - ''' - Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. - ''' - # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. - with ZipFile(wheel_path, 'r') as zf: - excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] - - if not excluded_members: - # Nothing to strip, so skip rewriting the wheel - return False - - with tempfile.TemporaryDirectory() as td: - td_path = Path(td) - - # Unpack the wheel into temp dir - unpack(wheel_path, dest=td_path) - unpacked_dir = next(td_path.iterdir()) - - # Remove excluded files/folders - for root, dirs, files in os.walk(td, topdown=False): - for d in list(dirs): - full_dir = Path(root) / d - rel = full_dir.relative_to(unpacked_dir).as_posix() - if is_excluded_from_wheel(rel): - shutil.rmtree(full_dir) - dirs.remove(d) - for f in files: - rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() - if is_excluded_from_wheel(rel): - os.remove(Path(root) / f) - - print(f'Tests removed from {wheel_path.name}') - - # Repack to same directory, regenerating RECORD - pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) - - return True - - -def is_excluded_from_wheel(path: str) -> bool: - ''' - These files are excluded from the wheel in the agent build: - https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb - In order to have more accurate results, this files are excluded when computing the size of the dependencies while - the wheels still include them. - ''' - excluded_test_paths = [ - os.path.normpath(path) - for path in [ - 'idlelib/idle_test', - 'bs4/tests', - 'Cryptodome/SelfTest', - 'gssapi/tests', - 'keystoneauth1/tests', - 'lazy_loader/tests', - 'openstack/tests', - 'os_service_types/tests', - 'pbr/tests', - 'pkg_resources/tests', - 'pip/_vendor/colorama/tests', - 'psutil/tests', - 'requests_unixsocket/tests', - 'securesystemslib/_vendor/ed25519/test_data', - 'setuptools/_distutils/compilers/C/tests', - 'setuptools/_vendor/packaging/tests', - 'setuptools/_distutils/tests', - 'setuptools/tests', - 'simplejson/tests', - 'stevedore/tests', - 'supervisor/tests', - 'test', # cm-client - 'vertica_python/tests', - 'websocket/tests', - 'win32com/test', - ] - ] - - type_annot_libraries = [ - 'krb5', - 'Cryptodome', - 'ddtrace', - 'pyVmomi', - 'gssapi', - ] - rel_path = Path(path).as_posix() - - # Test folders - for test_folder in excluded_test_paths: - if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): - return True - - # Python type annotations - path_parts = Path(rel_path).parts - if path_parts: - dependency_name = path_parts[0] - if dependency_name in type_annot_libraries: - if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': - return True - - return False \ No newline at end of file From 667e53cddd05ef422772214ee6c366947964bcd7 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 14:58:16 +0200 Subject: [PATCH 39/63] toml --- .builders/scripts/build_wheels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index df326455c5559..58fef1286d6f5 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -8,13 +8,13 @@ import subprocess import sys import time +import tomllib from functools import cache from hashlib import sha256 from pathlib import Path from tempfile import TemporaryDirectory from zipfile import ZipFile -import toml import urllib3 from dotenv import dotenv_values from utils import iter_wheels @@ -185,7 +185,7 @@ def is_excluded_from_wheel(path: str) -> bool: ''' files_to_remove_path = Path(__file__).parent / "files_to_remove.toml" with open(files_to_remove_path, "r", encoding="utf-8") as f: - config = toml.load(f) + config = tomllib.load(f) excluded_test_paths = [os.path.normpath(path) for path in config.get("excluded_test_paths", [])] type_annot_libraries = config.get("type_annot_libraries", []) From 46c9bfb57af8b25adf636d2b874bd8c4242bbef6 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 15:10:18 +0200 Subject: [PATCH 40/63] toml --- .builders/scripts/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 58fef1286d6f5..f1a608b093ae4 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -184,7 +184,7 @@ def is_excluded_from_wheel(path: str) -> bool: the wheels still include them. ''' files_to_remove_path = Path(__file__).parent / "files_to_remove.toml" - with open(files_to_remove_path, "r", encoding="utf-8") as f: + with open(files_to_remove_path, "rb", encoding="utf-8") as f: config = tomllib.load(f) excluded_test_paths = [os.path.normpath(path) for path in config.get("excluded_test_paths", [])] From 32abda2ecf3ee4cd66803ebc7963f6196c524215 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 23 Jul 2025 15:17:33 +0200 Subject: [PATCH 41/63] toml --- .builders/scripts/build_wheels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index f1a608b093ae4..fe4d7c2e64a59 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -148,7 +148,7 @@ def remove_test_files(wheel_path: Path) -> None: # Nothing to strip, so skip rewriting the wheel return False - with tempfile.TemporaryDirectory() as td: + with TemporaryDirectory() as td: td_path = Path(td) # Unpack the wheel into temp dir @@ -184,7 +184,7 @@ def is_excluded_from_wheel(path: str) -> bool: the wheels still include them. ''' files_to_remove_path = Path(__file__).parent / "files_to_remove.toml" - with open(files_to_remove_path, "rb", encoding="utf-8") as f: + with open(files_to_remove_path, "rb") as f: config = tomllib.load(f) excluded_test_paths = [os.path.normpath(path) for path in config.get("excluded_test_paths", [])] From 57dc620c524af8bee01425ebdd4d860a6aad9e66 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Tue, 19 Aug 2025 17:38:03 +0200 Subject: [PATCH 42/63] typo --- .builders/scripts/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index fe4d7c2e64a59..de4644dd6673b 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -180,7 +180,7 @@ def is_excluded_from_wheel(path: str) -> bool: ''' These files are excluded from the wheel in the agent build: https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb - In order to have more accurate results, this files are excluded when computing the size of the dependencies while + In order to have more accurate results, these files are excluded when computing the size of the dependencies while the wheels still include them. ''' files_to_remove_path = Path(__file__).parent / "files_to_remove.toml" From ef51ad4c6629e1d46d74deed4168cf28cc79547b Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 21 Aug 2025 11:25:02 +0200 Subject: [PATCH 43/63] change toml format to gitignore patterns --- .builders/images/runner_dependencies.txt | 1 + .builders/scripts/build_wheels.py | 54 ++++++++-------- .builders/scripts/files_to_remove.toml | 78 +++++++++++++----------- 3 files changed, 72 insertions(+), 61 deletions(-) diff --git a/.builders/images/runner_dependencies.txt b/.builders/images/runner_dependencies.txt index d15c3893dfbfb..18295cc266e2e 100644 --- a/.builders/images/runner_dependencies.txt +++ b/.builders/images/runner_dependencies.txt @@ -4,3 +4,4 @@ auditwheel==6.0.0; sys_platform == 'linux' delvewheel==1.5.2; sys_platform == 'win32' delocate==0.13.0; sys_platform == 'darwin' wheel==0.45.1 +pathspec==0.12.1 \ No newline at end of file diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index de4644dd6673b..a4067585a8675 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -15,6 +15,7 @@ from tempfile import TemporaryDirectory from zipfile import ZipFile +import pathspec import urllib3 from dotenv import dotenv_values from utils import iter_wheels @@ -161,11 +162,13 @@ def remove_test_files(wheel_path: Path) -> None: full_dir = Path(root) / d rel = full_dir.relative_to(unpacked_dir).as_posix() if is_excluded_from_wheel(rel): + print(f'Removing {rel}') shutil.rmtree(full_dir) dirs.remove(d) for f in files: rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() if is_excluded_from_wheel(rel): + print(f'Removing {rel}') os.remove(Path(root) / f) print(f'Tests removed from {wheel_path.name}') @@ -175,34 +178,31 @@ def remove_test_files(wheel_path: Path) -> None: return True - -def is_excluded_from_wheel(path: str) -> bool: - ''' - These files are excluded from the wheel in the agent build: - https://github.com/DataDog/datadog-agent/blob/main/omnibus/config/software/datadog-agent-integrations-py3.rb - In order to have more accurate results, these files are excluded when computing the size of the dependencies while - the wheels still include them. - ''' - files_to_remove_path = Path(__file__).parent / "files_to_remove.toml" - with open(files_to_remove_path, "rb") as f: +@cache +def _load_excluded_spec() -> pathspec.PathSpec: + """ + Load excluded paths from files_to_remove.toml and compile them + with .gitignore-style semantics. + """ + config_path = Path(__file__).parent / "files_to_remove.toml" + with open(config_path, "rb") as f: config = tomllib.load(f) - excluded_test_paths = [os.path.normpath(path) for path in config.get("excluded_test_paths", [])] - - type_annot_libraries = config.get("type_annot_libraries", []) - rel_path = Path(path).as_posix() - - # Test folders - for test_folder in excluded_test_paths: - if rel_path == test_folder or rel_path.startswith(test_folder + os.sep): - return True - - # Python type annotations - path_parts = Path(rel_path).parts - if path_parts: - dependency_name = path_parts[0] - if dependency_name in type_annot_libraries: - if path.endswith('.pyi') or os.path.basename(path) == 'py.typed': - return True + + patterns = config.get("excluded_paths", []) + return pathspec.PathSpec.from_lines("gitignore", patterns) + +def is_excluded_from_wheel(path: str | Path) -> bool: + """ + Return True if `path` (file or directory) should be excluded per files_to_remove.toml. + Matches: + - type annotation files: **/*.pyi, **/py.typed + - test directories listed with a trailing '/' + """ + spec = _load_excluded_spec() + rel = Path(path).as_posix() + + if spec.match_file(rel) or spec.match_file(rel + "/"): + return True return False diff --git a/.builders/scripts/files_to_remove.toml b/.builders/scripts/files_to_remove.toml index 125085bc5cc5e..f60a460f27010 100644 --- a/.builders/scripts/files_to_remove.toml +++ b/.builders/scripts/files_to_remove.toml @@ -1,35 +1,45 @@ -excluded_test_paths = [ - "idlelib/idle_test", - "bs4/tests", - "Cryptodome/SelfTest", - "gssapi/tests", - "keystoneauth1/tests", - "lazy_loader/tests", - "openstack/tests", - "os_service_types/tests", - "pbr/tests", - "pkg_resources/tests", - "pip/_vendor/colorama/tests", - "psutil/tests", - "requests_unixsocket/tests", - "securesystemslib/_vendor/ed25519/test_data", - "setuptools/_distutils/compilers/C/tests", - "setuptools/_vendor/packaging/tests", - "setuptools/_distutils/tests", - "setuptools/tests", - "simplejson/tests", - "stevedore/tests", - "supervisor/tests", - "test", - "vertica_python/tests", - "websocket/tests", - "win32com/test", -] +excluded_paths = [ + # --- Type annotation --- + "krb5/**/*.pyi", + "krb5/**/py.typed", + + "Cryptodome/**/*.pyi", + "Cryptodome/**/py.typed", + + "ddtrace/**/*.pyi", + "ddtrace/**/py.typed", -type_annot_libraries = [ - 'krb5', - 'Cryptodome', - 'ddtrace', - 'pyVmomi', - 'gssapi', -] \ No newline at end of file + "pyVmomi/**/*.pyi", + "pyVmomi/**/py.typed", + + "gssapi/**/*.pyi", + "gssapi/**/py.typed", + + # --- Tests --- + + "idlelib/idle_test/", + "bs4/tests/", + "Cryptodome/SelfTest/", + "gssapi/tests/", + "keystoneauth1/tests/", + "lazy_loader/tests/", + "openstack/tests/", + "os_service_types/tests/", + "pbr/tests/", + "pkg_resources/tests/", + "pip/_vendor/colorama/tests/", + "psutil/tests/", + "requests_unixsocket/tests/", + "securesystemslib/_vendor/ed25519/test_data/", + "setuptools/_distutils/compilers/C/tests/", + "setuptools/_vendor/packaging/tests/", + "setuptools/_distutils/tests/", + "setuptools/tests/", + "simplejson/tests/", + "stevedore/tests/", + "supervisor/tests/", + "/test/", + "vertica_python/tests/", + "websocket/tests/", + "win32com/test/", +] From 5aa5eb413093cae1c498c5cf0938baa734d25cd6 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 28 Aug 2025 10:49:28 +0200 Subject: [PATCH 44/63] publish wheels for testing --- .github/workflows/resolve-build-deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/resolve-build-deps.yaml b/.github/workflows/resolve-build-deps.yaml index 200da4aa8e8aa..a6a148276eb37 100644 --- a/.github/workflows/resolve-build-deps.yaml +++ b/.github/workflows/resolve-build-deps.yaml @@ -246,7 +246,7 @@ jobs: publish: name: Publish artifacts and update lockfiles via PR - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) + # if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) needs: - build - build-macos From df287e21cb97573ee66baa877299650cd706bdbc Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Thu, 28 Aug 2025 13:03:42 +0200 Subject: [PATCH 45/63] test --- .builders/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/upload.py b/.builders/upload.py index 4a39e4bdd0655..867ce1c558ba3 100644 --- a/.builders/upload.py +++ b/.builders/upload.py @@ -111,7 +111,7 @@ def upload(targets_dir): if not is_valid_project_name(project_name): message = f'Invalid project name `{project_name}` found in wheel: {wheel.name}' raise RuntimeError(message) - + print(f'Project name: {project_name}') upload_data.append((normalize_project_name(project_name), project_metadata, wheel)) queued = len(upload_data) From cb03b6aa3da39677a8b9865d09805fdcb498f042 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 29 Aug 2025 10:46:24 +0200 Subject: [PATCH 46/63] rename wheels --- .builders/scripts/build_wheels.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index a4067585a8675..3681574595db0 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -137,7 +137,7 @@ def wheel_was_built(wheel: Path) -> bool: return file_hash != wheel_hashes[wheel.name] -def remove_test_files(wheel_path: Path) -> None: +def remove_test_files(wheel_path: Path) -> bool: ''' Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. ''' @@ -148,33 +148,40 @@ def remove_test_files(wheel_path: Path) -> None: if not excluded_members: # Nothing to strip, so skip rewriting the wheel return False - with TemporaryDirectory() as td: td_path = Path(td) # Unpack the wheel into temp dir unpack(wheel_path, dest=td_path) unpacked_dir = next(td_path.iterdir()) - # Remove excluded files/folders for root, dirs, files in os.walk(td, topdown=False): for d in list(dirs): full_dir = Path(root) / d rel = full_dir.relative_to(unpacked_dir).as_posix() if is_excluded_from_wheel(rel): - print(f'Removing {rel}') shutil.rmtree(full_dir) dirs.remove(d) for f in files: rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() if is_excluded_from_wheel(rel): - print(f'Removing {rel}') os.remove(Path(root) / f) print(f'Tests removed from {wheel_path.name}') - + + dest_dir = wheel_path.parent + before = {p.resolve() for p in dest_dir.glob("*.whl")} # Repack to same directory, regenerating RECORD - pack(unpacked_dir, dest_dir=wheel_path.parent, build_number=None) + pack(unpacked_dir, dest_dir=dest_dir, build_number=None) + + # The wheel might not be platform-specific, so repacking restores its original name. + # We need to move the repacked wheel to wheel_path, which was changed to be platform-specific. + after = {p.resolve() for p in wheel_path.parent.glob("*.whl")} + new_files = sorted(after - before, key=lambda p: p.stat().st_mtime, reverse=True) + + if new_files: + shutil.move(str(new_files[0]), str(wheel_path)) + return True @@ -324,6 +331,7 @@ def main(): new_path = built_wheels_dir / wheel.name wheel.rename(new_path) wheel = new_path + print(f'Moved {wheel.name} to built directory') add_dependency(dependencies, wheel) From 76db4bf297ab90df8d1d4abd459a7d41df2e1d68 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 3 Sep 2025 16:39:35 +0200 Subject: [PATCH 47/63] uncomment --- .github/workflows/resolve-build-deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/resolve-build-deps.yaml b/.github/workflows/resolve-build-deps.yaml index a6a148276eb37..200da4aa8e8aa 100644 --- a/.github/workflows/resolve-build-deps.yaml +++ b/.github/workflows/resolve-build-deps.yaml @@ -246,7 +246,7 @@ jobs: publish: name: Publish artifacts and update lockfiles via PR - # if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) needs: - build - build-macos From 221352fa506b2e7f9fc2de07f6ce952a79c77916 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 10 Sep 2025 15:27:18 +0200 Subject: [PATCH 48/63] remove built extra index --- .builders/scripts/build_wheels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 95877f22f24f8..1d91c656ba14c 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -285,8 +285,8 @@ def main(): str(python_path), '-m', 'pip', 'wheel', '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', str(staged_wheel_dir), - # Temporarily removing extra index urls. See below. - # '--extra-index-url', CUSTOM_EXTERNAL_INDEX, + Temporarily removing extra index urls. See below. + '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] # Temporarily disable extra index urls. There are broken wheels in the gcloud bucket # while working on removing tests from them. Adding extra indices causes undefined behavior From 9579f04c7d519fbe72b5a704c3f7e3cca8e4237d Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Wed, 10 Sep 2025 16:04:02 +0200 Subject: [PATCH 49/63] remove built flag --- .builders/scripts/build_wheels.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 1d91c656ba14c..4cab834b56146 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -285,14 +285,8 @@ def main(): str(python_path), '-m', 'pip', 'wheel', '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', str(staged_wheel_dir), - Temporarily removing extra index urls. See below. '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] - # Temporarily disable extra index urls. There are broken wheels in the gcloud bucket - # while working on removing tests from them. Adding extra indices causes undefined behavior - # and can pull a broken image, preventing the building from running. - # if args.use_built_index: - # command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX]) check_process(command_args, env=env_vars) From bd853e14bc832e5cbf7aec2caba77801818aa0f8 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Mon, 29 Sep 2025 15:07:44 +0200 Subject: [PATCH 50/63] upload wheels --- .github/workflows/resolve-build-deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/resolve-build-deps.yaml b/.github/workflows/resolve-build-deps.yaml index 200da4aa8e8aa..a6a148276eb37 100644 --- a/.github/workflows/resolve-build-deps.yaml +++ b/.github/workflows/resolve-build-deps.yaml @@ -246,7 +246,7 @@ jobs: publish: name: Publish artifacts and update lockfiles via PR - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) + # if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) needs: - build - build-macos From 7f212feddbd505473f6d8545712f529f11202699 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 11:30:58 +0100 Subject: [PATCH 51/63] Uncomment condition to publish wheels --- .github/workflows/resolve-build-deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/resolve-build-deps.yaml b/.github/workflows/resolve-build-deps.yaml index a6a148276eb37..200da4aa8e8aa 100644 --- a/.github/workflows/resolve-build-deps.yaml +++ b/.github/workflows/resolve-build-deps.yaml @@ -246,7 +246,7 @@ jobs: publish: name: Publish artifacts and update lockfiles via PR - # if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))) needs: - build - build-macos From ca20180aa35df3b60fdbdf0a82321d62908bc08e Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 12:16:49 +0100 Subject: [PATCH 52/63] Use PEP 517 --- .builders/scripts/build_backend.py | 91 +++++++++++++ .builders/scripts/build_wheels.py | 204 ++++++++++++++--------------- 2 files changed, 193 insertions(+), 102 deletions(-) create mode 100644 .builders/scripts/build_backend.py diff --git a/.builders/scripts/build_backend.py b/.builders/scripts/build_backend.py new file mode 100644 index 0000000000000..30ace40ce40d2 --- /dev/null +++ b/.builders/scripts/build_backend.py @@ -0,0 +1,91 @@ +import inspect +import shutil +import sys +import tomllib +import zipfile +from functools import cache +from pathlib import Path + +import pathspec +from setuptools import build_meta as _orig + + +def remove_test_files(wheel_path: Path) -> None: + """ + Remove excluded files and directories from a built wheel. + """ + tmp_wheel = wheel_path.with_suffix(".tmp.whl") + + with ( + zipfile.ZipFile(wheel_path, "r") as zin, + zipfile.ZipFile(tmp_wheel, "w", compression=zipfile.ZIP_DEFLATED) as zout, + ): + for info in zin.infolist(): + rel = info.filename + if is_excluded_from_wheel(rel): + continue # skip excluded file or directory + + data = zin.read(rel) + zout.writestr(info, data) + + shutil.move(tmp_wheel, wheel_path) + + +def is_excluded_from_wheel(path: str | Path) -> bool: + """ + Return True if `path` (file or directory) should be excluded per files_to_remove.toml. + Matches: + - type annotation files: **/*.pyi, **/py.typed + - test directories listed with a trailing '/' + """ + spec = _load_excluded_spec() + rel = Path(path).as_posix() + + if spec.match_file(rel) or spec.match_file(rel + "/"): + return True + + return False + + +@cache +def _load_excluded_spec() -> pathspec.PathSpec: + """ + Load excluded paths from files_to_remove.toml and compile them + with .gitignore-style semantics. + """ + config_path = Path(__file__).parent / "files_to_remove.toml" + with open(config_path, "rb") as f: + config = tomllib.load(f) + + patterns = config.get("excluded_paths", []) + return pathspec.PathSpec.from_lines("gitignore", patterns) + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + """Intercept wheel building to strip test files.""" + wheel_file = _orig.build_wheel(wheel_directory, config_settings, metadata_directory) + + # Post-process the wheel to remove tests + wheel_path = Path(wheel_directory) / wheel_file + remove_test_files(wheel_path) + + return wheel_file + + +# Proxy all other PEP 517 hooks +# prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel +# build_sdist = _orig.build_sdist +# (better do by iterating over _orig methods instead) +for name, func in inspect.getmembers(_orig, inspect.isfunction): + # Only copy methods if they haven't been defined in the current module + # (i.e., don't overwrite your custom build_wheel) + print("Name: ", name, "Func: ", func, "Is in globals: ", name in globals()) + if name not in globals(): + globals()[name] = func + print("Added to globals: ", name) + +# for name in dir(_orig): +# # Check if the attribute name is a PEP 517 hook and not one we defined/overrode +# if name.startswith('build_') or 'requires_for' in name: +# if name not in globals(): +# setattr(sys.modules[__name__], name, getattr(_orig, name)) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0db5dd5711497..0964a600b4084 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -21,8 +21,6 @@ import urllib3 from dotenv import dotenv_values from utils import iter_wheels -from wheel.cli.pack import pack -from wheel.cli.unpack import unpack INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com' CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external' @@ -143,84 +141,72 @@ def wheel_was_built(wheel: Path) -> bool: return file_hash != wheel_hashes[wheel.name] -def remove_test_files(wheel_path: Path) -> bool: - ''' - Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. - ''' - # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. - with ZipFile(wheel_path, 'r') as zf: - excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] - - if not excluded_members: - # Nothing to strip, so skip rewriting the wheel - return False - with TemporaryDirectory() as td: - td_path = Path(td) - - # Unpack the wheel into temp dir - unpack(wheel_path, dest=td_path) - unpacked_dir = next(td_path.iterdir()) - # Remove excluded files/folders - for root, dirs, files in os.walk(td, topdown=False): - for d in list(dirs): - full_dir = Path(root) / d - rel = full_dir.relative_to(unpacked_dir).as_posix() - if is_excluded_from_wheel(rel): - shutil.rmtree(full_dir) - dirs.remove(d) - for f in files: - rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() - if is_excluded_from_wheel(rel): - os.remove(Path(root) / f) - - print(f'Tests removed from {wheel_path.name}') - - dest_dir = wheel_path.parent - before = {p.resolve() for p in dest_dir.glob("*.whl")} - # Repack to same directory, regenerating RECORD - pack(unpacked_dir, dest_dir=dest_dir, build_number=None) - - # The wheel might not be platform-specific, so repacking restores its original name. - # We need to move the repacked wheel to wheel_path, which was changed to be platform-specific. - after = {p.resolve() for p in wheel_path.parent.glob("*.whl")} - new_files = sorted(after - before, key=lambda p: p.stat().st_mtime, reverse=True) - - if new_files: - shutil.move(str(new_files[0]), str(wheel_path)) - - - return True - -@cache -def _load_excluded_spec() -> pathspec.PathSpec: - """ - Load excluded paths from files_to_remove.toml and compile them - with .gitignore-style semantics. - """ - config_path = Path(__file__).parent / "files_to_remove.toml" - with open(config_path, "rb") as f: - config = tomllib.load(f) - - patterns = config.get("excluded_paths", []) - return pathspec.PathSpec.from_lines("gitignore", patterns) - -def is_excluded_from_wheel(path: str | Path) -> bool: - """ - Return True if `path` (file or directory) should be excluded per files_to_remove.toml. - Matches: - - type annotation files: **/*.pyi, **/py.typed - - test directories listed with a trailing '/' - """ - spec = _load_excluded_spec() - rel = Path(path).as_posix() - - if spec.match_file(rel) or spec.match_file(rel + "/"): - return True - - return False - - -def add_dependency(dependencies: dict[str, str], wheel: Path) -> None: +# def remove_test_files(wheel_path: Path) -> bool: +# ''' +# Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. +# ''' +# # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. +# with ZipFile(wheel_path, 'r') as zf: +# excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] + +# if not excluded_members: +# # Nothing to strip, so skip rewriting the wheel +# return False +# with TemporaryDirectory() as td: +# td_path = Path(td) + +# # Unpack the wheel into temp dir +# unpack(wheel_path, dest=td_path) +# unpacked_dir = next(td_path.iterdir()) +# # Remove excluded files/folders +# for root, dirs, files in os.walk(td, topdown=False): +# for d in list(dirs): +# full_dir = Path(root) / d +# rel = full_dir.relative_to(unpacked_dir).as_posix() +# if is_excluded_from_wheel(rel): +# shutil.rmtree(full_dir) +# dirs.remove(d) +# for f in files: +# rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() +# if is_excluded_from_wheel(rel): +# os.remove(Path(root) / f) + +# print(f'Tests removed from {wheel_path.name}') + +# dest_dir = wheel_path.parent +# before = {p.resolve() for p in dest_dir.glob("*.whl")} +# # Repack to same directory, regenerating RECORD +# pack(unpacked_dir, dest_dir=dest_dir, build_number=None) + +# # The wheel might not be platform-specific, so repacking restores its original name. +# # We need to move the repacked wheel to wheel_path, which was changed to be platform-specific. +# after = {p.resolve() for p in wheel_path.parent.glob("*.whl")} +# new_files = sorted(after - before, key=lambda p: p.stat().st_mtime, reverse=True) + +# if new_files: +# shutil.move(str(new_files[0]), str(wheel_path)) + + +# return True + + +# def is_excluded_from_wheel(path: str | Path) -> bool: +# """ +# Return True if `path` (file or directory) should be excluded per files_to_remove.toml. +# Matches: +# - type annotation files: **/*.pyi, **/py.typed +# - test directories listed with a trailing '/' +# """ +# spec = _load_excluded_spec() +# rel = Path(path).as_posix() + +# if spec.match_file(rel) or spec.match_file(rel + "/"): +# return True + +# return False + + +def add_dependency(dependencies: dict[str, str], sizes: dict[str, WheelSizes], wheel: Path) -> None: project_metadata = extract_metadata(wheel) project_name = normalize_project_name(project_metadata['Name']) project_version = project_metadata['Version'] @@ -295,10 +281,18 @@ def main(): # Fetch or build wheels command_args = [ - str(python_path), '-m', 'pip', 'wheel', - '-r', str(MOUNT_DIR / 'requirements.in'), - '--wheel-dir', str(staged_wheel_dir), - '--extra-index-url', CUSTOM_EXTERNAL_INDEX, + str(python_path), + '-m', + 'pip', + 'wheel', + '--config-settings', + '--build-backend=.builders/scripts/build_backend.py', + '-r', + str(MOUNT_DIR / 'requirements.in'), + '--wheel-dir', + str(staged_wheel_dir), + '--extra-index-url', + CUSTOM_EXTERNAL_INDEX, ] check_process(command_args, env=env_vars) @@ -330,25 +324,31 @@ def main(): dependencies: dict[str, tuple[str, str]] = {} sizes: dict[str, WheelSizes] = {} - # Handle wheels currently in the external directory and move them to the built directory if they were modified - for wheel in iter_wheels(external_wheels_dir): - was_modified = remove_test_files(wheel) - if was_modified: - # A modified wheel is no longer external → move it to built directory - new_path = built_wheels_dir / wheel.name - wheel.rename(new_path) - wheel = new_path - print(f'Moved {wheel.name} to built directory') - - add_dependency(dependencies, sizes wheel) - - # Handle wheels already in the built directory - for wheel in iter_wheels(built_wheels_dir): - remove_test_files(wheel) - add_dependency(dependencies, sizes, wheel) - - + # # Handle wheels currently in the external directory and move them to the built directory if they were modified + # for wheel in iter_wheels(external_wheels_dir): + # was_modified = remove_test_files(wheel) + # if was_modified: + # # A modified wheel is no longer external → move it to built directory + # new_path = built_wheels_dir / wheel.name + # wheel.rename(new_path) + # wheel = new_path + # print(f'Moved {wheel.name} to built directory') + + # add_dependency(dependencies, sizes, wheel) + + # # Handle wheels already in the built directory + # for wheel in iter_wheels(built_wheels_dir): + # remove_test_files(wheel) + # add_dependency(dependencies, sizes, wheel) + + for wheel_dir in wheels_dir.iterdir(): + for wheel in wheel_dir.iterdir(): + project_metadata = extract_metadata(wheel) + project_name = normalize_project_name(project_metadata['Name']) + project_version = project_metadata['Version'] + dependencies[project_name] = project_version + sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)} output_path = MOUNT_DIR / 'sizes.json' with output_path.open('w', encoding='utf-8') as fp: From ba090177ef079a49a292693dfae809da4f34084d Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 12:33:08 +0100 Subject: [PATCH 53/63] Add debugging logs --- .builders/scripts/build_backend.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.builders/scripts/build_backend.py b/.builders/scripts/build_backend.py index 30ace40ce40d2..54a467ddfab0d 100644 --- a/.builders/scripts/build_backend.py +++ b/.builders/scripts/build_backend.py @@ -13,8 +13,10 @@ def remove_test_files(wheel_path: Path) -> None: """ Remove excluded files and directories from a built wheel. + Prints the number of files removed. """ tmp_wheel = wheel_path.with_suffix(".tmp.whl") + removed_count = 0 with ( zipfile.ZipFile(wheel_path, "r") as zin, @@ -23,12 +25,14 @@ def remove_test_files(wheel_path: Path) -> None: for info in zin.infolist(): rel = info.filename if is_excluded_from_wheel(rel): + removed_count += 1 continue # skip excluded file or directory data = zin.read(rel) zout.writestr(info, data) shutil.move(tmp_wheel, wheel_path) + print(f"Removed {removed_count} files from {wheel_path.name}") def is_excluded_from_wheel(path: str | Path) -> bool: @@ -76,6 +80,7 @@ def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): # prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel # build_sdist = _orig.build_sdist # (better do by iterating over _orig methods instead) +print("-> Inspecting _orig methods") for name, func in inspect.getmembers(_orig, inspect.isfunction): # Only copy methods if they haven't been defined in the current module # (i.e., don't overwrite your custom build_wheel) From b74f756e6899a105cad4b0682b731ec8d54b7e77 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 12:40:33 +0100 Subject: [PATCH 54/63] Add debugging logs --- .builders/scripts/build_wheels.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 0964a600b4084..b78da1045a6bd 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -278,7 +278,9 @@ def main(): if constraints_file := env_vars.get('PIP_CONSTRAINT'): env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) - + print("--------------------------------") + print("Building wheels") + print("--------------------------------") # Fetch or build wheels command_args = [ str(python_path), @@ -294,7 +296,9 @@ def main(): '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] - + print("--------------------------------") + print("Finished building wheels") + print("--------------------------------") check_process(command_args, env=env_vars) # Classify wheels From 804bc7b9a9a2e874d38b1b6c62af253bd12cd702 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 12:53:40 +0100 Subject: [PATCH 55/63] Change parameter format --- .builders/scripts/build_wheels.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index b78da1045a6bd..cf04ca1192b12 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -287,8 +287,7 @@ def main(): '-m', 'pip', 'wheel', - '--config-settings', - '--build-backend=.builders/scripts/build_backend.py', + '--config-settings=build-backend=.builders/scripts/build_backend.py', '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', @@ -351,8 +350,7 @@ def main(): project_name = normalize_project_name(project_metadata['Name']) project_version = project_metadata['Version'] dependencies[project_name] = project_version - - sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)} + sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)} output_path = MOUNT_DIR / 'sizes.json' with output_path.open('w', encoding='utf-8') as fp: From 661846cdcebe7a7ad8e7885b2924bc8f8179fe00 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 14:29:38 +0100 Subject: [PATCH 56/63] Add pyproject --- .builders/scripts/pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .builders/scripts/pyproject.toml diff --git a/.builders/scripts/pyproject.toml b/.builders/scripts/pyproject.toml new file mode 100644 index 0000000000000..e0c8494a41dae --- /dev/null +++ b/.builders/scripts/pyproject.toml @@ -0,0 +1,4 @@ +[build-system] +# Tells the frontend to import the 'build_backend' module +# and use its 'build_wheel' function. +build-backend = "build_backend" \ No newline at end of file From e3ff87a901289e9f938affc5722dc53b3220013f Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 14:48:27 +0100 Subject: [PATCH 57/63] remove flag --- .builders/scripts/build_wheels.py | 86 ++----------------------------- .builders/scripts/pyproject.toml | 3 +- 2 files changed, 5 insertions(+), 84 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index cf04ca1192b12..2cd2fbf112863 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -27,10 +27,12 @@ CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built' UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+') + class WheelSizes(TypedDict): compressed: int uncompressed: int + if sys.platform == 'win32': PY3_PATH = Path('C:\\py3\\Scripts\\python.exe') PY2_PATH = Path('C:\\py2\\Scripts\\python.exe') @@ -141,71 +143,6 @@ def wheel_was_built(wheel: Path) -> bool: return file_hash != wheel_hashes[wheel.name] -# def remove_test_files(wheel_path: Path) -> bool: -# ''' -# Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly. -# ''' -# # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched. -# with ZipFile(wheel_path, 'r') as zf: -# excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)] - -# if not excluded_members: -# # Nothing to strip, so skip rewriting the wheel -# return False -# with TemporaryDirectory() as td: -# td_path = Path(td) - -# # Unpack the wheel into temp dir -# unpack(wheel_path, dest=td_path) -# unpacked_dir = next(td_path.iterdir()) -# # Remove excluded files/folders -# for root, dirs, files in os.walk(td, topdown=False): -# for d in list(dirs): -# full_dir = Path(root) / d -# rel = full_dir.relative_to(unpacked_dir).as_posix() -# if is_excluded_from_wheel(rel): -# shutil.rmtree(full_dir) -# dirs.remove(d) -# for f in files: -# rel = Path(root).joinpath(f).relative_to(unpacked_dir).as_posix() -# if is_excluded_from_wheel(rel): -# os.remove(Path(root) / f) - -# print(f'Tests removed from {wheel_path.name}') - -# dest_dir = wheel_path.parent -# before = {p.resolve() for p in dest_dir.glob("*.whl")} -# # Repack to same directory, regenerating RECORD -# pack(unpacked_dir, dest_dir=dest_dir, build_number=None) - -# # The wheel might not be platform-specific, so repacking restores its original name. -# # We need to move the repacked wheel to wheel_path, which was changed to be platform-specific. -# after = {p.resolve() for p in wheel_path.parent.glob("*.whl")} -# new_files = sorted(after - before, key=lambda p: p.stat().st_mtime, reverse=True) - -# if new_files: -# shutil.move(str(new_files[0]), str(wheel_path)) - - -# return True - - -# def is_excluded_from_wheel(path: str | Path) -> bool: -# """ -# Return True if `path` (file or directory) should be excluded per files_to_remove.toml. -# Matches: -# - type annotation files: **/*.pyi, **/py.typed -# - test directories listed with a trailing '/' -# """ -# spec = _load_excluded_spec() -# rel = Path(path).as_posix() - -# if spec.match_file(rel) or spec.match_file(rel + "/"): -# return True - -# return False - - def add_dependency(dependencies: dict[str, str], sizes: dict[str, WheelSizes], wheel: Path) -> None: project_metadata = extract_metadata(wheel) project_name = normalize_project_name(project_metadata['Name']) @@ -213,6 +150,7 @@ def add_dependency(dependencies: dict[str, str], sizes: dict[str, WheelSizes], w dependencies[project_name] = project_version sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)} + def calculate_wheel_sizes(wheel_path: Path) -> WheelSizes: compressed_size = wheel_path.stat(follow_symlinks=True).st_size with ZipFile(wheel_path) as zf: @@ -287,7 +225,6 @@ def main(): '-m', 'pip', 'wheel', - '--config-settings=build-backend=.builders/scripts/build_backend.py', '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', @@ -327,23 +264,6 @@ def main(): dependencies: dict[str, tuple[str, str]] = {} sizes: dict[str, WheelSizes] = {} - # # Handle wheels currently in the external directory and move them to the built directory if they were modified - # for wheel in iter_wheels(external_wheels_dir): - # was_modified = remove_test_files(wheel) - # if was_modified: - # # A modified wheel is no longer external → move it to built directory - # new_path = built_wheels_dir / wheel.name - # wheel.rename(new_path) - # wheel = new_path - # print(f'Moved {wheel.name} to built directory') - - # add_dependency(dependencies, sizes, wheel) - - # # Handle wheels already in the built directory - # for wheel in iter_wheels(built_wheels_dir): - # remove_test_files(wheel) - # add_dependency(dependencies, sizes, wheel) - for wheel_dir in wheels_dir.iterdir(): for wheel in wheel_dir.iterdir(): project_metadata = extract_metadata(wheel) diff --git a/.builders/scripts/pyproject.toml b/.builders/scripts/pyproject.toml index e0c8494a41dae..7f3ebda546f81 100644 --- a/.builders/scripts/pyproject.toml +++ b/.builders/scripts/pyproject.toml @@ -1,4 +1,5 @@ [build-system] # Tells the frontend to import the 'build_backend' module # and use its 'build_wheel' function. -build-backend = "build_backend" \ No newline at end of file +requires = ["setuptools", "wheel", "pathspec"] +build-backend = "build_backend" From fd23319a83e016a785545287ba303c80fef4b75d Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 15:13:38 +0100 Subject: [PATCH 58/63] Change flag --- .builders/scripts/build_wheels.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 2cd2fbf112863..e39f2d60e03fb 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -225,6 +225,8 @@ def main(): '-m', 'pip', 'wheel', + '--config-settings', + f'--build-backend={MOUNT_DIR / "scripts" / "pyproject.toml"}' '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', From 5bf73608d9096b724d4f5ed9597166fa9dde63fb Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 15:19:03 +0100 Subject: [PATCH 59/63] Fix typo --- .builders/scripts/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index e39f2d60e03fb..2ac8989f1c95b 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -226,7 +226,7 @@ def main(): 'pip', 'wheel', '--config-settings', - f'--build-backend={MOUNT_DIR / "scripts" / "pyproject.toml"}' + f'--build-backend={MOUNT_DIR / "scripts" / "pyproject.toml"}', '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', From 6c49f37b3e077a7a9e7e0417a5e2a9feb3865dee Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 15:21:17 +0100 Subject: [PATCH 60/63] Add debug logs --- .builders/scripts/build_wheels.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 2ac8989f1c95b..b051e74528600 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -216,6 +216,7 @@ def main(): if constraints_file := env_vars.get('PIP_CONSTRAINT'): env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) + print("--------------------------------") print("Building wheels") print("--------------------------------") @@ -234,11 +235,11 @@ def main(): '--extra-index-url', CUSTOM_EXTERNAL_INDEX, ] + + check_process(command_args, env=env_vars) print("--------------------------------") print("Finished building wheels") print("--------------------------------") - check_process(command_args, env=env_vars) - # Classify wheels for wheel in iter_wheels(staged_wheel_dir): if wheel_was_built(wheel): From f2f00af17e36ef09e5693b7201b11fee1bb15245 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 15:34:20 +0100 Subject: [PATCH 61/63] remove pyproject --- .builders/scripts/pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .builders/scripts/pyproject.toml diff --git a/.builders/scripts/pyproject.toml b/.builders/scripts/pyproject.toml deleted file mode 100644 index 7f3ebda546f81..0000000000000 --- a/.builders/scripts/pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -[build-system] -# Tells the frontend to import the 'build_backend' module -# and use its 'build_wheel' function. -requires = ["setuptools", "wheel", "pathspec"] -build-backend = "build_backend" From b3d36c5c4bed005fd1cdb4a63e44f147199aed5d Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 15:39:55 +0100 Subject: [PATCH 62/63] Change flag --- .builders/scripts/build_wheels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index b051e74528600..484cd26db5ca2 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -216,7 +216,7 @@ def main(): if constraints_file := env_vars.get('PIP_CONSTRAINT'): env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file) - + print("--------------------------------") print("Building wheels") print("--------------------------------") @@ -227,7 +227,7 @@ def main(): 'pip', 'wheel', '--config-settings', - f'--build-backend={MOUNT_DIR / "scripts" / "pyproject.toml"}', + f'--build-backend={MOUNT_DIR / "scripts" / "build_backend.py"}', '-r', str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', From a8f32c290b6ee6fe7b4fb3cad22e5cabe9d70e86 Mon Sep 17 00:00:00 2001 From: Lucia Sanchez Bella Date: Fri, 31 Oct 2025 15:45:28 +0100 Subject: [PATCH 63/63] Remove flag --- .builders/scripts/build_wheels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index 484cd26db5ca2..cec4dcdd21491 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -232,8 +232,8 @@ def main(): str(MOUNT_DIR / 'requirements.in'), '--wheel-dir', str(staged_wheel_dir), - '--extra-index-url', - CUSTOM_EXTERNAL_INDEX, + # '--extra-index-url', + # CUSTOM_EXTERNAL_INDEX, ] check_process(command_args, env=env_vars)