Skip to content

Commit 3e608b5

Browse files
committed
Fix aarch64/arm64 Linux wheel build failure due to missing libatomic linking
Problem The tokenizers library fails to build on aarch64 Linux systems during CI with: /opt/rh/gcc-toolset-13/root/usr/libexec/gcc/aarch64-redhat-linux/13/ld: cannot find -latomic: No such file or directory Root Cause - On aarch64, certain atomic operations require explicit linking with libatomic - The sentencepiece dependency detects libatomic (Found atomic: /usr/lib64/libatomic.so.1) but incorrectly adds just the string "atomic" to link flags instead of the actual library path - This causes the linker to fail when building the Python extension Solution Added proper libatomic detection and linking for aarch64/arm64 systems in two places: 1. For the tokenizers static library 2. For the pytorch_tokenizers_cpp Python extension module The fix uses CMake's find_library to locate libatomic and explicitly links it when building on aarch64/arm64 architectures.
1 parent 348c748 commit 3e608b5

File tree

1 file changed

+21
-0
lines changed

1 file changed

+21
-0
lines changed

CMakeLists.txt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,15 @@ target_include_directories(
9494
)
9595
target_link_libraries(tokenizers PUBLIC sentencepiece-static re2::re2)
9696

97+
# Link with atomic library on aarch64/arm64 systems
98+
# Some aarch64 systems require explicit linking with libatomic for certain atomic operations
99+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
100+
find_library(ATOMIC_LIB NAMES atomic libatomic.so libatomic.so.1)
101+
if(ATOMIC_LIB)
102+
target_link_libraries(tokenizers PUBLIC ${ATOMIC_LIB})
103+
endif()
104+
endif()
105+
97106
if(SUPPORT_REGEX_LOOKAHEAD)
98107
set(PCRE2_STATIC_PIC ON)
99108
set(PCRE2_BUILD_PCRE2_8 ON)
@@ -160,6 +169,18 @@ if(TOKENIZERS_BUILD_PYTHON)
160169
# Link with the tokenizers library
161170
target_link_libraries(pytorch_tokenizers_cpp PRIVATE tokenizers)
162171

172+
# Link with atomic library on aarch64/arm64 systems
173+
# Some aarch64 systems require explicit linking with libatomic for certain atomic operations
174+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
175+
find_library(ATOMIC_LIB NAMES atomic libatomic.so libatomic.so.1)
176+
if(ATOMIC_LIB)
177+
target_link_libraries(pytorch_tokenizers_cpp PRIVATE ${ATOMIC_LIB})
178+
message(STATUS "Found libatomic: ${ATOMIC_LIB}")
179+
else()
180+
message(WARNING "libatomic not found on aarch64 - build may fail")
181+
endif()
182+
endif()
183+
163184
# Set properties for the Python extension
164185
target_compile_definitions(pytorch_tokenizers_cpp PRIVATE VERSION_INFO=${PROJECT_VERSION})
165186

0 commit comments

Comments
 (0)