stop using Requires.jl (use extensions instead) (#662)

* stop using Requires.jl (use extensions instead)

* update changelog

* don't build PyCall in tests

instead, add a case to CI where we use system python for both, ensuring we test the same case. we also add tests for the non-same case.

* typo

* only test PyCall when we can guarantee same interpreter

* Update PyCallExt.jl

---------

Co-authored-by: Christopher Doris <github.com/cjdoris>

Author: cjdoris

.github/workflows/tests.yml

@@ -12,14 +12,20 @@ on:
 
 jobs:
   julia:
-    name: Test Julia (${{ matrix.jlversion }}, ${{ matrix.os }})
+    name: Test Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         arch: [x64] # x86 unsupported by MicroMamba
         os: [ubuntu-latest, windows-latest, macos-latest]
         jlversion: ['1','1.9']
+        pythonexe: ['@CondaPkg']
+        include:
+        - arch: x64
+          os: ubuntu-latest
+          jlversion: '1'
+          pythonexe: python
 
     steps:
       - uses: actions/checkout@v5
@@ -34,12 +40,16 @@ jobs:
 
       - name: Build package
         uses: julia-actions/julia-buildpkg@v1
+        env:
+          PYTHON: python
 
       - name: Run tests
         uses: julia-actions/julia-runtest@v1
         env:
           JULIA_DEBUG: PythonCall
           JULIA_NUM_THREADS: '2'
+          PYTHON: python
+          JULIA_PYTHONCALL_EXE: ${{ matrix.pythonexe }}
 
       - name: Process coverage
         uses: julia-actions/julia-processcoverage@v1

Project.toml

@@ -10,33 +10,41 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
 
 [compat]
 Aqua = "0 - 999"
+CategoricalArrays = "0.10, 1"
 CondaPkg = "0.2.30"
 Dates = "1"
 Libdl = "1"
 MacroTools = "0.5"
 Markdown = "1"
 Pkg = "1"
 PyCall = "1"
-Requires = "1"
 Serialization = "1"
 Tables = "1"
 Test = "1"
 TestItemRunner = "0 - 999"
 UnsafePointers = "1"
 julia = "1.9"
 
+[extensions]
+PyCallExt = "PyCall"
+CategoricalArraysExt = "CategoricalArrays"
+
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 
 [targets]
 test = ["Aqua", "PyCall", "Test", "TestItemRunner"]
+
+[weakdeps]
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"

docs/src/releasenotes.md

@@ -1,5 +1,9 @@
 # Release Notes
 
+## Unreleased
+* Bug fixes.
+* Internal: switch from Requires.jl to package extensions.
+
 ## 0.9.27 (2025-08-19)
 * Internal: Use heap-allocated types (PyType_FromSpec) to improve ABI compatibility.
 * Minimum supported Python version is now 3.9.

ext/CategoricalArraysExt.jl

@@ -0,0 +1,14 @@
+module CategoricalArraysExt
+
+using PythonCall
+
+using CategoricalArrays: CategoricalArrays
+
+function PythonCall.Compat.aspandasvector(x::CategoricalArrays.CategoricalArray)
+    codes = map(x -> x === missing ? -1 : Int(CategoricalArrays.levelcode(x)) - 1, x)
+    cats = CategoricalArrays.levels(x)
+    ordered = x.pool.ordered
+    pyimport("pandas").Categorical.from_codes(codes, cats, ordered = ordered)
+end
+
+end

ext/PyCallExt.jl

@@ -0,0 +1,52 @@
+module PyCallExt
+
+# While this extension is precompilable, it can only be used without crashing if PyCall is
+# using the same Python interpreter. However Julia precompiles all available extensions
+# eagerly, which therefore crashes if PyCall is using a different interpreter. So for now
+# we disable precompilation of this extension entirely.
+__precompile__(false)
+
+using PythonCall
+using PythonCall.Core
+using PythonCall.C
+
+using PyCall: PyCall
+
+import PythonCall: Py
+
+# true if PyCall and PythonCall are using the same interpreter
+const SAME = Ref{Bool}(false)
+
+function __init__()
+    # see if PyCall and PythonCall are using the same interpreter by checking if a couple of memory addresses are the same
+    ptr1 = C.Py_GetVersion()
+    ptr2 = ccall(PyCall.@pysym(:Py_GetVersion), Ptr{Cchar}, ())
+    SAME[] = ptr1 == ptr2
+    if PythonCall.C.CTX.which == :PyCall
+        @assert SAME[]
+    end
+end
+
+# allow explicit conversion between PythonCall.Py and PyCall.PyObject
+# provided they are using the same interpretr
+const ERRMSG = """
+Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter.
+
+There are two ways to achieve this:
+- Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same
+  interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies.
+- Set the environment variable `PYTHON` to `PythonCall.python_executable_path()` and rebuild PyCall. This forces
+  PyCall to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment.
+"""
+
+function Py(x::PyCall.PyObject)
+    SAME[] || error(ERRMSG)
+    return pynew(C.PyPtr(PyCall.pyreturn(x)))
+end
+
+function PyCall.PyObject(x::Py)
+    SAME[] || error(ERRMSG)
+    return PyCall.PyObject(PyCall.PyPtr(getptr(incref(x))))
+end
+
+end

src/C/C.jl

@@ -9,11 +9,11 @@ using Base: @kwdef
 using UnsafePointers: UnsafePtr
 using CondaPkg: CondaPkg
 using Pkg: Pkg
-using Requires: @require
 using Libdl:
     dlpath, dlopen, dlopen_e, dlclose, dlsym, dlsym_e, RTLD_LAZY, RTLD_DEEPBIND, RTLD_GLOBAL
 
-import ..PythonCall: python_executable_path, python_library_path, python_library_handle, python_version
+import ..PythonCall:
+    python_executable_path, python_library_path, python_library_handle, python_version
 
 
 include("consts.jl")

src/C/context.jl

@@ -17,7 +17,6 @@ A handle to a loaded instance of libpython, its interpreter, function pointers,
     pyhome_w::Any = missing
     which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown
     version::Union{VersionNumber,Missing} = missing
-    matches_pycall::Union{Bool,Missing} = missing
 end
 
 const CTX = Context()
@@ -141,15 +140,9 @@ function init_context()
         # Get function pointers from the library
         init_pointers()
 
-        # Compare libpath with PyCall
-        @require PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall)
-
         # Initialize the interpreter
         CTX.is_preinitialized = Py_IsInitialized() != 0
-        if CTX.is_preinitialized
-            @assert CTX.which == :PyCall || CTX.matches_pycall isa Bool
-        else
-            @assert CTX.which != :PyCall
+        if !CTX.is_preinitialized
             # Find ProgramName and PythonHome
             script = if Sys.iswindows()
                 """
@@ -243,13 +236,3 @@ const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall")
 
 const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0")
 const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall")
-
-function init_pycall(PyCall::Module)
-    # see if PyCall and PythonCall are using the same interpreter by checking if a couple of memory addresses are the same
-    ptr1 = Py_GetVersion()
-    ptr2 = @eval PyCall ccall(@pysym(:Py_GetVersion), Ptr{Cchar}, ())
-    CTX.matches_pycall = ptr1 == ptr2
-    if CTX.which == :PyCall
-        @assert CTX.matches_pycall
-    end
-end

src/Compat/Compat.jl

@@ -13,7 +13,6 @@ using ..Wrap
 
 using Serialization: Serialization, AbstractSerializer, serialize, deserialize
 using Tables: Tables
-using Requires: @require
 
 import ..PythonCall: event_loop_on, event_loop_off, fix_qt_plugin_path, pytable
 
@@ -22,13 +21,10 @@ include("ipython.jl")
 include("multimedia.jl")
 include("serialization.jl")
 include("tables.jl")
-include("pycall.jl")
 
 function __init__()
     init_gui()
     init_pyshow()
-    init_tables()
-    @require PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall)
 end
 
 end

src/Compat/pycall.jl

@@ -63,14 +63,3 @@ function _pytable_pandas(src, cols = Tables.columns(src); opts...)
         opts...,
     )
 end
-
-function init_tables()
-    @require CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" @eval begin
-        aspandasvector(x::CategoricalArrays.CategoricalArray) = begin
-            codes = map(x -> x === missing ? -1 : Int(CategoricalArrays.levelcode(x)) - 1, x)
-            cats = CategoricalArrays.levels(x)
-            ordered = x.pool.ordered
-            pyimport("pandas").Categorical.from_codes(codes, cats, ordered = ordered)
-        end
-    end
-end