Skip to content

Commit c34d992

Browse files
scottsscotts
committed
Adding Mac to S3 non-GPL FFmpeg compile
1 parent ec24944 commit c34d992

File tree

3 files changed

+282
-29
lines changed

3 files changed

+282
-29
lines changed

‎.github/workflows/wheel.yaml‎ renamed to ‎.github/workflows/linux_wheel.yaml‎

File renamed without changes.

‎.github/workflows/macos_wheel.yaml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: Build and test MacOS
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
tags:
11+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
12+
workflow_dispatch:
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
16+
cancel-in-progress: true
17+
18+
permissions:
19+
id-token: write
20+
contents: write
21+
22+
defaults:
23+
run:
24+
shell: bash -l -eo pipefail {0}
25+
26+
jobs:
27+
# generate-matrix:
28+
# uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
29+
# with:
30+
# package-type: wheel
31+
# os: linux
32+
# test-infra-repository: pytorch/test-infra
33+
# test-infra-ref: main
34+
# with-xpu: disable
35+
# with-rocm: disable
36+
# with-cuda: disable
37+
# build:
38+
# needs: generate-matrix
39+
# strategy:
40+
# fail-fast: false
41+
# name: Build and Upload wheel
42+
# uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
43+
# with:
44+
# repository: pytorch/torchcodec
45+
# ref: ""
46+
# test-infra-repository: pytorch/test-infra
47+
# test-infra-ref: main
48+
# build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
49+
# post-script: packaging/post_build_script.sh
50+
# smoke-test-script: packaging/fake_smoke_test.py
51+
# package-name: torchcodec
52+
# trigger-event: ${{ github.event_name }}
53+
# build-platform: "python-build-package"
54+
# build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation"
55+
56+
install-and-test:
57+
runs-on: macos-12
58+
strategy:
59+
fail-fast: false
60+
matrix:
61+
python-version: ['3.9']
62+
ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
63+
if: ${{ always() }}
64+
# needs: build
65+
steps:
66+
# - uses: actions/download-artifact@v3
67+
# with:
68+
# name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 # FIXME should be arm64
69+
# path: pytorch/torchcodec/dist/
70+
- name: Setup conda env
71+
uses: conda-incubator/setup-miniconda@v2
72+
with:
73+
auto-update-conda: true
74+
miniconda-version: "latest"
75+
activate-environment: test
76+
python-version: ${{ matrix.python-version }}
77+
- name: Update pip
78+
run: python -m pip install --upgrade pip
79+
- name: Install PyTorch
80+
run: |
81+
python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
82+
# - name: Install torchcodec from the wheel
83+
# run: |
84+
# wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
85+
# echo Installing $wheel_path
86+
# python -m pip install $wheel_path -vvv
87+
#
88+
- name: Install torchcodec from source
89+
run: |
90+
BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 pip install -e “[.dev]” –no-build-isolation
91+
92+
- name: Check out repo
93+
uses: actions/checkout@v3
94+
- name: Install ffmpeg, post build
95+
run: |
96+
# Ideally we would have checked for that before installing the wheel,
97+
# but we need to checkout the repo to access this file, and we don't
98+
# want to checkout the repo before installing the wheel to avoid any
99+
# side-effect. It's OK.
100+
source packaging/helpers.sh
101+
assert_ffmpeg_not_installed
102+
103+
conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
104+
ffmpeg -version
105+
106+
- name: Install test dependencies
107+
run: |
108+
python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
109+
# Ideally we would find a way to get those dependencies from pyproject.toml
110+
python -m pip install numpy pytest pillow
111+
112+
- name: Delete the src/ folder just for fun
113+
run: |
114+
# The only reason we checked-out the repo is to get access to the
115+
# tests. We don't care about the rest. Out of precaution, we delete
116+
# the src/ folder to be extra sure that we're running the code from
117+
# the installed wheel rather than from the source.
118+
# This is just to be extra cautious and very overkill because a)
119+
# there's no way the `torchcodec` package from src/ can be found from
120+
# the PythonPath: the main point of `src/` is precisely to protect
121+
# against that and b) if we ever were to execute code from
122+
# `src/torchcodec`, it would fail loudly because the built .so files
123+
# aren't present there.
124+
rm -r src/
125+
ls
126+
- name: Smoke test
127+
run: |
128+
python test/decoders/manual_smoke_test.py
129+
- name: Run Python tests
130+
run: |
131+
pytest test

‎src/torchcodec/decoders/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake‎

Lines changed: 151 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,33 +8,150 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
88
endif()
99

1010
include(FetchContent)
11+
1112
set(
1213
base_url
13-
https://pytorch.s3.amazonaws.com/torchcodec/ffmpeg/2024-06-11/linux_x86_64
14+
https://pytorch.s3.amazonaws.com/torchcodec/ffmpeg/2024-09-08
1415
)
16+
17+
if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
18+
set(
19+
platform_url
20+
${base_url}/linux_x86_64
21+
)
22+
23+
set(
24+
f4_hash
25+
c920e8064878b8d10648ee440659878974363374d08f1edb17f1ac435ca3afbf
26+
)
27+
set(
28+
f5_hash
29+
39f70050338616dd0dee461bbb0e7ea1f68d80693310182efb8acceded5e84e8
30+
)
31+
set(
32+
f6_hash
33+
c1c08d796b177f566a9650d8e945e62925041f95d6016fd4266d6337ae782444
34+
)
35+
set(
36+
f7_hash
37+
18a00d44f45cd57a8fe1ca32397496f17c9f359492a536cb2d46c6494a1d4b17
38+
)
39+
40+
set(
41+
f4_library_file_names
42+
libavutil.so.56
43+
libavcodec.so.58
44+
libavformat.so.58
45+
libavdevice.so.58
46+
libavfilter.so.7
47+
)
48+
set(
49+
f5_library_file_names
50+
libavutil.so.57
51+
libavcodec.so.59
52+
libavformat.so.59
53+
libavdevice.so.59
54+
libavfilter.so.8
55+
)
56+
set(
57+
f6_library_file_names
58+
libavutil.so.58
59+
libavcodec.so.60
60+
libavformat.so.60
61+
libavdevice.so.60
62+
libavfilter.so.9
63+
)
64+
set(
65+
f7_library_file_names
66+
libavutil.so.59
67+
libavcodec.so.61
68+
libavformat.so.61
69+
libavdevice.so.61
70+
libavfilter.so.10
71+
)
72+
elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
73+
set(
74+
platform_url
75+
${base_url}/macos_arm64
76+
)
77+
set(
78+
f4_hash
79+
8d4e2ee26b6b65eb520402189aea11534fb39aecfb2f01232d96fedd5cd16e94
80+
)
81+
set(
82+
f5_hash
83+
79cdfe9c04b480dcaff0deb61aba60d88681918e8bc6f7de27c751c77622f43
84+
)
85+
set(
86+
f6_hash
87+
a5f37393ef17617bea4844a1423a945cbcff145c69281dc70dc31db09282cc8a
88+
)
89+
set(
90+
f7_hash
91+
e9ea739e5fc5f14741324f2b9ccbfe92f13e574f2cbad49488ba88a8ad8f4d49
92+
)
93+
set(
94+
f4_library_file_names
95+
libavutil.56.dylib
96+
libavcodec.58.dylib
97+
libavformat.58.dylib
98+
libavdevice.58.dylib
99+
libavfilter.7.dylib
100+
)
101+
set(
102+
f5_library_file_names
103+
libavutil.57.dylib
104+
libavcodec.59.dylib
105+
libavformat.59.dylib
106+
libavdevice.59.dylib
107+
libavfilter.8.dylib
108+
)
109+
set(
110+
f6_library_file_names
111+
libavutil.58.dylib
112+
libavcodec.60.dylib
113+
libavformat.60.dylib
114+
libavdevice.60.dylib
115+
libavfilter.9.dylib
116+
)
117+
set(
118+
f7_library_file_names
119+
libavutil.59.dylib
120+
libavcodec.61.dylib
121+
libavformat.61.dylib
122+
libavdevice.61.dylib
123+
libavfilter.10.dylib
124+
)
125+
else()
126+
message(
127+
FATAL_ERROR
128+
"Unsupported operating system: ${CMAKE_SYSTEM_NAME}"
129+
)
130+
endif()
131+
15132
FetchContent_Declare(
16133
f4
17-
URL ${base_url}/ffmpeg_4.4.4.tar.gz
134+
URL ${platform_url}/4.4.4.tar.gz
18135
URL_HASH
19-
SHA256=a564721e51038d01ead4bbc7a482398929101ca4c80e5ce5c42042637235a297
136+
SHA256=${f4_hash}
20137
)
21138
FetchContent_Declare(
22139
f5
23-
URL ${base_url}/ffmpeg_5.1.4.tar.gz
140+
URL ${platform_url}/5.1.4.tar.gz
24141
URL_HASH
25-
SHA256=d9c2d3a355c091ddc3205ae73426d0d6402ad8a31212dc920daabbaa5fdae944
142+
SHA256=${f5_hash}
26143
)
27144
FetchContent_Declare(
28145
f6
29-
URL ${base_url}/ffmpeg_6.1.1.tar.gz
146+
URL ${platform_url}/6.1.1.tar.gz
30147
URL_HASH
31-
SHA256=7ee5830dc09fed7270aa575650474ab16e18477551e5511f256ce92daa30b136
148+
SHA256=${f6_hash}
32149
)
33150
FetchContent_Declare(
34151
f7
35-
URL ${base_url}/ffmpeg_7.0.1.tar.gz
152+
URL ${platform_url}/7.0.1.tar.gz
36153
URL_HASH
37-
SHA256=fa4cda7aa67fcd58428017f7ebd2a981b0c6babba7ec89f71d6840877712ddcd
154+
SHA256=${f7_hash}
38155
)
39156

40157
FetchContent_MakeAvailable(f4 f5 f6 f7)
@@ -50,39 +167,44 @@ target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include)
50167
target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include)
51168
target_include_directories(ffmpeg7 INTERFACE ${f7_SOURCE_DIR}/include)
52169

170+
list(
171+
TRANSFORM f4_library_file_names
172+
PREPEND ${f4_SOURCE_DIR}/lib/
173+
OUTPUT_VARIABLE f4_library_paths
174+
)
175+
list(
176+
TRANSFORM f5_library_file_names
177+
PREPEND ${f5_SOURCE_DIR}/lib/
178+
OUTPUT_VARIABLE f5_library_paths
179+
)
180+
list(
181+
TRANSFORM f6_library_file_names
182+
PREPEND ${f6_SOURCE_DIR}/lib/
183+
OUTPUT_VARIABLE f6_library_paths
184+
)
185+
list(
186+
TRANSFORM f7_library_file_names
187+
PREPEND ${f7_SOURCE_DIR}/lib/
188+
OUTPUT_VARIABLE f7_library_paths
189+
)
190+
53191
target_link_libraries(
54192
ffmpeg4
55193
INTERFACE
56-
${f4_SOURCE_DIR}/lib/libavutil.so.56
57-
${f4_SOURCE_DIR}/lib/libavcodec.so.58
58-
${f4_SOURCE_DIR}/lib/libavformat.so.58
59-
${f4_SOURCE_DIR}/lib/libavdevice.so.58
60-
${f4_SOURCE_DIR}/lib/libavfilter.so.7
194+
${f4_library_paths}
61195
)
62196
target_link_libraries(
63197
ffmpeg5
64198
INTERFACE
65-
${f5_SOURCE_DIR}/lib/libavutil.so.57
66-
${f5_SOURCE_DIR}/lib/libavcodec.so.59
67-
${f5_SOURCE_DIR}/lib/libavformat.so.59
68-
${f5_SOURCE_DIR}/lib/libavdevice.so.59
69-
${f5_SOURCE_DIR}/lib/libavfilter.so.8
199+
${f5_library_paths}
70200
)
71201
target_link_libraries(
72202
ffmpeg6
73203
INTERFACE
74-
${f6_SOURCE_DIR}/lib/libavutil.so.58
75-
${f6_SOURCE_DIR}/lib/libavcodec.so.60
76-
${f6_SOURCE_DIR}/lib/libavformat.so.60
77-
${f6_SOURCE_DIR}/lib/libavdevice.so.60
78-
${f6_SOURCE_DIR}/lib/libavfilter.so.9
204+
${f6_library_paths}
79205
)
80206
target_link_libraries(
81207
ffmpeg7
82208
INTERFACE
83-
${f7_SOURCE_DIR}/lib/libavutil.so.59
84-
${f7_SOURCE_DIR}/lib/libavcodec.so.61
85-
${f7_SOURCE_DIR}/lib/libavformat.so.61
86-
${f7_SOURCE_DIR}/lib/libavdevice.so.61
87-
${f7_SOURCE_DIR}/lib/libavfilter.so.10
209+
${f7_library_paths}
88210
)

0 commit comments

Comments
 (0)