Merge branch 'main' of github.com:pytorch/torchcodec into nvdec_receive_send

Author: NicolasHug

src/torchcodec/_core/CpuDeviceInterface.cpp

@@ -15,6 +15,18 @@ static bool g_cpu = registerDeviceInterface(
 
 } // namespace
 
+CpuDeviceInterface::SwsFrameContext::SwsFrameContext(
+    int inputWidth,
+    int inputHeight,
+    AVPixelFormat inputFormat,
+    int outputWidth,
+    int outputHeight)
+    : inputWidth(inputWidth),
+      inputHeight(inputHeight),
+      inputFormat(inputFormat),
+      outputWidth(outputWidth),
+      outputHeight(outputHeight) {}
+
 bool CpuDeviceInterface::SwsFrameContext::operator==(
     const CpuDeviceInterface::SwsFrameContext& other) const {
   return inputWidth == other.inputWidth && inputHeight == other.inputHeight &&
@@ -97,13 +109,12 @@ void CpuDeviceInterface::convertAVFrameToFrameOutput(
     // And we sometimes re-create them because it's possible for frame
     // resolution to change mid-stream. Finally, we want to reuse the colorspace
     // conversion objects as much as possible for performance reasons.
-    SwsFrameContext swsFrameContext;
-
-    swsFrameContext.inputWidth = avFrame->width;
-    swsFrameContext.inputHeight = avFrame->height;
-    swsFrameContext.inputFormat = frameFormat;
-    swsFrameContext.outputWidth = expectedOutputWidth;
-    swsFrameContext.outputHeight = expectedOutputHeight;
+    SwsFrameContext swsFrameContext(
+        avFrame->width,
+        avFrame->height,
+        frameFormat,
+        expectedOutputWidth,
+        expectedOutputHeight);
 
     outputTensor = preAllocatedOutputTensor.value_or(allocateEmptyHWCTensor(
         expectedOutputHeight, expectedOutputWidth, torch::kCPU));
@@ -128,22 +139,20 @@ void CpuDeviceInterface::convertAVFrameToFrameOutput(
   } else if (colorConversionLibrary == ColorConversionLibrary::FILTERGRAPH) {
     // See comment above in swscale branch about the filterGraphContext_
     // creation. creation
-    FiltersContext filtersContext;
-
-    filtersContext.inputWidth = avFrame->width;
-    filtersContext.inputHeight = avFrame->height;
-    filtersContext.inputFormat = frameFormat;
-    filtersContext.inputAspectRatio = avFrame->sample_aspect_ratio;
-    filtersContext.outputWidth = expectedOutputWidth;
-    filtersContext.outputHeight = expectedOutputHeight;
-    filtersContext.outputFormat = AV_PIX_FMT_RGB24;
-    filtersContext.timeBase = timeBase;
-
     std::stringstream filters;
     filters << "scale=" << expectedOutputWidth << ":" << expectedOutputHeight;
     filters << ":sws_flags=bilinear";
 
-    filtersContext.filtergraphStr = filters.str();
+    FiltersContext filtersContext(
+        avFrame->width,
+        avFrame->height,
+        frameFormat,
+        avFrame->sample_aspect_ratio,
+        expectedOutputWidth,
+        expectedOutputHeight,
+        AV_PIX_FMT_RGB24,
+        filters.str(),
+        timeBase);
 
     if (!filterGraphContext_ || prevFiltersContext_ != filtersContext) {
       filterGraphContext_ =

src/torchcodec/_core/CpuDeviceInterface.h

@@ -43,11 +43,19 @@ class CpuDeviceInterface : public DeviceInterface {
       const UniqueAVFrame& avFrame);
 
   struct SwsFrameContext {
-    int inputWidth;
-    int inputHeight;
-    AVPixelFormat inputFormat;
-    int outputWidth;
-    int outputHeight;
+    int inputWidth = 0;
+    int inputHeight = 0;
+    AVPixelFormat inputFormat = AV_PIX_FMT_NONE;
+    int outputWidth = 0;
+    int outputHeight = 0;
+
+    SwsFrameContext() = default;
+    SwsFrameContext(
+        int inputWidth,
+        int inputHeight,
+        AVPixelFormat inputFormat,
+        int outputWidth,
+        int outputHeight);
     bool operator==(const SwsFrameContext&) const;
     bool operator!=(const SwsFrameContext&) const;
   };

src/torchcodec/_core/DeviceInterface.h

@@ -36,14 +36,6 @@ struct DeviceInterfaceKey {
       : deviceType(type), variant(var) {}
 };
 
-// Note that all these device functions should only be called if the device is
-// not a CPU device. CPU device functions are already implemented in the
-// SingleStreamDecoder implementation.
-// These functions should only be called from within an if block like this:
-// if (device.type() != torch::kCPU) {
-//   deviceFunction(device, ...);
-// }
-
 class DeviceInterface {
  public:
   DeviceInterface(const torch::Device& device) : device_(device) {}

src/torchcodec/_core/FilterGraph.cpp

@@ -13,6 +13,26 @@ extern "C" {
 
 namespace facebook::torchcodec {
 
+FiltersContext::FiltersContext(
+    int inputWidth,
+    int inputHeight,
+    AVPixelFormat inputFormat,
+    AVRational inputAspectRatio,
+    int outputWidth,
+    int outputHeight,
+    AVPixelFormat outputFormat,
+    const std::string& filtergraphStr,
+    AVRational timeBase)
+    : inputWidth(inputWidth),
+      inputHeight(inputHeight),
+      inputFormat(inputFormat),
+      inputAspectRatio(inputAspectRatio),
+      outputWidth(outputWidth),
+      outputHeight(outputHeight),
+      outputFormat(outputFormat),
+      filtergraphStr(filtergraphStr),
+      timeBase(timeBase) {}
+
 bool operator==(const AVRational& lhs, const AVRational& rhs) {
   return lhs.num == rhs.num && lhs.den == rhs.den;
 }

src/torchcodec/_core/FilterGraph.h

@@ -19,11 +19,24 @@ struct FiltersContext {
   int outputWidth = 0;
   int outputHeight = 0;
   AVPixelFormat outputFormat = AV_PIX_FMT_NONE;
-
   std::string filtergraphStr;
   AVRational timeBase = {0, 0};
   UniqueAVBufferRef hwFramesCtx;
 
+  FiltersContext() = default;
+  FiltersContext(FiltersContext&&) = default;
+  FiltersContext& operator=(FiltersContext&&) = default;
+  FiltersContext(
+      int inputWidth,
+      int inputHeight,
+      AVPixelFormat inputFormat,
+      AVRational inputAspectRatio,
+      int outputWidth,
+      int outputHeight,
+      AVPixelFormat outputFormat,
+      const std::string& filtergraphStr,
+      AVRational timeBase);
+
   bool operator==(const FiltersContext&) const;
   bool operator!=(const FiltersContext&) const;
 };

src/torchcodec/_core/SingleStreamDecoder.cpp

@@ -244,13 +244,6 @@ void SingleStreamDecoder::scanFileAndUpdateMetadataAndIndex() {
     return;
   }
 
-  for (unsigned int i = 0; i < formatContext_->nb_streams; ++i) {
-    // We want to scan and update the metadata of all streams.
-    TORCH_CHECK(
-        formatContext_->streams[i]->discard != AVDISCARD_ALL,
-        "Did you add a stream before you called for a scan?");
-  }
-
   AutoAVPacket autoAVPacket;
   while (true) {
     ReferenceAVPacket packet(autoAVPacket);
@@ -1439,7 +1432,11 @@ FrameOutput SingleStreamDecoder::convertAVFrameToFrameOutput(
       formatContext_->streams[activeStreamIndex_]->time_base);
   if (streamInfo.avMediaType == AVMEDIA_TYPE_AUDIO) {
     convertAudioAVFrameToFrameOutputOnCPU(avFrame, frameOutput);
-  } else if (deviceInterface_) {
+  } else {
+    TORCH_CHECK(
+        deviceInterface_ != nullptr,
+        "No device interface available for video decoding. This ",
+        "shouldn't happen, please report.");
     deviceInterface_->convertAVFrameToFrameOutput(
         streamInfo.videoStreamOptions,
         streamInfo.timeBase,

src/torchcodec/decoders/_video_decoder.py

@@ -247,6 +247,12 @@ def get_frames_at(self, indices: list[int]) -> FrameBatch:
         Returns:
             FrameBatch: The frames at the given indices.
         """
+        if isinstance(indices, torch.Tensor):
+            # TODO we should avoid converting tensors to lists and just let the
+            # core ops and C++ code natively accept tensors.  See
+            # https://github.com/pytorch/torchcodec/issues/879
+            indices = indices.to(torch.int).tolist()
+
         data, pts_seconds, duration_seconds = core.get_frames_at_indices(
             self._decoder, frame_indices=indices
         )
@@ -322,6 +328,12 @@ def get_frames_played_at(self, seconds: list[float]) -> FrameBatch:
         Returns:
             FrameBatch: The frames that are played at ``seconds``.
         """
+        if isinstance(seconds, torch.Tensor):
+            # TODO we should avoid converting tensors to lists and just let the
+            # core ops and C++ code natively accept tensors.  See
+            # https://github.com/pytorch/torchcodec/issues/879
+            seconds = seconds.to(torch.float).tolist()
+
         data, pts_seconds, duration_seconds = core.get_frames_by_pts(
             self._decoder, timestamps=seconds
         )

test/test_decoders.py

@@ -1407,6 +1407,17 @@ def test_custom_frame_mappings_init_fails_invalid_json(self, tmp_path, device):
                         custom_frame_mappings=custom_frame_mappings,
                     )
 
+    def test_get_frames_at_tensor_indices(self):
+        # Non-regression test for tensor support in get_frames_at() and
+        # get_frames_played_at()
+        decoder = VideoDecoder(NASA_VIDEO.path)
+
+        decoder.get_frames_at(torch.tensor([0, 10], dtype=torch.int))
+        decoder.get_frames_at(torch.tensor([0, 10], dtype=torch.float))
+
+        decoder.get_frames_played_at(torch.tensor([0, 1], dtype=torch.int))
+        decoder.get_frames_played_at(torch.tensor([0, 1], dtype=torch.float))
+
 
 class TestAudioDecoder:
     @pytest.mark.parametrize("asset", (NASA_AUDIO, NASA_AUDIO_MP3, SINE_MONO_S32))