From 084b2f0ad0a12f039ff2b9dfae2d1f4501474fc0 Mon Sep 17 00:00:00 2001
From: FaranMustafa <faran.mustafa@bytecorp.io>
Date: Fri, 18 Apr 2025 04:17:30 +0500
Subject: [PATCH] Added gocv to webrtc example

---
 gocv-to-webrtc/README.md         | 130 +++++++++++++++++
 gocv-to-webrtc/main.go           | 230 +++++++++++++++++++++++++++++++
 gocv-to-webrtc/static/index.html |  83 +++++++++++
 3 files changed, 443 insertions(+)
 create mode 100644 gocv-to-webrtc/README.md
 create mode 100644 gocv-to-webrtc/main.go
 create mode 100644 gocv-to-webrtc/static/index.html

diff --git a/gocv-to-webrtc/README.md b/gocv-to-webrtc/README.md
new file mode 100644
index 00000000..c58022f1
--- /dev/null
+++ b/gocv-to-webrtc/README.md
@@ -0,0 +1,130 @@
+# GoCV + FFmpeg + Pion WebRTC
+
+This project demonstrates how to stream live webcam video to a browser using GoCV for camera capture, FFmpeg for real‑time VP8 encoding, and Pion WebRTC for media transport.
+
+---
+
+## Overview
+
+- **Capture**: Uses [GoCV](https://gocv.io/) to access a webcam and read raw BGR frames.
+- **Encode**: Pipes raw frames into `ffmpeg` for VP8 encoding in IVF format.
+- **Stream**: Uses [Pion WebRTC](https://github.com/pion/webrtc) to send encoded video frames to a browser client.
+- **Frontend**: Minimal HTML/JS page that negotiates WebRTC Offer/Answer and displays incoming video.
+
+---
+
+## Prerequisites
+
+- Go 1.20 or newer
+- FFmpeg installed with `libvpx` support
+- OpenCV 4.x installed
+- GoCV installed (`go get -u -d gocv.io/x/gocv`)
+- A working webcam (USB or internal)
+
+---
+
+## Download & Install the Example
+
+Install and run the `gocv-to-webrtc` example directly:
+
+```bash
+go install github.com/pion/webrtc/v4/examples/govc-to-webrtc@latest
+```
+
+On macOS, set the camera index to `0` (instead of `2`) by editing `main.go`:
+
+```go
+webcam, _ := gocv.OpenVideoCapture(0)
+```
+
+Then run:
+
+```bash
+gocv-to-webrtc
+```
+
+---
+
+## Usage
+
+1. Run the example:
+   ```bash
+   gocv-to-webrtc
+   ```
+2. Open your browser at `http://localhost:8080`.
+3. Click **Start Session** to initiate WebRTC negotiation.
+4. After ICE connects, you should see your webcam video in the page.
+
+---
+
+## How It Works
+
+### Server (`main.go`)
+
+1. **HTTP Server**
+   - Serves static files (`static/`) on port 8080.
+   - Handles `/offer` endpoint for SDP exchange.
+
+2. **WebRTC Setup**
+   - Reads the browser’s SDP Offer.
+   - Creates a Pion `PeerConnection` with a VP8 track (`TrackLocalStaticSample`).
+   - Sets remote description, creates Answer, and returns it once ICE gathering completes.
+   - Starts the camera stream after ICE connection.
+
+3. **Video Pipeline** (`startCameraAndStream`)
+   - Opens webcam via GoCV (`gocv.OpenVideoCapture`).
+   - Pipes raw BGR frames into FFmpeg:
+     ```bash
+     ffmpeg -y \
+       -f rawvideo -pixel_format bgr24 -video_size 640x480 -framerate 30 -i pipe:0 \
+       -c:v libvpx -b:v 1M -f ivf pipe:1
+     ```
+   - Reads VP8 IVF frames from FFmpeg’s stdout with `ivfreader`.
+   - Writes frames into the WebRTC track.
+
+### Frontend (`static/index.html`)
+
+1. Creates an `RTCPeerConnection` with STUN.
+2. Adds a `recvonly` video transceiver.
+3. Sends SDP Offer to server.
+4. Sets remote Answer.
+5. Attaches incoming stream to a `<video>` element.
+
+---
+
+## Configuration
+
+- **Camera Device**: Change `gocv.OpenVideoCapture(2)` to the appropriate index (e.g., `0`).
+- **Resolution & Frame Rate**: Adjust GoCV settings and FFmpeg flags (`-video_size`, `-framerate`).
+- **Frame Rate**: Adjust `-framerate 30` in FFmpeg and ticker interval in Go.
+- **Bitrate & Codec**: Modify `-b:v 1M` or swap codecs (H264/Opus).
+
+---
+
+## Troubleshooting
+
+- **No Video**: Check camera index and FFmpeg installation.
+- **ICE Fails**: Verify STUN server and network/firewall.
+- **High CPU**: Lower resolution/bitrate or tune FFmpeg CPU usage.
+
+---
+
+## Related Libraries
+
+- [GoCV](https://gocv.io/)
+- [FFmpeg](https://ffmpeg.org/)
+- [Pion WebRTC](https://github.com/pion/webrtc)
+- [ivfreader](https://pkg.go.dev/github.com/pion/webrtc/v4/pkg/media/ivfreader)
+
+---
+
+## License
+
+MIT License.
+
+---
+
+## Acknowledgements
+
+Inspired by [pion/webrtc examples](https://github.com/pion/webrtc/tree/master/examples).
+
diff --git a/gocv-to-webrtc/main.go b/gocv-to-webrtc/main.go
new file mode 100644
index 00000000..dafd8a8a
--- /dev/null
+++ b/gocv-to-webrtc/main.go
@@ -0,0 +1,230 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os/exec"
+	"time"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
+	"github.com/pion/webrtc/v4/pkg/media/ivfreader"
+	gocv "gocv.io/x/gocv"
+)
+
+func main() {
+	// Serve the static/ folder on http://localhost:8080
+	http.Handle("/", http.FileServer(http.Dir("static")))
+
+	// POST /offer will handle the browser's WebRTC offer
+	http.HandleFunc("/offer", handleOffer)
+
+	fmt.Println("Listening on http://localhost:8080")
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}
+
+func handleOffer(w http.ResponseWriter, r *http.Request) {
+	// 1) Read the Offer from the browser
+	var offer webrtc.SessionDescription
+	if err := json.NewDecoder(r.Body).Decode(&offer); err != nil {
+		http.Error(w, "invalid offer", http.StatusBadRequest)
+		return
+	}
+
+	// 2) Create a new PeerConnection
+	peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{URLs: []string{"stun:stun.l.google.com:19302"}},
+		},
+	})
+	if err != nil {
+		http.Error(w, "failed to create PeerConnection", http.StatusInternalServerError)
+		return
+	}
+
+	// 3) Create a video track for VP8
+	videoTrack, err := webrtc.NewTrackLocalStaticSample(
+		webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8},
+		"video", // track id
+		"gocv",  // stream id
+	)
+	if err != nil {
+		http.Error(w, "failed to create video track", http.StatusInternalServerError)
+		return
+	}
+
+	// Add the track to the PeerConnection
+	rtpSender, err := peerConnection.AddTrack(videoTrack)
+	if err != nil {
+		http.Error(w, "failed to add track", http.StatusInternalServerError)
+		return
+	}
+
+	// Read RTCP (for NACK, etc.) in a separate goroutine
+	go func() {
+		rtcpBuf := make([]byte, 1500)
+		for {
+			if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
+				return
+			}
+		}
+	}()
+
+	// 4) Watch for ICE connection state
+	iceConnectedCtx, iceConnectedCancel := context.WithCancel(context.Background())
+	peerConnection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) {
+		log.Printf("ICE state: %s\n", state)
+		if state == webrtc.ICEConnectionStateConnected {
+			iceConnectedCancel()
+		}
+	})
+
+	// 5) Set the remote description (the browser's Offer)
+	if err := peerConnection.SetRemoteDescription(offer); err != nil {
+		http.Error(w, "failed to set remote desc", http.StatusInternalServerError)
+		return
+	}
+
+	// 6) Create an Answer
+	answer, err := peerConnection.CreateAnswer(nil)
+	if err != nil {
+		http.Error(w, "failed to create answer", http.StatusInternalServerError)
+		return
+	}
+
+	// 7) Gather ICE candidates
+	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
+	if err := peerConnection.SetLocalDescription(answer); err != nil {
+		http.Error(w, "failed to set local desc", http.StatusInternalServerError)
+		return
+	}
+	<-gatherComplete
+
+	// 8) Write the Answer back to the browser
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(peerConnection.LocalDescription())
+
+	// 9) Once ICE is connected, start reading frames from the camera via GoCV,
+	//    pipe them into FFmpeg for VP8 encoding, and push the IVF frames into the track.
+	go func() {
+		<-iceConnectedCtx.Done()
+
+		if err := startCameraAndStream(videoTrack); err != nil {
+			log.Printf("camera streaming error: %v\n", err)
+		}
+	}()
+}
+
+// startCameraAndStream opens the webcam with GoCV, sends raw frames to FFmpeg (via stdin),
+// reads IVF from FFmpeg (via stdout), and writes them into the WebRTC video track.
+func startCameraAndStream(videoTrack *webrtc.TrackLocalStaticSample) error {
+	// Open default camera with GoCV
+	webcam, err := gocv.OpenVideoCapture(2)
+	if err != nil {
+		return fmt.Errorf("cannot open camera: %w", err)
+	}
+	defer webcam.Close()
+
+	// Set some camera settings if needed
+	// e.g. webcam.Set(gocv.VideoCaptureFrameWidth, 640)
+	//      webcam.Set(gocv.VideoCaptureFrameHeight, 480)
+	// Or rely on defaults
+
+	// Prepare FFmpeg cmd:
+	//   -f rawvideo: We feed raw frames
+	//   -pixel_format bgr24: Our GoCV frames come in BGR format
+	//   -video_size 640x480: must match your actual capture size
+	//   -i pipe:0 : read from stdin
+	//   Then encode with libvpx -> IVF on stdout
+	ffmpeg := exec.Command(
+		"ffmpeg",
+		"-y",
+		"-f", "rawvideo",
+		"-pixel_format", "bgr24",
+		"-video_size", "640x480",
+		"-framerate", "30", // assume ~30fps
+		"-i", "pipe:0",
+		"-c:v", "libvpx",
+		"-b:v", "1M",
+		"-f", "ivf",
+		"pipe:1",
+	)
+
+	stdin, err := ffmpeg.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("ffmpeg stdin error: %w", err)
+	}
+	stdout, err := ffmpeg.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("ffmpeg stdout error: %w", err)
+	}
+
+	// Start FFmpeg
+	if err := ffmpeg.Start(); err != nil {
+		return fmt.Errorf("failed to start ffmpeg: %w", err)
+	}
+
+	// Goroutine to write raw frames to FFmpeg stdin
+	go func() {
+		defer stdin.Close()
+
+		frame := gocv.NewMat()
+		defer frame.Close()
+
+		ticker := time.NewTicker(time.Millisecond * 33) // ~30fps
+		defer ticker.Stop()
+
+		for range ticker.C {
+			if ok := webcam.Read(&frame); !ok {
+				log.Println("cannot read frame from camera")
+				continue
+			}
+			if frame.Empty() {
+				continue
+			}
+
+			// (Optional) do any OpenCV processing on `frame` here
+
+			// Write raw BGR bytes to FFmpeg
+			// frame.DataPtrUint8() points to the underlying byte array
+			_, _ = stdin.Write(frame.ToBytes())
+		}
+	}()
+
+	// Read IVF from FFmpeg stdout; parse frames with ivfreader
+	ivf, _, err := ivfreader.NewWith(stdout)
+	if err != nil {
+		return fmt.Errorf("ivfreader init error: %w", err)
+	}
+	// Loop reading IVF frames; push them to the video track
+	for {
+		frame, _, err := ivf.ParseNextFrame()
+		if errors.Is(err, io.EOF) {
+			log.Println("ffmpeg ended (EOF)")
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("ivf parse error: %w", err)
+		}
+		// Deliver the VP8 frame
+		writeErr := videoTrack.WriteSample(media.Sample{
+			Data:     frame,
+			Duration: time.Second / 30,
+		})
+		if writeErr != nil {
+			return fmt.Errorf("write sample error: %w", writeErr)
+		}
+	}
+
+	// Wait for ffmpeg to exit
+	if err := ffmpeg.Wait(); err != nil {
+		return fmt.Errorf("ffmpeg wait error: %w", err)
+	}
+
+	return nil
+}
diff --git a/gocv-to-webrtc/static/index.html b/gocv-to-webrtc/static/index.html
new file mode 100644
index 00000000..f2bbbb93
--- /dev/null
+++ b/gocv-to-webrtc/static/index.html
@@ -0,0 +1,83 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>GoCV + FFmpeg + Pion WebRTC</title>
+  </head>
+  <body>
+    <h2>GoCV + WebRTC Demo</h2>
+    <button id="btnStart">Start Session</button>
+
+    <h3>Remote Video</h3>
+    <div id="remoteVideos"></div>
+
+    <h3>Logs</h3>
+    <div id="log"></div>
+
+    <script>
+      const logDiv = document.getElementById("log");
+      function log(msg) {
+        logDiv.innerHTML += msg + "<br>";
+      }
+
+      document.getElementById("btnStart").onclick = async () => {
+        const pc = new RTCPeerConnection({
+          iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+        });
+
+        pc.oniceconnectionstatechange = () => {
+          log("ICE state: " + pc.iceConnectionState);
+        };
+
+        // When the server sends a video track, display it
+        pc.ontrack = (event) => {
+          const el = document.createElement(event.track.kind);
+          el.srcObject = event.streams[0];
+          el.autoplay = true;
+          el.controls = true;
+          document.getElementById("remoteVideos").appendChild(el);
+        };
+
+        // We want to receive (only) a video track from the server
+        pc.addTransceiver("video", { direction: "recvonly" });
+
+        // 1) Create an offer
+        log("Creating offer...");
+        const offer = await pc.createOffer();
+        await pc.setLocalDescription(offer);
+
+        // 2) Wait for ICE gathering to finish
+        await new Promise((resolve) => {
+          if (pc.iceGatheringState === "complete") {
+            resolve();
+          } else {
+            pc.onicegatheringstatechange = () => {
+              if (pc.iceGatheringState === "complete") {
+                resolve();
+              }
+            };
+          }
+        });
+
+        // 3) POST the offer to the server
+        log("Sending offer to server...");
+        const resp = await fetch("/offer", {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(pc.localDescription),
+        });
+        if (!resp.ok) {
+          const errText = await resp.text();
+          throw new Error("Error from server: " + errText);
+        }
+
+        // 4) Get the server's answer and set it
+        const answer = await resp.json();
+        log("Received answer, setting remote description...");
+        await pc.setRemoteDescription(answer);
+
+        log("WebRTC negotiation complete! Waiting for remote video...");
+      };
+    </script>
+  </body>
+</html>