V2

Makefile

@@ -1,5 +1,10 @@
+FFLIBS=`pkg-config --libs libavformat libavcodec libavutil`
+FFFLAGS=`pkg-config --cflags libavformat libavcodec libavutil`
 all:
-	gcc -Wall -g live_segmenter.c -o live_segmenter -lavformat -lavcodec -lavutil -lbz2 -lm -lz -lfaac -lmp3lame -lx264 -lfaad -lpthread
+	gcc -Wall -g live_segmenter.c -o live_segmenter ${FFFLAGS} ${FFLIBS}
+
+old_ffmpeg:
+	gcc -Wall -g live_segmenter.c -o live_segmenter -D USE_OLD_FFMPEG -lavformat -lavcodec -lavutil -lbz2 -lm -lz -lfaac -lmp3lame -lx264 -lfaad -lpthread
 
 clean:
 	rm -f live_segmenter

README.md

@@ -13,6 +13,7 @@ The project includes a ruby script and a C program that use FFMpeg to encode and
 
 - Creates both single and variable bitrate outputs
 - Transfer encoded segments via copy, FTP, SCP or transfer to AWS S3
+- Implements load balance and failover protection when distributing to multiple media servers
 - Sending the INT signal to the segmenter process will cause it to terminate gracefully 
 
 ## REQUIREMENTS
@@ -24,6 +25,9 @@ FFMpeg is the primary external requirement for the ruby script. The segmenter ne
 
 - *RightScale AWS*
 	See http://rubyforge.org/projects/rightscale for more information. To install run gem install right_aws
+
+RubyGems must be installed, even if the previous modules are not needed. 
+
 
 ## INSTALL
 
@@ -62,7 +66,7 @@ A quick overview of the configuration options:
 - *encoding_profile* 
 	Specifies what encoding profile to use. It can be either a single entry, 'ep_128k', or an array, [ 'ep_128k', 'ep_386k', 'ep_512k' ], for multi-bitrate outputs.
 - *transfer_profile* 
-	The transfer profile to use after each segment is produced
+	The transfer profile to use after each segment is produced. It can be either a single entry, 'scp://foo.com/media', or an array, [ 'scp://foo.com/media', 'scp://bar.com/media' ], for load balancing/failover support.
 
 Encoding profiles are given a name and have two options following that name:
 
@@ -84,6 +88,8 @@ Transfer profiles are given a name in the same way encoding profiles are and hav
 	The AWS api key for S3
 - *aws_api_secret* 
 	The AWS api secret for S3
+- *url_prefix* 
+	This is the URL where the stream (ts) files will end up if multi-server/failover is enabled
 
 ### For FTP based transfers:
 - *transfer_type*
@@ -96,6 +102,8 @@ Transfer profiles are given a name in the same way encoding profiles are and hav
 	The password to use to log into the ftp site
 - *directory*
 	The directory to change to before starting the ftp upload
+- *url_prefix* 
+	This is the URL where the stream (ts) files will end up if multi-server/failover is enabled
 
 ### For SCP based transfers:
 - *transfer_type*
@@ -108,12 +116,17 @@ Transfer profiles are given a name in the same way encoding profiles are and hav
 	The password to use for scp. This is optional, if it isn't provided the scp will be done using a previously generated private key.
 - *directory*
 	The directory to change to before uploading the segment
+- *url_prefix* 
+	This is the URL where the stream (ts) files will end up if multi-server/failover is enabled
 
 ### For copy based transfers:
 - *transfer_type*
 	Must be set to 'copy'
 - *directory*
 	The destination directory to copy the segment to
+- *url_prefix* 
+	This is the URL where the stream (ts) files will end up if multi-server/failover is enabled
+
 
 ## LICENSE
 

example-configs/config-simple-failover.yml

@@ -0,0 +1,90 @@
+temp_dir: '/tmp/'
+segment_prefix: 'sample'
+index_prefix: 'stream'
+
+# type of logging: STDOUT, FILE
+log_type: 'STDOUT'
+#log_type: 'FILE'
+#log_file: '/tmp/streamer.log'
+# levels: DEBUG, INFO, WARN, ERROR
+log_level: 'DEBUG'
+
+# where the origin video is going to come from
+input_location: '/tmp/test.avi'
+
+# segment length in seconds
+segment_length: 10 
+
+# this is the URL where the stream (ts) files will end up
+# if this location is the same as the location where the m3u8 
+# index is then leave it blank
+# url_prefix: ''
+url_prefix: 'http://192.168.1.1/streamingvideo/'
+
+# how many segments to keep in the index
+#index_segment_count: 15
+index_segment_count: 100000
+
+# this command is used for multirate encodings and is what pushes the encoders
+source_command: 'cat %s'
+
+# This is the location of the segmenter
+segmenter_binary: './live_segmenter'
+
+# the encoding profile to use
+encoding_profile: 'ep_128k'
+
+# The upload profile to use
+transfer_profile: ['copy_dev', 'copy_dev_mirror']
+
+#
+# Encoding profiles
+#
+ep_128k: 
+  ffmpeg_command: "ffmpeg -er 4 -y -i %s -f mpegts -acodec libmp3lame -ar 48000 -ab 64k -s 320x240 -vcodec libx264 -b 128k -flags +loop -cmp +chroma -partitions +parti4x4+partp8x8+partb8x8 -subq 5 -trellis 1 -refs 1 -coder 0 -me_range 16 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -bt 128k -maxrate 128k -bufsize 128k -rc_eq 'blurCplx^(1-qComp)' -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -level 30 -aspect 320:240 -g 30 -async 2 - | %s %s %s %s %s"
+  bandwidth: 128000
+
+ep_386k: 
+  ffmpeg_command: "ffmpeg -er 4 -y -i %s -f mpegts -acodec libmp3lame -ar 48000 -ab 64k -s 320x240 -vcodec libx264 -b 386k -flags +loop -cmp +chroma -partitions +parti4x4+partp8x8+partb8x8 -subq 5 -trellis 1 -refs 1 -coder 0 -me_range 16 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -bt 386k -maxrate 386k -bufsize 386k -rc_eq 'blurCplx^(1-qComp)' -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -level 30 -aspect 320:240 -g 30 -async 2 - | %s %s %s %s %s"
+  bandwidth: 386000
+
+ep_512k: 
+  ffmpeg_command: "ffmpeg -er 4 -y -i %s -f mpegts -acodec libmp3lame -ar 48000 -ab 64k -s 320x240 -vcodec libx264 -b 512k -flags +loop -cmp +chroma -partitions +parti4x4+partp8x8+partb8x8 -subq 5 -trellis 1 -refs 1 -coder 0 -me_range 16 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -bt 512k -maxrate 512k -bufsize 512k -rc_eq 'blurCplx^(1-qComp)' -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -level 30 -aspect 320:240 -g 30 -async 2 - | %s %s %s %s %s"
+  bandwidth: 512000
+
+#
+# Transfer profiles
+#
+s3_dev:
+  transfer_type: 's3'
+  bucket_name: 'bucket'
+  key_prefix: 'stream0001'
+  aws_api_key: 'key'
+  aws_api_secret: 'secret'
+  url_prefix: 'http://192.168.1.1/video/'
+
+ftp_dev:
+  transfer_type: 'ftp'
+  remote_host: '192.168.1.1'
+  user_name: 'user'
+  password: 'pass'
+  directory: 'html/streamingvideo'
+  url_prefix: 'http://192.168.1.1/video/'
+
+scp_dev:
+  transfer_type: 'scp'
+  remote_host: '192.168.1.1'
+  user_name: 'root'
+  #password: 'pass'
+  directory: '/web/sites/test.com/html/streamingvideo'
+  url_prefix: 'http://192.168.1.1/video/'
+
+copy_dev:
+  transfer_type: 'copy'
+  directory: '/var/www/html/streamingvideo'
+  url_prefix: 'http://192.168.1.1/video/'
+
+copy_dev_mirror:
+  transfer_type: 'copy'
+  directory: '/var/www/html/mirror/streamingvideo'
+  url_prefix: 'http://192.168.1.2/video/'

hs_config.rb

@@ -98,30 +98,39 @@ def sanity_check(config)
       end
     end
 
-    if config[config['transfer_profile']].nil?
-      log.error("The given transfer profile was not found in the config: #{config['transfer_profile']}")
-      raise 
+    tp = config['transfer_profile']
+    if tp.is_a?(Array)
+      tps = tp
+    else
+      tps = [tpName]
     end
 
-    if config[config['transfer_profile']]['transfer_type'] != 'ftp' and config[config['transfer_profile']]['transfer_type'] != 'scp' and
-       config[config['transfer_profile']]['transfer_type'] != 's3' and config[config['transfer_profile']]['transfer_type'] != 'copy'
-      log.error("The given transfer type is not known: #{config[config['transfer_profile']]['transfer_type']}")
-      raise 
-    end
+    tps.each do |tpName|
+      if config[tpName].nil?
+        log.error("The given transfer profile was not found in the config: #{tp}")
+        raise
+      end
 
-    if !HSTransfer::can_ftp and config[config['transfer_profile']]['transfer_type'] == 'ftp'
-      log.error("The given transfer type is not available: #{config[config['transfer_profile']]['transfer_type']}")
-      raise 
-    end
+      if config[tpName]['transfer_type'] != 'ftp' and config[tpName]['transfer_type'] != 'scp' and
+      config[tpName]['transfer_type'] != 's3' and config[tpName]['transfer_type'] != 'copy'
+        log.error("The given transfer type is not known: #{config[tpName]['transfer_type']}")
+        raise
+      end
 
-    if !HSTransfer::can_scp and config[config['transfer_profile']]['transfer_type'] == 'scp'
-      log.error("The given transfer type is not available: #{config[config['transfer_profile']]['transfer_type']}")
-      raise 
-    end
+      if !HSTransfer::can_ftp and config[tpName]['transfer_type'] == 'ftp'
+        log.error("The given transfer type is not available: #{config[tpName]['transfer_type']}")
+        raise
+      end
 
-    if !HSTransfer::can_s3 and config[config['transfer_profile']]['transfer_type'] == 's3'
-      log.error("The given transfer type is not available: #{config[config['transfer_profile']]['transfer_type']}")
-      raise 
+      if !HSTransfer::can_scp and config[tpName]['transfer_type'] == 'scp'
+        log.error("The given transfer type is not available: #{config[tpName]['transfer_type']}")
+        raise
+      end
+
+      if !HSTransfer::can_s3 and config[tpName]['transfer_type'] == 's3'
+        log.error("The given transfer type is not available: #{config[tpName]['transfer_type']}")
+        raise
+      end
     end
 
   end

hs_transfer.rb

@@ -104,9 +104,18 @@ def create_and_transfer_multirate_index
 
       @config['encoding_profile'].each do |encoding_profile_name|
         encoding_profile = @config[encoding_profile_name]
-        index_file.write("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=#{encoding_profile['bandwidth']}\n")
-        index_name = "%s_%s.m3u8" % [@config['index_prefix'], encoding_profile_name]
-        index_file.write("#{@config['url_prefix']}#{index_name}\n")
+        if @config['transfer_profile'].is_a?(Array)
+          @config['transfer_profile'].each do |tpname|
+            tp = @config[tpname]
+            index_file.write("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=#{encoding_profile['bandwidth']}\n")
+            index_name = "%s_%s.m3u8" % [@config['index_prefix'], encoding_profile_name]
+            index_file.write("#{tp['url_prefix']}#{index_name}\n")
+          end
+        else
+          index_file.write("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=#{encoding_profile['bandwidth']}\n")
+          index_name = "%s_%s.m3u8" % [@config['index_prefix'], encoding_profile_name]
+          index_file.write("#{@config['url_prefix']}#{index_name}\n")
+        end
       end
     end
 
@@ -150,39 +159,49 @@ def create_index_and_run_transfer(value)
   end
 
   def transfer_file(source_file, destination_file)
-     transfer_config = @config[@config['transfer_profile']]
-
-     case transfer_config['transfer_type']
-       when 'copy'
-         File.copy(source_file, transfer_config['directory'] + '/' + destination_file)
-       when 'ftp'
-         require 'net/ftp'
-         Net::FTP.open(transfer_config['remote_host']) do |ftp|
-           ftp.login(transfer_config['user_name'], transfer_config['password'])
-           files = ftp.chdir(transfer_config['directory'])
-           ftp.putbinaryfile(source_file, destination_file)
-         end
-       when 'scp'
-         require 'net/scp'
-         if transfer_config.has_key?('password')
-           Net::SCP.upload!(transfer_config['remote_host'], transfer_config['user_name'], source_file, transfer_config['directory'] + '/' + destination_file, :password => transfer_config['password'])
-         else
-           Net::SCP.upload!(transfer_config['remote_host'], transfer_config['user_name'], source_file, transfer_config['directory'] + '/' + destination_file)
-         end
-       when 's3'
-         require 'right_aws'
-         s3 = RightAws::S3Interface.new(transfer_config['aws_api_key'], transfer_config['aws_api_secret'])
-
-         content_type = source_file =~ /.*\.m3u8$/ ? 'application/vnd.apple.mpegurl' : 'video/MP2T'
-
-         @log.debug("Content type: #{content_type}")
-
-         s3.put(transfer_config['bucket_name'], "#{transfer_config['key_prefix']}/#{destination_file}", File.open(source_file), {'x-amz-acl' => 'public-read', 'content-type' => content_type})
-       else
-         @log.error("Unknown transfer type: #{transfer_config['transfer_type']}")
-     end
-
-     File.unlink(source_file)
-  end
+    transfer_configs = []
+    transfer_profile = @config['transfer_profile']
+    if transfer_profile.is_a?(Array) then
+      transfer_profile.each do |tp|
+        transfer_configs << @config[tp]
+      end
+    else
+      transfer_configs << @config[transfer_profile]
+    end
+
+    transfer_configs.each do |transfer_config|
+      case transfer_config['transfer_type']
+      when 'copy'
+        File.copy(source_file, transfer_config['directory'] + '/' + destination_file)
+      when 'ftp'
+        require 'net/ftp'
+        Net::FTP.open(transfer_config['remote_host']) do |ftp|
+          ftp.login(transfer_config['user_name'], transfer_config['password'])
+          files = ftp.chdir(transfer_config['directory'])
+          ftp.putbinaryfile(source_file, destination_file)
+        end
+      when 'scp'
+        require 'net/scp'
+        if transfer_config.has_key?('password')
+          Net::SCP.upload!(transfer_config['remote_host'], transfer_config['user_name'], source_file, transfer_config['directory'] + '/' + destination_file, :password => transfer_config['password'])
+        else
+          Net::SCP.upload!(transfer_config['remote_host'], transfer_config['user_name'], source_file, transfer_config['directory'] + '/' + destination_file)
+        end
+      when 's3'
+        require 'right_aws'
+        s3 = RightAws::S3Interface.new(transfer_config['aws_api_key'], transfer_config['aws_api_secret'])
+
+        content_type = source_file =~ /.*\.m3u8$/ ? 'application/vnd.apple.mpegurl' : 'video/MP2T'
+
+        @log.debug("Content type: #{content_type}")
 
+        s3.put(transfer_config['bucket_name'], "#{transfer_config['key_prefix']}/#{destination_file}", File.open(source_file), {'x-amz-acl' => 'public-read', 'content-type' => content_type})
+      else
+        @log.error("Unknown transfer type: #{transfer_config['transfer_type']}")
+      end
+
+    end
+
+    File.unlink(source_file)
+  end
 end