Various recording script fixes & cleanups

This is just a bundle of a few things I've been fixing up in the past
while.

= Workaround for BBB 1.1 beta deskshare timestamp bug

This is unlikely to be used, but I have the code for it, might as
well merge it in.

= Rework video tiling code for ffmpeg

Render video using the 'hstack' and 'vstack' filters rather than the
'overlay' filter. This is somewhat faster, particularly with lots of
videos.

= Etc.

- Remove usage of the streamio-ffmpeg gem.
  The video rendering code has some stuff to directly read 'ffprobe'
  output, so re-use that instead of this gem (which is kind of old and
  has issues with newer ffmpeg versions).

- Don't hardcode the deskshare video area size, pull it from the
  properties file

- Remove some code that worked around missing video end events.
  In some cases this could cause flickering or strange video issues.
  It's no longer strictly needed, the new tiling code doesn't break if
  the seekpoint is after the end of the video.

record-and-playback/core/lib/recordandplayback.rb

@@ -33,7 +33,6 @@ require 'recordandplayback/generators/audio'
 require 'recordandplayback/generators/video'
 require 'recordandplayback/generators/audio_processor'
 require 'recordandplayback/generators/presentation'
-require 'custom_hash'
 require 'open4'
 require 'pp'
 require 'absolute_time'

record-and-playback/core/lib/recordandplayback/edl.rb

@@ -44,7 +44,7 @@ module BigBlueButton
           end
           ffmpeg_cmd += ['-i', audio]
         end
-        ffmpeg_cmd += [*pass, lastoutput]
+        ffmpeg_cmd += [*pass, '-passlogfile', output_basename, lastoutput]
         Dir.chdir(File.dirname(output)) do
           exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd)
           raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0

record-and-playback/core/lib/recordandplayback/edl/video.rb

@@ -24,8 +24,8 @@ module BigBlueButton
   module EDL
     module Video
       FFMPEG_WF_CODEC = 'mpeg2video'
-      FFMPEG_WF_FRAMERATE = '24'
+      FFMPEG_WF_FRAMERATE = 24
-      FFMPEG_WF_ARGS = ['-an', '-codec', FFMPEG_WF_CODEC, '-q:v', '2', '-g', '240', '-pix_fmt', 'yuv420p', '-r', FFMPEG_WF_FRAMERATE, '-f', 'mpegts']
+      FFMPEG_WF_ARGS = ['-an', '-codec', FFMPEG_WF_CODEC.to_s, '-q:v', '2', '-g', (FFMPEG_WF_FRAMERATE * 10).to_s, '-pix_fmt', 'yuv420p', '-r', FFMPEG_WF_FRAMERATE.to_s, '-f', 'mpegts']
       WF_EXT = 'ts'
 
       def self.dump(edl)
@@ -37,7 +37,7 @@ module BigBlueButton
           entry[:areas].each do |name, videos|
             BigBlueButton.logger.debug "    #{name}"
             videos.each do |video|
-              BigBlueButton.logger.debug "      #{video[:filename]} at #{video[:timestamp]}"
+              BigBlueButton.logger.debug "      #{video[:filename]} at #{video[:timestamp]} (original duration: #{video[:original_duration]})"
             end
           end
         end
@@ -124,7 +124,8 @@ module BigBlueButton
               merged_entry[:areas][area] = videos.map do |video|
                 {
                   :filename => video[:filename],
-                  :timestamp => video[:timestamp] + merged_entry[:timestamp] - last_entry[:timestamp]
+                  :timestamp => video[:timestamp] + merged_entry[:timestamp] - last_entry[:timestamp],
+                  :original_duration => video[:original_duration]
                 }
               end
             end
@@ -189,10 +190,13 @@ module BigBlueButton
           BigBlueButton.logger.debug "  #{videofile}"
           info = video_info(videofile)
           BigBlueButton.logger.debug "    width: #{info[:width]}, height: #{info[:height]}, duration: #{info[:duration]}"
-
           if !info[:video]
             BigBlueButton.logger.warn "    This video file is corrupt! It will be removed from the output."
             corrupt_videos << videofile
+          else
+            if info[:video][:deskshare_timestamp_bug]
+              BigBlueButton.logger.debug("    has early 1.1 deskshare timestamp bug")
+            end
           end
 
           videoinfo[videofile] = info
@@ -207,49 +211,6 @@ module BigBlueButton
           end
         end
 
-        BigBlueButton.logger.info "Generating missing video end events"
-        videoinfo.each do |filename, info|
-
-          edl.each_with_index do |event, index|
-
-            new_entry = { :areas => {} }
-            add_new_entry = false
-            event[:areas].each do |area, videos|
-              videos.each do |video|
-                if video[:filename] == filename
-                  if video[:timestamp] > info[:duration]
-                    videos.delete(video)
-                  # Note that I'm using a 5-second fuzz factor here.
-                  # If there's a stop event within 5 seconds of the video ending, don't bother to generate
-                  # an extra event.
-                  elsif video[:timestamp] + (event[:next_timestamp] - event[:timestamp]) > info[:duration] + 5000
-                    BigBlueButton.logger.warn "Over-long video #{video[:filename]}, synthesizing stop event"
-                    new_entry[:timestamp] = event[:timestamp] + info[:duration] - video[:timestamp]
-                    new_entry[:next_timestamp] = event[:next_timestamp]
-                    event[:next_timestamp] = new_entry[:timestamp]
-                    add_new_entry = true
-                  end
-                end
-              end
-            end
-
-            if add_new_entry
-              event[:areas].each do |area, videos|
-                new_entry[:areas][area] = videos.select do |video|
-                  video[:filename] != filename
-                end.map do |video|
-                  {
-                    :filename => video[:filename],
-                    :timestamp => video[:timestamp] + new_entry[:timestamp] - event[:timestamp]
-                  }
-                end
-              end
-              edl.insert(index + 1, new_entry)
-            end
-
-          end
-
-        end
         dump(edl)
 
         BigBlueButton.logger.info "Compositing cuts"
@@ -296,6 +257,10 @@ module BigBlueButton
             info[:aspect_ratio] = Rational(info[:width], info[:height])
           end
 
+          if info[:format][:format_name] == 'flv' and info[:video][:codec_name] == 'h264'
+            info[:video][:deskshare_timestamp_bug] = self.check_deskshare_timestamp_bug(filename)
+          end
+
           # Convert the duration to milliseconds
           info[:duration] = (info[:format][:duration].to_r * 1000).to_i
 
@@ -304,6 +269,31 @@ module BigBlueButton
         {}
       end
 
+      def self.check_deskshare_timestamp_bug(filename)
+        IO.popen([*FFPROBE, '-select_streams', 'v:0', '-show_frames', '-read_intervals', '%+#10', filename]) do |probe|
+          info = JSON.parse(probe.read, symbolize_names: true)
+          return false if !info
+
+          if !info[:frames]
+            return false
+          end
+
+          # First frame in broken stream always has pts=1
+          if info[:frames][0][:pkt_pts] != 1
+            return false
+          end
+
+          # Remaining frames start at 200, and go up by exactly 200 each frame
+          for i in 1...info[:frames].length
+            if info[:frames][i][:pkt_pts] != i * 200
+              return false
+            end
+          end
+
+          return true
+        end
+      end
+
       def self.ms_to_s(timestamp)
         s = timestamp / 1000
         ms = timestamp % 1000
@@ -312,29 +302,30 @@ module BigBlueButton
 
       def self.aspect_scale(old_width, old_height, new_width, new_height)
         if old_width.to_f / old_height > new_width.to_f / new_height
-          [new_width, old_height * new_width / old_width]
+          new_height = (2 * (old_height.to_f * new_width / old_width / 2).round).to_i
         else
-          [old_width * new_height / old_height, new_height]
+          new_width = (2 * (old_width.to_f * new_height / old_height / 2).round).to_i
         end
+        [new_width, new_height]
       end
 
       def self.pad_offset(video_width, video_height, area_width, area_height)
-        [(area_width - video_width) / 2, (area_height - video_height) / 2]
+        pad_x = (2 * ((area_width - video_width).to_f / 4).round).to_i
+        pad_y = (2 * ((area_height - video_height).to_f / 4).round).to_i
+        [pad_x, pad_y]
       end
 
       def self.composite_cut(output, cut, layout, videoinfo)
         duration = cut[:next_timestamp] - cut[:timestamp]
         BigBlueButton.logger.info "  Cut start time #{cut[:timestamp]}, duration #{duration}"
 
-        ffmpeg_inputs = []
         ffmpeg_filter = "color=c=white:s=#{layout[:width]}x#{layout[:height]}:r=24"
 
-        index = 0
-
         layout[:areas].each do |layout_area|
           area = cut[:areas][layout_area[:name]]
           video_count = area.length
           BigBlueButton.logger.debug "  Laying out #{video_count} videos in #{layout_area[:name]}"
+          next if video_count == 0
 
           tile_offset_x = layout_area[:x]
           tile_offset_y = layout_area[:y]
@@ -348,8 +339,8 @@ module BigBlueButton
           # Do an exhaustive search to maximize video areas
           for tmp_tiles_v in 1..video_count
             tmp_tiles_h = (video_count / tmp_tiles_v.to_f).ceil
-            tmp_tile_width = layout_area[:width] / tmp_tiles_h
+            tmp_tile_width = (2 * (layout_area[:width].to_f / tmp_tiles_h / 2).floor).to_i
-            tmp_tile_height = layout_area[:height] / tmp_tiles_v
+            tmp_tile_height = (2 * (layout_area[:height].to_f / tmp_tiles_v / 2).floor).to_i
             next if tmp_tile_width <= 0 or tmp_tile_height <= 0
 
             tmp_total_area = 0
@@ -374,8 +365,9 @@ module BigBlueButton
 
           BigBlueButton.logger.debug "    Tiling in a #{tiles_h}x#{tiles_v} grid"
 
+          ffmpeg_filter << "[#{layout_area[:name]}_in];"
+
           area.each do |video|
-            BigBlueButton.logger.debug "    clip ##{index}"
             BigBlueButton.logger.debug "    tile location (#{tile_x}, #{tile_y})"
             video_width = videoinfo[video[:filename]][:width]
             video_height = videoinfo[video[:filename]][:height]
@@ -385,18 +377,17 @@ module BigBlueButton
             BigBlueButton.logger.debug "      scaled size: #{scale_width}x#{scale_height}"
 
             offset_x, offset_y = pad_offset(scale_width, scale_height, tile_width, tile_height)
-            offset_x += tile_offset_x + (tile_x * tile_width)
-            offset_y += tile_offset_y + (tile_y * tile_height)
             BigBlueButton.logger.debug "      offset: left: #{offset_x}, top: #{offset_y}"
 
             BigBlueButton.logger.debug "      start timestamp: #{video[:timestamp]}"
             BigBlueButton.logger.debug("      codec: #{videoinfo[video[:filename]][:video][:codec_name].inspect}")
+            BigBlueButton.logger.debug("      duration: #{videoinfo[video[:filename]][:duration]}, original duration: #{video[:original_duration]}")
 
             if videoinfo[video[:filename]][:video][:codec_name] == "flashsv2"
               # Desktop sharing videos in flashsv2 do not have regular
               # keyframes, so seeking in them doesn't really work.
               # To make processing more reliable, always decode them from the
-              # start in each cut.
+              # start in each cut. (Slow!)
               seek = 0
             else
               # Webcam videos are variable, low fps; it might be that there's
@@ -406,33 +397,66 @@ module BigBlueButton
               seek = 0 if seek < 0
             end
 
-            ffmpeg_inputs << {
+            # Workaround early 1.1 deskshare timestamp bug
-              :filename => video[:filename],
+            # It resulted in video files that were too short. To workaround, we
-              :seek => seek
+            # assume that the framerate was constant throughout (it might not
-            }
+            # actually be...) and scale the video length.
-            ffmpeg_filter << "[in#{index}]; [#{index}]fps=24,trim=start=#{ms_to_s(video[:timestamp])},setpts=PTS-STARTPTS,scale=#{scale_width}:#{scale_height}"
+            scale = nil
-            if layout_area[:pad]
+            if !video[:original_duration].nil? and
-              ffmpeg_filter << ",pad=w=#{tile_width}:h=#{tile_height}:x=#{offset_x}:y=#{offset_y}:color=white"
+                  videoinfo[video[:filename]][:video][:deskshare_timestamp_bug]
-              offset_x = 0
+              scale = video[:original_duration].to_f / videoinfo[video[:filename]][:duration]
-              offset_y = 0
+              # Rather than attempt to recalculate seek...
+              seek = 0
+              BigBlueButton.logger.debug("      Early 1.1 deskshare timestamp bug: scaling video length by #{scale}")
             end
-            ffmpeg_filter << "[mv#{index}]; [in#{index}][mv#{index}] overlay=#{offset_x}:#{offset_y}" 
+
+            pad_name = "#{layout_area[:name]}_x#{tile_x}_y#{tile_y}"
+
+            ffmpeg_filter << "movie=#{video[:filename]}:sp=#{ms_to_s(seek)}"
+            if !scale.nil?
+              ffmpeg_filter << ",setpts=PTS*#{scale}"
+            end
+            ffmpeg_filter << ",fps=#{FFMPEG_WF_FRAMERATE}:start_time=#{ms_to_s(video[:timestamp])}"
+            ffmpeg_filter << ",setpts=PTS-STARTPTS,scale=#{scale_width}:#{scale_height}"
+            ffmpeg_filter << ",pad=w=#{tile_width}:h=#{tile_height}:x=#{offset_x}:y=#{offset_y}:color=white"
+            ffmpeg_filter << "[#{pad_name}];"
 
             tile_x += 1
             if tile_x >= tiles_h
               tile_x = 0
               tile_y += 1
             end
-            index += 1
           end
+
+          remaining = video_count
+          (0...tiles_v).each do |tile_y|
+            this_tiles_h = [tiles_h, remaining].min
+            remaining -= this_tiles_h
+
+            (0...this_tiles_h).each do |tile_x|
+              ffmpeg_filter << "[#{layout_area[:name]}_x#{tile_x}_y#{tile_y}]"
+            end
+            if this_tiles_h > 1
+              ffmpeg_filter << "hstack=inputs=#{this_tiles_h},"
+            end
+            ffmpeg_filter << "pad=w=#{layout_area[:width]}:h=#{tile_height}:color=white"
+            ffmpeg_filter << "[#{layout_area[:name]}_y#{tile_y}];"
+          end
+
+          (0...tiles_v).each do |tile_y|
+            ffmpeg_filter << "[#{layout_area[:name]}_y#{tile_y}]"
+          end
+          if tiles_v > 1
+            ffmpeg_filter << "vstack=inputs=#{tiles_v},"
+          end
+          ffmpeg_filter << "pad=w=#{layout_area[:width]}:h=#{layout_area[:height]}:color=white"
+          ffmpeg_filter << "[#{layout_area[:name]}];"
+          ffmpeg_filter << "[#{layout_area[:name]}_in][#{layout_area[:name]}]overlay=x=#{layout_area[:x]}:y=#{layout_area[:y]}"
         end
 
         ffmpeg_filter << ",trim=end=#{ms_to_s(duration)}"
 
         ffmpeg_cmd = [*FFMPEG]
-        ffmpeg_inputs.each do |input|
-          ffmpeg_cmd += ['-ss', ms_to_s(input[:seek]), '-itsoffset', ms_to_s(input[:seek]), '-i', input[:filename]]
-        end
         ffmpeg_cmd += ['-filter_complex', ffmpeg_filter, *FFMPEG_WF_ARGS, '-']
 
         File.open(output, 'a') do |outio|

record-and-playback/core/lib/recordandplayback/generators/events.rb

@@ -292,6 +292,24 @@ module BigBlueButton
             }
           }
         when 'DeskshareStoppedEvent'
+          # Fill in the original/expected video duration when available
+          duration = event.at_xpath('duration')
+          if !duration.nil?
+            duration = duration.text.to_i
+            filename = event.at_xpath('file').text
+            filename = "#{archive_dir}/deskshare/#{File.basename(filename)}"
+            deskshare_edl.each do |entry|
+              if !entry[:areas][:deskshare].nil?
+                entry[:areas][:deskshare].each do |file|
+                  if file[:filename] == filename
+                    file[:original_duration] = duration * 1000
+                  end
+                end
+              end
+            end
+          end
+
+          # Terminating entry
           deskshare_edl << {
             :timestamp => timestamp,
             :areas => { :deskshare => [] }
@@ -336,7 +354,8 @@ module BigBlueButton
           videos.each do |video|
             new_entry[:areas][area] << {
               :filename => video[:filename],
-              :timestamp => video[:timestamp] + offset
+              :timestamp => video[:timestamp] + offset,
+              :original_duration => video[:original_duration]
             }
           end
         end

record-and-playback/core/lib/recordandplayback/generators/video.rb

@@ -21,23 +21,11 @@
 
 
 require 'rubygems'
-require 'streamio-ffmpeg'
 
 require File.expand_path('../../edl', __FILE__)
 
 module BigBlueButton
 
-  def self.get_video_height(video)
-    FFMPEG::Movie.new(video).height
-  end
-
-  def self.get_video_width(video)
-    FFMPEG::Movie.new(video).width
-  end
-
-  def self.is_video_valid?(video)
-    FFMPEG::Movie.new(video).valid?
-  end
 
   def BigBlueButton.process_webcam_videos(target_dir, temp_dir, meeting_id, output_width, output_height, audio_offset, processed_audio_file)
     BigBlueButton.logger.info("Processing webcam videos")

record-and-playback/presentation/scripts/presentation.yml

@@ -1,5 +1,3 @@
-
-
 video_output_width: 640
 video_output_height: 480
 # Alternate output size to use when deskshare videos are present

record-and-playback/presentation/scripts/process/presentation.rb

@@ -210,12 +210,14 @@ if not FileTest.directory?(target_dir)
       File.open("#{target_dir}/presentation_text.json","w") { |f| f.puts presentation_text.to_json }
     end
 
-    # We have to decide whether to actually generate the video file
+    # We have to decide whether to actually generate the webcams video file
     # We do so if any of the following conditions are true:
     # - There is webcam video present, or
     # - There's broadcast video present, or
     # - There are closed captions present (they need a video stream to be rendered on top of)
-    if !Dir["#{raw_archive_dir}/video/*"].empty? or !Dir["#{raw_archive_dir}/video-broadcast/*"].empty? or captions.length > 0
+    if !Dir["#{raw_archive_dir}/video/*"].empty? or
+        !Dir["#{raw_archive_dir}/video-broadcast/*"].empty? or
+        captions.length > 0
       webcam_width = presentation_props['video_output_width']
       webcam_height = presentation_props['video_output_height']
 

record-and-playback/presentation/scripts/publish/presentation.rb

@@ -29,6 +29,7 @@ require 'builder'
 require 'fastimage' # require fastimage to get the image size of the slides (gem install fastimage)
 
 
+# This script lives in scripts/archive/steps while properties.yaml lives in scripts/
 bbb_props = YAML::load(File.open('../../core/scripts/bigbluebutton.yml'))
 presentation_props = YAML::load(File.open('presentation.yml'))
 
@@ -37,8 +38,8 @@ presentation_props = YAML::load(File.open('presentation.yml'))
 $magic_mystery_number = 2
 
 def scaleToDeskshareVideo(width, height)
-  deskshare_video_height = 720.to_f
+  deskshare_video_height = presentation_props['deskshare_output_height'].to_f
-  deskshare_video_width = 1280.to_f
+  deskshare_video_width = presentation_props['deskshare_output_height'].to_f
 
   scale = [deskshare_video_width/width, deskshare_video_height/height]
   video_width = width * scale.min
@@ -48,14 +49,13 @@ def scaleToDeskshareVideo(width, height)
 end
 
 def getDeskshareVideoDimension(deskshare_stream_name)
-  video_width = 1280
+  video_width = presentation_props['deskshare_output_height'].to_f
-  video_height = 720
+  video_height = presentation_props['deskshare_output_height'].to_f
   deskshare_video_filename = "#{$deskshare_dir}/#{deskshare_stream_name}"
 
   if File.exist?(deskshare_video_filename)
-    video_width = BigBlueButton.get_video_width(deskshare_video_filename)
+    video_info = BigBlueButton::EDL::Video.video_info(deskshare_video_filename)
-    video_height = BigBlueButton.get_video_height(deskshare_video_filename)
+    video_width, video_height = scaleToDeskshareVideo(video_info[:width], video_info[:height])
-    video_width, video_height = scaleToDeskshareVideo(video_width, video_height)
   else
     BigBlueButton.logger.error("Could not find deskshare video: #{deskshare_video_filename}")
   end
@@ -1088,7 +1088,8 @@ def processDeskshareEvents
         start_timestamp = (translateTimestamp(event[:start_timestamp].to_f) / 1000).round(1)
         stop_timestamp = (translateTimestamp(event[:stop_timestamp].to_f) / 1000).round(1)
         if (start_timestamp != stop_timestamp)
-          if !BigBlueButton.is_video_valid?("#{$deskshare_dir}/#{event[:stream]}")
+          video_info = BigBlueButton::EDL::Video.video_info("#{$deskshare_dir}/#{event[:stream]}")
+          if !video_info[:video]
             BigBlueButton.logger.warn("#{event[:stream]} is not a valid video file, skipping...")
             next
           end
@@ -1125,8 +1126,6 @@ begin
 
   if ($playback == "presentation")
 
-    # This script lives in scripts/archive/steps while properties.yaml lives in scripts/
-
     log_dir = bbb_props['log_dir']
 
     logger = Logger.new("#{log_dir}/presentation/publish-#{$meeting_id}.log", 'daily' )