From dba5cd9196249aa72bb19666ba232f4ac5211579 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 18 Aug 2017 15:24:54 -0400 Subject: [PATCH] Various recording script fixes & cleanups This is just a bundle of a few things I've been fixing up in the past while. = Workaround for BBB 1.1 beta deskshare timestamp bug This is unlikely to be used, but I have the code for it, might as well merge it in. = Rework video tiling code for ffmpeg Render video using the 'hstack' and 'vstack' filters rather than the 'overlay' filter. This is somewhat faster, particularly with lots of videos. = Etc. - Remove usage of the streamio-ffmpeg gem. The video rendering code has some stuff to directly read 'ffprobe' output, so re-use that instead of this gem (which is kind of old and has issues with newer ffmpeg versions). - Don't hardcode the deskshare video area size, pull it from the properties file - Remove some code that worked around missing video end events. In some cases this could cause flickering or strange video issues. It's no longer strictly needed, the new tiling code doesn't break if the seekpoint is after the end of the video. --- .../core/lib/recordandplayback.rb | 1 - .../core/lib/recordandplayback/edl.rb | 2 +- .../core/lib/recordandplayback/edl/video.rb | 174 ++++++++++-------- .../recordandplayback/generators/events.rb | 21 ++- .../lib/recordandplayback/generators/video.rb | 12 -- .../presentation/scripts/presentation.yml | 2 - .../scripts/process/presentation.rb | 6 +- .../scripts/publish/presentation.rb | 19 +- 8 files changed, 133 insertions(+), 104 deletions(-) diff --git a/record-and-playback/core/lib/recordandplayback.rb b/record-and-playback/core/lib/recordandplayback.rb index 80a7ef355f..a00fa8a46c 100755 --- a/record-and-playback/core/lib/recordandplayback.rb +++ b/record-and-playback/core/lib/recordandplayback.rb @@ -33,7 +33,6 @@ require 'recordandplayback/generators/audio' require 'recordandplayback/generators/video' require 'recordandplayback/generators/audio_processor' require 'recordandplayback/generators/presentation' -require 'custom_hash' require 'open4' require 'pp' require 'absolute_time' diff --git a/record-and-playback/core/lib/recordandplayback/edl.rb b/record-and-playback/core/lib/recordandplayback/edl.rb index 5c4778306f..e74c03d0cb 100644 --- a/record-and-playback/core/lib/recordandplayback/edl.rb +++ b/record-and-playback/core/lib/recordandplayback/edl.rb @@ -44,7 +44,7 @@ module BigBlueButton end ffmpeg_cmd += ['-i', audio] end - ffmpeg_cmd += [*pass, lastoutput] + ffmpeg_cmd += [*pass, '-passlogfile', output_basename, lastoutput] Dir.chdir(File.dirname(output)) do exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd) raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0 diff --git a/record-and-playback/core/lib/recordandplayback/edl/video.rb b/record-and-playback/core/lib/recordandplayback/edl/video.rb index 8ed5cd8ef1..898d86a65c 100644 --- a/record-and-playback/core/lib/recordandplayback/edl/video.rb +++ b/record-and-playback/core/lib/recordandplayback/edl/video.rb @@ -24,8 +24,8 @@ module BigBlueButton module EDL module Video FFMPEG_WF_CODEC = 'mpeg2video' - FFMPEG_WF_FRAMERATE = '24' - FFMPEG_WF_ARGS = ['-an', '-codec', FFMPEG_WF_CODEC, '-q:v', '2', '-g', '240', '-pix_fmt', 'yuv420p', '-r', FFMPEG_WF_FRAMERATE, '-f', 'mpegts'] + FFMPEG_WF_FRAMERATE = 24 + FFMPEG_WF_ARGS = ['-an', '-codec', FFMPEG_WF_CODEC.to_s, '-q:v', '2', '-g', (FFMPEG_WF_FRAMERATE * 10).to_s, '-pix_fmt', 'yuv420p', '-r', FFMPEG_WF_FRAMERATE.to_s, '-f', 'mpegts'] WF_EXT = 'ts' def self.dump(edl) @@ -37,7 +37,7 @@ module BigBlueButton entry[:areas].each do |name, videos| BigBlueButton.logger.debug " #{name}" videos.each do |video| - BigBlueButton.logger.debug " #{video[:filename]} at #{video[:timestamp]}" + BigBlueButton.logger.debug " #{video[:filename]} at #{video[:timestamp]} (original duration: #{video[:original_duration]})" end end end @@ -124,7 +124,8 @@ module BigBlueButton merged_entry[:areas][area] = videos.map do |video| { :filename => video[:filename], - :timestamp => video[:timestamp] + merged_entry[:timestamp] - last_entry[:timestamp] + :timestamp => video[:timestamp] + merged_entry[:timestamp] - last_entry[:timestamp], + :original_duration => video[:original_duration] } end end @@ -189,10 +190,13 @@ module BigBlueButton BigBlueButton.logger.debug " #{videofile}" info = video_info(videofile) BigBlueButton.logger.debug " width: #{info[:width]}, height: #{info[:height]}, duration: #{info[:duration]}" - if !info[:video] BigBlueButton.logger.warn " This video file is corrupt! It will be removed from the output." corrupt_videos << videofile + else + if info[:video][:deskshare_timestamp_bug] + BigBlueButton.logger.debug(" has early 1.1 deskshare timestamp bug") + end end videoinfo[videofile] = info @@ -207,49 +211,6 @@ module BigBlueButton end end - BigBlueButton.logger.info "Generating missing video end events" - videoinfo.each do |filename, info| - - edl.each_with_index do |event, index| - - new_entry = { :areas => {} } - add_new_entry = false - event[:areas].each do |area, videos| - videos.each do |video| - if video[:filename] == filename - if video[:timestamp] > info[:duration] - videos.delete(video) - # Note that I'm using a 5-second fuzz factor here. - # If there's a stop event within 5 seconds of the video ending, don't bother to generate - # an extra event. - elsif video[:timestamp] + (event[:next_timestamp] - event[:timestamp]) > info[:duration] + 5000 - BigBlueButton.logger.warn "Over-long video #{video[:filename]}, synthesizing stop event" - new_entry[:timestamp] = event[:timestamp] + info[:duration] - video[:timestamp] - new_entry[:next_timestamp] = event[:next_timestamp] - event[:next_timestamp] = new_entry[:timestamp] - add_new_entry = true - end - end - end - end - - if add_new_entry - event[:areas].each do |area, videos| - new_entry[:areas][area] = videos.select do |video| - video[:filename] != filename - end.map do |video| - { - :filename => video[:filename], - :timestamp => video[:timestamp] + new_entry[:timestamp] - event[:timestamp] - } - end - end - edl.insert(index + 1, new_entry) - end - - end - - end dump(edl) BigBlueButton.logger.info "Compositing cuts" @@ -296,6 +257,10 @@ module BigBlueButton info[:aspect_ratio] = Rational(info[:width], info[:height]) end + if info[:format][:format_name] == 'flv' and info[:video][:codec_name] == 'h264' + info[:video][:deskshare_timestamp_bug] = self.check_deskshare_timestamp_bug(filename) + end + # Convert the duration to milliseconds info[:duration] = (info[:format][:duration].to_r * 1000).to_i @@ -304,6 +269,31 @@ module BigBlueButton {} end + def self.check_deskshare_timestamp_bug(filename) + IO.popen([*FFPROBE, '-select_streams', 'v:0', '-show_frames', '-read_intervals', '%+#10', filename]) do |probe| + info = JSON.parse(probe.read, symbolize_names: true) + return false if !info + + if !info[:frames] + return false + end + + # First frame in broken stream always has pts=1 + if info[:frames][0][:pkt_pts] != 1 + return false + end + + # Remaining frames start at 200, and go up by exactly 200 each frame + for i in 1...info[:frames].length + if info[:frames][i][:pkt_pts] != i * 200 + return false + end + end + + return true + end + end + def self.ms_to_s(timestamp) s = timestamp / 1000 ms = timestamp % 1000 @@ -312,29 +302,30 @@ module BigBlueButton def self.aspect_scale(old_width, old_height, new_width, new_height) if old_width.to_f / old_height > new_width.to_f / new_height - [new_width, old_height * new_width / old_width] + new_height = (2 * (old_height.to_f * new_width / old_width / 2).round).to_i else - [old_width * new_height / old_height, new_height] + new_width = (2 * (old_width.to_f * new_height / old_height / 2).round).to_i end + [new_width, new_height] end def self.pad_offset(video_width, video_height, area_width, area_height) - [(area_width - video_width) / 2, (area_height - video_height) / 2] + pad_x = (2 * ((area_width - video_width).to_f / 4).round).to_i + pad_y = (2 * ((area_height - video_height).to_f / 4).round).to_i + [pad_x, pad_y] end def self.composite_cut(output, cut, layout, videoinfo) duration = cut[:next_timestamp] - cut[:timestamp] BigBlueButton.logger.info " Cut start time #{cut[:timestamp]}, duration #{duration}" - ffmpeg_inputs = [] ffmpeg_filter = "color=c=white:s=#{layout[:width]}x#{layout[:height]}:r=24" - index = 0 - layout[:areas].each do |layout_area| area = cut[:areas][layout_area[:name]] video_count = area.length BigBlueButton.logger.debug " Laying out #{video_count} videos in #{layout_area[:name]}" + next if video_count == 0 tile_offset_x = layout_area[:x] tile_offset_y = layout_area[:y] @@ -348,8 +339,8 @@ module BigBlueButton # Do an exhaustive search to maximize video areas for tmp_tiles_v in 1..video_count tmp_tiles_h = (video_count / tmp_tiles_v.to_f).ceil - tmp_tile_width = layout_area[:width] / tmp_tiles_h - tmp_tile_height = layout_area[:height] / tmp_tiles_v + tmp_tile_width = (2 * (layout_area[:width].to_f / tmp_tiles_h / 2).floor).to_i + tmp_tile_height = (2 * (layout_area[:height].to_f / tmp_tiles_v / 2).floor).to_i next if tmp_tile_width <= 0 or tmp_tile_height <= 0 tmp_total_area = 0 @@ -374,9 +365,10 @@ module BigBlueButton BigBlueButton.logger.debug " Tiling in a #{tiles_h}x#{tiles_v} grid" + ffmpeg_filter << "[#{layout_area[:name]}_in];" + area.each do |video| - BigBlueButton.logger.debug " clip ##{index}" - BigBlueButton.logger.debug " tile location (#{tile_x}, #{tile_y})" + BigBlueButton.logger.debug " tile location (#{tile_x}, #{tile_y})" video_width = videoinfo[video[:filename]][:width] video_height = videoinfo[video[:filename]][:height] BigBlueButton.logger.debug " original size: #{video_width}x#{video_height}" @@ -385,18 +377,17 @@ module BigBlueButton BigBlueButton.logger.debug " scaled size: #{scale_width}x#{scale_height}" offset_x, offset_y = pad_offset(scale_width, scale_height, tile_width, tile_height) - offset_x += tile_offset_x + (tile_x * tile_width) - offset_y += tile_offset_y + (tile_y * tile_height) BigBlueButton.logger.debug " offset: left: #{offset_x}, top: #{offset_y}" BigBlueButton.logger.debug " start timestamp: #{video[:timestamp]}" BigBlueButton.logger.debug(" codec: #{videoinfo[video[:filename]][:video][:codec_name].inspect}") + BigBlueButton.logger.debug(" duration: #{videoinfo[video[:filename]][:duration]}, original duration: #{video[:original_duration]}") if videoinfo[video[:filename]][:video][:codec_name] == "flashsv2" # Desktop sharing videos in flashsv2 do not have regular # keyframes, so seeking in them doesn't really work. # To make processing more reliable, always decode them from the - # start in each cut. + # start in each cut. (Slow!) seek = 0 else # Webcam videos are variable, low fps; it might be that there's @@ -406,33 +397,66 @@ module BigBlueButton seek = 0 if seek < 0 end - ffmpeg_inputs << { - :filename => video[:filename], - :seek => seek - } - ffmpeg_filter << "[in#{index}]; [#{index}]fps=24,trim=start=#{ms_to_s(video[:timestamp])},setpts=PTS-STARTPTS,scale=#{scale_width}:#{scale_height}" - if layout_area[:pad] - ffmpeg_filter << ",pad=w=#{tile_width}:h=#{tile_height}:x=#{offset_x}:y=#{offset_y}:color=white" - offset_x = 0 - offset_y = 0 + # Workaround early 1.1 deskshare timestamp bug + # It resulted in video files that were too short. To workaround, we + # assume that the framerate was constant throughout (it might not + # actually be...) and scale the video length. + scale = nil + if !video[:original_duration].nil? and + videoinfo[video[:filename]][:video][:deskshare_timestamp_bug] + scale = video[:original_duration].to_f / videoinfo[video[:filename]][:duration] + # Rather than attempt to recalculate seek... + seek = 0 + BigBlueButton.logger.debug(" Early 1.1 deskshare timestamp bug: scaling video length by #{scale}") end - ffmpeg_filter << "[mv#{index}]; [in#{index}][mv#{index}] overlay=#{offset_x}:#{offset_y}" + + pad_name = "#{layout_area[:name]}_x#{tile_x}_y#{tile_y}" + + ffmpeg_filter << "movie=#{video[:filename]}:sp=#{ms_to_s(seek)}" + if !scale.nil? + ffmpeg_filter << ",setpts=PTS*#{scale}" + end + ffmpeg_filter << ",fps=#{FFMPEG_WF_FRAMERATE}:start_time=#{ms_to_s(video[:timestamp])}" + ffmpeg_filter << ",setpts=PTS-STARTPTS,scale=#{scale_width}:#{scale_height}" + ffmpeg_filter << ",pad=w=#{tile_width}:h=#{tile_height}:x=#{offset_x}:y=#{offset_y}:color=white" + ffmpeg_filter << "[#{pad_name}];" tile_x += 1 if tile_x >= tiles_h tile_x = 0 tile_y += 1 end - index += 1 end + + remaining = video_count + (0...tiles_v).each do |tile_y| + this_tiles_h = [tiles_h, remaining].min + remaining -= this_tiles_h + + (0...this_tiles_h).each do |tile_x| + ffmpeg_filter << "[#{layout_area[:name]}_x#{tile_x}_y#{tile_y}]" + end + if this_tiles_h > 1 + ffmpeg_filter << "hstack=inputs=#{this_tiles_h}," + end + ffmpeg_filter << "pad=w=#{layout_area[:width]}:h=#{tile_height}:color=white" + ffmpeg_filter << "[#{layout_area[:name]}_y#{tile_y}];" + end + + (0...tiles_v).each do |tile_y| + ffmpeg_filter << "[#{layout_area[:name]}_y#{tile_y}]" + end + if tiles_v > 1 + ffmpeg_filter << "vstack=inputs=#{tiles_v}," + end + ffmpeg_filter << "pad=w=#{layout_area[:width]}:h=#{layout_area[:height]}:color=white" + ffmpeg_filter << "[#{layout_area[:name]}];" + ffmpeg_filter << "[#{layout_area[:name]}_in][#{layout_area[:name]}]overlay=x=#{layout_area[:x]}:y=#{layout_area[:y]}" end ffmpeg_filter << ",trim=end=#{ms_to_s(duration)}" ffmpeg_cmd = [*FFMPEG] - ffmpeg_inputs.each do |input| - ffmpeg_cmd += ['-ss', ms_to_s(input[:seek]), '-itsoffset', ms_to_s(input[:seek]), '-i', input[:filename]] - end ffmpeg_cmd += ['-filter_complex', ffmpeg_filter, *FFMPEG_WF_ARGS, '-'] File.open(output, 'a') do |outio| diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb index 2c53812e53..c0b76a8b9d 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/events.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb @@ -292,6 +292,24 @@ module BigBlueButton } } when 'DeskshareStoppedEvent' + # Fill in the original/expected video duration when available + duration = event.at_xpath('duration') + if !duration.nil? + duration = duration.text.to_i + filename = event.at_xpath('file').text + filename = "#{archive_dir}/deskshare/#{File.basename(filename)}" + deskshare_edl.each do |entry| + if !entry[:areas][:deskshare].nil? + entry[:areas][:deskshare].each do |file| + if file[:filename] == filename + file[:original_duration] = duration * 1000 + end + end + end + end + end + + # Terminating entry deskshare_edl << { :timestamp => timestamp, :areas => { :deskshare => [] } @@ -336,7 +354,8 @@ module BigBlueButton videos.each do |video| new_entry[:areas][area] << { :filename => video[:filename], - :timestamp => video[:timestamp] + offset + :timestamp => video[:timestamp] + offset, + :original_duration => video[:original_duration] } end end diff --git a/record-and-playback/core/lib/recordandplayback/generators/video.rb b/record-and-playback/core/lib/recordandplayback/generators/video.rb index 1ac9343037..0caa329505 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/video.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/video.rb @@ -21,23 +21,11 @@ require 'rubygems' -require 'streamio-ffmpeg' require File.expand_path('../../edl', __FILE__) module BigBlueButton - def self.get_video_height(video) - FFMPEG::Movie.new(video).height - end - - def self.get_video_width(video) - FFMPEG::Movie.new(video).width - end - - def self.is_video_valid?(video) - FFMPEG::Movie.new(video).valid? - end def BigBlueButton.process_webcam_videos(target_dir, temp_dir, meeting_id, output_width, output_height, audio_offset, processed_audio_file) BigBlueButton.logger.info("Processing webcam videos") diff --git a/record-and-playback/presentation/scripts/presentation.yml b/record-and-playback/presentation/scripts/presentation.yml index a96fa19eb8..f8dbcbfa23 100755 --- a/record-and-playback/presentation/scripts/presentation.yml +++ b/record-and-playback/presentation/scripts/presentation.yml @@ -1,5 +1,3 @@ - - video_output_width: 640 video_output_height: 480 # Alternate output size to use when deskshare videos are present diff --git a/record-and-playback/presentation/scripts/process/presentation.rb b/record-and-playback/presentation/scripts/process/presentation.rb index 996b893e3d..ec912f3610 100755 --- a/record-and-playback/presentation/scripts/process/presentation.rb +++ b/record-and-playback/presentation/scripts/process/presentation.rb @@ -210,12 +210,14 @@ if not FileTest.directory?(target_dir) File.open("#{target_dir}/presentation_text.json","w") { |f| f.puts presentation_text.to_json } end - # We have to decide whether to actually generate the video file + # We have to decide whether to actually generate the webcams video file # We do so if any of the following conditions are true: # - There is webcam video present, or # - There's broadcast video present, or # - There are closed captions present (they need a video stream to be rendered on top of) - if !Dir["#{raw_archive_dir}/video/*"].empty? or !Dir["#{raw_archive_dir}/video-broadcast/*"].empty? or captions.length > 0 + if !Dir["#{raw_archive_dir}/video/*"].empty? or + !Dir["#{raw_archive_dir}/video-broadcast/*"].empty? or + captions.length > 0 webcam_width = presentation_props['video_output_width'] webcam_height = presentation_props['video_output_height'] diff --git a/record-and-playback/presentation/scripts/publish/presentation.rb b/record-and-playback/presentation/scripts/publish/presentation.rb index 2cab0d043b..9d803f0e12 100755 --- a/record-and-playback/presentation/scripts/publish/presentation.rb +++ b/record-and-playback/presentation/scripts/publish/presentation.rb @@ -29,6 +29,7 @@ require 'builder' require 'fastimage' # require fastimage to get the image size of the slides (gem install fastimage) +# This script lives in scripts/archive/steps while properties.yaml lives in scripts/ bbb_props = YAML::load(File.open('../../core/scripts/bigbluebutton.yml')) presentation_props = YAML::load(File.open('presentation.yml')) @@ -37,8 +38,8 @@ presentation_props = YAML::load(File.open('presentation.yml')) $magic_mystery_number = 2 def scaleToDeskshareVideo(width, height) - deskshare_video_height = 720.to_f - deskshare_video_width = 1280.to_f + deskshare_video_height = presentation_props['deskshare_output_height'].to_f + deskshare_video_width = presentation_props['deskshare_output_height'].to_f scale = [deskshare_video_width/width, deskshare_video_height/height] video_width = width * scale.min @@ -48,14 +49,13 @@ def scaleToDeskshareVideo(width, height) end def getDeskshareVideoDimension(deskshare_stream_name) - video_width = 1280 - video_height = 720 + video_width = presentation_props['deskshare_output_height'].to_f + video_height = presentation_props['deskshare_output_height'].to_f deskshare_video_filename = "#{$deskshare_dir}/#{deskshare_stream_name}" if File.exist?(deskshare_video_filename) - video_width = BigBlueButton.get_video_width(deskshare_video_filename) - video_height = BigBlueButton.get_video_height(deskshare_video_filename) - video_width, video_height = scaleToDeskshareVideo(video_width, video_height) + video_info = BigBlueButton::EDL::Video.video_info(deskshare_video_filename) + video_width, video_height = scaleToDeskshareVideo(video_info[:width], video_info[:height]) else BigBlueButton.logger.error("Could not find deskshare video: #{deskshare_video_filename}") end @@ -1088,7 +1088,8 @@ def processDeskshareEvents start_timestamp = (translateTimestamp(event[:start_timestamp].to_f) / 1000).round(1) stop_timestamp = (translateTimestamp(event[:stop_timestamp].to_f) / 1000).round(1) if (start_timestamp != stop_timestamp) - if !BigBlueButton.is_video_valid?("#{$deskshare_dir}/#{event[:stream]}") + video_info = BigBlueButton::EDL::Video.video_info("#{$deskshare_dir}/#{event[:stream]}") + if !video_info[:video] BigBlueButton.logger.warn("#{event[:stream]} is not a valid video file, skipping...") next end @@ -1125,8 +1126,6 @@ begin if ($playback == "presentation") - # This script lives in scripts/archive/steps while properties.yaml lives in scripts/ - log_dir = bbb_props['log_dir'] logger = Logger.new("#{log_dir}/presentation/publish-#{$meeting_id}.log", 'daily' )