diff --git a/record-and-playback/core/lib/recordandplayback.rb b/record-and-playback/core/lib/recordandplayback.rb index 80a7ef355f..a00fa8a46c 100755 --- a/record-and-playback/core/lib/recordandplayback.rb +++ b/record-and-playback/core/lib/recordandplayback.rb @@ -33,7 +33,6 @@ require 'recordandplayback/generators/audio' require 'recordandplayback/generators/video' require 'recordandplayback/generators/audio_processor' require 'recordandplayback/generators/presentation' -require 'custom_hash' require 'open4' require 'pp' require 'absolute_time' diff --git a/record-and-playback/core/lib/recordandplayback/edl.rb b/record-and-playback/core/lib/recordandplayback/edl.rb index 5c4778306f..e74c03d0cb 100644 --- a/record-and-playback/core/lib/recordandplayback/edl.rb +++ b/record-and-playback/core/lib/recordandplayback/edl.rb @@ -44,7 +44,7 @@ module BigBlueButton end ffmpeg_cmd += ['-i', audio] end - ffmpeg_cmd += [*pass, lastoutput] + ffmpeg_cmd += [*pass, '-passlogfile', output_basename, lastoutput] Dir.chdir(File.dirname(output)) do exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd) raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0 diff --git a/record-and-playback/core/lib/recordandplayback/edl/video.rb b/record-and-playback/core/lib/recordandplayback/edl/video.rb index 8ed5cd8ef1..898d86a65c 100644 --- a/record-and-playback/core/lib/recordandplayback/edl/video.rb +++ b/record-and-playback/core/lib/recordandplayback/edl/video.rb @@ -24,8 +24,8 @@ module BigBlueButton module EDL module Video FFMPEG_WF_CODEC = 'mpeg2video' - FFMPEG_WF_FRAMERATE = '24' - FFMPEG_WF_ARGS = ['-an', '-codec', FFMPEG_WF_CODEC, '-q:v', '2', '-g', '240', '-pix_fmt', 'yuv420p', '-r', FFMPEG_WF_FRAMERATE, '-f', 'mpegts'] + FFMPEG_WF_FRAMERATE = 24 + FFMPEG_WF_ARGS = ['-an', '-codec', FFMPEG_WF_CODEC.to_s, '-q:v', '2', '-g', (FFMPEG_WF_FRAMERATE * 10).to_s, '-pix_fmt', 'yuv420p', '-r', FFMPEG_WF_FRAMERATE.to_s, '-f', 'mpegts'] WF_EXT = 'ts' def self.dump(edl) @@ -37,7 +37,7 @@ module BigBlueButton entry[:areas].each do |name, videos| BigBlueButton.logger.debug " #{name}" videos.each do |video| - BigBlueButton.logger.debug " #{video[:filename]} at #{video[:timestamp]}" + BigBlueButton.logger.debug " #{video[:filename]} at #{video[:timestamp]} (original duration: #{video[:original_duration]})" end end end @@ -124,7 +124,8 @@ module BigBlueButton merged_entry[:areas][area] = videos.map do |video| { :filename => video[:filename], - :timestamp => video[:timestamp] + merged_entry[:timestamp] - last_entry[:timestamp] + :timestamp => video[:timestamp] + merged_entry[:timestamp] - last_entry[:timestamp], + :original_duration => video[:original_duration] } end end @@ -189,10 +190,13 @@ module BigBlueButton BigBlueButton.logger.debug " #{videofile}" info = video_info(videofile) BigBlueButton.logger.debug " width: #{info[:width]}, height: #{info[:height]}, duration: #{info[:duration]}" - if !info[:video] BigBlueButton.logger.warn " This video file is corrupt! It will be removed from the output." corrupt_videos << videofile + else + if info[:video][:deskshare_timestamp_bug] + BigBlueButton.logger.debug(" has early 1.1 deskshare timestamp bug") + end end videoinfo[videofile] = info @@ -207,49 +211,6 @@ module BigBlueButton end end - BigBlueButton.logger.info "Generating missing video end events" - videoinfo.each do |filename, info| - - edl.each_with_index do |event, index| - - new_entry = { :areas => {} } - add_new_entry = false - event[:areas].each do |area, videos| - videos.each do |video| - if video[:filename] == filename - if video[:timestamp] > info[:duration] - videos.delete(video) - # Note that I'm using a 5-second fuzz factor here. - # If there's a stop event within 5 seconds of the video ending, don't bother to generate - # an extra event. - elsif video[:timestamp] + (event[:next_timestamp] - event[:timestamp]) > info[:duration] + 5000 - BigBlueButton.logger.warn "Over-long video #{video[:filename]}, synthesizing stop event" - new_entry[:timestamp] = event[:timestamp] + info[:duration] - video[:timestamp] - new_entry[:next_timestamp] = event[:next_timestamp] - event[:next_timestamp] = new_entry[:timestamp] - add_new_entry = true - end - end - end - end - - if add_new_entry - event[:areas].each do |area, videos| - new_entry[:areas][area] = videos.select do |video| - video[:filename] != filename - end.map do |video| - { - :filename => video[:filename], - :timestamp => video[:timestamp] + new_entry[:timestamp] - event[:timestamp] - } - end - end - edl.insert(index + 1, new_entry) - end - - end - - end dump(edl) BigBlueButton.logger.info "Compositing cuts" @@ -296,6 +257,10 @@ module BigBlueButton info[:aspect_ratio] = Rational(info[:width], info[:height]) end + if info[:format][:format_name] == 'flv' and info[:video][:codec_name] == 'h264' + info[:video][:deskshare_timestamp_bug] = self.check_deskshare_timestamp_bug(filename) + end + # Convert the duration to milliseconds info[:duration] = (info[:format][:duration].to_r * 1000).to_i @@ -304,6 +269,31 @@ module BigBlueButton {} end + def self.check_deskshare_timestamp_bug(filename) + IO.popen([*FFPROBE, '-select_streams', 'v:0', '-show_frames', '-read_intervals', '%+#10', filename]) do |probe| + info = JSON.parse(probe.read, symbolize_names: true) + return false if !info + + if !info[:frames] + return false + end + + # First frame in broken stream always has pts=1 + if info[:frames][0][:pkt_pts] != 1 + return false + end + + # Remaining frames start at 200, and go up by exactly 200 each frame + for i in 1...info[:frames].length + if info[:frames][i][:pkt_pts] != i * 200 + return false + end + end + + return true + end + end + def self.ms_to_s(timestamp) s = timestamp / 1000 ms = timestamp % 1000 @@ -312,29 +302,30 @@ module BigBlueButton def self.aspect_scale(old_width, old_height, new_width, new_height) if old_width.to_f / old_height > new_width.to_f / new_height - [new_width, old_height * new_width / old_width] + new_height = (2 * (old_height.to_f * new_width / old_width / 2).round).to_i else - [old_width * new_height / old_height, new_height] + new_width = (2 * (old_width.to_f * new_height / old_height / 2).round).to_i end + [new_width, new_height] end def self.pad_offset(video_width, video_height, area_width, area_height) - [(area_width - video_width) / 2, (area_height - video_height) / 2] + pad_x = (2 * ((area_width - video_width).to_f / 4).round).to_i + pad_y = (2 * ((area_height - video_height).to_f / 4).round).to_i + [pad_x, pad_y] end def self.composite_cut(output, cut, layout, videoinfo) duration = cut[:next_timestamp] - cut[:timestamp] BigBlueButton.logger.info " Cut start time #{cut[:timestamp]}, duration #{duration}" - ffmpeg_inputs = [] ffmpeg_filter = "color=c=white:s=#{layout[:width]}x#{layout[:height]}:r=24" - index = 0 - layout[:areas].each do |layout_area| area = cut[:areas][layout_area[:name]] video_count = area.length BigBlueButton.logger.debug " Laying out #{video_count} videos in #{layout_area[:name]}" + next if video_count == 0 tile_offset_x = layout_area[:x] tile_offset_y = layout_area[:y] @@ -348,8 +339,8 @@ module BigBlueButton # Do an exhaustive search to maximize video areas for tmp_tiles_v in 1..video_count tmp_tiles_h = (video_count / tmp_tiles_v.to_f).ceil - tmp_tile_width = layout_area[:width] / tmp_tiles_h - tmp_tile_height = layout_area[:height] / tmp_tiles_v + tmp_tile_width = (2 * (layout_area[:width].to_f / tmp_tiles_h / 2).floor).to_i + tmp_tile_height = (2 * (layout_area[:height].to_f / tmp_tiles_v / 2).floor).to_i next if tmp_tile_width <= 0 or tmp_tile_height <= 0 tmp_total_area = 0 @@ -374,9 +365,10 @@ module BigBlueButton BigBlueButton.logger.debug " Tiling in a #{tiles_h}x#{tiles_v} grid" + ffmpeg_filter << "[#{layout_area[:name]}_in];" + area.each do |video| - BigBlueButton.logger.debug " clip ##{index}" - BigBlueButton.logger.debug " tile location (#{tile_x}, #{tile_y})" + BigBlueButton.logger.debug " tile location (#{tile_x}, #{tile_y})" video_width = videoinfo[video[:filename]][:width] video_height = videoinfo[video[:filename]][:height] BigBlueButton.logger.debug " original size: #{video_width}x#{video_height}" @@ -385,18 +377,17 @@ module BigBlueButton BigBlueButton.logger.debug " scaled size: #{scale_width}x#{scale_height}" offset_x, offset_y = pad_offset(scale_width, scale_height, tile_width, tile_height) - offset_x += tile_offset_x + (tile_x * tile_width) - offset_y += tile_offset_y + (tile_y * tile_height) BigBlueButton.logger.debug " offset: left: #{offset_x}, top: #{offset_y}" BigBlueButton.logger.debug " start timestamp: #{video[:timestamp]}" BigBlueButton.logger.debug(" codec: #{videoinfo[video[:filename]][:video][:codec_name].inspect}") + BigBlueButton.logger.debug(" duration: #{videoinfo[video[:filename]][:duration]}, original duration: #{video[:original_duration]}") if videoinfo[video[:filename]][:video][:codec_name] == "flashsv2" # Desktop sharing videos in flashsv2 do not have regular # keyframes, so seeking in them doesn't really work. # To make processing more reliable, always decode them from the - # start in each cut. + # start in each cut. (Slow!) seek = 0 else # Webcam videos are variable, low fps; it might be that there's @@ -406,33 +397,66 @@ module BigBlueButton seek = 0 if seek < 0 end - ffmpeg_inputs << { - :filename => video[:filename], - :seek => seek - } - ffmpeg_filter << "[in#{index}]; [#{index}]fps=24,trim=start=#{ms_to_s(video[:timestamp])},setpts=PTS-STARTPTS,scale=#{scale_width}:#{scale_height}" - if layout_area[:pad] - ffmpeg_filter << ",pad=w=#{tile_width}:h=#{tile_height}:x=#{offset_x}:y=#{offset_y}:color=white" - offset_x = 0 - offset_y = 0 + # Workaround early 1.1 deskshare timestamp bug + # It resulted in video files that were too short. To workaround, we + # assume that the framerate was constant throughout (it might not + # actually be...) and scale the video length. + scale = nil + if !video[:original_duration].nil? and + videoinfo[video[:filename]][:video][:deskshare_timestamp_bug] + scale = video[:original_duration].to_f / videoinfo[video[:filename]][:duration] + # Rather than attempt to recalculate seek... + seek = 0 + BigBlueButton.logger.debug(" Early 1.1 deskshare timestamp bug: scaling video length by #{scale}") end - ffmpeg_filter << "[mv#{index}]; [in#{index}][mv#{index}] overlay=#{offset_x}:#{offset_y}" + + pad_name = "#{layout_area[:name]}_x#{tile_x}_y#{tile_y}" + + ffmpeg_filter << "movie=#{video[:filename]}:sp=#{ms_to_s(seek)}" + if !scale.nil? + ffmpeg_filter << ",setpts=PTS*#{scale}" + end + ffmpeg_filter << ",fps=#{FFMPEG_WF_FRAMERATE}:start_time=#{ms_to_s(video[:timestamp])}" + ffmpeg_filter << ",setpts=PTS-STARTPTS,scale=#{scale_width}:#{scale_height}" + ffmpeg_filter << ",pad=w=#{tile_width}:h=#{tile_height}:x=#{offset_x}:y=#{offset_y}:color=white" + ffmpeg_filter << "[#{pad_name}];" tile_x += 1 if tile_x >= tiles_h tile_x = 0 tile_y += 1 end - index += 1 end + + remaining = video_count + (0...tiles_v).each do |tile_y| + this_tiles_h = [tiles_h, remaining].min + remaining -= this_tiles_h + + (0...this_tiles_h).each do |tile_x| + ffmpeg_filter << "[#{layout_area[:name]}_x#{tile_x}_y#{tile_y}]" + end + if this_tiles_h > 1 + ffmpeg_filter << "hstack=inputs=#{this_tiles_h}," + end + ffmpeg_filter << "pad=w=#{layout_area[:width]}:h=#{tile_height}:color=white" + ffmpeg_filter << "[#{layout_area[:name]}_y#{tile_y}];" + end + + (0...tiles_v).each do |tile_y| + ffmpeg_filter << "[#{layout_area[:name]}_y#{tile_y}]" + end + if tiles_v > 1 + ffmpeg_filter << "vstack=inputs=#{tiles_v}," + end + ffmpeg_filter << "pad=w=#{layout_area[:width]}:h=#{layout_area[:height]}:color=white" + ffmpeg_filter << "[#{layout_area[:name]}];" + ffmpeg_filter << "[#{layout_area[:name]}_in][#{layout_area[:name]}]overlay=x=#{layout_area[:x]}:y=#{layout_area[:y]}" end ffmpeg_filter << ",trim=end=#{ms_to_s(duration)}" ffmpeg_cmd = [*FFMPEG] - ffmpeg_inputs.each do |input| - ffmpeg_cmd += ['-ss', ms_to_s(input[:seek]), '-itsoffset', ms_to_s(input[:seek]), '-i', input[:filename]] - end ffmpeg_cmd += ['-filter_complex', ffmpeg_filter, *FFMPEG_WF_ARGS, '-'] File.open(output, 'a') do |outio| diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb index 2c53812e53..c0b76a8b9d 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/events.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb @@ -292,6 +292,24 @@ module BigBlueButton } } when 'DeskshareStoppedEvent' + # Fill in the original/expected video duration when available + duration = event.at_xpath('duration') + if !duration.nil? + duration = duration.text.to_i + filename = event.at_xpath('file').text + filename = "#{archive_dir}/deskshare/#{File.basename(filename)}" + deskshare_edl.each do |entry| + if !entry[:areas][:deskshare].nil? + entry[:areas][:deskshare].each do |file| + if file[:filename] == filename + file[:original_duration] = duration * 1000 + end + end + end + end + end + + # Terminating entry deskshare_edl << { :timestamp => timestamp, :areas => { :deskshare => [] } @@ -336,7 +354,8 @@ module BigBlueButton videos.each do |video| new_entry[:areas][area] << { :filename => video[:filename], - :timestamp => video[:timestamp] + offset + :timestamp => video[:timestamp] + offset, + :original_duration => video[:original_duration] } end end diff --git a/record-and-playback/core/lib/recordandplayback/generators/video.rb b/record-and-playback/core/lib/recordandplayback/generators/video.rb index 1ac9343037..0caa329505 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/video.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/video.rb @@ -21,23 +21,11 @@ require 'rubygems' -require 'streamio-ffmpeg' require File.expand_path('../../edl', __FILE__) module BigBlueButton - def self.get_video_height(video) - FFMPEG::Movie.new(video).height - end - - def self.get_video_width(video) - FFMPEG::Movie.new(video).width - end - - def self.is_video_valid?(video) - FFMPEG::Movie.new(video).valid? - end def BigBlueButton.process_webcam_videos(target_dir, temp_dir, meeting_id, output_width, output_height, audio_offset, processed_audio_file) BigBlueButton.logger.info("Processing webcam videos") diff --git a/record-and-playback/presentation/scripts/presentation.yml b/record-and-playback/presentation/scripts/presentation.yml index a96fa19eb8..f8dbcbfa23 100755 --- a/record-and-playback/presentation/scripts/presentation.yml +++ b/record-and-playback/presentation/scripts/presentation.yml @@ -1,5 +1,3 @@ - - video_output_width: 640 video_output_height: 480 # Alternate output size to use when deskshare videos are present diff --git a/record-and-playback/presentation/scripts/process/presentation.rb b/record-and-playback/presentation/scripts/process/presentation.rb index 996b893e3d..ec912f3610 100755 --- a/record-and-playback/presentation/scripts/process/presentation.rb +++ b/record-and-playback/presentation/scripts/process/presentation.rb @@ -210,12 +210,14 @@ if not FileTest.directory?(target_dir) File.open("#{target_dir}/presentation_text.json","w") { |f| f.puts presentation_text.to_json } end - # We have to decide whether to actually generate the video file + # We have to decide whether to actually generate the webcams video file # We do so if any of the following conditions are true: # - There is webcam video present, or # - There's broadcast video present, or # - There are closed captions present (they need a video stream to be rendered on top of) - if !Dir["#{raw_archive_dir}/video/*"].empty? or !Dir["#{raw_archive_dir}/video-broadcast/*"].empty? or captions.length > 0 + if !Dir["#{raw_archive_dir}/video/*"].empty? or + !Dir["#{raw_archive_dir}/video-broadcast/*"].empty? or + captions.length > 0 webcam_width = presentation_props['video_output_width'] webcam_height = presentation_props['video_output_height'] diff --git a/record-and-playback/presentation/scripts/publish/presentation.rb b/record-and-playback/presentation/scripts/publish/presentation.rb index 2cab0d043b..9d803f0e12 100755 --- a/record-and-playback/presentation/scripts/publish/presentation.rb +++ b/record-and-playback/presentation/scripts/publish/presentation.rb @@ -29,6 +29,7 @@ require 'builder' require 'fastimage' # require fastimage to get the image size of the slides (gem install fastimage) +# This script lives in scripts/archive/steps while properties.yaml lives in scripts/ bbb_props = YAML::load(File.open('../../core/scripts/bigbluebutton.yml')) presentation_props = YAML::load(File.open('presentation.yml')) @@ -37,8 +38,8 @@ presentation_props = YAML::load(File.open('presentation.yml')) $magic_mystery_number = 2 def scaleToDeskshareVideo(width, height) - deskshare_video_height = 720.to_f - deskshare_video_width = 1280.to_f + deskshare_video_height = presentation_props['deskshare_output_height'].to_f + deskshare_video_width = presentation_props['deskshare_output_height'].to_f scale = [deskshare_video_width/width, deskshare_video_height/height] video_width = width * scale.min @@ -48,14 +49,13 @@ def scaleToDeskshareVideo(width, height) end def getDeskshareVideoDimension(deskshare_stream_name) - video_width = 1280 - video_height = 720 + video_width = presentation_props['deskshare_output_height'].to_f + video_height = presentation_props['deskshare_output_height'].to_f deskshare_video_filename = "#{$deskshare_dir}/#{deskshare_stream_name}" if File.exist?(deskshare_video_filename) - video_width = BigBlueButton.get_video_width(deskshare_video_filename) - video_height = BigBlueButton.get_video_height(deskshare_video_filename) - video_width, video_height = scaleToDeskshareVideo(video_width, video_height) + video_info = BigBlueButton::EDL::Video.video_info(deskshare_video_filename) + video_width, video_height = scaleToDeskshareVideo(video_info[:width], video_info[:height]) else BigBlueButton.logger.error("Could not find deskshare video: #{deskshare_video_filename}") end @@ -1088,7 +1088,8 @@ def processDeskshareEvents start_timestamp = (translateTimestamp(event[:start_timestamp].to_f) / 1000).round(1) stop_timestamp = (translateTimestamp(event[:stop_timestamp].to_f) / 1000).round(1) if (start_timestamp != stop_timestamp) - if !BigBlueButton.is_video_valid?("#{$deskshare_dir}/#{event[:stream]}") + video_info = BigBlueButton::EDL::Video.video_info("#{$deskshare_dir}/#{event[:stream]}") + if !video_info[:video] BigBlueButton.logger.warn("#{event[:stream]} is not a valid video file, skipping...") next end @@ -1125,8 +1126,6 @@ begin if ($playback == "presentation") - # This script lives in scripts/archive/steps while properties.yaml lives in scripts/ - log_dir = bbb_props['log_dir'] logger = Logger.new("#{log_dir}/presentation/publish-#{$meeting_id}.log", 'daily' )