From 0acb3b5750357a37749531ffabcad60b1e5b388d Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Mon, 30 Oct 2017 13:38:55 -0400 Subject: [PATCH 01/13] Adapt rap-archive-worker to segmented recording done file format --- .../core/scripts/rap-archive-worker.rb | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/record-and-playback/core/scripts/rap-archive-worker.rb b/record-and-playback/core/scripts/rap-archive-worker.rb index 8f764582db..953acb73b2 100755 --- a/record-and-playback/core/scripts/rap-archive-worker.rb +++ b/record-and-playback/core/scripts/rap-archive-worker.rb @@ -31,27 +31,43 @@ def archive_recorded_meetings(recording_dir) FileUtils.mkdir_p("#{recording_dir}/status/archived") recorded_done_files.each do |recorded_done| - match = /([^\/]*).done$/.match(recorded_done) - meeting_id = match[1] + recorded_done_base = File.basename(recorded_done, '.done') + meeting_id = nil + break_timestamp = nil - if File.mtime(recorded_done) + ARCHIVE_DELAY_SECONDS > Time.now - BigBlueButton.logger.info("Temporarily skipping #{meeting_id} for Red5 race workaround") + if match = /^([0-9a-f]+-[0-9]+)$/.match(recorded_done_base) + meeting_id = match[1] + elsif match = /^([0-9a-f]+-[0-9]+)-([0-9]+)$/.match(recorded_done_base) + meeting_id = match[1] + break_timestamp = match[2] + else + BigBlueButton.logger.warn("Recording done file for #{recorded_done_base} has invalid format") next end - archived_done = "#{recording_dir}/status/archived/#{meeting_id}.done" + if File.mtime(recorded_done) + ARCHIVE_DELAY_SECONDS > Time.now + BigBlueButton.logger.info("Temporarily skipping #{recorded_done_base} for Red5 race workaround") + next + end + + archived_done = "#{recording_dir}/status/archived/#{recorded_done_base}.done" next if File.exists?(archived_done) - archived_norecord = "#{recording_dir}/status/archived/#{meeting_id}.norecord" + archived_norecord = "#{recording_dir}/status/archived/#{recorded_done_base}.norecord" next if File.exists?(archived_norecord) - archived_fail = "#{recording_dir}/status/archived/#{meeting_id}.fail" + archived_fail = "#{recording_dir}/status/archived/#{recorded_done_base}.fail" next if File.exists?(archived_fail) + # TODO: define redis messages for recording segments... BigBlueButton.redis_publisher.put_archive_started(meeting_id) step_start_time = BigBlueButton.monotonic_clock - ret = BigBlueButton.exec_ret("ruby", "archive/archive.rb", "-m", meeting_id) + if !break_timestamp.nil? + ret = BigBlueButton.exec_ret("ruby", "archive/archive.rb", "-m", meeting_id, '-b', break_timestamp) + else + ret = BigBlueButton.exec_ret("ruby", "archive/archive.rb", "-m", meeting_id) + end step_stop_time = BigBlueButton.monotonic_clock step_time = step_stop_time - step_start_time @@ -65,10 +81,10 @@ def archive_recorded_meetings(recording_dir) }) if step_succeeded - BigBlueButton.logger.info("Successfully archived #{meeting_id}") + BigBlueButton.logger.info("Successfully archived #{recorded_done_base}") FileUtils.rm_f(recorded_done) else - BigBlueButton.logger.error("Failed to archive #{meeting_id}") + BigBlueButton.logger.error("Failed to archive #{recorded_done_base}") FileUtils.touch(archived_fail) end end From 0affb6a36ed9f48432ac056678448eabad93806a Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Mon, 30 Oct 2017 15:04:30 -0400 Subject: [PATCH 02/13] Make progressive archiving of media files work. Switch to using rsync to archive video directories, with the -t option so it'll only update modified/new files. --- .../core/lib/recordandplayback.rb | 4 - .../lib/recordandplayback/audio_archiver.rb | 37 ------- .../recordandplayback/deskshare_archiver.rb | 38 -------- .../presentation_archiver.rb | 39 -------- .../lib/recordandplayback/video_archiver.rb | 38 -------- .../core/scripts/archive/archive.rb | 96 +++++++------------ 6 files changed, 33 insertions(+), 219 deletions(-) delete mode 100755 record-and-playback/core/lib/recordandplayback/audio_archiver.rb delete mode 100755 record-and-playback/core/lib/recordandplayback/deskshare_archiver.rb delete mode 100755 record-and-playback/core/lib/recordandplayback/presentation_archiver.rb delete mode 100755 record-and-playback/core/lib/recordandplayback/video_archiver.rb diff --git a/record-and-playback/core/lib/recordandplayback.rb b/record-and-playback/core/lib/recordandplayback.rb index a00fa8a46c..3b4a46ce80 100755 --- a/record-and-playback/core/lib/recordandplayback.rb +++ b/record-and-playback/core/lib/recordandplayback.rb @@ -23,11 +23,7 @@ path = File.expand_path(File.join(File.dirname(__FILE__), '../lib')) $LOAD_PATH << path -require 'recordandplayback/audio_archiver' require 'recordandplayback/events_archiver' -require 'recordandplayback/video_archiver' -require 'recordandplayback/presentation_archiver' -require 'recordandplayback/deskshare_archiver' require 'recordandplayback/generators/events' require 'recordandplayback/generators/audio' require 'recordandplayback/generators/video' diff --git a/record-and-playback/core/lib/recordandplayback/audio_archiver.rb b/record-and-playback/core/lib/recordandplayback/audio_archiver.rb deleted file mode 100755 index d60c4e5e42..0000000000 --- a/record-and-playback/core/lib/recordandplayback/audio_archiver.rb +++ /dev/null @@ -1,37 +0,0 @@ -# Set encoding to utf-8 -# encoding: UTF-8 -# -# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ -# -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). -# -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. -# -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with BigBlueButton; if not, see . -# - - -require 'fileutils' - -module BigBlueButton - class AudioArchiver - def self.archive(meeting_id, from_dir, to_dir) - raise MissingDirectoryException, "Directory not found: [#{from_dir}]" if not BigBlueButton.dir_exists?(from_dir) - raise MissingDirectoryException, "Directory not found: [#{to_dir}]" if not BigBlueButton.dir_exists?(to_dir) - raise FileNotFoundException, "No recording for #{meeting_id} in #{from_dir}" if Dir.glob("#{from_dir}/**/#{meeting_id}*.wav").empty? - - Dir.glob("#{from_dir}/**/#{meeting_id}*.wav").each do |file| - BigBlueButton.logger.debug("Copying #{file} to #{to_dir}") - FileUtils.cp(file, to_dir) - end - end - end -end diff --git a/record-and-playback/core/lib/recordandplayback/deskshare_archiver.rb b/record-and-playback/core/lib/recordandplayback/deskshare_archiver.rb deleted file mode 100755 index ee880067ac..0000000000 --- a/record-and-playback/core/lib/recordandplayback/deskshare_archiver.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Set encoding to utf-8 -# encoding: UTF-8 - -# -# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ -# -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). -# -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. -# -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with BigBlueButton; if not, see . -# - - -require 'fileutils' - -module BigBlueButton - class DeskshareArchiver - def self.archive(meeting_id, from_dir, to_dir) - raise MissingDirectoryException, "Directory not found #{from_dir}" if not BigBlueButton.dir_exists?(from_dir) - raise MissingDirectoryException, "Directory not found #{to_dir}" if not BigBlueButton.dir_exists?(to_dir) - raise FileNotFoundException, "No recording for #{meeting_id} in #{from_dir}" if Dir.glob("#{from_dir}").empty? - - Dir.glob("#{from_dir}/#{meeting_id}-*").each { |file| - puts "deskshare #{file} to #{to_dir}" - FileUtils.cp(file, to_dir) - } - end - end -end diff --git a/record-and-playback/core/lib/recordandplayback/presentation_archiver.rb b/record-and-playback/core/lib/recordandplayback/presentation_archiver.rb deleted file mode 100755 index b1e8710bc6..0000000000 --- a/record-and-playback/core/lib/recordandplayback/presentation_archiver.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Set encoding to utf-8 -# encoding: UTF-8 - -# -# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ -# -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). -# -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. -# -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with BigBlueButton; if not, see . -# - - -require 'fileutils' - -module BigBlueButton - class PresentationArchiver - def self.archive(meeting_id, from_dir, to_dir) - raise MissingDirectoryException, "Directory not found #{from_dir}" if not BigBlueButton.dir_exists?(from_dir) - raise MissingDirectoryException, "Directory not found #{to_dir}" if not BigBlueButton.dir_exists?(to_dir) - raise FileNotFoundException, "No presentation for #{meeting_id} in #{from_dir}" if Dir.glob("#{from_dir}").empty? - - Dir.glob("#{from_dir}/*").each { |file| - puts "Presentation from #{file} to #{to_dir}" - FileUtils.cp_r(file, to_dir) - } - end - end -end - \ No newline at end of file diff --git a/record-and-playback/core/lib/recordandplayback/video_archiver.rb b/record-and-playback/core/lib/recordandplayback/video_archiver.rb deleted file mode 100755 index d1e40ae2d4..0000000000 --- a/record-and-playback/core/lib/recordandplayback/video_archiver.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Set encoding to utf-8 -# encoding: UTF-8 - -# -# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ -# -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). -# -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. -# -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with BigBlueButton; if not, see . -# - - -require 'fileutils' - -module BigBlueButton - class VideoArchiver - def self.archive(meeting_id, from_dir, to_dir) - raise MissingDirectoryException, "Directory not found: [#{from_dir}]" if not BigBlueButton.dir_exists?(from_dir) - raise MissingDirectoryException, "Directory not found: [#{to_dir}]" if not BigBlueButton.dir_exists?(to_dir) - raise FileNotFoundException, "Video for #{meeting_id} not found." if Dir.glob("#{from_dir}").empty? - - Dir.glob("#{from_dir}").each do |file| - FileUtils.cp_r(file, to_dir) - end - end - end -end - \ No newline at end of file diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index 31bdd4625d..a815b932b8 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -3,16 +3,17 @@ # # BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ # -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). +# Copyright (c) 2017 BigBlueButton Inc. and by respective authors (see below). # -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. # -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. # # You should have received a copy of the GNU Lesser General Public License along # with BigBlueButton; if not, see . @@ -25,17 +26,6 @@ require 'trollop' require 'yaml' -def archive_audio(meeting_id, audio_dir, raw_archive_dir) - BigBlueButton.logger.info("Archiving audio #{audio_dir}/#{meeting_id}*.wav.") - begin - audio_dest_dir = "#{raw_archive_dir}/#{meeting_id}/audio" - FileUtils.mkdir_p audio_dest_dir - BigBlueButton::AudioArchiver.archive(meeting_id, audio_dir, audio_dest_dir) - rescue => e - BigBlueButton.logger.warn("Failed to archive audio for #{meeting_id}. " + e.to_s) - end -end - def archive_events(meeting_id, redis_host, redis_port, raw_archive_dir) BigBlueButton.logger.info("Archiving events for #{meeting_id}.") begin @@ -48,47 +38,23 @@ def archive_events(meeting_id, redis_host, redis_port, raw_archive_dir) end end -def archive_video(meeting_id, video_dir, raw_archive_dir) - BigBlueButton.logger.info("Archiving video for #{meeting_id}.") - begin - video_dest_dir = "#{raw_archive_dir}/#{meeting_id}/video" - FileUtils.mkdir_p video_dest_dir - BigBlueButton::VideoArchiver.archive(meeting_id, "#{video_dir}/#{meeting_id}", video_dest_dir) - rescue => e - BigBlueButton.logger.warn("Failed to archive video for #{meeting_id}. " + e.to_s) +def archive_audio(meeting_id, audio_dir, raw_archive_dir) + BigBlueButton.logger.info("Archiving audio #{audio_dir}/#{meeting_id}-*.wav") + audio_dest_dir = "#{raw_archive_dir}/#{meeting_id}/audio" + ret = BigBlueButton.exec_ret('rsync', '-rstv', + *Dir.glob("#{audio_dir}/#{meeting_id}-*.wav"), + "#{raw_archive_dir}/#{meeting_id}/audio/") + if ret != 0 + BigBlueButton.logger.warn("Failed to archive audio for #{meeting_id}") end end -def archive_deskshare(meeting_id, deskshare_dir, raw_archive_dir) - BigBlueButton.logger.info("Archiving deskshare for #{meeting_id}.") - begin - deskshare_dest_dir = "#{raw_archive_dir}/#{meeting_id}/deskshare" - FileUtils.mkdir_p deskshare_dest_dir - BigBlueButton::DeskshareArchiver.archive(meeting_id, deskshare_dir, deskshare_dest_dir) - rescue => e - BigBlueButton.logger.warn("Failed to archive deskshare for #{meeting_id}. " + e.to_s) - end -end - -def archive_screenshare(meeting_id, deskshare_dir, raw_archive_dir) - BigBlueButton.logger.info("Archiving screenshare for #{meeting_id}.") - begin - deskshare_dest_dir = "#{raw_archive_dir}/#{meeting_id}/deskshare" - FileUtils.mkdir_p deskshare_dest_dir - BigBlueButton::DeskshareArchiver.archive(meeting_id, "#{deskshare_dir}/#{meeting_id}", deskshare_dest_dir) - rescue => e - BigBlueButton.logger.warn("Failed to archive screenshare for #{meeting_id}. " + e.to_s) - end -end - -def archive_presentation(meeting_id, presentation_dir, raw_archive_dir) - BigBlueButton.logger.info("Archiving presentation for #{meeting_id}.") - begin - presentation_dest_dir = "#{raw_archive_dir}/#{meeting_id}/presentation" - FileUtils.mkdir_p presentation_dest_dir - BigBlueButton::PresentationArchiver.archive(meeting_id, "#{presentation_dir}/#{meeting_id}/#{meeting_id}", presentation_dest_dir) - rescue => e - BigBlueButton.logger.warn("Failed to archive presentations for #{meeting_id}. " + e.to_s) +def archive_directory(source, dest) + BigBlueButton.logger.info("Archiving contents of #{source}") + ret = BigBlueButton.exec_ret('resync', '-rstv', + "#{source}/", "#{dest}/") + if ret != 0 + BigBlueButton.logger.warn("Failed to archive contents of #{source}") end end @@ -109,11 +75,13 @@ end ################## START ################################ opts = Trollop::options do - opt :meeting_id, "Meeting id to archive", :default => '58f4a6b3-cd07-444d-8564-59116cb53974', :type => String + opt :meeting_id, "Meeting id to archive", type: :string + opt :break_timestamp, "Chapter break end timestamp", type: :string end +Trollop::die :meeting_id, "must be provided" if opts[:meeting_id].nil? meeting_id = opts[:meeting_id] - +break_timestamp = opts[:break_timestamp] # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) @@ -136,10 +104,12 @@ if not FileTest.directory?(target_dir) FileUtils.mkdir_p target_dir archive_events(meeting_id, redis_host, redis_port, raw_archive_dir) archive_audio(meeting_id, audio_dir, raw_archive_dir) - archive_presentation(meeting_id, presentation_dir, raw_archive_dir) - archive_deskshare(meeting_id, deskshare_dir, raw_archive_dir) - archive_screenshare(meeting_id, screenshare_dir, raw_archive_dir) - archive_video(meeting_id, video_dir, raw_archive_dir) + archive_directory("#{presentation_dir}/#{meeting_id}/#{meeting_id}", + "#{target_dir}/presentation") + archive_directory("#{screenshare_dir}/#{meeting_id}", + "#{target_dir}/deskshare") + archive_directory("#{video_dir}/#{meeting_id}", + "#{target_dir}/video/#{meeting_id}") if not archive_has_recording_marks?(meeting_id, raw_archive_dir) BigBlueButton.logger.info("There's no recording marks for #{meeting_id}, not processing recording.") From a78997d1a19058c16addcb53826497dc0e061a6f Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Wed, 1 Nov 2017 12:31:07 -0400 Subject: [PATCH 03/13] Work in progress on segment archiving --- .../lib/recordandplayback/events_archiver.rb | 174 +++++++++++++----- .../core/scripts/archive/archive.rb | 2 +- 2 files changed, 130 insertions(+), 46 deletions(-) diff --git a/record-and-playback/core/lib/recordandplayback/events_archiver.rb b/record-and-playback/core/lib/recordandplayback/events_archiver.rb index ac8f0f781e..446dc2b923 100755 --- a/record-and-playback/core/lib/recordandplayback/events_archiver.rb +++ b/record-and-playback/core/lib/recordandplayback/events_archiver.rb @@ -4,16 +4,17 @@ # # BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ # -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). +# Copyright (c) 2017 BigBlueButton Inc. and by respective authors (see below). # -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. # -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. +# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. # # You should have received a copy of the GNU Lesser General Public License along # with BigBlueButton; if not, see . @@ -209,54 +210,137 @@ module BigBlueButton def initialize(redis) @redis = redis end + + def store_events(meeting_id, events_file, break_timestamp) + version = YAML::load(File.open('../../core/scripts/bigbluebutton.yml'))["bbb_version"] + + if File.exist?(events_file) + io = File.open(events_file, 'rb') + events_doc = Nokogiri::XML::Document.parse(events_file) + io.close + recording = events_doc.create_element('recording', + 'meeting_id' => meeting_id, + 'bbb_version' => version) + events_doc.root = recording + else + events_doc = Nokogiri::XML::Document.new() + recording = events_doc.at_xpath('/recording') + meeting = events_doc.at_xpath('/recording/meeting') + metadata = events_doc.at_xpath('/recording/metadata') + breakout = events_doc.at_xpath('/recording/breakout') + breakoutRooms = events_doc.at_xpath('/recording/breakoutRooms') + end + + meeting_metadata = @redis.metadata_for(meeting_id) + return if meeting_metadata.nil? + + # Fill in/update the top-level meeting element + if meeting.nil? + meeting = events_doc.create_element('meeting') + recording << meeting + end + meeting['id'] = meeting_id + meeting['externalId'] = meeting_metadata[MEETINGID] + meeting['name'] = meeting_metadata[MEETINGNAME] + meeting['breakout'] = meeting_metadata[ISBREAKOUT] + + # Fill in/update the top-level metadata element + if metadata.nil? + metadata = events_doc.create_element('metadata') + recording << metadata + end + meeting_metadata.each do |k, v| + metadata[k] = v + end + + # Fill in/update the top-level breakout element + if @redis.has_breakout_metadata_for(meeting_id) + if breakout.nil? + breakout = events_doc.create_element('breakout') + recording << breakout + end + breakout_metadata = @redis.breakout_metadata_for(meeting_id) + breakout_metadata.each do |k, v| + breakout[k] = v + end + end + + # Fill in/update the top-level breakoutRooms list + if @redis.has_breakout_rooms_for(meeting_id) + newBreakoutRooms = events_doc.create_element('breakoutRooms') + breakout_rooms = @redis.breakout_rooms_for(meeting_id) + breakout_rooms.each do |breakout_room| + newBreakoutRooms << events_doc.create_element('breakoutRoom', breakout_room) + end + if !breakoutRooms.nil? + breakoutRooms.replace(newBreakoutRooms) + else + recording << newBreakoutRooms + end + end + + # Append event elements, up until the break_timestamp if provided + msgs = @redis.events_for(meeting_id) + last_index = -1 + msgs.each_with_index do |msg, i| + res = @redis.event_info_for(meeting_id, msg) + event = events_doc.create_element('event', + 'timestamp' => res[TIMESTAMP], + 'module' => res[MODULE], + 'eventname' => res[EVENTNAME]) + res.each do |k, v| + if ![TIMESTAMP, MODULE, EVENTNAME, MEETINGID].include?(k) + if res[MODULE] == 'PRESENTATION' and key == 'slidesInfo' + # The slidesInfo value is XML serialized info, just insert it + # directly into the event + event << v + elsif res[MODULE] == 'CHAT' and res[EVENTNAME] == 'PublicChatEvent' and k == 'message' + # Apply a cleanup that removes certain ranges of special + # characters from chat messages + event << events_doc.create_element(k, v.tr("\u0000-\u001f\u007f\u2028",'')) + else + event << events_doc.create_element(k, v) + end + end + end + + # Stop reading events if we've reached the recording break for this + # segment + if res[MODULE] == 'PARTICIPANT' and res[EVENTNAME] == 'RecordChapterBreakEvent' and res['breakTimestamp'] == break_timestamp + last_index = i + break + end + end + + # And, finally, write the events file. + io = File.open(events_file, 'wb') + io.write(events_doc.to_xml(indent: 2, encoding: 'UTF-8')) + io.close + end + def store_events(meeting_id) Encoding.default_external="UTF-8" xml = Builder::XmlMarkup.new( :indent => 2 ) - result = xml.instruct! :xml, :version => "1.0", :encoding=>"UTF-8" - meeting_metadata = @redis.metadata_for(meeting_id) - version = YAML::load(File.open('../../core/scripts/bigbluebutton.yml'))["bbb_version"] - - if (meeting_metadata != nil) - xml.recording(:meeting_id => meeting_id, :bbb_version => version) { - xml.meeting(:id => meeting_id, :externalId => meeting_metadata[MEETINGID], :name => meeting_metadata[MEETINGNAME], :breakout => meeting_metadata[ISBREAKOUT]) - xml.metadata(meeting_metadata) - - if (@redis.has_breakout_metadata_for(meeting_id)) - breakout_metadata = @redis.breakout_metadata_for(meeting_id) - xml.breakout(breakout_metadata) - end - - if (@redis.has_breakout_rooms_for(meeting_id)) - breakout_rooms = @redis.breakout_rooms_for(meeting_id) - if (breakout_rooms != nil) - xml.breakoutRooms() { - breakout_rooms.each do |breakout_room| - xml.breakoutRoom(breakout_room) - end - } - end - end - msgs = @redis.events_for(meeting_id) msgs.each do |msg| res = @redis.event_info_for(meeting_id, msg) xml.event(:timestamp => res[TIMESTAMP], :module => res[MODULE], :eventname => res[EVENTNAME]) { res.each do |key, val| if not [TIMESTAMP, MODULE, EVENTNAME, MEETINGID].include?(key) - # a temporary solution for enable a good display of the xml in the presentation module and for add CDATA to chat - if res[MODULE] == "PRESENTATION" && key == "slidesInfo" - xml.method_missing(key){ - xml << val - } - elsif res[MODULE] == "CHAT" && res[EVENTNAME] == "PublicChatEvent" && key == "message" - xml.method_missing(key){ - xml.cdata!(val.tr("\u0000-\u001f\u007f\u2028",'')) - } - else - xml.method_missing(key, val) - end + # a temporary solution for enable a good display of the xml in the presentation module and for add CDATA to chat + if res[MODULE] == "PRESENTATION" && key == "slidesInfo" + xml.method_missing(key){ + xml << val + } + elsif res[MODULE] == "CHAT" && res[EVENTNAME] == "PublicChatEvent" && key == "message" + xml.method_missing(key){ + xml.cdata!(val.tr("\u0000-\u001f\u007f\u2028",'')) + } + else + xml.method_missing(key, val) + end end end } diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index a815b932b8..6e1c0b0858 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -27,7 +27,7 @@ require 'yaml' def archive_events(meeting_id, redis_host, redis_port, raw_archive_dir) - BigBlueButton.logger.info("Archiving events for #{meeting_id}.") + BigBlueButton.logger.info("Archiving events for #{meeting_id}") begin redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) events_archiver = BigBlueButton::RedisEventsArchiver.new redis From 2c4cadf459525fd6f1279929d8802aa0765f8cf2 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 3 Nov 2017 10:27:52 -0400 Subject: [PATCH 04/13] Delete event data from redis after archive writes xml Todo: we also need to remove the events from the meeting event list --- .../core/lib/recordandplayback/events_archiver.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/record-and-playback/core/lib/recordandplayback/events_archiver.rb b/record-and-playback/core/lib/recordandplayback/events_archiver.rb index 446dc2b923..b655650a24 100755 --- a/record-and-playback/core/lib/recordandplayback/events_archiver.rb +++ b/record-and-playback/core/lib/recordandplayback/events_archiver.rb @@ -312,10 +312,19 @@ module BigBlueButton end end - # And, finally, write the events file. + # Write the events file. io = File.open(events_file, 'wb') io.write(events_doc.to_xml(indent: 2, encoding: 'UTF-8')) io.close + + # Once the events file has been written, we can delete this segment's + # events from redis. + + msgs.each_with_index do |msg, i| + @redis.delete_event_info_for(meeting_id, msg) + break if i >= 0 and i >= last_index + end + end From f5cd1628e0a8549ea821836cedf212538ada5294 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 3 Nov 2017 11:16:44 -0400 Subject: [PATCH 05/13] Trim events list after archive, make archive fail block future segments --- .../lib/recordandplayback/events_archiver.rb | 16 +++++++++++++++- .../core/scripts/rap-archive-worker.rb | 4 +++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/record-and-playback/core/lib/recordandplayback/events_archiver.rb b/record-and-playback/core/lib/recordandplayback/events_archiver.rb index b655650a24..67852c5050 100755 --- a/record-and-playback/core/lib/recordandplayback/events_archiver.rb +++ b/record-and-playback/core/lib/recordandplayback/events_archiver.rb @@ -87,6 +87,20 @@ module BigBlueButton @redis.del("recording:#{meeting_id}:#{event}") end + # Trim the events list for a meeting to remove the events up to including + # the one at last_index. After this is done, the event at last_index + 1 + # will be the new first event. + def trim_events_for(meeting_id, last_index) + if last_index < 0 + # Interpret negative last_index as meaning delete all events (this + # will happen after the last segment of a multi-segment recording). + delete_events_for(meeting_id) + else + @redis.ltrim("meeting:#{meeting_id}:recordings", + last_index + 1, -1) + end + end + def delete_events_for(meeting_id) @redis.del("meeting:#{meeting_id}:recordings") end @@ -319,7 +333,7 @@ module BigBlueButton # Once the events file has been written, we can delete this segment's # events from redis. - + @redis.trim_events_for(meeting_id, last_index) msgs.each_with_index do |msg, i| @redis.delete_event_info_for(meeting_id, msg) break if i >= 0 and i >= last_index diff --git a/record-and-playback/core/scripts/rap-archive-worker.rb b/record-and-playback/core/scripts/rap-archive-worker.rb index 953acb73b2..e26aa1ceea 100755 --- a/record-and-playback/core/scripts/rap-archive-worker.rb +++ b/record-and-playback/core/scripts/rap-archive-worker.rb @@ -56,7 +56,9 @@ def archive_recorded_meetings(recording_dir) archived_norecord = "#{recording_dir}/status/archived/#{recorded_done_base}.norecord" next if File.exists?(archived_norecord) - archived_fail = "#{recording_dir}/status/archived/#{recorded_done_base}.fail" + # The fail filename doesn't contain the break timestamp, because we need an + # archive failure to block archiving of future segments. + archived_fail = "#{recording_dir}/status/archived/#{meeting_id}.fail" next if File.exists?(archived_fail) # TODO: define redis messages for recording segments... From 037014db94bc2dcff9077767e0267e1fb9349b06 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 3 Nov 2017 11:27:00 -0400 Subject: [PATCH 06/13] Adjust sanity rap worker to support segment breaks --- .../core/scripts/rap-sanity-worker.rb | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/record-and-playback/core/scripts/rap-sanity-worker.rb b/record-and-playback/core/scripts/rap-sanity-worker.rb index 4fe30ebe41..2fd2ac990f 100755 --- a/record-and-playback/core/scripts/rap-sanity-worker.rb +++ b/record-and-playback/core/scripts/rap-sanity-worker.rb @@ -28,19 +28,36 @@ def sanity_archived_meetings(recording_dir) FileUtils.mkdir_p("#{recording_dir}/status/sanity") archived_done_files.each do |archived_done| - match = /([^\/]*).done$/.match(archived_done) - meeting_id = match[1] + archived_done_base = File.basename(archived_done, '.done') + meeting_id = nil + break_timestamp = nil - sanity_done = "#{recording_dir}/status/sanity/#{meeting_id}.done" + if match = /^([0-9a-f]+-[0-9]+)$/.match(archived_done_base) + meeting_id = match[1] + elsif match = /^([0-9a-f]+-[0-9]+)-([0-9]+)$/.match(archived_done_base) + meeting_id = match[1] + break_timestamp = match[2] + else + BigBlueButton.logger.warn("Archive done file for #{archived_done_base} has invalid format") + next + end + + sanity_done = "#{recording_dir}/status/sanity/#{archived_done_base}.done" next if File.exists?(sanity_done) - sanity_fail = "#{recording_dir}/status/sanity/#{meeting_id}.fail" + sanity_fail = "#{recording_dir}/status/sanity/#{archived_done_base}.fail" next if File.exists?(sanity_fail) + # TODO: define redis messages for recording segments... BigBlueButton.redis_publisher.put_sanity_started(meeting_id) step_start_time = BigBlueButton.monotonic_clock - ret = BigBlueButton.exec_ret("ruby", "sanity/sanity.rb", "-m", meeting_id) + if !break_timestamp.nil? + ret = BigBlueButton.exec_ret('ruby', 'sanity/sanity.rb', + '-m' meeting_id, '-b', break_timestamp) + else + ret = BigBlueButton.exec_ret('ruby', 'sanity/sanity.rb', '-m', meeting_id) + end step_stop_time = BigBlueButton.monotonic_clock step_time = step_stop_time - step_start_time From 0e2785205ad632aa31d2958078d1ef792f38fea6 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 3 Nov 2017 12:02:36 -0400 Subject: [PATCH 07/13] Update sanity script for recording segments, fix archive done files --- .../core/scripts/archive/archive.rb | 28 +++++--- .../core/scripts/sanity/sanity.rb | 70 ++++++++++++------- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index 6e1c0b0858..a292dcdbe9 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -83,6 +83,15 @@ Trollop::die :meeting_id, "must be provided" if opts[:meeting_id].nil? meeting_id = opts[:meeting_id] break_timestamp = opts[:break_timestamp] +# Determine the filenames for the done and fail files +if !break_timestamp.nil? + done_base = "#{meeting_id}-#{break_timestamp}" +else + done_base = meeting_id +end +archive_done_file = "#{recording_dir}/status/archived/#{done_base}.done" +archive_norecord_file = "#{recording_dir}/status/archived/#{done_base}.norecord" + # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) @@ -111,22 +120,25 @@ if not FileTest.directory?(target_dir) archive_directory("#{video_dir}/#{meeting_id}", "#{target_dir}/video/#{meeting_id}") + # TODO we need to check for recording marks in the segment being archived if not archive_has_recording_marks?(meeting_id, raw_archive_dir) BigBlueButton.logger.info("There's no recording marks for #{meeting_id}, not processing recording.") - # we need to delete the keys here because the sanity phase won't - # automatically happen for this recording - BigBlueButton.logger.info("Deleting redis keys") - redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) - events_archiver = BigBlueButton::RedisEventsArchiver.new redis - events_archiver.delete_events(meeting_id) + if break_timestamp.nil? + # we need to delete the keys here because the sanity phase might not + # automatically happen for this recording + BigBlueButton.logger.info("Deleting redis keys") + redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) + events_archiver = BigBlueButton::RedisEventsArchiver.new redis + events_archiver.delete_events(meeting_id) + end - File.open("#{recording_dir}/status/archived/#{meeting_id}.norecord", "w") do |archive_norecord| + File.open(archive_norecord_file, "w") do |archive_norecord| archive_norecord.write("Archived #{meeting_id} (no recording marks") end else - File.open("#{recording_dir}/status/archived/#{meeting_id}.done", "w") do |archive_done| + File.open(archive_done_file, "w") do |archive_done| archive_done.write("Archived #{meeting_id}") end end diff --git a/record-and-playback/core/scripts/sanity/sanity.rb b/record-and-playback/core/scripts/sanity/sanity.rb index cf11b5e8aa..abbe4c7c1f 100755 --- a/record-and-playback/core/scripts/sanity/sanity.rb +++ b/record-and-playback/core/scripts/sanity/sanity.rb @@ -130,10 +130,22 @@ end opts = Trollop::options do - opt :meeting_id, "Meeting id to archive", :default => '58f4a6b3-cd07-444d-8564-59116cb53974', :type => String + opt :meeting_id, "Meeting id to archive", type: :string + opt :break_timestamp, "Chapter break end timestamp", type: :string end +Trollop::die :meeting_id, "must be provided" if opts[:meeting_id].nil? meeting_id = opts[:meeting_id] +break_timestamp = opts[:break_timestamp] + +# Determine the filenames for the done and fail files +if !break_timestamp.nil? + done_base = "#{meeting_id}-#{break_timestamp}" +else + done_base = meeting_id +end +sanity_done_file = "#{recording_dir}/status/sanity/#{done_base}.done" +sanity_fail_file = "#{recording_dir}/status/sanity/#{done_base}.fail" # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) @@ -147,31 +159,41 @@ redis_port = props['redis_port'] BigBlueButton.logger = Logger.new("#{log_dir}/sanity.log", 'daily' ) begin - BigBlueButton.logger.info("Starting sanity check for recording #{meeting_id}.") - BigBlueButton.logger.info("Checking events.xml") - check_events_xml(raw_archive_dir,meeting_id) - BigBlueButton.logger.info("Checking audio") - check_audio_files(raw_archive_dir,meeting_id) - BigBlueButton.logger.info("Checking webcam videos") - check_webcam_files(raw_archive_dir,meeting_id) - BigBlueButton.logger.info("Checking deskshare videos") - check_deskshare_files(raw_archive_dir,meeting_id) - #delete keys - BigBlueButton.logger.info("Deleting keys") - redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) - events_archiver = BigBlueButton::RedisEventsArchiver.new redis - events_archiver.delete_events(meeting_id) + BigBlueButton.logger.info("Starting sanity check for recording #{meeting_id}") + if !break_timestamp.nil? + BigBlueButton.logger.info("Break timestamp is #{break_timestamp}") + end - #create done files for sanity - BigBlueButton.logger.info("creating sanity done files") - sanity_done = File.new("#{recording_dir}/status/sanity/#{meeting_id}.done", "w") - sanity_done.write("sanity check #{meeting_id}") - sanity_done.close + BigBlueButton.logger.info("Checking events.xml") + check_events_xml(raw_archive_dir,meeting_id) + + BigBlueButton.logger.info("Checking audio") + check_audio_files(raw_archive_dir,meeting_id) + + BigBlueButton.logger.info("Checking webcam videos") + check_webcam_files(raw_archive_dir,meeting_id) + + BigBlueButton.logger.info("Checking deskshare videos") + check_deskshare_files(raw_archive_dir,meeting_id) + + if break_timestamp.nil? + # Either this recording isn't segmented, or we are working on the last + # segment, so go ahead and clean up all the redis data. + BigBlueButton.logger.info("Deleting keys") + redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) + events_archiver = BigBlueButton::RedisEventsArchiver.new(redis) + events_archiver.delete_events(meeting_id) + end + + BigBlueButton.logger.info("creating sanity done files") + File.open(sanity_done_file, "w") do |sanity_done| + sanity_done.write("sanity check #{meeting_id}") + end rescue Exception => e - BigBlueButton.logger.error("error in sanity check: " + e.message) - sanity_done = File.new("#{recording_dir}/status/sanity/#{meeting_id}.fail", "w") - sanity_done.write("error: " + e.message) - sanity_done.close + BigBlueButton.logger.error("error in sanity check: " + e.message) + File.open(sanity_fail_file, "w") do |sanity_fail| + sanity_fail.write("error: " + e.message) + end end From f9c4843f284616ca69cb3eed4669d4cc5b5df942 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 3 Nov 2017 15:17:04 -0400 Subject: [PATCH 08/13] Cleanups, initial work on checking for segment recorded status. --- .../core/lib/recordandplayback/generators/events.rb | 10 +++++----- record-and-playback/core/scripts/archive/archive.rb | 10 +++++++--- .../presentation/scripts/publish/presentation.rb | 5 +++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb index 2100984d0f..743b24ad17 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/events.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb @@ -343,11 +343,11 @@ module BigBlueButton end def self.get_start_stop_events_for_edl(archive_dir) + doc = Nokogiri::XML(File.open("#{archive_dir}/events.xml")) initial_timestamp = BigBlueButton::Events.first_event_timestamp( "#{archive_dir}/events.xml") start_stop_events = BigBlueButton::Events.match_start_and_stop_rec_events( - BigBlueButton::Events.get_start_and_stop_rec_events( - "#{archive_dir}/events.xml")) + BigBlueButton::Events.get_start_and_stop_rec_events(doc)) start_stop_events.each do |record_event| record_event[:start_timestamp] -= initial_timestamp record_event[:stop_timestamp] -= initial_timestamp @@ -439,9 +439,8 @@ module BigBlueButton def self.get_record_status_events(events_xml) BigBlueButton.logger.info "Getting record status events" - doc = Nokogiri::XML(File.open(events_xml)) rec_events = [] - doc.xpath("//event[@eventname='RecordStatusEvent']").each do |event| + events_xml.xpath("//event[@eventname='RecordStatusEvent']").each do |event| s = { :timestamp => event['timestamp'].to_i } rec_events << s end @@ -481,8 +480,9 @@ module BigBlueButton # Calculate the length of the final recording from the start/stop events def self.get_recording_length(rec_events) duration = 0 + doc = Nokogiri::XML(File.open(rec_events)) start_stop_events = BigBlueButton::Events.match_start_and_stop_rec_events( - BigBlueButton::Events.get_start_and_stop_rec_events(rec_events)) + BigBlueButton::Events.get_start_and_stop_rec_events(doc)) start_stop_events.each do |start_stop| duration += start_stop[:stop_timestamp] - start_stop[:start_timestamp] end diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index a292dcdbe9..6f382e23ea 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -58,11 +58,15 @@ def archive_directory(source, dest) end end -def archive_has_recording_marks?(meeting_id, raw_archive_dir) +def archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) + doc = Nokogiri::XML(File.open("#{raw_archive_dir}/#{meeting_id}/events.xml")) + if !break_timestamp.nil? + # Locate the start time for this recording segment + end BigBlueButton.logger.info("Fetching the recording marks for #{meeting_id}.") has_recording_marks = true begin - record_events = BigBlueButton::Events.get_record_status_events("#{raw_archive_dir}/#{meeting_id}/events.xml") + record_events = BigBlueButton::Events.get_record_status_events(doc) BigBlueButton.logger.info("record_events:\n#{BigBlueButton.hash_to_str(record_events)}") has_recording_marks = (not record_events.empty?) rescue => e @@ -121,7 +125,7 @@ if not FileTest.directory?(target_dir) "#{target_dir}/video/#{meeting_id}") # TODO we need to check for recording marks in the segment being archived - if not archive_has_recording_marks?(meeting_id, raw_archive_dir) + if not archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) BigBlueButton.logger.info("There's no recording marks for #{meeting_id}, not processing recording.") if break_timestamp.nil? diff --git a/record-and-playback/presentation/scripts/publish/presentation.rb b/record-and-playback/presentation/scripts/publish/presentation.rb index 7f15539460..b5564387cc 100755 --- a/record-and-playback/presentation/scripts/publish/presentation.rb +++ b/record-and-playback/presentation/scripts/publish/presentation.rb @@ -1240,14 +1240,15 @@ begin processing_time = File.read("#{$process_dir}/processing_time") + @doc = Nokogiri::XML(File.open("#{$process_dir}/events.xml")) + # Retrieve record events and calculate total recording duration. $rec_events = BigBlueButton::Events.match_start_and_stop_rec_events( - BigBlueButton::Events.get_start_and_stop_rec_events("#{$process_dir}/events.xml")) + BigBlueButton::Events.get_start_and_stop_rec_events(@doc)) recording_time = BigBlueButton::Events.get_recording_length("#{$process_dir}/events.xml") # presentation_url = "/slides/" + $meeting_id + "/presentation" - @doc = Nokogiri::XML(File.open("#{$process_dir}/events.xml")) $meeting_start = @doc.xpath("//event")[0][:timestamp] $meeting_end = @doc.xpath("//event").last()[:timestamp] From 5bb5cbcd0e6c83326b8c4ac874efa94536574d34 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Fri, 3 Nov 2017 16:12:05 -0400 Subject: [PATCH 09/13] Have archive script check whether a particular segment is recorded It previously checked whether any part of the entire meeting was recorded. Helper functions are added to look up the time of segment start and end (which handle non-segmented recordings correctly too). Part of the events handling code was rewritten to reduce the number of times that the events.xml file gets parsed. --- .../recordandplayback/generators/events.rb | 56 ++++++++++++++++--- .../core/scripts/archive/archive.rb | 42 +++++++++----- .../recordandplayback/audio_processor_spec.rb | 38 ------------- 3 files changed, 75 insertions(+), 61 deletions(-) delete mode 100755 record-and-playback/core/spec/recordandplayback/audio_processor_spec.rb diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb index 743b24ad17..3046386c6e 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/events.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb @@ -72,16 +72,14 @@ module BigBlueButton # Get the timestamp of the first event. def self.first_event_timestamp(events_xml) - BigBlueButton.logger.info("Task: Getting the timestamp of the first event.") - doc = Nokogiri::XML(File.open(events_xml)) - doc.xpath("recording/event").first["timestamp"].to_i + first_event = events_xml.at_xpath('/recording/event[position() = 1]') + first_event['timestamp'].to_i end # Get the timestamp of the last event. def self.last_event_timestamp(events_xml) - BigBlueButton.logger.info("Task: Getting the timestamp of the last event") - doc = Nokogiri::XML(File.open(events_xml)) - doc.xpath("recording/event").last["timestamp"].to_i + last_event = events_xml.at_xpath('/recording/event[position() = last()]') + last_event['timestamp'].to_i end # Determine if the start and stop event matched. @@ -189,7 +187,8 @@ module BigBlueButton end def self.get_matched_start_and_stop_deskshare_events(events_path) - last_timestamp = BigBlueButton::Events.last_event_timestamp(events_path) + doc = Nokogiri::XML(File.open(events_path)) + last_timestamp = BigBlueButton::Events.last_event_timestamp(doc) deskshare_start_events = BigBlueButton::Events.get_start_deskshare_events(events_path) deskshare_stop_events = BigBlueButton::Events.get_stop_deskshare_events(events_path) return BigBlueButton::Events.match_start_and_stop_deskshare_events( @@ -344,8 +343,7 @@ module BigBlueButton def self.get_start_stop_events_for_edl(archive_dir) doc = Nokogiri::XML(File.open("#{archive_dir}/events.xml")) - initial_timestamp = BigBlueButton::Events.first_event_timestamp( - "#{archive_dir}/events.xml") + initial_timestamp = BigBlueButton::Events.first_event_timestamp(doc) start_stop_events = BigBlueButton::Events.match_start_and_stop_rec_events( BigBlueButton::Events.get_start_and_stop_rec_events(doc)) start_stop_events.each do |record_event| @@ -543,6 +541,46 @@ module BigBlueButton return false end + # Get the start timestamp for a recording segment with a given break + # timestamp (end of segment timestamp). Pass nil to get the start timestamp + # of the last segment in a recording. + def self.get_segment_start_timestamp(events_xml, break_timestamp) + chapter_breaks = events_xml.xpath('/recording/event[@module="PARTICIPANT" and @eventname="RecordChapterBreakEvent"]') + + # Locate the chapter break event for the end of this segment + segment_i = chapter_breaks.length + chapter_breaks.each_with_index do |event, i| + timestamp = event.at_xpath('timestamp').text.to_i + if timestamp == break_timestamp + segment_i = i + break + end + end + + if segment_i > 0 + # Get the timestamp of the previous chapter break event + event = chapter_breaks[segment_i - 1] + return event.at_xpath('timestamp').text.to_i + else + # This is the first (or only) segment, so return the timestamp of + # recording start (first event) + return BigBlueButton::Events.first_event_timestamp(events_xml) + end + end + + # Get the end timestamp for a recording segment with a given break + # timestamp. + # In most cases, the break timestamp *is* the recording segment end, but + # for the last segment (which has no break timestamp), we return the + # recording end timestamp (last event) instead. + def self.get_segment_end_timestamp(events_xml, break_timestamp) + if !break_timestamp.nil? + return break_timestamp + else + return BigBlueButton::Events.last_event_timestamp(events_xml) + end + end + # Version of the bbb server where it was recorded def self.bbb_version(events_xml) events = Nokogiri::XML(File.open(events_xml)) diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index 6f382e23ea..7b6ac3858d 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -60,18 +60,33 @@ end def archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) doc = Nokogiri::XML(File.open("#{raw_archive_dir}/#{meeting_id}/events.xml")) - if !break_timestamp.nil? - # Locate the start time for this recording segment - end - BigBlueButton.logger.info("Fetching the recording marks for #{meeting_id}.") - has_recording_marks = true - begin - record_events = BigBlueButton::Events.get_record_status_events(doc) - BigBlueButton.logger.info("record_events:\n#{BigBlueButton.hash_to_str(record_events)}") - has_recording_marks = (not record_events.empty?) - rescue => e - BigBlueButton.logger.warn("Failed to fetch the recording marks for #{meeting_id}. " + e.to_s) + + # Find the start and stop timestamps for the current recording segment + start_timestamp = BigBlueButton::Events.get_segment_start_timestamp( + doc, break_timestamp) + end_timestamp = BigBlueButton::Events.get_segment_end_timestamp( + doc, break_timestamp) + BigBlueButton.logger.info("Segment start: #{start_timestamp}, end: #{end_timestamp}") + + BigBlueButton.logger.info("Checking for recording marks for #{meeting_id} segment #{break_timestamp}") + rec_events = BigBlueButton::Events.match_start_and_stop_rec_events( + BigBlueButton::Events.get_start_and_stop_rec_events(doc)) + has_recording_marks = false + # Scan for a set of recording start/stop events which fits any of these cases: + # - Recording started during segment + # - Recording stopped during segment + # - Recording started before segment and stopped after segment + rec_events.each do |rec_event| + if (rec_event[:start_timestamp] > start_timestamp + and rec_event[:start_timestamp] < end_timestamp) + or (rec_event[:end_timestamp] > start_timestamp + and rec_event[:end_timestamp] < end_timestamp) + or (rec_event[:start_timestamp] <= start_timestamp + and rec_event[:end_timestamp] >= end_timestamp) + has_recording_marks = true + end end + BigBlueButton.logger.info("Recording marks found: #{has_recording_marks}") has_recording_marks end @@ -80,7 +95,7 @@ end opts = Trollop::options do opt :meeting_id, "Meeting id to archive", type: :string - opt :break_timestamp, "Chapter break end timestamp", type: :string + opt :break_timestamp, "Chapter break end timestamp", type: :integer end Trollop::die :meeting_id, "must be provided" if opts[:meeting_id].nil? @@ -124,7 +139,6 @@ if not FileTest.directory?(target_dir) archive_directory("#{video_dir}/#{meeting_id}", "#{target_dir}/video/#{meeting_id}") - # TODO we need to check for recording marks in the segment being archived if not archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) BigBlueButton.logger.info("There's no recording marks for #{meeting_id}, not processing recording.") @@ -133,7 +147,7 @@ if not FileTest.directory?(target_dir) # automatically happen for this recording BigBlueButton.logger.info("Deleting redis keys") redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) - events_archiver = BigBlueButton::RedisEventsArchiver.new redis + events_archiver = BigBlueButton::RedisEventsArchiver.new(redis) events_archiver.delete_events(meeting_id) end diff --git a/record-and-playback/core/spec/recordandplayback/audio_processor_spec.rb b/record-and-playback/core/spec/recordandplayback/audio_processor_spec.rb deleted file mode 100755 index 49e6b19c82..0000000000 --- a/record-and-playback/core/spec/recordandplayback/audio_processor_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/ -# -# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below). -# -# This program is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free Software -# Foundation; either version 3.0 of the License, or (at your option) any later -# version. -# -# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with BigBlueButton; if not, see . -# - -require 'spec_helper' -require 'digest/md5' - -module BigBlueButton - describe AudioEvents do - context "#success" do - - it "should find the timestamp of the first event" do - events_xml = 'resources/raw/good_audio_events.xml' - BigBlueButton::Events.first_event_timestamp(events_xml).should == 50 - end - - it "should find the timestamp of the last event" do - events_xml = 'resources/raw/good_audio_events.xml' - BigBlueButton::Events.last_event_timestamp(events_xml).should == 1000 - end - - end - end -end From 9add60d81bb61468919a1471df8496341140b996 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Mon, 6 Nov 2017 15:22:43 -0500 Subject: [PATCH 10/13] Fix a bunch of bugs found when testing the archive code --- .../lib/recordandplayback/events_archiver.rb | 57 ++-------- .../recordandplayback/generators/events.rb | 4 +- .../core/scripts/archive/archive.rb | 103 +++++++++--------- 3 files changed, 66 insertions(+), 98 deletions(-) diff --git a/record-and-playback/core/lib/recordandplayback/events_archiver.rb b/record-and-playback/core/lib/recordandplayback/events_archiver.rb index 67852c5050..6bebb7a4e1 100755 --- a/record-and-playback/core/lib/recordandplayback/events_archiver.rb +++ b/record-and-playback/core/lib/recordandplayback/events_archiver.rb @@ -230,19 +230,22 @@ module BigBlueButton if File.exist?(events_file) io = File.open(events_file, 'rb') - events_doc = Nokogiri::XML::Document.parse(events_file) + events_doc = Nokogiri::XML::Document.parse(io) io.close - recording = events_doc.create_element('recording', - 'meeting_id' => meeting_id, - 'bbb_version' => version) - events_doc.root = recording - else - events_doc = Nokogiri::XML::Document.new() recording = events_doc.at_xpath('/recording') + if recording.nil? + raise "recording is nil" + end meeting = events_doc.at_xpath('/recording/meeting') metadata = events_doc.at_xpath('/recording/metadata') breakout = events_doc.at_xpath('/recording/breakout') breakoutRooms = events_doc.at_xpath('/recording/breakoutRooms') + else + events_doc = Nokogiri::XML::Document.new() + recording = events_doc.create_element('recording', + 'meeting_id' => meeting_id, + 'bbb_version' => version) + events_doc.root = recording end meeting_metadata = @redis.metadata_for(meeting_id) @@ -304,7 +307,7 @@ module BigBlueButton 'eventname' => res[EVENTNAME]) res.each do |k, v| if ![TIMESTAMP, MODULE, EVENTNAME, MEETINGID].include?(k) - if res[MODULE] == 'PRESENTATION' and key == 'slidesInfo' + if res[MODULE] == 'PRESENTATION' and k == 'slidesInfo' # The slidesInfo value is XML serialized info, just insert it # directly into the event event << v @@ -317,6 +320,7 @@ module BigBlueButton end end end + recording << event # Stop reading events if we've reached the recording break for this # segment @@ -341,38 +345,6 @@ module BigBlueButton end - - def store_events(meeting_id) - Encoding.default_external="UTF-8" - xml = Builder::XmlMarkup.new( :indent => 2 ) - - msgs = @redis.events_for(meeting_id) - msgs.each do |msg| - res = @redis.event_info_for(meeting_id, msg) - xml.event(:timestamp => res[TIMESTAMP], :module => res[MODULE], :eventname => res[EVENTNAME]) { - res.each do |key, val| - if not [TIMESTAMP, MODULE, EVENTNAME, MEETINGID].include?(key) - # a temporary solution for enable a good display of the xml in the presentation module and for add CDATA to chat - if res[MODULE] == "PRESENTATION" && key == "slidesInfo" - xml.method_missing(key){ - xml << val - } - elsif res[MODULE] == "CHAT" && res[EVENTNAME] == "PublicChatEvent" && key == "message" - xml.method_missing(key){ - xml.cdata!(val.tr("\u0000-\u001f\u007f\u2028",'')) - } - else - xml.method_missing(key, val) - end - end - end - } - end - } - end - xml.target! - end - def delete_events(meeting_id) meeting_metadata = @redis.metadata_for(meeting_id) if (meeting_metadata != nil) @@ -387,10 +359,5 @@ module BigBlueButton @redis.delete_breakout_rooms_for(meeting_id) end - def save_events_to_file(directory, result) - File.open("#{directory}/events.xml", 'w') do |f2| - f2.puts result - end - end end end diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb index 3046386c6e..b49a280257 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/events.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb @@ -446,10 +446,10 @@ module BigBlueButton end # Get events when the moderator wants the recording to start or stop - def self.get_start_and_stop_rec_events(events_xml) + def self.get_start_and_stop_rec_events(events_xml, allow_empty_events=false) BigBlueButton.logger.info "Getting start and stop rec button events" rec_events = BigBlueButton::Events.get_record_status_events(events_xml) - if rec_events.empty? + if !allow_empty_events and rec_events.empty? # old recording generated in a version without the record button rec_events << { :timestamp => BigBlueButton::Events.first_event_timestamp(events_xml) } end diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index 7b6ac3858d..d359237956 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -26,21 +26,23 @@ require 'trollop' require 'yaml' -def archive_events(meeting_id, redis_host, redis_port, raw_archive_dir) +def archive_events(meeting_id, redis_host, redis_port, raw_archive_dir, break_timestamp) BigBlueButton.logger.info("Archiving events for #{meeting_id}") - begin + #begin redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) events_archiver = BigBlueButton::RedisEventsArchiver.new redis - events = events_archiver.store_events(meeting_id) - events_archiver.save_events_to_file("#{raw_archive_dir}/#{meeting_id}", events ) - rescue => e - BigBlueButton.logger.warn("Failed to archive events for #{meeting_id}. " + e.to_s) - end + events = events_archiver.store_events(meeting_id, + "#{raw_archive_dir}/#{meeting_id}/events.xml", + break_timestamp) + #rescue => e + # BigBlueButton.logger.warn("Failed to archive events for #{meeting_id}. " + e.to_s) + #end end def archive_audio(meeting_id, audio_dir, raw_archive_dir) BigBlueButton.logger.info("Archiving audio #{audio_dir}/#{meeting_id}-*.wav") audio_dest_dir = "#{raw_archive_dir}/#{meeting_id}/audio" + FileUtils.mkdir_p(audio_dest_dir) ret = BigBlueButton.exec_ret('rsync', '-rstv', *Dir.glob("#{audio_dir}/#{meeting_id}-*.wav"), "#{raw_archive_dir}/#{meeting_id}/audio/") @@ -51,7 +53,8 @@ end def archive_directory(source, dest) BigBlueButton.logger.info("Archiving contents of #{source}") - ret = BigBlueButton.exec_ret('resync', '-rstv', + FileUtils.mkdir_p(dest) + ret = BigBlueButton.exec_ret('rsync', '-rstv', "#{source}/", "#{dest}/") if ret != 0 BigBlueButton.logger.warn("Failed to archive contents of #{source}") @@ -70,19 +73,19 @@ def archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) BigBlueButton.logger.info("Checking for recording marks for #{meeting_id} segment #{break_timestamp}") rec_events = BigBlueButton::Events.match_start_and_stop_rec_events( - BigBlueButton::Events.get_start_and_stop_rec_events(doc)) + BigBlueButton::Events.get_start_and_stop_rec_events(doc, true)) has_recording_marks = false # Scan for a set of recording start/stop events which fits any of these cases: # - Recording started during segment # - Recording stopped during segment # - Recording started before segment and stopped after segment rec_events.each do |rec_event| - if (rec_event[:start_timestamp] > start_timestamp - and rec_event[:start_timestamp] < end_timestamp) - or (rec_event[:end_timestamp] > start_timestamp - and rec_event[:end_timestamp] < end_timestamp) - or (rec_event[:start_timestamp] <= start_timestamp - and rec_event[:end_timestamp] >= end_timestamp) + if (rec_event[:start_timestamp] > start_timestamp and + rec_event[:start_timestamp] < end_timestamp) or + (rec_event[:stop_timestamp] > start_timestamp and + rec_event[:stop_timestamp] < end_timestamp) or + (rec_event[:start_timestamp] <= start_timestamp and + rec_event[:stop_timestamp] >= end_timestamp) has_recording_marks = true end end @@ -102,15 +105,6 @@ Trollop::die :meeting_id, "must be provided" if opts[:meeting_id].nil? meeting_id = opts[:meeting_id] break_timestamp = opts[:break_timestamp] -# Determine the filenames for the done and fail files -if !break_timestamp.nil? - done_base = "#{meeting_id}-#{break_timestamp}" -else - done_base = meeting_id -end -archive_done_file = "#{recording_dir}/status/archived/#{done_base}.done" -archive_norecord_file = "#{recording_dir}/status/archived/#{done_base}.norecord" - # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) @@ -125,39 +119,46 @@ presentation_dir = props['raw_presentation_src'] video_dir = props['raw_video_src'] log_dir = props['log_dir'] +# Determine the filenames for the done and fail files +if !break_timestamp.nil? + done_base = "#{meeting_id}-#{break_timestamp}" +else + done_base = meeting_id +end +archive_done_file = "#{recording_dir}/status/archived/#{done_base}.done" +archive_norecord_file = "#{recording_dir}/status/archived/#{done_base}.norecord" + BigBlueButton.logger = Logger.new("#{log_dir}/archive-#{meeting_id}.log", 'daily' ) target_dir = "#{raw_archive_dir}/#{meeting_id}" -if not FileTest.directory?(target_dir) - FileUtils.mkdir_p target_dir - archive_events(meeting_id, redis_host, redis_port, raw_archive_dir) - archive_audio(meeting_id, audio_dir, raw_archive_dir) - archive_directory("#{presentation_dir}/#{meeting_id}/#{meeting_id}", - "#{target_dir}/presentation") - archive_directory("#{screenshare_dir}/#{meeting_id}", - "#{target_dir}/deskshare") - archive_directory("#{video_dir}/#{meeting_id}", - "#{target_dir}/video/#{meeting_id}") +FileUtils.mkdir_p target_dir +archive_events(meeting_id, redis_host, redis_port, raw_archive_dir, break_timestamp) +archive_audio(meeting_id, audio_dir, raw_archive_dir) +archive_directory("#{presentation_dir}/#{meeting_id}/#{meeting_id}", + "#{target_dir}/presentation") +archive_directory("#{screenshare_dir}/#{meeting_id}", + "#{target_dir}/deskshare") +archive_directory("#{video_dir}/#{meeting_id}", + "#{target_dir}/video/#{meeting_id}") - if not archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) - BigBlueButton.logger.info("There's no recording marks for #{meeting_id}, not processing recording.") +if not archive_has_recording_marks?(meeting_id, raw_archive_dir, break_timestamp) + BigBlueButton.logger.info("There's no recording marks for #{meeting_id}, not processing recording.") - if break_timestamp.nil? - # we need to delete the keys here because the sanity phase might not - # automatically happen for this recording - BigBlueButton.logger.info("Deleting redis keys") - redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) - events_archiver = BigBlueButton::RedisEventsArchiver.new(redis) - events_archiver.delete_events(meeting_id) - end + if break_timestamp.nil? + # we need to delete the keys here because the sanity phase might not + # automatically happen for this recording + BigBlueButton.logger.info("Deleting redis keys") + redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) + events_archiver = BigBlueButton::RedisEventsArchiver.new(redis) + events_archiver.delete_events(meeting_id) + end - File.open(archive_norecord_file, "w") do |archive_norecord| - archive_norecord.write("Archived #{meeting_id} (no recording marks") - end + File.open(archive_norecord_file, "w") do |archive_norecord| + archive_norecord.write("Archived #{meeting_id} (no recording marks") + end - else - File.open(archive_done_file, "w") do |archive_done| - archive_done.write("Archived #{meeting_id}") - end +else + File.open(archive_done_file, "w") do |archive_done| + archive_done.write("Archived #{meeting_id}") end end From 35f7f425e288d2962c6b305282a7edcdd7dcf4aa Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Tue, 7 Nov 2017 09:52:50 -0500 Subject: [PATCH 11/13] Fix break detection in events --- .../core/lib/recordandplayback/events_archiver.rb | 2 ++ .../core/lib/recordandplayback/generators/events.rb | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/record-and-playback/core/lib/recordandplayback/events_archiver.rb b/record-and-playback/core/lib/recordandplayback/events_archiver.rb index 6bebb7a4e1..c2d0020e42 100755 --- a/record-and-playback/core/lib/recordandplayback/events_archiver.rb +++ b/record-and-playback/core/lib/recordandplayback/events_archiver.rb @@ -297,6 +297,8 @@ module BigBlueButton end # Append event elements, up until the break_timestamp if provided + # TODO: check if the break we're looking for is already in the events + # file msgs = @redis.events_for(meeting_id) last_index = -1 msgs.each_with_index do |msg, i| diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb index b49a280257..2a8d7bb52c 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/events.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb @@ -550,7 +550,7 @@ module BigBlueButton # Locate the chapter break event for the end of this segment segment_i = chapter_breaks.length chapter_breaks.each_with_index do |event, i| - timestamp = event.at_xpath('timestamp').text.to_i + timestamp = event.at_xpath('breakTimestamp').text.to_i if timestamp == break_timestamp segment_i = i break @@ -560,7 +560,7 @@ module BigBlueButton if segment_i > 0 # Get the timestamp of the previous chapter break event event = chapter_breaks[segment_i - 1] - return event.at_xpath('timestamp').text.to_i + return event.at_xpath('breakTimestamp').text.to_i else # This is the first (or only) segment, so return the timestamp of # recording start (first event) From 50f2c1b90b529694470f8e99881cd20eb04b3517 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Tue, 7 Nov 2017 09:53:08 -0500 Subject: [PATCH 12/13] Fix syntax issues in sanity scripts --- .../core/scripts/rap-sanity-worker.rb | 2 +- .../core/scripts/sanity/sanity.rb | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/record-and-playback/core/scripts/rap-sanity-worker.rb b/record-and-playback/core/scripts/rap-sanity-worker.rb index 2fd2ac990f..4a67e0388e 100755 --- a/record-and-playback/core/scripts/rap-sanity-worker.rb +++ b/record-and-playback/core/scripts/rap-sanity-worker.rb @@ -54,7 +54,7 @@ def sanity_archived_meetings(recording_dir) step_start_time = BigBlueButton.monotonic_clock if !break_timestamp.nil? ret = BigBlueButton.exec_ret('ruby', 'sanity/sanity.rb', - '-m' meeting_id, '-b', break_timestamp) + '-m', meeting_id, '-b', break_timestamp) else ret = BigBlueButton.exec_ret('ruby', 'sanity/sanity.rb', '-m', meeting_id) end diff --git a/record-and-playback/core/scripts/sanity/sanity.rb b/record-and-playback/core/scripts/sanity/sanity.rb index abbe4c7c1f..3269617812 100755 --- a/record-and-playback/core/scripts/sanity/sanity.rb +++ b/record-and-playback/core/scripts/sanity/sanity.rb @@ -138,15 +138,6 @@ Trollop::die :meeting_id, "must be provided" if opts[:meeting_id].nil? meeting_id = opts[:meeting_id] break_timestamp = opts[:break_timestamp] -# Determine the filenames for the done and fail files -if !break_timestamp.nil? - done_base = "#{meeting_id}-#{break_timestamp}" -else - done_base = meeting_id -end -sanity_done_file = "#{recording_dir}/status/sanity/#{done_base}.done" -sanity_fail_file = "#{recording_dir}/status/sanity/#{done_base}.fail" - # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) log_dir = props['log_dir'] @@ -156,6 +147,15 @@ raw_archive_dir = "#{recording_dir}/raw" redis_host = props['redis_host'] redis_port = props['redis_port'] +# Determine the filenames for the done and fail files +if !break_timestamp.nil? + done_base = "#{meeting_id}-#{break_timestamp}" +else + done_base = meeting_id +end +sanity_done_file = "#{recording_dir}/status/sanity/#{done_base}.done" +sanity_fail_file = "#{recording_dir}/status/sanity/#{done_base}.fail" + BigBlueButton.logger = Logger.new("#{log_dir}/sanity.log", 'daily' ) begin From 4e243536c7b6ea2b51a9f6acfd03d04b174218f5 Mon Sep 17 00:00:00 2001 From: Calvin Walton Date: Tue, 7 Nov 2017 10:03:20 -0500 Subject: [PATCH 13/13] Handle the no audio files case a bit better If no audio files were found, it was running rsync with one argument, which is a bit unexpected. It just printed a file list in this case, but we can provide a cleaner error message instead. --- record-and-playback/core/scripts/archive/archive.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index d359237956..ff041e7c6b 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -43,8 +43,12 @@ def archive_audio(meeting_id, audio_dir, raw_archive_dir) BigBlueButton.logger.info("Archiving audio #{audio_dir}/#{meeting_id}-*.wav") audio_dest_dir = "#{raw_archive_dir}/#{meeting_id}/audio" FileUtils.mkdir_p(audio_dest_dir) - ret = BigBlueButton.exec_ret('rsync', '-rstv', - *Dir.glob("#{audio_dir}/#{meeting_id}-*.wav"), + audio_files = Dir.glob("#{audio_dir}/#{meeting_id}-*.wav") + if audio_files.empty? + BigBlueButton.logger.warn("No audio found for #{meeting_id}") + return + end + ret = BigBlueButton.exec_ret('rsync', '-rstv', *audio_files, "#{raw_archive_dir}/#{meeting_id}/audio/") if ret != 0 BigBlueButton.logger.warn("Failed to archive audio for #{meeting_id}")