From 460dfb52bf31d9c4124f8130c0d66fbe0b0d9ee0 Mon Sep 17 00:00:00 2001 From: Richard Alam Date: Thu, 3 Apr 2014 20:23:49 +0000 Subject: [PATCH 1/3] - modify scripts so we can run it manually --- .../lib/recordandplayback/generators/audio.rb | 2 +- record-and-playback/core/scripts/README | 77 +++++++++++++++++++ .../core/scripts/archive/archive.rb | 3 +- .../core/scripts/bigbluebutton.yml | 17 +++- .../core/scripts/rap-worker.rb | 11 ++- .../core/scripts/sanity/sanity.rb | 14 ++-- .../presentation/scripts/presentation.yml | 9 ++- .../scripts/process/presentation.rb | 12 ++- .../scripts/publish/presentation.rb | 9 ++- 9 files changed, 132 insertions(+), 22 deletions(-) create mode 100755 record-and-playback/core/scripts/README mode change 100644 => 100755 record-and-playback/presentation/scripts/presentation.yml mode change 100644 => 100755 record-and-playback/presentation/scripts/publish/presentation.rb diff --git a/record-and-playback/core/lib/recordandplayback/generators/audio.rb b/record-and-playback/core/lib/recordandplayback/generators/audio.rb index 0b066bc819..48bcff35de 100755 --- a/record-and-playback/core/lib/recordandplayback/generators/audio.rb +++ b/record-and-playback/core/lib/recordandplayback/generators/audio.rb @@ -98,7 +98,7 @@ module BigBlueButton # Volume adjustment: 1.215 command = "sox #{file} -n stat 2>&1" # Try "sox --i -D file" as it is much shorter - BigBlueButton.logger.info("Task: Getting length of audio") + BigBlueButton.logger.info("Task: Getting length of audio [#{command}]") output = BigBlueButton.execute(command).output if output.to_s =~ /Length(.+)/ stats = $1 diff --git a/record-and-playback/core/scripts/README b/record-and-playback/core/scripts/README new file mode 100755 index 0000000000..df8adf4e6c --- /dev/null +++ b/record-and-playback/core/scripts/README @@ -0,0 +1,77 @@ +README + +This instructions below are for testing by running scripts manually: + +1. Create some temp scratch dirs: + mkdir -p ~/temp/log/presentation ~/temp/recording/{process,publish,raw} ~/temp/recording/status/{recorded,archived,processed,sanity} ~/temp/published + +2. Edit core/scripts/bigbluebutton.yml and comment out the PRODUCTION dirs while uncommenting the DEVELOPMENT dir. The dir should match what you created above. + +raw_audio_src: /var/freeswitch/meetings +raw_video_src: /usr/share/red5/webapps/video/streams +raw_deskshare_src: /var/bigbluebutton/deskshare +raw_presentation_src: /var/bigbluebutton +redis_host: 127.0.0.1 +redis_port: 6379 + + +# For PRODUCTION +log_dir: /var/log/bigbluebutton +recording_dir: /var/bigbluebutton/recording +published_dir: /var/bigbluebutton/published +playback_host: 10.0.3.203 + +# For DEVELOPMENT +# This allows us to run the scripts manually +#scripts_dir: /home/ubuntu/dev/bigbluebutton/record-and-playback/core/scripts +#log_dir: /home/ubuntu/temp/log +#recording_dir: /home/ubuntu/temp/recording +#published_dir: /home/ubuntu/temp/published +#playback_host: 192.168.22.137 + +3. Create a recording using BigBlueButton. After logging out, it should have created a .done file in + /var/bigbluebutton/recording/status/recorded dir. Make note of this meeting-id as we use that to tell the script + which recording to process. + +4. Before running the scripts, we have to make sure our scripts have the PATHs setup correctly. + Edit presentation/scripts/process/presentation.rb and uncomment the DEVELOPMENT PATH while + commenting the PRODUCTION PATH. We need to do this so the script will be able to find the + core library. + +5. Now we run the archive step. Go to record-and-playback/core/scripts dir and type + ruby archive/archive.rb -m + +6. If everything goes well, you should have the raw files in ~/temp/recording/raw/ + You can also check the logs at ~/temp/log/archive-.log + + You should also have an entry in ~/temp/recording/status/archived dir + +7. Then we need to do a sanity check if the raw recordings are complete. Type + ruby sanity/sanity.rb -m + + Check the log in ~/temp/log/sanity.log + + You should also have an entry in ~/temp/recording/status/sanity dir + +8. Assuming the recording passed the sanity check, it's time to process the recording. + cd record-and-playback/presentation/scripts + ruby process/presentation.rb -m + + You can monitor the progress by tailing the log at ~/temp/log/presentation/process-.log + +9. Assuming that everything goes well. We can now run the publish script. However, we need to cheat a little bit. + The publish script will be looking for a "processing_time" file which contains information on how long the + processing took. Unfortunately, that file is created by the rap-worker.rb script which we don't run. + + So we create that file manually at + vi ~/temp/recording/process/presentation//processing_time + + Enter any number (e.g. 46843) and save the file. + +10. Now run the publish script + ruby publish/presentation.rb -m -presentation + + Notice we appended "presentation" to the meeting-id, this will tell the script to publish using the "presentation" format. + + + diff --git a/record-and-playback/core/scripts/archive/archive.rb b/record-and-playback/core/scripts/archive/archive.rb index 8e059d1b1c..939b8c02cf 100755 --- a/record-and-playback/core/scripts/archive/archive.rb +++ b/record-and-playback/core/scripts/archive/archive.rb @@ -103,7 +103,6 @@ end meeting_id = opts[:meeting_id] -BigBlueButton.logger = Logger.new("/var/log/bigbluebutton/archive-#{meeting_id}.log", 'daily' ) # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) @@ -116,7 +115,9 @@ redis_host = props['redis_host'] redis_port = props['redis_port'] presentation_dir = props['raw_presentation_src'] video_dir = props['raw_video_src'] +log_dir = props['log_dir'] +BigBlueButton.logger = Logger.new("#{log_dir}/archive-#{meeting_id}.log", 'daily' ) target_dir = "#{raw_archive_dir}/#{meeting_id}" if not FileTest.directory?(target_dir) diff --git a/record-and-playback/core/scripts/bigbluebutton.yml b/record-and-playback/core/scripts/bigbluebutton.yml index 1881997586..5e65a3488e 100755 --- a/record-and-playback/core/scripts/bigbluebutton.yml +++ b/record-and-playback/core/scripts/bigbluebutton.yml @@ -1,11 +1,22 @@ -scripts_dir: /home/firstuser/dev/source/bigbluebutton/record-and-playback/rap/scripts -recording_dir: /var/bigbluebutton/recording -published_dir: /var/bigbluebutton/published + raw_audio_src: /var/freeswitch/meetings raw_video_src: /usr/share/red5/webapps/video/streams raw_deskshare_src: /var/bigbluebutton/deskshare raw_presentation_src: /var/bigbluebutton redis_host: 127.0.0.1 redis_port: 6379 + + +# For PRODUCTION +log_dir: /var/log/bigbluebutton +recording_dir: /var/bigbluebutton/recording +published_dir: /var/bigbluebutton/published playback_host: 10.0.3.203 +# For DEVELOPMENT +# This allows us to run the scripts manually +#scripts_dir: /home/ubuntu/dev/bigbluebutton/record-and-playback/core/scripts +#log_dir: /home/ubuntu/temp/log +#recording_dir: /home/ubuntu/temp/recording +#published_dir: /home/ubuntu/temp/published +#playback_host: 192.168.22.137 diff --git a/record-and-playback/core/scripts/rap-worker.rb b/record-and-playback/core/scripts/rap-worker.rb index a5996ff148..b0ba28641a 100755 --- a/record-and-playback/core/scripts/rap-worker.rb +++ b/record-and-playback/core/scripts/rap-worker.rb @@ -25,10 +25,6 @@ require 'rubygems' require 'yaml' require 'fileutils' -logger = Logger.new("/var/log/bigbluebutton/bbb-rap-worker.log",'daily' ) -logger.level = Logger::ERROR -BigBlueButton.logger = logger - def archive_recorded_meeting(recording_dir) recorded_done_files = Dir.glob("#{recording_dir}/status/recorded/*.done") archived_dirs = Dir.entries("#{recording_dir}/raw/") - ['.','..'] @@ -147,7 +143,14 @@ def publish_processed_meeting(recording_dir) end props = YAML::load(File.open('bigbluebutton.yml')) + +log_dir = props['log_dir'] recording_dir = props['recording_dir'] + +logger = Logger.new("#{log_dir}/bbb-rap-worker.log",'daily' ) +logger.level = Logger::ERROR +BigBlueButton.logger = logger + archive_recorded_meeting(recording_dir) sanity_archived_meeting(recording_dir) process_archived_meeting(recording_dir) diff --git a/record-and-playback/core/scripts/sanity/sanity.rb b/record-and-playback/core/scripts/sanity/sanity.rb index 71e0c847f6..729d6b6a19 100755 --- a/record-and-playback/core/scripts/sanity/sanity.rb +++ b/record-and-playback/core/scripts/sanity/sanity.rb @@ -94,7 +94,6 @@ def check_deskshare_files(raw_dir, meeting_id) end end -BigBlueButton.logger = Logger.new('/var/log/bigbluebutton/sanity.log', 'daily' ) opts = Trollop::options do opt :meeting_id, "Meeting id to archive", :default => '58f4a6b3-cd07-444d-8564-59116cb53974', :type => String @@ -104,28 +103,29 @@ meeting_id = opts[:meeting_id] # This script lives in scripts/archive/steps while bigbluebutton.yml lives in scripts/ props = YAML::load(File.open('bigbluebutton.yml')) - +log_dir = props['log_dir'] audio_dir = props['raw_audio_src'] recording_dir = props['recording_dir'] raw_archive_dir = "#{recording_dir}/raw" redis_host = props['redis_host'] redis_port = props['redis_port'] +BigBlueButton.logger = Logger.new("#{log_dir}/sanity.log", 'daily' ) begin BigBlueButton.logger.info("Starting sanity check for recording #{meeting_id}.") BigBlueButton.logger.info("Checking events.xml") check_events_xml(raw_archive_dir,meeting_id) BigBlueButton.logger.info("Checking audio") check_audio_files(raw_archive_dir,meeting_id) - BigBlueButton.logger.info("Checking webcam videos") - check_webcam_files(raw_archive_dir,meeting_id) - BigBlueButton.logger.info("Checking deskshare videos") - check_deskshare_files(raw_archive_dir,meeting_id) + BigBlueButton.logger.info("Checking webcam videos") + check_webcam_files(raw_archive_dir,meeting_id) + BigBlueButton.logger.info("Checking deskshare videos") + check_deskshare_files(raw_archive_dir,meeting_id) #delete keys BigBlueButton.logger.info("Deleting keys") redis = BigBlueButton::RedisWrapper.new(redis_host, redis_port) events_archiver = BigBlueButton::RedisEventsArchiver.new redis - events_archiver.delete_events(meeting_id) + events_archiver.delete_events(meeting_id) #create done files for sanity BigBlueButton.logger.info("creating sanity done files") diff --git a/record-and-playback/presentation/scripts/presentation.yml b/record-and-playback/presentation/scripts/presentation.yml old mode 100644 new mode 100755 index ac3bafd5bb..a96fa19eb8 --- a/record-and-playback/presentation/scripts/presentation.yml +++ b/record-and-playback/presentation/scripts/presentation.yml @@ -1,5 +1,5 @@ -publish_dir: /var/bigbluebutton/published/presentation + video_output_width: 640 video_output_height: 480 # Alternate output size to use when deskshare videos are present @@ -10,3 +10,10 @@ deskshare_output_height: 720 # audio_offset = 1200 means that the audio will be delayed by 1200ms audio_offset: 0 include_deskshare: true + +# For PRODUCTION +publish_dir: /var/bigbluebutton/published/presentation + +# For DEVELOPMENT +#publish_dir: /home/ubuntu/temp/published/presentation + diff --git a/record-and-playback/presentation/scripts/process/presentation.rb b/record-and-playback/presentation/scripts/process/presentation.rb index cc2c7e1f09..ed81c3f7e4 100755 --- a/record-and-playback/presentation/scripts/process/presentation.rb +++ b/record-and-playback/presentation/scripts/process/presentation.rb @@ -19,7 +19,13 @@ # with BigBlueButton; if not, see . # +# For DEVELOPMENT +# Allows us to run the script manually +# require File.expand_path('../../../../core/lib/recordandplayback', __FILE__) + +# For PRODUCTION require File.expand_path('../../../lib/recordandplayback', __FILE__) + require 'rubygems' require 'trollop' require 'yaml' @@ -38,13 +44,13 @@ presentation_props['include_deskshare'] = false if presentation_props['include_d recording_dir = props['recording_dir'] raw_archive_dir = "#{recording_dir}/raw/#{meeting_id}" - +log_dir = props['log_dir'] target_dir = "#{recording_dir}/process/presentation/#{meeting_id}" if not FileTest.directory?(target_dir) - FileUtils.mkdir_p "/var/log/bigbluebutton/presentation" - logger = Logger.new("/var/log/bigbluebutton/presentation/process-#{meeting_id}.log", 'daily' ) + FileUtils.mkdir_p "#{log_dir}/presentation" + logger = Logger.new("#{log_dir}/presentation/process-#{meeting_id}.log", 'daily' ) BigBlueButton.logger = logger BigBlueButton.logger.info("Processing script presentation.rb") FileUtils.mkdir_p target_dir diff --git a/record-and-playback/presentation/scripts/publish/presentation.rb b/record-and-playback/presentation/scripts/publish/presentation.rb old mode 100644 new mode 100755 index 2fd2c5f923..bcd2b3ad11 --- a/record-and-playback/presentation/scripts/publish/presentation.rb +++ b/record-and-playback/presentation/scripts/publish/presentation.rb @@ -826,11 +826,16 @@ $playback = match[2] puts $meeting_id puts $playback if ($playback == "presentation") - logger = Logger.new("/var/log/bigbluebutton/presentation/publish-#{$meeting_id}.log", 'daily' ) - BigBlueButton.logger = logger + # This script lives in scripts/archive/steps while properties.yaml lives in scripts/ bbb_props = YAML::load(File.open('../../core/scripts/bigbluebutton.yml')) simple_props = YAML::load(File.open('presentation.yml')) + + log_dir = bbb_props['log_dir'] + + logger = Logger.new("#{log_dir}/presentation/publish-#{$meeting_id}.log", 'daily' ) + BigBlueButton.logger = logger + BigBlueButton.logger.info("Setting recording dir") recording_dir = bbb_props['recording_dir'] BigBlueButton.logger.info("Setting process dir") From f8d8079f594fa919214153c98d2928654ead3bcf Mon Sep 17 00:00:00 2001 From: Richard Alam Date: Fri, 4 Apr 2014 14:51:14 +0000 Subject: [PATCH 2/3] - make sure to record the EndAndKickAllEvent --- .../service/recorder/RecorderApplication.java | 16 ++++++++-------- .../bigbluebutton/core/BigBlueButtonActor.scala | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/bigbluebutton-apps/src/main/java/org/bigbluebutton/conference/service/recorder/RecorderApplication.java b/bigbluebutton-apps/src/main/java/org/bigbluebutton/conference/service/recorder/RecorderApplication.java index c3c1890849..7e25d1d70b 100755 --- a/bigbluebutton-apps/src/main/java/org/bigbluebutton/conference/service/recorder/RecorderApplication.java +++ b/bigbluebutton-apps/src/main/java/org/bigbluebutton/conference/service/recorder/RecorderApplication.java @@ -42,12 +42,12 @@ public class RecorderApplication { private BlockingQueue messages; private volatile boolean recordEvents = false; - private final Map recordingSessions; +// private final Map recordingSessions; private Recorder recorder; public RecorderApplication() { messages = new LinkedBlockingQueue(); - recordingSessions = new ConcurrentHashMap(); +// recordingSessions = new ConcurrentHashMap(); } public void start() { @@ -75,25 +75,25 @@ public class RecorderApplication { } public void destroyRecordSession(String meetingID) { - recordingSessions.remove(meetingID); +// recordingSessions.remove(meetingID); } public void createRecordSession(String meetingID) { - recordingSessions.put(meetingID, meetingID); +// recordingSessions.put(meetingID, meetingID); } public void record(String meetingID, RecordEvent message) { messages.offer(message); - if (recordingSessions.containsKey(meetingID)) { +// if (recordingSessions.containsKey(meetingID)) { recorder.record(meetingID, message); - } +// } } private void recordEvent(RecordEvent message) { - if (recordingSessions.containsKey(message.getMeetingID())) { +// if (recordingSessions.containsKey(message.getMeetingID())) { recorder.record(message.getMeetingID(), message); - } +// } } public void setRecorder(Recorder recorder) { diff --git a/bigbluebutton-apps/src/main/scala/org/bigbluebutton/core/BigBlueButtonActor.scala b/bigbluebutton-apps/src/main/scala/org/bigbluebutton/core/BigBlueButtonActor.scala index 545840340f..bcb0dc0ef2 100755 --- a/bigbluebutton-apps/src/main/scala/org/bigbluebutton/core/BigBlueButtonActor.scala +++ b/bigbluebutton-apps/src/main/scala/org/bigbluebutton/core/BigBlueButtonActor.scala @@ -46,7 +46,10 @@ class BigBlueButtonActor(outGW: MessageOutGateway) extends Actor { case None => println("Could not find meeting id[" + msg.meetingID + "] to destroy.") case Some(m) => { m ! StopMeetingActor - meetings -= msg.meetingID + meetings -= msg.meetingID + println("Kinc everyone out on meeting id[" + msg.meetingID + "].") + outGW.send(new EndAndKickAll(msg.meetingID, m.recorded)) + println("Destroyed meeting id[" + msg.meetingID + "].") outGW.send(new MeetingDestroyed(msg.meetingID)) } From bafedc0c07f4f60b4aae888120a0ffef75de69b7 Mon Sep 17 00:00:00 2001 From: Richard Alam Date: Fri, 4 Apr 2014 15:28:30 +0000 Subject: [PATCH 3/3] - enable recording when there is only 1 user in fs --- .../config/freeswitch/conf/autoload_configs/conference.conf.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/bbb-voice-conference/config/freeswitch/conf/autoload_configs/conference.conf.xml b/bbb-voice-conference/config/freeswitch/conf/autoload_configs/conference.conf.xml index 64be8e1f0f..e7e84135cc 100644 --- a/bbb-voice-conference/config/freeswitch/conf/autoload_configs/conference.conf.xml +++ b/bbb-voice-conference/config/freeswitch/conf/autoload_configs/conference.conf.xml @@ -123,6 +123,7 @@ +