Merge pull request #428 from mconf/rap-publish-events

Rap publish events
This commit is contained in:
Calvin Walton 2014-11-14 12:08:44 -02:00
commit 213dc07c19
11 changed files with 269 additions and 256 deletions

View File

@ -19,12 +19,9 @@
source "http://rubygems.org"
gem "rspec", "2.0.0", :require => "spec"
gem "cucumber", "0.9.2"
gem "redis", "2.1.1"
#gem "SystemTimer", "1.2.3"
gem "nokogiri", "1.4.4"
gem "resque", "1.15.0"
gem "redis"
gem "nokogiri"
gem "resque"
gem "mime-types"
gem "streamio-ffmpeg"
gem "rubyzip"
@ -32,3 +29,5 @@ gem "curb"
gem "builder"
gem "trollop"
gem "open4"
gem "fastimage"
gem "absolute_time"

View File

@ -1,65 +1,53 @@
GEM
remote: http://rubygems.org/
specs:
SystemTimer (1.2.3)
builder (2.1.2)
cucumber (0.9.2)
builder (~> 2.1.2)
diff-lcs (~> 1.1.2)
gherkin (~> 2.2.5)
json (~> 1.4.6)
term-ansicolor (~> 1.0.5)
curb (0.7.15)
diff-lcs (1.1.2)
gherkin (2.2.9)
json (~> 1.4.6)
term-ansicolor (~> 1.0.5)
json (1.4.6)
mime-types (1.16)
nokogiri (1.4.4)
rack (1.2.2)
redis (2.1.1)
redis-namespace (0.10.0)
redis (< 3.0.0)
resque (1.15.0)
json (~> 1.4.6)
redis-namespace (>= 0.10.0)
addressable (2.3.6)
builder (3.2.2)
curb (0.8.6)
fastimage (1.6.4)
addressable (~> 2.3, >= 2.3.5)
mime-types (2.4.3)
mini_portile (0.6.1)
mono_logger (1.1.0)
multi_json (1.10.1)
nokogiri (1.6.4.1)
mini_portile (~> 0.6.0)
open4 (1.3.4)
rack (1.5.2)
rack-protection (1.5.3)
rack
redis (3.1.0)
redis-namespace (1.5.1)
redis (~> 3.0, >= 3.0.4)
resque (1.25.2)
mono_logger (~> 1.0)
multi_json (~> 1.0)
redis-namespace (~> 1.3)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
rspec (2.0.0)
rspec-core (= 2.0.0)
rspec-expectations (= 2.0.0)
rspec-mocks (= 2.0.0)
rspec-core (2.0.0)
rspec-expectations (2.0.0)
diff-lcs (>= 1.1.2)
rspec-mocks (2.0.0)
rspec-core (= 2.0.0)
rspec-expectations (= 2.0.0)
rubyzip (0.9.4)
sinatra (1.2.1)
rack (~> 1.1)
tilt (< 2.0, >= 1.2.2)
streamio-ffmpeg (0.7.8)
term-ansicolor (1.0.5)
tilt (1.2.2)
trollop (1.16.2)
vegas (0.1.8)
rubyzip (1.1.6)
sinatra (1.4.5)
rack (~> 1.4)
rack-protection (~> 1.4)
tilt (~> 1.3, >= 1.3.4)
streamio-ffmpeg (1.0.0)
tilt (1.4.1)
trollop (2.0)
vegas (0.1.11)
rack (>= 1.0.0)
PLATFORMS
ruby
DEPENDENCIES
SystemTimer (= 1.2.3)
builder
cucumber (= 0.9.2)
curb
fastimage
mime-types
nokogiri (= 1.4.4)
redis (= 2.1.1)
resque (= 1.15.0)
rspec (= 2.0.0)
nokogiri
open4
redis
resque
rubyzip
streamio-ffmpeg
trollop

View File

@ -1,15 +0,0 @@
This directory will contain god scripts to watch and manage BigBlueButton processes.
To deploy
1. Copy god dir to /etc/bigbluebutton
cp -R god /etc/bigbluebutton/
2. Copy initd.god as to /etc/init.d/god
sudo cp initd.god /etc/init.d/god
chmod +x /etc/init.d/god
sudo /usr/sbin/update-rc.d -f god defaults
3. Start god
sudo /etc/init.d/god start

View File

@ -1 +0,0 @@
Location for config files that will be managed by god.

View File

@ -1,56 +0,0 @@
#
# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/
#
# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below).
#
# This program is free software; you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free Software
# Foundation; either version 3.0 of the License, or (at your option) any later
# version.
#
# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with BigBlueButton; if not, see <http://www.gnu.org/licenses/>.
#
# NOTE:
# Copy into /etc/bigbluebutton/god/conf
# sudo cp rap-god-conf.rb /etc/bigbluebutton/god/conf/rap-conf.rb
#
# Monitors the BigBlueButton archive, ingesting, processing, publishing process
God.watch do |w|
# The name of the watcher
w.name = "bbb-rap-proc"
# The default time for reporting the state of the monitored process
w.interval = 1.minute
# Start the process
w.start = "sudo -u tomcat6 ruby rap-worker.rb"
# Start your process in this directory
w.dir = "/usr/local/bigbluebutton/core/scripts/"
# Time to wait before monitoring, after starting the process
w.start_grace = 30.seconds
# Cleans the pid file before starting the process.
# god will daemonizes the process
w.behavior(:clean_pid_file)
# Start the process if it is not running
# And report its status every 30 seconds
# In other words god revives the process every time it dies
w.start_if do |start|
start.condition(:process_running) do |c|
c.interval = 30.seconds
c.running = false
end
end
end

View File

@ -1,22 +0,0 @@
#
# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/
#
# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below).
#
# This program is free software; you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free Software
# Foundation; either version 3.0 of the License, or (at your option) any later
# version.
#
# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with BigBlueButton; if not, see <http://www.gnu.org/licenses/>.
#
# load in all god configs
God.load "/etc/bigbluebutton/god/conf/*.rb"

View File

@ -1,67 +0,0 @@
#!/bin/bash
#
# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/
#
# Copyright (c) 2012 BigBlueButton Inc. and by respective authors (see below).
#
# This program is free software; you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free Software
# Foundation; either version 3.0 of the License, or (at your option) any later
# version.
#
# BigBlueButton is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with BigBlueButton; if not, see <http://www.gnu.org/licenses/>.
#
#
# Record and Playback God init.d script
# http://god.rubyforge.org/
#
### BEGIN INIT INFO
# Provides: bbb-record-core
# Required-Start: $syslog
# Required-Stop: $syslog
# Should-Start: $local_fs
# Should-Stop: $local_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: bbb-record-core - BigBlueButton Record and Playback core
# Description: bbb-record-core - BigBlueButton Record and Playback core
### END INIT INFO
set -e
RETVAL=0
case "$1" in
start)
god -c /etc/bigbluebutton/god/god.rb -P /var/run/god.pid -l /var/log/god.log
RETVAL=$?
echo "God started"
;;
stop)
kill `cat /var/run/god.pid`
RETVAL=$?
echo "God stopped"
;;
restart)
kill `cat /var/run/god.pid`
god -c /etc/bigbluebutton/god/god.rb -P /var/run/god.pid -l /var/log/god.log
RETVAL=$?
echo "God restarted"
;;
status)
RETVAL=$?
;;
*)
echo "Usage: god {start|stop|restart|status}"
exit 1
;;
esac
exit $RETVAL

View File

@ -36,6 +36,7 @@ require 'recordandplayback/generators/audio_processor'
require 'recordandplayback/generators/presentation'
require 'open4'
require 'pp'
require 'absolute_time'
module BigBlueButton
class MissingDirectoryException < RuntimeError
@ -86,6 +87,14 @@ module BigBlueButton
logger.level = Logger::INFO
@logger = logger
end
def self.redis_publisher=(publisher)
@redis_publisher = publisher
end
def self.redis_publisher
return @redis_publisher
end
def self.dir_exists?(dir)
FileTest.directory?(dir)
@ -142,4 +151,8 @@ module BigBlueButton
def self.hash_to_str(hash)
return PP.pp(hash, "")
end
def self.monotonic_clock()
return (AbsoluteTime.now * 1000).to_i
end
end

View File

@ -72,7 +72,94 @@ module BigBlueButton
def delete_metadata_for(meeting_id)
@redis.del("meeting:info:#{meeting_id}")
end
end
def build_header(message_type)
return {
"timestamp" => BigBlueButton.monotonic_clock, #
"name" => message_type,
"current_time" => Time.now.to_i, # unix timestamp
"version" => "0.0.1"
}
end
def build_message(header, payload)
return {
"header" => header,
"payload" => payload
}
end
RECORDINGS_CHANNEL = "bigbluebutton:from-rap"
def put_message(message_type, meeting_id, additional_payload = {})
msg = build_message build_header(message_type), additional_payload.merge({
"meeting_id" => meeting_id
})
@redis.publish RECORDINGS_CHANNEL, msg.to_json
end
def put_message_workflow(message_type, workflow, meeting_id, additional_payload = {})
put_message message_type, meeting_id, additional_payload.merge({
"workflow" => workflow
})
end
def put_archive_started(meeting_id, additional_payload = {})
put_message "archive_started", meeting_id, additional_payload
end
def put_archive_ended(meeting_id, additional_payload = {})
put_message "archive_ended", meeting_id, additional_payload
end
def put_sanity_started(meeting_id, additional_payload = {})
put_message "sanity_started", meeting_id, additional_payload
end
def put_sanity_ended(meeting_id, additional_payload = {})
put_message "sanity_ended", meeting_id, additional_payload
end
def put_process_started(workflow, meeting_id, additional_payload = {})
put_message_workflow "process_started", workflow, meeting_id, additional_payload
end
def put_process_ended(workflow, meeting_id, additional_payload = {})
put_message_workflow "process_ended", workflow, meeting_id, additional_payload
end
def put_publish_started(workflow, meeting_id, additional_payload = {})
put_message_workflow "publish_started", workflow, meeting_id, additional_payload
end
def put_publish_ended(workflow, meeting_id, additional_payload = {})
put_message_workflow "publish_ended", workflow, meeting_id, additional_payload
end
def put_post_archive_started(workflow, meeting_id, additional_payload = {})
put_message_workflow "post_archive_started", workflow, meeting_id, additional_payload
end
def put_post_archive_ended(workflow, meeting_id, additional_payload = {})
put_message_workflow "post_archive_ended", workflow, meeting_id, additional_payload
end
def put_post_process_started(workflow, meeting_id, additional_payload = {})
put_message_workflow "post_process_started", workflow, meeting_id, additional_payload
end
def put_post_process_ended(workflow, meeting_id, additional_payload = {})
put_message_workflow "post_process_ended", workflow, meeting_id, additional_payload
end
def put_post_publish_started(workflow, meeting_id, additional_payload = {})
put_message_workflow "post_publish_started", workflow, meeting_id, additional_payload
end
def put_post_publish_ended(workflow, meeting_id, additional_payload = {})
put_message_workflow "post_publish_ended", workflow, meeting_id, additional_payload
end
end
class RedisEventsArchiver

View File

@ -48,9 +48,21 @@ def archive_recorded_meeting(recording_dir)
archived_fail = "#{recording_dir}/status/archived/#{meeting_id}.fail"
next if File.exists?(archived_fail)
ret = BigBlueButton.exec_ret("ruby", "archive/archive.rb", "-m", meeting_id)
BigBlueButton.redis_publisher.put_archive_started meeting_id
if ret == 0 && File.exists?(archived_done)
step_start_time = BigBlueButton.monotonic_clock
ret = BigBlueButton.exec_ret("ruby", "archive/archive.rb", "-m", meeting_id)
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
step_succeeded = (ret == 0 && File.exists?(archived_done))
BigBlueButton.redis_publisher.put_archive_ended meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if step_succeeded
BigBlueButton.logger.info("Successfully archived #{meeting_id}")
FileUtils.rm(recorded_done)
else
@ -74,9 +86,21 @@ def sanity_archived_meeting(recording_dir)
sanity_fail = "#{recording_dir}/status/sanity/#{meeting_id}.fail"
next if File.exists?(sanity_fail)
ret = BigBlueButton.exec_ret("ruby", "sanity/sanity.rb", "-m", meeting_id)
BigBlueButton.redis_publisher.put_sanity_started meeting_id
if ret == 0 && File.exists?(sanity_done)
step_start_time = BigBlueButton.monotonic_clock
ret = BigBlueButton.exec_ret("ruby", "sanity/sanity.rb", "-m", meeting_id)
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
step_succeeded = (ret == 0 && File.exists?(sanity_done))
BigBlueButton.redis_publisher.put_sanity_ended meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if step_succeeded
BigBlueButton.logger.info("Successfully sanity checked #{meeting_id}")
post_archive(meeting_id)
FileUtils.rm(archived_done)
@ -96,7 +120,7 @@ def process_archived_meeting(recording_dir)
match = /([^\/]*).done$/.match(sanity_done)
meeting_id = match[1]
process_succeeded = true
step_succeeded = true
# Iterate over the list of recording processing scripts to find available types
# For now, we look for the ".rb" extension - TODO other scripting languages?
@ -109,31 +133,41 @@ def process_archived_meeting(recording_dir)
processed_fail = "#{recording_dir}/status/processed/#{meeting_id}-#{process_type}.fail"
if File.exists?(processed_fail)
process_succeeded = false
step_succeeded = false
next
end
BigBlueButton.redis_publisher.put_process_started process_type, meeting_id
# If the process directory doesn't exist, the script does nothing
FileUtils.rm_rf("#{recording_dir}/process/#{process_type}/#{meeting_id}")
process_start = Time.now
step_start_time = BigBlueButton.monotonic_clock
ret = BigBlueButton.exec_ret("ruby", process_script, "-m", meeting_id)
process_stop = Time.now
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
process_time = ((process_stop - process_start) * 1000).to_i
IO.write("#{recording_dir}/process/#{process_type}/#{meeting_id}/processing_time", process_time)
if ret == 0 and File.exists?(processed_done)
IO.write("#{recording_dir}/process/#{process_type}/#{meeting_id}/processing_time", step_time)
step_succeeded = (ret == 0 and File.exists?(processed_done))
BigBlueButton.redis_publisher.put_process_ended process_type, meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if step_succeeded
BigBlueButton.logger.info("Process format #{process_type} succeeded for #{meeting_id}")
BigBlueButton.logger.info("Process took #{process_time}ms")
BigBlueButton.logger.info("Process took #{step_time}ms")
else
BigBlueButton.logger.info("Process format #{process_type} failed for #{meeting_id}")
BigBlueButton.logger.info("Process took #{process_time}ms")
BigBlueButton.logger.info("Process took #{step_time}ms")
FileUtils.touch(processed_fail)
process_succeeded = false
step_succeeded = false
end
end
if process_succeeded
if step_succeeded
post_process(meeting_id)
FileUtils.rm(sanity_done)
end
@ -171,14 +205,26 @@ def publish_processed_meeting(recording_dir)
next
end
BigBlueButton.redis_publisher.put_publish_started publish_type, meeting_id
# If the publish directory doesn't exist, the script does nothing
FileUtils.rm_rf("#{recording_dir}/publish/#{publish_type}/#{meeting_id}")
step_start_time = BigBlueButton.monotonic_clock
# For legacy reasons, the meeting ID passed to the publish script contains
# the playback format name.
ret = BigBlueButton.exec_ret("ruby", publish_script, "-m", "#{meeting_id}-#{publish_type}")
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
if ret == 0 and File.exists?(published_done)
step_succeeded = (ret == 0 and File.exists?(published_done))
BigBlueButton.redis_publisher.put_publish_ended publish_type, meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if step_succeeded
BigBlueButton.logger.info("Publish format #{publish_type} succeeded for #{meeting_id}")
else
BigBlueButton.logger.info("Publish format #{publish_type} failed for #{meeting_id}")
@ -210,9 +256,24 @@ end
def post_archive(meeting_id)
Dir.glob("post_archive/*.rb").sort.each do |post_archive_script|
BigBlueButton.logger.info("Running post archive script #{post_archive_script}")
match = /([^\/]*).rb$/.match(post_archive_script)
post_type = match[1]
BigBlueButton.logger.info("Running post archive script #{post_type}")
BigBlueButton.redis_publisher.put_post_archive_started post_type, meeting_id
step_start_time = BigBlueButton.monotonic_clock
ret = BigBlueButton.exec_ret("ruby", post_archive_script, "-m", meeting_id)
if ret != 0
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
step_succeeded = (ret == 0)
BigBlueButton.redis_publisher.put_post_archive_ended post_type, meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if not step_succeeded
BigBlueButton.logger.warn("Post archive script #{post_archive_script} failed")
end
end
@ -220,9 +281,24 @@ end
def post_process(meeting_id)
Dir.glob("post_process/*.rb").sort.each do |post_process_script|
BigBlueButton.logger.info("Running post process script #{post_process_script}")
match = /([^\/]*).rb$/.match(post_process_script)
post_type = match[1]
BigBlueButton.logger.info("Running post process script #{post_type}")
BigBlueButton.redis_publisher.put_post_process_started post_type, meeting_id
step_start_time = BigBlueButton.monotonic_clock
ret = BigBlueButton.exec_ret("ruby", post_process_script, "-m", meeting_id)
if ret != 0
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
step_succeeded = (ret == 0)
BigBlueButton.redis_publisher.put_post_process_ended post_type, meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if not step_succeeded
BigBlueButton.logger.warn("Post process script #{post_process_script} failed")
end
end
@ -230,9 +306,24 @@ end
def post_publish(meeting_id)
Dir.glob("post_publish/*.rb").sort.each do |post_publish_script|
BigBlueButton.logger.info("Running post publish script #{post_publish_script}")
match = /([^\/]*).rb$/.match(post_publish_script)
post_type = match[1]
BigBlueButton.logger.info("Running post publish script #{post_type}")
BigBlueButton.redis_publisher.put_post_publish_started post_type, meeting_id
step_start_time = BigBlueButton.monotonic_clock
ret = BigBlueButton.exec_ret("ruby", post_publish_script, "-m", meeting_id)
if ret != 0
step_stop_time = BigBlueButton.monotonic_clock
step_time = step_stop_time - step_start_time
step_succeeded = (ret == 0)
BigBlueButton.redis_publisher.put_post_publish_ended post_type, meeting_id, {
"success" => step_succeeded,
"step_time" => step_time
}
if not step_succeeded
BigBlueButton.logger.warn("Post publish script #{post_publish_script} failed")
end
end
@ -240,6 +331,9 @@ end
begin
props = YAML::load(File.open('bigbluebutton.yml'))
redis_host = props['redis_host']
redis_port = props['redis_port']
BigBlueButton.redis_publisher = BigBlueButton::RedisWrapper.new(redis_host, redis_port)
log_dir = props['log_dir']
recording_dir = props['recording_dir']

View File

@ -1,5 +1,4 @@
#!/bin/bash
#
# BigBlueButton open source conferencing system - http://www.bigbluebutton.org/
#
@ -18,33 +17,27 @@
# with BigBlueButton; if not, see <http://www.gnu.org/licenses/>.
#
set -e
set -xe
sudo cp core/Gemfile /usr/local/bigbluebutton/core/Gemfile
sudo rm -rf /usr/local/bigbluebutton/core/lib
sudo cp -r core/lib /usr/local/bigbluebutton/core/
sudo rm -rf /usr/local/bigbluebutton/core/scripts
sudo cp -r core/scripts /usr/local/bigbluebutton/core/
sudo rm -rf /etc/bigbluebutton/god
sudo cp -r core/god/god /etc/bigbluebutton/
sudo rm -f /etc/init.d/bbb-record-core
sudo cp core/god/initd.god /etc/init.d/bbb-record-core
sudo chmod 0755 /etc/init.d/bbb-record-core
sudo rm -rf /var/bigbluebutton/playback/*
function deploy_format() {
local formats=$1
for format in $formats
do
playback_dir="$format/playback/$format"
scripts_dir="$format/scripts"
if [ -d $playback_dir ]; then sudo cp -r $playback_dir /var/bigbluebutton/playback/; fi
if [ -d $scripts_dir ]; then sudo cp -r $scripts_dir/* /usr/local/bigbluebutton/core/scripts/; fi
sudo mkdir -p /var/log/bigbluebutton/$format
done
local formats=$1
for format in $formats
do
playback_dir="$format/playback/$format"
scripts_dir="$format/scripts"
if [ -d $playback_dir ]; then sudo cp -r $playback_dir /var/bigbluebutton/playback/; fi
if [ -d $scripts_dir ]; then sudo cp -r $scripts_dir/* /usr/local/bigbluebutton/core/scripts/; fi
sudo mkdir -p /var/log/bigbluebutton/$format /var/bigbluebutton/published/$format /var/bigbluebutton/recording/publish/$format
done
}
# deploy_format "slides"
deploy_format "presentation"
sudo mkdir -p /var/bigbluebutton/playback/
@ -57,7 +50,7 @@ sudo mkdir -p /var/bigbluebutton/recording/status/processed/
sudo mkdir -p /var/bigbluebutton/recording/status/sanity/
sudo mv /usr/local/bigbluebutton/core/scripts/*.nginx /etc/bigbluebutton/nginx/
sudo chown -R tomcat6:tomcat6 /var/bigbluebutton/ /var/log/bigbluebutton/
sudo chown -R tomcat7:tomcat7 /var/bigbluebutton/ /var/log/bigbluebutton/
sudo chown -R red5:red5 /var/bigbluebutton/deskshare/
sudo chown -R freeswitch:daemon /var/bigbluebutton/meetings/