diff --git a/deskshare-app/.classpath b/deskshare-app/.classpath index c2f00c0546..e1dc378048 100644 --- a/deskshare-app/.classpath +++ b/deskshare-app/.classpath @@ -6,17 +6,17 @@ - - - + + + diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/CaptureUpdateEvent.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/CaptureUpdateEvent.java index bdfcee18da..80fe97e628 100644 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/CaptureUpdateEvent.java +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/CaptureUpdateEvent.java @@ -23,27 +23,29 @@ import java.awt.image.BufferedImage; public class CaptureUpdateEvent implements ICaptureEvent { - private final BufferedImage screen; + private final BufferedImage tile; private final String room; private final int width; private final int height; private final int x; private final int y; + private final int position; public CaptureUpdateEvent(BufferedImage screen, String room, int width, - int height, int x, int y) { + int height, int x, int y, int position) { - this.screen = screen; + this.tile = screen; this.room = room; this.width = width; this.height = height; this.x = x; this.y = y; + this.position = position; } - public BufferedImage getScreen() { - return screen; + public BufferedImage getTile() { + return tile; } public String getRoom() { @@ -66,8 +68,18 @@ public class CaptureUpdateEvent implements ICaptureEvent { return y; } + public int getPosition() { + return position; + } + @Override public CaptureMessage getMessageType() { return CaptureMessage.CAPTURE_UPDATE; } + + public static CaptureUpdateEvent copy(CaptureUpdateEvent event) { + return new CaptureUpdateEvent(event.getTile(), event.getRoom(), + event.getWidth(), event.getHeight(), + event.getX(), event.getY(), event.getPosition()); + } } diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/ChangedTileProcessor.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/ChangedTileProcessor.java new file mode 100644 index 0000000000..347ca2e05b --- /dev/null +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/ChangedTileProcessor.java @@ -0,0 +1,77 @@ +package org.bigbluebutton.deskshare; + +import java.awt.Graphics2D; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; + +import javax.imageio.ImageIO; + +import org.red5.logging.Red5LoggerFactory; +import org.slf4j.Logger; + +public class ChangedTileProcessor { + final private Logger log = Red5LoggerFactory.getLogger(ChangedTileProcessor.class, "deskshare"); + + private final Executor exec = Executors.newSingleThreadExecutor(); + private BlockingQueue queue = new LinkedBlockingQueue(); + private Runnable eventHandler; + private volatile boolean handleEvent = false; + + private BufferedImage image; + private Graphics2D graphics; + + public ChangedTileProcessor(int width, int height){ + this.image = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR); + this.graphics = this.image.createGraphics(); + } + + public void appendTile(BufferedImage tile, int x, int y){ + graphics.drawImage(tile, x, y, null); + } + + public void stop() { + handleEvent = false; + } + + public void start() { + handleEvent = true; + eventHandler = new Runnable() { + public void run() { + while (handleEvent) { + try { + CaptureUpdateEvent event = queue.take(); + handleCaptureEvent(event); + } catch (InterruptedException e) { + log.warn("InterruptedExeption while taking event."); + } + } + } + }; + exec.execute(eventHandler); + } + + private void handleCaptureEvent(CaptureUpdateEvent event) { + log.debug("Handling captured event " + event.getPosition()); + BufferedImage image = event.getTile(); + appendTile(image, event.getX(), event.getY()); + } + + public BufferedImage getImage(){ + return image.getSubimage(0, 0, image.getWidth(), image.getHeight()); + } + + public void accept(CaptureUpdateEvent event) { + try { + // Make a copy so we can process safely on our own thread. + CaptureUpdateEvent copy = CaptureUpdateEvent.copy(event); + queue.put(copy); + } catch (InterruptedException e) { + log.warn("InterruptedException while putting event into queue."); + } + } +} diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/DeskShareStream.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/DeskShareStream.java index ab64cde847..c9c43ef131 100644 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/DeskShareStream.java +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/DeskShareStream.java @@ -62,14 +62,14 @@ public class DeskShareStream { private BlockingQueue queue = new LinkedBlockingQueue(); private final Executor exec = Executors.newSingleThreadExecutor(); - private Runnable eventHandler; - private volatile boolean handleEvent = false; + private Runnable capturedScreenSender; + private volatile boolean sendCapturedScreen = false; private IStreamCoder outStreamCoder; private BroadcastStream broadcastStream; private IContainer outContainer; private IStream outStream; - private ImageProcessor imageProcessor; + private ChangedTileProcessor changedTileProcessor; private static final Red5HandlerFactory factory = Red5HandlerFactory.getFactory(); private final IRTMPEventIOHandler outputHandler; @@ -78,6 +78,7 @@ public class DeskShareStream { private int width, height, frameRate, timestampBase; private String outStreamName; private IScope scope; + private long lastUpdate; /** * The default constructor @@ -91,7 +92,11 @@ public class DeskShareStream { this.height = height; this.frameRate = frameRate; this.timestampBase = 1000000 / this.frameRate; - this.imageProcessor = new ImageProcessor(width, height); + this.changedTileProcessor = new ChangedTileProcessor(width, height); + + changedTileProcessor.start(); + + lastUpdate = System.currentTimeMillis(); outputHandler = new IRTMPEventIOHandler(){ public Red5Message read() throws InterruptedException{ @@ -115,42 +120,42 @@ public class DeskShareStream { } public void stop() { - handleEvent = false; + sendCapturedScreen = false; streamEnded(); + changedTileProcessor.stop(); } public void start() { startPublishing(scope); setupStreams(); - handleEvent = true; + sendCapturedScreen = true; log.debug("Starting stream {}", outStreamName); - eventHandler = new Runnable() { + capturedScreenSender = new Runnable() { public void run() { - while (handleEvent) { - try { - CaptureUpdateEvent event = queue.take(); - handleCaptureEvent(event); - } catch (InterruptedException e) { - log.warn("InterruptedExeption while taking event."); - } + while (sendCapturedScreen) { +// try { +// CaptureUpdateEvent event = queue.take(); + sendCapturedScreen(); +// } catch (InterruptedException e) { +// log.warn("InterruptedExeption while taking event."); +// } } } }; - exec.execute(eventHandler); + exec.execute(capturedScreenSender); } - private void handleCaptureEvent(CaptureUpdateEvent event) { - BufferedImage image = event.getScreen(); - imageProcessor.appendTile(image, event.getX(), event.getY()); - imageReceived(imageProcessor.getImage()); + private void sendCapturedScreen() { + long now = System.currentTimeMillis(); + if ((now - lastUpdate) > 30000) { + log.debug("Sending image to XUGGLER"); + imageReceived(changedTileProcessor.getImage()); + lastUpdate = now; + } } public void accept(CaptureUpdateEvent event) { - try { - queue.put(event); - } catch (InterruptedException e) { - log.warn("InterruptedException while putting event into queue."); - } + changedTileProcessor.accept(event); } private void imageReceived(BufferedImage image) { diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/ImageProcessor.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/ImageProcessor.java deleted file mode 100644 index b2b0e7659d..0000000000 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/ImageProcessor.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.bigbluebutton.deskshare; - -import java.awt.Graphics2D; -import java.awt.image.BufferedImage; - -public class ImageProcessor { - private BufferedImage image; - private Graphics2D graphics; - - public ImageProcessor(int width, int height){ - this.image = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR); - this.graphics = this.image.createGraphics(); - } - - public void appendTile(BufferedImage tile, int x, int y){ - graphics.drawImage(tile, x, y, null); - } - - public BufferedImage getImage(){ - return this.image; - } -} diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/StreamerGateway.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/StreamerGateway.java index baf5f66d01..8e69911ba5 100644 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/StreamerGateway.java +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/StreamerGateway.java @@ -23,9 +23,13 @@ import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.red5.logging.Red5LoggerFactory; import org.red5.server.api.so.ISharedObject; +import org.slf4j.Logger; public class StreamerGateway { + final private Logger log = Red5LoggerFactory.getLogger(StreamerGateway.class, "deskshare"); + private final Map streamsMap; private StreamFactory streamFactory; private DeskShareApplication deskShareApp; @@ -35,6 +39,8 @@ public class StreamerGateway { } public void onCaptureStartEvent(CaptureStartEvent event) { + log.debug("Creating stream " + event.getRoom()); + DeskShareStream stream = streamFactory.createStream(event); streamsMap.put(event.getRoom(), stream); diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/servlet/TunnelController.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/servlet/TunnelController.java index c2d7175f89..a5a2696fdf 100644 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/servlet/TunnelController.java +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/servlet/TunnelController.java @@ -95,12 +95,12 @@ public class TunnelController extends MultiActionController { } private void sendCaptureEvent(CapturedScreen cs) { - streamerGateway.onCaptureEvent(new CaptureUpdateEvent(cs)); +// streamerGateway.onCaptureEvent(new CaptureUpdateEvent(cs)); } private void sendCaptureStartEvent(CapturedScreen cs) { - streamerGateway.onCaptureStartEvent(new CaptureStartEvent(cs)); +// streamerGateway.onCaptureStartEvent(new CaptureStartEvent(cs)); } private StreamerGateway getStreamerGateway() { diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureMessageHandler.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureMessageHandler.java index 1bf14de511..4367b0aa2e 100644 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureMessageHandler.java +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureMessageHandler.java @@ -23,9 +23,10 @@ import org.apache.mina.core.service.IoHandlerAdapter; import org.apache.mina.core.session.IdleStatus; import org.apache.mina.core.session.IoSession; import org.bigbluebutton.deskshare.CaptureEndEvent; +import org.bigbluebutton.deskshare.CaptureMessage; import org.bigbluebutton.deskshare.CaptureUpdateEvent; import org.bigbluebutton.deskshare.CaptureStartEvent; -import org.bigbluebutton.deskshare.CapturedScreen; +import org.bigbluebutton.deskshare.ICaptureEvent; import org.bigbluebutton.deskshare.StreamerGateway; import org.red5.logging.Red5LoggerFactory; import org.slf4j.Logger; @@ -38,23 +39,22 @@ public class ScreenCaptureMessageHandler extends IoHandlerAdapter { @Override public void exceptionCaught( IoSession session, Throwable cause ) throws Exception { - log.warn(cause.toString()); + log.warn(cause.toString() + " \n " + cause.getMessage()); cause.printStackTrace(); } @Override public void messageReceived( IoSession session, Object message ) throws Exception { - CapturedScreen cs = (CapturedScreen) message; - String room = cs.getRoom(); - log.debug("Got room {}", room); - - if (session.containsAttribute("room")) { - sendCaptureEvent(cs); - } else { - session.setAttribute("room", room); - sendCaptureStartEvent(cs); - } + CaptureMessage msgType = ((ICaptureEvent) message).getMessageType(); + + if (msgType == CaptureMessage.CAPTURE_START) { + log.debug("Received CAPTURE_START"); + sendCaptureStartEvent((CaptureStartEvent) message); + } else if (msgType == CaptureMessage.CAPTURE_UPDATE) { + // log.debug("Received CAPTURE_UPDATE"); + sendCaptureUpdateEvent((CaptureUpdateEvent) message); + } } @Override @@ -76,7 +76,7 @@ public class ScreenCaptureMessageHandler extends IoHandlerAdapter { @Override public void sessionClosed(IoSession session) throws Exception { log.debug("Session Closed."); - String room = (String) session.getAttribute("room"); + String room = (String) session.getAttribute("ROOM"); sendCaptureEndEvent(room); } @@ -84,13 +84,12 @@ public class ScreenCaptureMessageHandler extends IoHandlerAdapter { streamerGateway.onCaptureEndEvent(new CaptureEndEvent(room)); } - private void sendCaptureEvent(CapturedScreen cs) { - streamerGateway.onCaptureEvent(new CaptureUpdateEvent(cs)); + private void sendCaptureUpdateEvent(CaptureUpdateEvent event) { + streamerGateway.onCaptureEvent(event); } - private void sendCaptureStartEvent(CapturedScreen cs) { - - streamerGateway.onCaptureStartEvent(new CaptureStartEvent(cs)); + private void sendCaptureStartEvent(CaptureStartEvent event) { + streamerGateway.onCaptureStartEvent(event); } public void setStreamerGateway(StreamerGateway sg) { diff --git a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureProtocolDecoder.java b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureProtocolDecoder.java index a8040f6ef3..4b96002b30 100644 --- a/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureProtocolDecoder.java +++ b/deskshare-app/src/main/java/org/bigbluebutton/deskshare/socket/ScreenCaptureProtocolDecoder.java @@ -32,6 +32,7 @@ import org.apache.mina.core.session.IoSession; import org.apache.mina.filter.codec.CumulativeProtocolDecoder; import org.apache.mina.filter.codec.ProtocolDecoderOutput; import org.bigbluebutton.deskshare.CaptureStartEvent; +import org.bigbluebutton.deskshare.CaptureUpdateEvent; import org.bigbluebutton.deskshare.CapturedScreen; import org.red5.logging.Red5LoggerFactory; import org.slf4j.Logger; @@ -66,7 +67,7 @@ public class ScreenCaptureProtocolDecoder extends CumulativeProtocolDecoder { } else { if (in.remaining() < 20) return false; int message = in.getInt(); - log.debug("Got message " + message); + // log.debug("Got message " + message); session.setAttribute(MESSAGE_TYPE, new Integer(message)); return true; @@ -76,27 +77,25 @@ public class ScreenCaptureProtocolDecoder extends CumulativeProtocolDecoder { private boolean decodeCaptureStart(IoSession session, IoBuffer in, ProtocolDecoderOutput out) { - if (! session.containsAttribute(ROOM)) { - return decodeRoom(session, in); - } else { + if (session.containsAttribute(ROOM)) { if (decodeVideoInfo(session, in)) { sendCaptureStartMessage(session, out); return true; - } + } + } else { + decodeRoom(session, in); } return false; } - private boolean decodeRoom(IoSession session, IoBuffer in) { - if (in.remaining() < 200) return false; + private void decodeRoom(IoSession session, IoBuffer in) { + if (in.remaining() < 200) return; int len = in.getInt(); String room = decodeString(len, in); if (room != "") { session.setAttribute(ROOM, room); - return true; } - return false; } private boolean decodeVideoInfo(IoSession session, IoBuffer in) { @@ -128,14 +127,15 @@ public class ScreenCaptureProtocolDecoder extends CumulativeProtocolDecoder { String room = (String) session.getAttribute(ROOM); String videoInfo = (String) session.getAttribute(VIDEO_INFO); log.debug("Room " + room + " videoInfo " + videoInfo); + //Get the screen dimensions, i.e. the resolution of the video we need to create String[] screenDimensions = videoInfo.split("x"); int width = Integer.parseInt(screenDimensions[0]); int height = Integer.parseInt(screenDimensions[1]); int frameRate = Integer.parseInt(screenDimensions[2]); - CaptureStartEvent cse = new CaptureStartEvent(); - out.write("CAPTURE START"); + CaptureStartEvent cse = new CaptureStartEvent(room, width, height, frameRate); + out.write(cse); clearMessage(session); } @@ -144,10 +144,7 @@ public class ScreenCaptureProtocolDecoder extends CumulativeProtocolDecoder { return decodeTileInfo(session, in); } else { if (decodeTile(session, in)) { - String tileInfo = (String) session.getAttribute(TILE_INFO); - log.debug("TILE INFO " + tileInfo); - out.write("CAPTURE UPDATE"); - reset(session); + sendCaptureUpdateMessage(session, out); return true; } } @@ -177,12 +174,12 @@ public class ScreenCaptureProtocolDecoder extends CumulativeProtocolDecoder { if (in.remaining() >= tileLength) { byte[] bytes = new byte[tileLength]; - log.debug("Reading image with length {}", bytes.length); +// log.debug("Reading image with length {}", bytes.length); in.get(bytes); ByteArrayInputStream bais = new ByteArrayInputStream(bytes); try { - BufferedImage image = ImageIO.read(bais);; + BufferedImage image = ImageIO.read(bais); session.setAttribute(TILE_IMAGE, image); } catch (IOException e) { log.error("Failed to get captured screen for room "); @@ -190,102 +187,30 @@ public class ScreenCaptureProtocolDecoder extends CumulativeProtocolDecoder { return true; } - log.debug("Can't process image yet . " + tileLength); +// log.debug("Can't process image yet . " + tileLength); in.position(start); return false; } -/* - private void sendDecodedMessage(IoSession session, ProtocolDecoderOutput out) { - BufferedImage screen = (BufferedImage) session.getAttribute(CAPTURED_SCREEN); + private void sendCaptureUpdateMessage(IoSession session, ProtocolDecoderOutput out) { String room = (String) session.getAttribute(ROOM); - String videoInfo = (String) session.getAttribute(VIDEO_INFO); + String tileInfo = (String) session.getAttribute(TILE_INFO); +// log.debug("TILE INFO " + tileInfo); - //Get the screen dimensions, i.e. the resolution of the video we need to create - String[] screenDimensions = videoInfo.split("x"); - int width = Integer.parseInt(screenDimensions[0]); - int height = Integer.parseInt(screenDimensions[1]); - int frameRate = Integer.parseInt(screenDimensions[2]); - - CapturedScreen cs = new CapturedScreen(screen, room, width, height, frameRate); - out.write(cs); + //Get the tile dimensions, i.e. the resolution of the video we need to create + String[] tileDim = tileInfo.split("x"); + int width = Integer.parseInt(tileDim[0]); + int height = Integer.parseInt(tileDim[1]); + int x = Integer.parseInt(tileDim[2]); + int y = Integer.parseInt(tileDim[3]); + int pos = Integer.parseInt(tileDim[4]); + + BufferedImage tile = (BufferedImage) session.getAttribute(TILE_IMAGE); + CaptureUpdateEvent cse = new CaptureUpdateEvent(tile, room, width, height, x, y, pos); + out.write(cse); + reset(session); } - - private boolean canDecodeCapturedScreen(IoSession session, IoBuffer in) { - if (in.prefixedDataAvailable(4, MAX_IMAGE_SIZE)) { - return true; - } - return false; - } - - private int getLength(IoBuffer in) { - return in.getInt(); - } - - private boolean decodeRoom(IoSession session, IoBuffer in) { - return getCrLfTerminatedString(session, in); - } - - private boolean getCrLfTerminatedString(IoSession session, IoBuffer in) { - // Remember the initial position. - int start = in.position(); - - // Now find the first CRLF in the buffer. - byte previous = 0; - while (in.hasRemaining()) { - byte current = in.get(); - - if (previous == '\r' && current == '\n') { - // Remember the current position and limit. - int position = in.position(); - int limit = in.limit(); - try { - in.position(start); - in.limit(position); - // The bytes between in.position() and in.limit() - // now contain a full CRLF terminated line. - parseLine(session, in.slice()); - } finally { - // Set the position to point right after the - // detected line and set the limit to the old - // one. - in.position(position); - in.limit(limit); - } - return true; - } - - previous = current; - } - - // Could not find CRLF in the buffer. Reset the initial - // position to the one we recorded above. - in.position(start); - - return false; - } - - private void parseLine(IoSession session, IoBuffer in) { - try { - String line = in.getString(Charset.forName( "UTF-8" ).newDecoder()); - if (!session.containsAttribute(ROOM)) { - session.setAttribute(ROOM, line.trim()); - } else { - session.setAttribute(VIDEO_INFO, line.trim()); - } - } catch (CharacterCodingException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - } - - private void reset(IoSession session) { - session.removeAttribute(ROOM); - session.removeAttribute(VIDEO_INFO); - session.removeAttribute(CAPTURED_SCREEN); - } -*/ - + private void clearMessage(IoSession session) { session.removeAttribute(MESSAGE_TYPE); } diff --git a/deskshare-app/src/main/webapp/WEB-INF/red5-web.xml b/deskshare-app/src/main/webapp/WEB-INF/red5-web.xml index 73d47c5a5a..ae3204c60d 100644 --- a/deskshare-app/src/main/webapp/WEB-INF/red5-web.xml +++ b/deskshare-app/src/main/webapp/WEB-INF/red5-web.xml @@ -31,7 +31,7 @@ - + @@ -47,9 +47,5 @@ - - - - - +