From a802ab7e5a040f43012e98dcd76af694ff751205 Mon Sep 17 00:00:00 2001 From: Richard Alam Date: Thu, 10 May 2012 14:10:01 -0700 Subject: [PATCH] - make some improvements to dekstop sharing by 1. Convert the initial image that is sent to the server as grayscale 2. Add some debug logs on the applet to determine which step is taking a long time. As suspected, it is on the network, esp. using http. 3. Fix a bug in the server side. Turns out we are not incrementing the timestamps in constant interval like we do with the voice. Not doing so, doesn't allow the Flash Player to drop frames when packets arrive late. --- .../server/sessions/SessionSVC.scala | 4 ++- .../server/stream/DeskshareStream.scala | 10 +++++- .../deskshare/server/stream/Stream.scala | 2 +- .../deskshare/server/svc1/Block.scala | 2 +- .../deskshare/client/blocks/Block.java | 6 +++- .../client/net/NetworkHttpStreamSender.java | 32 +++++++++-------- .../client/net/NetworkSocketStreamSender.java | 28 ++++++++++----- .../client/net/NetworkStreamSender.java | 2 +- .../deskshare/common/ScreenVideoEncoder.java | 35 ++++++++++--------- 9 files changed, 75 insertions(+), 46 deletions(-) diff --git a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/sessions/SessionSVC.scala b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/sessions/SessionSVC.scala index ef1b947869..ee284bc10f 100755 --- a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/sessions/SessionSVC.scala +++ b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/sessions/SessionSVC.scala @@ -47,6 +47,7 @@ class SessionSVC(sessionManager:SessionManagerSVC, room: String, screenDim: Dime private var stop = true private var mouseLoc:Point = new Point(100,100) private var pendingGenKeyFrameRequest = false + private var timestamp = 0L; /* * Schedule to generate a key frame after 30seconds of a request. @@ -135,7 +136,8 @@ class SessionSVC(sessionManager:SessionManagerSVC, room: String, screenDim: Dime sessionManager ! new RemoveSession(room) } else { if (blockManager != null) { - stream ! new UpdateStream(room, blockManager.generateFrame(keyframe)) + timestamp += 50; + stream ! new UpdateStream(room, blockManager.generateFrame(keyframe), timestamp) stream ! new UpdateStreamMouseLocation(room, mouseLoc) } } diff --git a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/DeskshareStream.scala b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/DeskshareStream.scala index befbd714a5..d719389e4c 100755 --- a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/DeskshareStream.scala +++ b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/DeskshareStream.scala @@ -28,6 +28,7 @@ import org.red5.server.api.{IContext, IScope} import org.red5.server.api.so.ISharedObject import org.red5.server.net.rtmp.event.VideoData; import org.red5.server.stream.{BroadcastScope, IBroadcastScope, IProviderService} +import org.red5.server.net.rtmp.message.Constants; import org.apache.mina.core.buffer.IoBuffer import java.util.ArrayList import scala.actors.Actor @@ -110,7 +111,14 @@ class DeskshareStream(app: DeskshareApplication, name: String, val width: Int, v } val data: VideoData = new VideoData(buffer) - data.setTimestamp((System.currentTimeMillis() - startTimestamp).toInt) + data.setSourceType(Constants.SOURCE_TYPE_LIVE); + /* + * Use timestamp increments. This will force + * Flash Player to playback at proper timestamp. If we calculate timestamp using + * System.currentTimeMillis() - startTimestamp, the video has tendency to drift and + * introduce delay. See how we do the voice. (ralam may 10, 2012) + */ + data.setTimestamp(us.timestamp.toInt); broadcastStream.dispatchEvent(data) data.release() diff --git a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/Stream.scala b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/Stream.scala index 2791a92eda..5ca5e9e31d 100755 --- a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/Stream.scala +++ b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/stream/Stream.scala @@ -27,7 +27,7 @@ import java.awt.Point object StopStream object StartStream -class UpdateStream(val room: String, val videoData: Array[Byte]) +class UpdateStream(val room: String, val videoData: Array[Byte], val timestamp: Long) class UpdateStreamMouseLocation(val room: String, val loc: Point) abstract class Stream extends Actor \ No newline at end of file diff --git a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/svc1/Block.scala b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/svc1/Block.scala index 2977e3b016..d549717cbe 100755 --- a/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/svc1/Block.scala +++ b/deskshare/app/src/main/scala/org/bigbluebutton/deskshare/server/svc1/Block.scala @@ -46,7 +46,7 @@ class Block(val dim: Dimension, val position: Int) { for (i: Int <- 0 until blankPixels.length) { blankPixels(i) = 0xCECECE; } - val encodedBlankPixels = ScreenVideoEncoder.encodePixels(blankPixels, dim.width, dim.height) + val encodedBlankPixels = ScreenVideoEncoder.encodePixels(blankPixels, dim.width, dim.height, false) def update(videoData: Array[Byte], isKeyFrame: Boolean, seqNum: Int): Unit = { firstBlockReceived = true; diff --git a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/blocks/Block.java b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/blocks/Block.java index dfec91ae38..1ff1d62e82 100755 --- a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/blocks/Block.java +++ b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/blocks/Block.java @@ -25,6 +25,8 @@ import java.awt.Point; import java.awt.image.BufferedImage; import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + import org.bigbluebutton.deskshare.client.net.EncodedBlockData; import org.bigbluebutton.deskshare.common.PixelExtractException; import org.bigbluebutton.deskshare.common.ScreenVideoEncoder; @@ -40,6 +42,7 @@ public final class Block { private final Object pixelsLock = new Object(); private AtomicBoolean dirtyBlock = new AtomicBoolean(false); private long lastSent = System.currentTimeMillis(); + private AtomicLong sentCount = new AtomicLong(); Block(Dimension dim, int position, Point location) { checksum = new BlockChecksum(); @@ -86,6 +89,7 @@ public final class Block { } public void sent() { + sentCount.incrementAndGet(); dirtyBlock.set(false); } @@ -96,7 +100,7 @@ public final class Block { System.arraycopy(capturedPixels, 0, pixelsCopy, 0, capturedPixels.length); } - byte[] encodedBlock = ScreenVideoEncoder.encodePixels(pixelsCopy, getWidth(), getHeight()); + byte[] encodedBlock = ScreenVideoEncoder.encodePixels(pixelsCopy, getWidth(), getHeight(), (sentCount.longValue() > 5) /* send grayscale image */); return new EncodedBlockData(position, encodedBlock); } diff --git a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkHttpStreamSender.java b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkHttpStreamSender.java index b5923ffafa..d503211b7c 100755 --- a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkHttpStreamSender.java +++ b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkHttpStreamSender.java @@ -90,6 +90,7 @@ public class NetworkHttpStreamSender implements Runnable { * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4382944 * */ + long start = System.currentTimeMillis(); try { url = new URL("http://" + host + SCREEN_CAPTURE__URL); conn = url.openConnection(); @@ -100,10 +101,13 @@ public class NetworkHttpStreamSender implements Runnable { e.printStackTrace(); throw new ConnectionException("IOException while connecting to " + url.toString()); } + long end = System.currentTimeMillis(); + System.out.println("Http[" + id + "] Open connection took [" + (end-start) + " ms]"); } public void sendStartStreamMessage() { try { + System.out.println("Http[" + id + "] Open connection. In sendStartStreamMessage"); openConnection(); sendCaptureStartEvent(screenDim, blockDim); } catch (ConnectionException e) { @@ -116,17 +120,12 @@ public class NetworkHttpStreamSender implements Runnable { ClientHttpRequest chr; try { chr = new ClientHttpRequest(conn); - chr.setParameter(ROOM, room); - + chr.setParameter(ROOM, room); chr.setParameter(SEQ_NUM, seqNumGenerator.getNext()); - String screenInfo = Integer.toString(screen.getWidth()) - + "x" + Integer.toString(screen.getHeight()); - chr.setParameter(SCREEN, screenInfo); - - String blockInfo = Integer.toString(block.getWidth()) - + "x" + Integer.toString(block.getHeight()); + String screenInfo = Integer.toString(screen.getWidth()) + "x" + Integer.toString(screen.getHeight()); + chr.setParameter(SCREEN, screenInfo); + String blockInfo = Integer.toString(block.getWidth()) + "x" + Integer.toString(block.getHeight()); chr.setParameter(BLOCK, blockInfo); - chr.setParameter(EVENT, CaptureEvents.CAPTURE_START.getEvent()); chr.post(); } catch (IOException e) { @@ -138,13 +137,13 @@ public class NetworkHttpStreamSender implements Runnable { public void disconnect() throws ConnectionException { try { + System.out.println("Http[" + id + "] Open connection. In disconnect"); openConnection(); sendCaptureEndEvent(); } catch (ConnectionException e) { e.printStackTrace(); notifyNetworkStreamListener(ExitCode.DESKSHARE_SERVICE_UNAVAILABLE); - throw e; - + throw e; } finally { processBlocks = false; } @@ -168,8 +167,8 @@ public class NetworkHttpStreamSender implements Runnable { if (message.getMessageType() == Message.MessageType.BLOCK) { long start = System.currentTimeMillis(); Integer[] changedBlocks = ((BlockMessage)message).getBlocks(); - String blockSize = "Block length ["; - String encodeTime = "Encode times ["; + String blockSize = "Http[" + id + "] Block length ["; + String encodeTime = "Http[" + id + "]Encode times ["; long encStart = 0; long encEnd = 0; int totalBytes = 0; @@ -191,7 +190,7 @@ public class NetworkHttpStreamSender implements Runnable { retriever.blockSent((Integer)changedBlocks[i]); } long end = System.currentTimeMillis(); - System.out.println("[HTTP Thread " + id + "] Sending " + changedBlocks.length + " blocks took " + (end - start) + " millis"); + System.out.println("[HTTP " + id + "] Sending " + changedBlocks.length + " blocks took " + (end - start) + " millis"); } else if (message.getMessageType() == Message.MessageType.CURSOR) { CursorMessage msg = (CursorMessage)message; sendCursor(msg.getMouseLocation(), msg.getRoom()); @@ -217,6 +216,7 @@ public class NetworkHttpStreamSender implements Runnable { private void sendCursor(Point mouseLoc, String room) { ClientHttpRequest chr; try { + System.out.println("Http[" + id + "] Open connection. In sendCursor"); openConnection(); chr = new ClientHttpRequest(conn); chr.setParameter(ROOM, room); @@ -233,8 +233,10 @@ public class NetworkHttpStreamSender implements Runnable { } private void sendBlockData(BlockVideoData blockData) { + long start = System.currentTimeMillis(); ClientHttpRequest chr; try { + System.out.println("Http[" + id + "] Open connection. In sendBlockData"); openConnection(); chr = new ClientHttpRequest(conn); chr.setParameter(ROOM, blockData.getRoom()); @@ -250,5 +252,7 @@ public class NetworkHttpStreamSender implements Runnable { } catch (ConnectionException e) { System.out.println("ERROR: Failed to send block data."); } + long end = System.currentTimeMillis(); + System.out.println("[HTTP " + id + "] Sending " + blockData.getVideoData().length + " bytes took " + (end - start) + " ms"); } } diff --git a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkSocketStreamSender.java b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkSocketStreamSender.java index 95e48c1b29..9c9d64a04e 100755 --- a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkSocketStreamSender.java +++ b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkSocketStreamSender.java @@ -117,29 +117,37 @@ public class NetworkSocketStreamSender implements Runnable { public void disconnect() throws ConnectionException { System.out.println("Disconnecting socket stream"); if (!processMessages) return; - - } private void processNextMessageToSend(Message message) throws IOException { if (message.getMessageType() == Message.MessageType.BLOCK) { + long start = System.currentTimeMillis(); ByteArrayOutputStream dataToSend = new ByteArrayOutputStream(); dataToSend.reset(); - BlockStreamProtocolEncoder.encodeRoomAndSequenceNumber(room, seqNumGenerator.getNext(), dataToSend); - + BlockStreamProtocolEncoder.encodeRoomAndSequenceNumber(room, seqNumGenerator.getNext(), dataToSend); Integer[] changedBlocks = ((BlockMessage)message).getBlocks(); - BlockStreamProtocolEncoder.numBlocksChanged(changedBlocks.length, dataToSend); - - String blocksStr = "Encoding "; + + String blockSize = "Block length ["; + String encodeTime = "Encode times ["; + long encStart = 0; + long encEnd = 0; + int totalBytes = 0; + long totalMillis = 0; for (int i = 0; i < changedBlocks.length; i++) { - blocksStr += " " + (Integer)changedBlocks[i]; + encStart = System.currentTimeMillis(); EncodedBlockData block = retriever.getBlockToSend((Integer)changedBlocks[i]); + totalBytes += block.getVideoData().length; + blockSize += block.getVideoData().length + ","; + encEnd = System.currentTimeMillis(); + totalMillis += (encEnd - encStart); + encodeTime += (encEnd - encStart) + ","; BlockVideoData bv = new BlockVideoData(room, block.getPosition(), block.getVideoData(), false /* should remove later */); BlockStreamProtocolEncoder.encodeBlock(bv, dataToSend); } -// System.out.println(blocksStr); + System.out.println(blockSize + "] total=" + totalBytes + " bytes"); + System.out.println(encodeTime + "] total=" + totalMillis + " ms"); BlockStreamProtocolEncoder.encodeDelimiter(dataToSend); sendHeader(BlockStreamProtocolEncoder.encodeHeaderAndLength(dataToSend)); @@ -147,6 +155,8 @@ public class NetworkSocketStreamSender implements Runnable { for (int i = 0; i< changedBlocks.length; i++) { retriever.blockSent((Integer)changedBlocks[i]); } + long end = System.currentTimeMillis(); + System.out.println("[Socket Thread " + id + "] Sending " + changedBlocks.length + " blocks took " + (end - start) + " millis"); } else if (message.getMessageType() == Message.MessageType.CURSOR) { CursorMessage msg = (CursorMessage)message; sendCursor(msg.getMouseLocation(), msg.getRoom()); diff --git a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkStreamSender.java b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkStreamSender.java index 788748aac2..10984f9074 100755 --- a/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkStreamSender.java +++ b/deskshare/applet/src/main/java/org/bigbluebutton/deskshare/client/net/NetworkStreamSender.java @@ -65,7 +65,7 @@ public class NetworkStreamSender implements NextBlockRetriever, NetworkStreamLis this.blockDim = blockDim; this.httpTunnel = httpTunnel; - numThreads = Runtime.getRuntime().availableProcessors() * 2; + numThreads = Runtime.getRuntime().availableProcessors() * 3; System.out.println(NAME + "Starting up " + numThreads + " sender threads."); executor = Executors.newFixedThreadPool(numThreads); } diff --git a/deskshare/common/src/main/java/org/bigbluebutton/deskshare/common/ScreenVideoEncoder.java b/deskshare/common/src/main/java/org/bigbluebutton/deskshare/common/ScreenVideoEncoder.java index 53691a8414..faa918660c 100755 --- a/deskshare/common/src/main/java/org/bigbluebutton/deskshare/common/ScreenVideoEncoder.java +++ b/deskshare/common/src/main/java/org/bigbluebutton/deskshare/common/ScreenVideoEncoder.java @@ -82,11 +82,11 @@ public final class ScreenVideoEncoder { return pixels; } - public static byte[] encodePixels(int pixels[], int width, int height) { + public static byte[] encodePixels(int pixels[], int width, int height, boolean grayscale) { changePixelScanFromBottomLeftToTopRight(pixels, width, height); - byte[] bgrPixels = convertFromRGBtoBGR(pixels); + byte[] bgrPixels = convertFromRGBtoBGR(pixels, grayscale); byte[] compressedPixels = compressUsingZlib(bgrPixels); @@ -163,8 +163,8 @@ public final class ScreenVideoEncoder { * @param pixels * @return pixels in BGR order */ - private static byte[] convertFromRGBtoBGR(int[] pixels) { - long start = System.currentTimeMillis(); + private static byte[] convertFromRGBtoBGR(int[] pixels, boolean grayscale) { +// long start = System.currentTimeMillis(); byte[] rgbPixels = new byte[pixels.length * 3]; int position = 0; @@ -173,19 +173,20 @@ public final class ScreenVideoEncoder { byte green = (byte) ((pixels[i] >> 8) & 0xff); byte blue = (byte) (pixels[i] & 0xff); - // Sequence should be BGR - rgbPixels[position++] = blue; - rgbPixels[position++] = green; - rgbPixels[position++] = red; -/* - * If we want to send grayscale images. - byte brightness = convertToGrayScale(red, green, blue); - - // Sequence should be BGR - rgbPixels[position++] = brightness; - rgbPixels[position++] = brightness; - rgbPixels[position++] = brightness; -*/ } + if (grayscale) { + byte brightness = convertToGrayScale(red, green, blue); + + // Sequence should be BGR + rgbPixels[position++] = brightness; + rgbPixels[position++] = brightness; + rgbPixels[position++] = brightness; + } else { + // Sequence should be BGR + rgbPixels[position++] = blue; + rgbPixels[position++] = green; + rgbPixels[position++] = red; + } + } long end = System.currentTimeMillis(); // System.out.println("Extracting pixels[" + pixels.length + "] took " + (end-start) + " ms.");