use web assembly and web worker for data decoder

apps/assets/coffee/WVNC.coffee

@@ -6,9 +6,8 @@ class WVNC extends window.classes.BaseObject
         @uri = @args[0] if @args and @args.length > 0
         @canvas = undefined
         @canvas = ($ @args[1])[0] if @args and @args.length > 1
-        @buffer = $("<canvas>")[0]
         @lastPose = { x: 0, y: 0 }
-        @scale = 0.8
+        @scale = 1.0
         @decoder = new Worker('/assets/scripts/decoder.js')
         me = @
         @mouseMask = 0
@@ -40,7 +39,7 @@ class WVNC extends window.classes.BaseObject
             me.sendPointEvent p.x, p.y, me.mouseMask
 
         return unless me.canvas
-        ($ me.canvas).css "cursor", "none"
+        #($ me.canvas).css "cursor", "none"
         ($ me.canvas).contextmenu (e) ->
             e.preventDefault()
             return false
@@ -91,47 +90,30 @@ class WVNC extends window.classes.BaseObject
     initCanvas: (w, h , d) ->
         me = @
         @depth = d
-        @buffer.width = w
-        @buffer.height = h
+        @canvas.width = w
+        @canvas.height = h
         @resolution =
             w: w,
             h: h,
             depth: @depth
         @decoder.postMessage @resolution
-        ctx = @buffer.getContext('2d')
-        data = ctx.createImageData w, h
-        ctx.putImageData data, 0, 0
 
-    process: (data) ->
-        data.pixels = new Uint8ClampedArray data.pixels
-        data.pixels = data.pixels.subarray 10 if data.flag is 0 and @resolution.depth is 32
-        ctx = @buffer.getContext('2d')
-        imgData = ctx.createImageData data.w, data.h
-        imgData.data.set data.pixels
-        ctx.putImageData imgData, data.x, data.y
-        
-        @draw()  if data.x isnt @lastPose.x or data.y > @resolution.h - 10
-        @lastPose = { x: data.x, y: data.y }
+    process: (msg) ->
+        if not @socket
+            return
+        data = new Uint8Array msg.buffer
+        #w = @buffer.width * @scale
+        #h = @buffer.height * @scale
+        ctx = @canvas.getContext "2d"
+        imgData = ctx.createImageData  msg.w, msg.h
+        imgData.data.set data
+        ctx.putImageData imgData, msg.x, msg.y
         
 
     setScale: (n) ->
         @scale = n
         @draw()
 
-    draw: () ->
-        if not @socket
-            return
-
-        w = @buffer.width * @scale
-        h = @buffer.height * @scale
-        @canvas.width = w
-        @canvas.height = h
-        ctx = @canvas.getContext "2d"
-        ctx.save()
-        ctx.scale @scale, @scale
-        ctx.clearRect 0, 0, w, h
-        ctx.drawImage @buffer, 0, 0
-        ctx.restore()
 
     openSession: () ->
         me = @
@@ -152,7 +134,7 @@ class WVNC extends window.classes.BaseObject
     initConnection: () ->
         vncserver = "localhost:5901"
         data = new Uint8Array vncserver.length + 5
-        data[0] = 16 # bbp
+        data[0] = 32 # bbp
         ###
         flag:
             0: raw data no compress
@@ -160,7 +142,7 @@ class WVNC extends window.classes.BaseObject
             2: raw data compressed by zlib
             3: jpeg data compressed by zlib
         ###
-        data[1] = 2
+        data[1] = 3
         data[2] = 50 # jpeg quality
         ## rate in milisecond
         rate = 30
@@ -237,11 +219,8 @@ class WVNC extends window.classes.BaseObject
                 # status command for ack
                 @socket.send(@buildCommand 0x04, 1)
             when 0x84
-                #console.log "update"
+                # send data to web assembly for decoding
                 @decoder.postMessage data.buffer, [data.buffer]
-                #@decodeFB d
-                # ack
-                #@socket.send(@buildCommand 0x04, 1)
             else
                 console.log cmd
         

apps/assets/coffee/decoder.coffee

@@ -1,84 +1,48 @@
 #zlib library
-importScripts('pako.min.js')
-# jpeg library
-importScripts('jpeg-decoder.js')
+importScripts('wvnc_asm.js')
+api = {}
 resolution = undefined
-pixelValue = (value, depth) ->
-    pixel =
-        r: 255
-        g: 255
-        b: 255
-        a: 255
-    #console.log("len is" + arr.length)
-    if depth is 24 or depth is 32
-        pixel.r = value & 0xFF
-        pixel.g = (value >> 8) & 0xFF
-        pixel.b = (value >> 16) & 0xFF
-    else if depth is 16
-        pixel.r = (value & 0x1F) * (255 / 31)
-        pixel.g = ((value >> 5) & 0x3F) * (255 / 63)
-        pixel.b = ((value >> 11) & 0x1F) * (255 / 31)
-    #console.log pixel
-    return pixel
-
-getImageData = (d) ->
-    return d.pixels if resolution.depth is 32
-    step = resolution.depth / 8
-    npixels = d.pixels.length / step
-    data = new Uint8ClampedArray d.w * d.h * 4
-    for i in [0..npixels - 1]
-        value = 0
-        value = value | d.pixels[i * step + j] << (j * 8) for j in [0..step - 1]
-        pixel = pixelValue value, resolution.depth
-        data[i * 4] = pixel.r
-        data[i * 4 + 1] = pixel.g
-        data[i * 4 + 2] = pixel.b
-        data[i * 4 + 3] = pixel.a
-    return data
-
-decodeRaw = (d) ->
-    d.pixels = getImageData d
-    return d
-
-decodeJPEG = (d) ->
-    raw = decode d.pixels, { useTArray: true, colorTransform: true }
-    d.pixels = raw.data
-    return d
-    ###
-    blob = new Blob [d.pixels], { type: "image/jpeg" }
-    reader = new FileReader()
-    reader.onloadend = () ->
-        d.pixels = reader.result
-        postMessage d
-    reader.readAsDataURL blob
-    ###
-
-update = (msg) ->
-    d = {}
-    data = new Uint8Array msg
-    d.x = data[1] | (data[2]<<8)
-    d.y = data[3] | (data[4]<<8)
-    d.w = data[5] | (data[6]<<8)
-    d.h = data[7] | (data[8]<<8)
-    d.flag = data[9]
-    d.pixels = data.subarray 10
-    # the zlib is slower than expected
-    switch d.flag
-        when 0x0 # raw data
-            raw = decodeRaw d
-        when 0x1 # jpeg data
-            raw = decodeJPEG(d)
-        when 0x2 # raw compress in zlib format
-            d.pixels =  pako.inflate(d.pixels)
-            raw = decodeRaw d
-        when 0x3 # jpeg compress in zlib format
-            d.pixels = pako.inflate(d.pixels)
-            raw = decodeJPEG(d)
-    return unless raw
-    raw.pixels = raw.pixels.buffer
-    # fill the rectangle
-    postMessage raw, [raw.pixels]
+#frame_buffer = undefined
+Module.onRuntimeInitialized = () ->
+    api =
+    {
+        createBuffer: Module.cwrap('create_buffer', 'number', ['number', 'number']),
+        destroyBuffer: Module.cwrap('destroy_buffer', '', ['number']),
+        updateBuffer: Module.cwrap("update", 'number', ['number', 'number', 'number', 'number', 'number', 'number']),
+        decodeBuffer: Module.cwrap("decode",'number', ['number', 'number', 'number','number'] )
+    }
 
 onmessage = (e) ->
-    return resolution = e.data if e.data.depth
-    update e.data
+    if e.data.depth
+        resolution = e.data
+        #api.destroyBuffer frame_buffer if frame_buffer
+        #frame_buffer = api.createBuffer resolution.w * resolution.h * 4
+    #else if e.data.cleanup
+    #    api.destroyBuffer frame_buffer if frame_buffer
+    else
+        datain = new Uint8Array e.data
+        x = datain[1] | (datain[2] << 8)
+        y = datain[3] | (datain[4] << 8)
+        w = datain[5] | (datain[6] << 8)
+        h = datain[7] | (datain[8] << 8)
+        flag = datain[9]
+        p = api.createBuffer datain.length
+        Module.HEAP8.set datain, p
+        size = w * h * 4
+        po = api.decodeBuffer p, datain.length, resolution.depth, size
+        #api.updateBuffer frame_buffer, p, datain.length, resolution.w, resolution.h, resolution.depth
+        # create buffer array and send back to main
+        dataout = new Uint8Array Module.HEAP8.buffer, po, size
+        # console.log dataout
+        msg = {}
+        tmp = new Uint8Array size
+        tmp.set dataout, 0
+        msg.buffer = tmp.buffer
+        msg.x = x
+        msg.y = y
+        msg.w = w
+        msg.h = h
+        postMessage msg, [msg.buffer]
+        api.destroyBuffer p
+        if flag isnt 0x0 or resolution.depth isnt 32
+            api.destroyBuffer po

apps/assets/scripts/decoder.js

@@ -1,114 +1,50 @@
 // Generated by CoffeeScript 1.9.3
-  var decodeJPEG, decodeRaw, getImageData, onmessage, pixelValue, resolution, update;
+  var api, onmessage, resolution;
 
-  importScripts('pako.min.js');
+  importScripts('wvnc_asm.js');
 
-  importScripts('jpeg-decoder.js');
+  api = {};
 
   resolution = void 0;
 
-  pixelValue = function(value, depth) {
-    var pixel;
-    pixel = {
-      r: 255,
-      g: 255,
-      b: 255,
-      a: 255
+  Module.onRuntimeInitialized = function() {
+    return api = {
+      createBuffer: Module.cwrap('create_buffer', 'number', ['number', 'number']),
+      destroyBuffer: Module.cwrap('destroy_buffer', '', ['number']),
+      updateBuffer: Module.cwrap("update", 'number', ['number', 'number', 'number', 'number', 'number', 'number']),
+      decodeBuffer: Module.cwrap("decode", 'number', ['number', 'number', 'number', 'number'])
     };
-    if (depth === 24 || depth === 32) {
-      pixel.r = value & 0xFF;
-      pixel.g = (value >> 8) & 0xFF;
-      pixel.b = (value >> 16) & 0xFF;
-    } else if (depth === 16) {
-      pixel.r = (value & 0x1F) * (255 / 31);
-      pixel.g = ((value >> 5) & 0x3F) * (255 / 63);
-      pixel.b = ((value >> 11) & 0x1F) * (255 / 31);
-    }
-    return pixel;
-  };
-
-  getImageData = function(d) {
-    var data, i, j, k, l, npixels, pixel, ref, ref1, step, value;
-    if (resolution.depth === 32) {
-      return d.pixels;
-    }
-    step = resolution.depth / 8;
-    npixels = d.pixels.length / step;
-    data = new Uint8ClampedArray(d.w * d.h * 4);
-    for (i = k = 0, ref = npixels - 1; 0 <= ref ? k <= ref : k >= ref; i = 0 <= ref ? ++k : --k) {
-      value = 0;
-      for (j = l = 0, ref1 = step - 1; 0 <= ref1 ? l <= ref1 : l >= ref1; j = 0 <= ref1 ? ++l : --l) {
-        value = value | d.pixels[i * step + j] << (j * 8);
-      }
-      pixel = pixelValue(value, resolution.depth);
-      data[i * 4] = pixel.r;
-      data[i * 4 + 1] = pixel.g;
-      data[i * 4 + 2] = pixel.b;
-      data[i * 4 + 3] = pixel.a;
-    }
-    return data;
-  };
-
-  decodeRaw = function(d) {
-    d.pixels = getImageData(d);
-    return d;
-  };
-
-  decodeJPEG = function(d) {
-    var raw;
-    raw = decode(d.pixels, {
-      useTArray: true,
-      colorTransform: true
-    });
-    d.pixels = raw.data;
-    return d;
-
-    /*
-    blob = new Blob [d.pixels], { type: "image/jpeg" }
-    reader = new FileReader()
-    reader.onloadend = () ->
-        d.pixels = reader.result
-        postMessage d
-    reader.readAsDataURL blob
-     */
-  };
-
-  update = function(msg) {
-    var d, data, raw;
-    d = {};
-    data = new Uint8Array(msg);
-    d.x = data[1] | (data[2] << 8);
-    d.y = data[3] | (data[4] << 8);
-    d.w = data[5] | (data[6] << 8);
-    d.h = data[7] | (data[8] << 8);
-    d.flag = data[9];
-    d.pixels = data.subarray(10);
-    switch (d.flag) {
-      case 0x0:
-        raw = decodeRaw(d);
-        break;
-      case 0x1:
-        raw = decodeJPEG(d);
-        break;
-      case 0x2:
-        d.pixels = pako.inflate(d.pixels);
-        raw = decodeRaw(d);
-        break;
-      case 0x3:
-        d.pixels = pako.inflate(d.pixels);
-        raw = decodeJPEG(d);
-    }
-    if (!raw) {
-      return;
-    }
-    raw.pixels = raw.pixels.buffer;
-    return postMessage(raw, [raw.pixels]);
   };
 
   onmessage = function(e) {
+    var datain, dataout, flag, h, msg, p, po, size, tmp, w, x, y;
     if (e.data.depth) {
       return resolution = e.data;
+    } else {
+      datain = new Uint8Array(e.data);
+      x = datain[1] | (datain[2] << 8);
+      y = datain[3] | (datain[4] << 8);
+      w = datain[5] | (datain[6] << 8);
+      h = datain[7] | (datain[8] << 8);
+      flag = datain[9];
+      p = api.createBuffer(datain.length);
+      Module.HEAP8.set(datain, p);
+      size = w * h * 4;
+      po = api.decodeBuffer(p, datain.length, resolution.depth, size);
+      dataout = new Uint8Array(Module.HEAP8.buffer, po, size);
+      msg = {};
+      tmp = new Uint8Array(size);
+      tmp.set(dataout, 0);
+      msg.buffer = tmp.buffer;
+      msg.x = x;
+      msg.y = y;
+      msg.w = w;
+      msg.h = h;
+      postMessage(msg, [msg.buffer]);
+      api.destroyBuffer(p);
+      if (flag !== 0x0 || resolution.depth !== 32) {
+        return api.destroyBuffer(po);
+      }
     }
-    return update(e.data);
   };
 

apps/assets/scripts/main.js

@@ -176,12 +176,11 @@
       if (this.args && this.args.length > 1) {
         this.canvas = ($(this.args[1]))[0];
       }
-      this.buffer = $("<canvas>")[0];
       this.lastPose = {
         x: 0,
         y: 0
       };
-      this.scale = 0.8;
+      this.scale = 1.0;
       this.decoder = new Worker('/assets/scripts/decoder.js');
       me = this;
       this.mouseMask = 0;
@@ -228,7 +227,6 @@
       if (!me.canvas) {
         return;
       }
-      ($(me.canvas)).css("cursor", "none");
       ($(me.canvas)).contextmenu(function(e) {
         e.preventDefault();
         return false;
@@ -283,39 +281,29 @@
     };
 
     WVNC.prototype.initCanvas = function(w, h, d) {
-      var ctx, data, me;
+      var me;
       me = this;
       this.depth = d;
-      this.buffer.width = w;
-      this.buffer.height = h;
+      this.canvas.width = w;
+      this.canvas.height = h;
       this.resolution = {
         w: w,
         h: h,
         depth: this.depth
       };
-      this.decoder.postMessage(this.resolution);
-      ctx = this.buffer.getContext('2d');
-      data = ctx.createImageData(w, h);
-      return ctx.putImageData(data, 0, 0);
+      return this.decoder.postMessage(this.resolution);
     };
 
-    WVNC.prototype.process = function(data) {
-      var ctx, imgData;
-      data.pixels = new Uint8ClampedArray(data.pixels);
-      if (data.flag === 0 && this.resolution.depth === 32) {
-        data.pixels = data.pixels.subarray(10);
+    WVNC.prototype.process = function(msg) {
+      var ctx, data, imgData;
+      if (!this.socket) {
+        return;
       }
-      ctx = this.buffer.getContext('2d');
-      imgData = ctx.createImageData(data.w, data.h);
-      imgData.data.set(data.pixels);
-      ctx.putImageData(imgData, data.x, data.y);
-      if (data.x !== this.lastPose.x || data.y > this.resolution.h - 10) {
-        this.draw();
-      }
-      return this.lastPose = {
-        x: data.x,
-        y: data.y
-      };
+      data = new Uint8Array(msg.buffer);
+      ctx = this.canvas.getContext("2d");
+      imgData = ctx.createImageData(msg.w, msg.h);
+      imgData.data.set(data);
+      return ctx.putImageData(imgData, msg.x, msg.y);
     };
 
     WVNC.prototype.setScale = function(n) {
@@ -323,23 +311,6 @@
       return this.draw();
     };
 
-    WVNC.prototype.draw = function() {
-      var ctx, h, w;
-      if (!this.socket) {
-        return;
-      }
-      w = this.buffer.width * this.scale;
-      h = this.buffer.height * this.scale;
-      this.canvas.width = w;
-      this.canvas.height = h;
-      ctx = this.canvas.getContext("2d");
-      ctx.save();
-      ctx.scale(this.scale, this.scale);
-      ctx.clearRect(0, 0, w, h);
-      ctx.drawImage(this.buffer, 0, 0);
-      return ctx.restore();
-    };
-
     WVNC.prototype.openSession = function() {
       var me;
       me = this;
@@ -368,7 +339,7 @@
       var data, rate, vncserver;
       vncserver = "localhost:5901";
       data = new Uint8Array(vncserver.length + 5);
-      data[0] = 16;
+      data[0] = 32;
 
       /*
       flag:
@@ -377,7 +348,7 @@
           2: raw data compressed by zlib
           3: jpeg data compressed by zlib
        */
-      data[1] = 2;
+      data[1] = 3;
       data[2] = 50;
       rate = 30;
       data[3] = rate & 0xFF;

apps/router.lua

@@ -17,7 +17,7 @@ LOG_ROOT = WWW_ROOT..DIR_SEP.."logs"
 
 -- require needed library
 require(BASE_FRW.."silk.api")
-
+POLICY.mimes["application/wasm"] = true
 -- registry object store global variables
 local REGISTRY = {}
 -- set logging level