RemoteDesktop: use asm decoder, remove dependencies on libjpg, improvement rendering perf

This commit is contained in:
DanyLE 2022-08-16 21:30:08 +02:00
parent 0dcfcc9afd
commit 86e2c1246c
16 changed files with 517 additions and 44 deletions

87
RemoteDesktop/build.json Normal file
View File

@ -0,0 +1,87 @@
{
"name": "RemoteDesktop",
"targets": {
"init": {
"jobs": [
{
"name": "vfs-mkdir",
"data": [
"build",
"build/debug",
"build/release"
]
}
]
},
"coffee": {
"require": [
"coffee"
],
"jobs": [
{
"name": "coffee-compile",
"data": {
"src": [
"coffees/main.coffee",
"coffees/WVNC.coffee"
],
"dest": "build/debug/main.js"
}
}
]
},
"uglify": {
"require": [
"terser"
],
"jobs": [
{
"name": "terser-uglify",
"data": [
"build/debug/main.js"
]
}
]
},
"copy": {
"jobs": [
{
"name": "vfs-cp",
"data": {
"src": [
"assets/scheme.html",
"package.json",
"README.md",
"assets/icon.png",
"assets/main.css",
"javascript/decoder_asm.js",
"javascript/wvnc_asm.js",
"javascript/wvnc_asm.wasm"
],
"dest": "build/debug"
}
}
]
},
"release": {
"require": [
"zip"
],
"depend": [
"init",
"coffee",
"uglify",
"copy"
],
"jobs": [
{
"name": "zip-mk",
"data": {
"src": "build/debug",
"dest": "build/release/RemoteDesktop.zip"
}
}
]
}
}
}

View File

@ -1 +0,0 @@
var api,onmessage,resolution,wasm_update;importScripts("wvnc_asm.js"),api={},resolution=void 0,Module.onRuntimeInitialized=function(){return api={createBuffer:Module.cwrap("create_buffer","number",["number","number"]),destroyBuffer:Module.cwrap("destroy_buffer","",["number"]),updateBuffer:Module.cwrap("update","number",["number","number","number","number","number","number"]),decodeBuffer:Module.cwrap("decode","number",["number","number","number","number"])}},wasm_update=function(e){var r,u,n,t,a,o,d,f,i,s,m;if(s=(r=new Uint8Array(e))[1]|r[2]<<8,m=r[3]|r[4]<<8,i=r[5]|r[6]<<8,t=r[7]|r[8]<<8,n=r[9],a=api.createBuffer(r.length),Module.HEAP8.set(r,a),d=i*t*4,o=api.decodeBuffer(a,r.length,resolution.depth,d),u=new Uint8Array(Module.HEAP8.buffer,o,d),e={},(f=new Uint8Array(d)).set(u,0),e.pixels=f.buffer,e.x=s,e.y=m,e.w=i,e.h=t,postMessage(e,[e.pixels]),api.destroyBuffer(a),0!==n||32!==resolution.depth)return api.destroyBuffer(o)},onmessage=function(e){return e.data.depth?resolution=e.data:wasm_update(e.data)};

View File

@ -0,0 +1,52 @@
// Generated by CoffeeScript 1.12.8
var api, onmessage, resolution, wasm_update;
importScripts('wvnc_asm.js');
api = {};
resolution = void 0;
Module.onRuntimeInitialized = function() {
return api = {
createBuffer: Module.cwrap('create_buffer', 'number', ['number', 'number']),
destroyBuffer: Module.cwrap('destroy_buffer', '', ['number']),
updateBuffer: Module.cwrap("update", 'number', ['number', 'number', 'number', 'number', 'number', 'number']),
decodeBuffer: Module.cwrap("decode", 'number', ['number', 'number', 'number', 'number'])
};
};
wasm_update = function(msg) {
var datain, dataout, flag, h, p, po, size, tmp, w, x, y;
datain = new Uint8Array(msg);
x = datain[1] | (datain[2] << 8);
y = datain[3] | (datain[4] << 8);
w = datain[5] | (datain[6] << 8);
h = datain[7] | (datain[8] << 8);
flag = datain[9];
p = api.createBuffer(datain.length);
Module.HEAP8.set(datain, p);
size = w * h * 4;
po = api.decodeBuffer(p, datain.length, size);
dataout = new Uint8Array(Module.HEAP8.buffer, po, size);
msg = {};
tmp = new Uint8Array(size);
tmp.set(dataout, 0);
msg.pixels = tmp.buffer;
msg.x = x;
msg.y = y;
msg.w = w;
msg.h = h;
postMessage(msg, [msg.pixels]);
api.destroyBuffer(po);
return api.destroyBuffer(p);
};
onmessage = function(e) {
if (e.data.depth) {
return resolution = e.data;
} else {
return wasm_update(e.data);
}
};

View File

@ -1,4 +1,3 @@
afx-app-window[data-id="RemoteDesktop"] div[data-id="container"]
{
background-color: #272822;

File diff suppressed because one or more lines are too long

View File

@ -1,13 +1,14 @@
{
"pkgname":"RemoteDesktop",
"app":"RemoteDesktop",
"name":"WVNC remote desktop",
"description":"",
"info":{
"author": "",
"email": ""
"author": "Dany LE",
"email": "contact@iohub.dev"
},
"version":"0.1.6-a",
"dependencies": ["libwvnc@0.1.2-a"],
"version":"0.1.7-b",
"dependencies": [],
"category":"Internet",
"icon": "icon.png",
"mimes":["none"]

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,306 @@
class WVNC
constructor: (args) ->
@socket = undefined
@ws = undefined
@canvas = undefined
worker = "pkg://RemoteDesktop/decoder_asm.js".asFileHandle().getlink()
@scale = 1.0
@ws = args.ws if args.ws
@canvas = args.element
@canvas = document.getElementById @canvas if typeof @canvas is 'string'
@decoder = new Worker worker
@enableEvent = true
me = @
@mouseMask = 0
@decoder.onmessage = (e) ->
me.process e.data
init: () ->
me = @
return new Promise (r, e) ->
return e('Canvas is not set') if not me.canvas
# fix keyboard event problem
$(me.canvas).attr 'tabindex', '1'
me.initInputEvent()
r()
initInputEvent: () ->
me = @
return unless @canvas
getMousePos = (e) ->
rect = me.canvas.getBoundingClientRect()
pos=
x: Math.floor((e.clientX - rect.left) / me.scale)
y: Math.floor((e.clientY - rect.top) / me.scale)
return pos
sendMouseLocation = (e) ->
return unless me.enableEvent
p = getMousePos e
me.sendPointEvent p.x, p.y, me.mouseMask
return unless me.canvas
me.canvas.oncontextmenu = (e) ->
e.preventDefault()
return false
me.canvas.onmousemove = (e) ->
sendMouseLocation e
me.canvas.onmousedown = (e) ->
state = 1 << e.button
me.mouseMask = me.mouseMask | state
sendMouseLocation e
#e.preventDefault()
me.canvas.onmouseup = (e) ->
state = 1 << e.button
me.mouseMask = me.mouseMask & (~state)
sendMouseLocation e
#e.preventDefault()
me.canvas.onkeydown = me.canvas.onkeyup = (e) ->
# get the key code
keycode = e.keyCode
#console.log e
switch keycode
when 8 then code = 0xFF08 #back space
when 9 then code = 0xff89 #0xFF09 # tab ?
when 13 then code = 0xFF0D # return
when 27 then code = 0xFF1B # esc
when 46 then code = 0xFFFF # delete to verify
when 38 then code = 0xFF52 # up
when 40 then code = 0xFF54 # down
when 37 then code = 0xFF51 # left
when 39 then code = 0xFF53 # right
when 91 then code = 0xFFE7 # meta left
when 93 then code = 0xFFE8 # meta right
when 16 then code = 0xFFE1 # shift left
when 17 then code = 0xFFE3 # ctrl left
when 18 then code = 0xFFE9 # alt left
when 20 then code = 0xFFE5 # capslock
when 113 then code = 0xFFBF # f2
when 112 then code = 0xFFBE # f1
when 114 then code = 0xFFC0 # f3
when 115 then code = 0xFFC1 # f4
when 116 then code = 0xFFC2 # f5
when 117 then code = 0xFFC3 # f6
when 118 then code = 0xFFC4 # f7
when 119 then code = 0xFFC5 # f8
when 120 then code = 0xFFC6 # f9
when 121 then code = 0xFFC7 # f10
when 122 then code = 0xFFC8 # f11
when 123 then code = 0xFFC9 # f12
else
code = e.key.charCodeAt(0) #if not e.ctrlKey and not e.altKey
#if ((keycode > 47 and keycode < 58) or (keycode > 64 and keycode < 91) or (keycode > 95 and keycode < 112) or (keycode > 185 and keycode < 193) or (keycode > 218 && keycode < 223))
# code = e.key.charCodeAt(0)
#else
# code = keycode
e.preventDefault()
return unless code
if e.type is "keydown"
me.sendKeyEvent code, 1
else if e.type is "keyup"
me.sendKeyEvent code, 0
# mouse wheel event
@canvas.addEventListener 'wheel', (e) ->
return unless me.enableEvent
#if (e.deltaY < 0) # up
p = getMousePos e
e.preventDefault()
if e.deltaY < 0
me.sendPointEvent p.x, p.y, 8
me.sendPointEvent p.x, p.y, 0
return
me.sendPointEvent p.x, p.y, 16
me.sendPointEvent p.x, p.y, 0
# paste event
@canvas.onpaste = (e) ->
return unless me.enableEvent
pastedText = undefined
if window.clipboardData and window.clipboardData.getData #IE
pastedText = window.clipboardData.getData 'Text'
else if e.clipboardData and e.clipboardData.getData
pastedText = e.clipboardData.getData 'text/plain'
return false unless pastedText
e.preventDefault()
me.sendTextAsClipboard pastedText
# global event
fn = (e) =>
@disconnect(true)
window.addEventListener "unload", fn
window.addEventListener "beforeunload", fn
initCanvas: (w, h , d) ->
me = @
@depth = d
@canvas.width = w
@canvas.height = h
@resolution =
w: w,
h: h,
depth: @depth
@decoder.postMessage @resolution
#me.canvas.style.cursor = "none"
@setScale @scale
process: (msg) ->
if not @socket
return
data = new Uint8Array msg.pixels
#w = @buffer.width * @scale
#h = @buffer.height * @scale
ctx = @canvas.getContext "2d", { alpha: false }
imgData = ctx.createImageData msg.w, msg.h
imgData.data.set data
ctx.putImageData imgData,msg.x, msg.y
setScale: (n) ->
@scale = n
return unless @canvas
@canvas.style.transformOrigin = '0 0'
@canvas.style.transform = 'scale(' + n + ')'
connect: (url, args) ->
me = @
@disconnect(false)
return unless @ws
@socket = new WebSocket @ws
@socket.binaryType = "arraybuffer"
@socket.onopen = () ->
console.log "socket opened"
me.initConnection(url, args)
@socket.onmessage = (e) ->
me.consume e
@socket.onclose = () ->
me.socket = null
me.canvas.style.cursor = "auto"
me.canvas.getContext('2d').clearRect 0,0, me.resolution.w, me.resolution.h if me.canvas and me.resolution
console.log "socket closed"
disconnect: (close_worker) ->
@socket.close() if @socket
@socket = undefined
@decoder.terminate() if close_worker
initConnection: (vncserver, params) ->
#vncserver = "192.168.1.20:5901"
data = new Uint8Array vncserver.length + 1
data[0] = 50 # jpeg quality
if params
data[0] = params.quality if params.quality
## rate in milisecond
data.set (new TextEncoder()).encode(vncserver), 1
@socket.send(@buildCommand 0x01, data)
sendPointEvent: (x, y, mask) ->
return unless @socket
data = new Uint8Array 5
data[0] = x & 0xFF
data[1] = x >> 8
data[2] = y & 0xFF
data[3] = y >> 8
data[4] = mask
@socket.send( @buildCommand 0x05, data )
sendKeyEvent: (code, v) ->
#console.log code, v
return unless @socket
return unless @enableEvent
data = new Uint8Array 3
data[0] = code & 0xFF
data[1] = code >> 8
data[2] = v
@socket.send( @buildCommand 0x06, data )
buildCommand: (hex, o) ->
data = undefined
switch typeof o
when 'string'
data = (new TextEncoder()).encode(o)
when 'number'
data = new Uint8Array [o]
else
data = o
cmd = new Uint8Array data.length + 3
cmd[0] = hex
cmd[2] = data.length >> 8
cmd[1] = data.length & 0x0F
cmd.set data, 3
#console.log "the command is", cmd.buffer
return cmd.buffer
oncopy: (text) ->
console.log "Get clipboard text: " + text
onpassword: () ->
return new Promise (resolve, reject) ->
reject("onpassword is not implemented")
sendTextAsClipboard: (text) ->
return unless @socket
console.log "send ", text
@socket.send (@buildCommand 0x07, text)
oncredential: () ->
return new Promise (resolve, reject) ->
reject("oncredential is not implemented")
onerror: (m) ->
console.log "Error", m
onresize: () ->
console.log "resize"
consume: (e) ->
data = new Uint8Array e.data
cmd = data[0]
me = @
switch cmd
when 0xFE #error
data = data.subarray 1, data.length - 1
dec = new TextDecoder("utf-8")
@onerror dec.decode(data)
when 0x81
console.log "Request for password"
@enableEvent = false
@onpassword().then (pass) ->
me.socket.send (me.buildCommand 0x02, pass)
me.enableEvent = true
when 0x82
console.log "Request for login"
@enableEvent = false
@oncredential().then (user, pass) ->
arr = new Uint8Array user.length + pass.length + 1
arr.set (new TextEncoder()).encode(user), 0
arr.set ['\0'], user.length
arr.set (new TextEncoder()).encode(pass), user.length + 1
me.socket.send(me.buildCommand 0x03, arr)
me.enableEvent = true
when 0x83
w = data[1] | (data[2]<<8)
h = data[3] | (data[4]<<8)
depth = 32
@initCanvas w, h, depth
# status command for ack
@socket.send(@buildCommand 0x04, 1)
@onresize()
when 0x84
# send data to web assembly for decoding
@decoder.postMessage data.buffer, [data.buffer]
@socket.send(@buildCommand 0x04, 1)
when 0x85
# clipboard data from server
data = data.subarray 1
dec = new TextDecoder "utf-8"
@oncopy dec.decode data
@socket.send(@buildCommand 0x04, 1)
else
console.log cmd
window.WVNC = WVNC

View File

@ -5,22 +5,12 @@ class ConnectionDialog extends this.OS.GUI.BasicDialog
main: () ->
super.main()
@find("bbp").data = [
{ text: "16 bits", value: 16, selected: true },
{ text: "32 bits", value: 32 }
]
@find("compression").data = [
{text: "No compression", value:0},
{text: "JPEG", value:1, selected: true}
]
@find("jq").value = 40
@find("bt-ok").onbtclick = (e) =>
return unless @handle
data =
wvnc: (@find "txtWVNC").value
server: (@find "txtServer").value
bbp: (@find "bbp").selectedItem.data.value,
flag: (@find "compression").selectedItem.data.value,
quality:(@find "jq").value
@handle data
@quit()
@ -29,20 +19,14 @@ class ConnectionDialog extends this.OS.GUI.BasicDialog
@quit()
ConnectionDialog.scheme = """
<afx-app-window width='350' height='270'>
<afx-app-window width='350' height='220'>
<afx-hbox>
<div data-width="5"></div>
<afx-vbox>
<afx-label text="__(WVNC Websocket)" data-height="25" class="header" ></afx-label>
<input data-height="25" data-id="txtWVNC" value="wss://localhost/wvnc"></input>
<input data-height="25" data-id="txtWVNC" value="wss://app.iohub.dev/wbs/wvnc"></input>
<afx-label text="__(VNC Server)" data-height="25" class="header" ></afx-label>
<input data-height="25" data-id="txtServer" value="192.168.1.27:5901"></input>
<div data-height="5"></div>
<afx-label text="__(Bits per pixel)" data-height="25" class="header" ></afx-label>
<afx-list-view dropdown = "true" data-id ="bbp" data-height="25" ></afx-list-view>
<div data-height="5"></div>
<afx-label text="__(Compression)" data-height="25" class="header" ></afx-label>
<afx-list-view dropdown = "true" data-id ="compression" data-height="25" ></afx-list-view>
<input data-height="25" data-id="txtServer" value="192.168.1.27:5900"></input>
<div data-height="5"></div>
<afx-label text="__(JPEG quality)" data-height="25" class="header" ></afx-label>
<afx-slider data-id ="jq" data-height="25" ></afx-slider>
@ -92,7 +76,6 @@ CredentialDialog.scheme = """
</afx-hbox>
</afx-vbox>
</afx-app-window>
"""
class RemoteDesktop extends this.OS.application.BaseApplication
@ -103,8 +86,7 @@ class RemoteDesktop extends this.OS.application.BaseApplication
@canvas = @find "screen"
@container = @find "container"
@client = new WVNC {
element: @canvas,
libjpeg: "pkg://libjpeg/jpg.js".asFileHandle().getlink()
element: @canvas
}
@client.onerror = (m) =>
@error m
@ -167,7 +149,4 @@ class RemoteDesktop extends this.OS.application.BaseApplication
cleanup: () ->
@client.disconnect(true) if @client
RemoteDesktop.dependencies = [
"pkg://libwvnc/main.js"
]
this.OS.register "RemoteDesktop", RemoteDesktop

View File

@ -0,0 +1,52 @@
// Generated by CoffeeScript 1.12.8
var api, onmessage, resolution, wasm_update;
importScripts('wvnc_asm.js');
api = {};
resolution = void 0;
Module.onRuntimeInitialized = function() {
return api = {
createBuffer: Module.cwrap('create_buffer', 'number', ['number', 'number']),
destroyBuffer: Module.cwrap('destroy_buffer', '', ['number']),
updateBuffer: Module.cwrap("update", 'number', ['number', 'number', 'number', 'number', 'number', 'number']),
decodeBuffer: Module.cwrap("decode", 'number', ['number', 'number', 'number', 'number'])
};
};
wasm_update = function(msg) {
var datain, dataout, flag, h, p, po, size, tmp, w, x, y;
datain = new Uint8Array(msg);
x = datain[1] | (datain[2] << 8);
y = datain[3] | (datain[4] << 8);
w = datain[5] | (datain[6] << 8);
h = datain[7] | (datain[8] << 8);
flag = datain[9];
p = api.createBuffer(datain.length);
Module.HEAP8.set(datain, p);
size = w * h * 4;
po = api.decodeBuffer(p, datain.length, size);
dataout = new Uint8Array(Module.HEAP8.buffer, po, size);
msg = {};
tmp = new Uint8Array(size);
tmp.set(dataout, 0);
msg.pixels = tmp.buffer;
msg.x = x;
msg.y = y;
msg.w = w;
msg.h = h;
postMessage(msg, [msg.pixels]);
api.destroyBuffer(po);
return api.destroyBuffer(p);
};
onmessage = function(e) {
if (e.data.depth) {
return resolution = e.data;
} else {
return wasm_update(e.data);
}
};

File diff suppressed because one or more lines are too long

Binary file not shown.

View File

@ -1,13 +1,14 @@
{
"pkgname":"RemoteDesktop",
"app":"RemoteDesktop",
"name":"WVNC remote desktop",
"description":"",
"info":{
"author": "",
"email": ""
"author": "Dany LE",
"email": "contact@iohub.dev"
},
"version":"0.1.6-a",
"dependencies": ["libwvnc@0.1.2-a"],
"version":"0.1.7-b",
"dependencies": [],
"category":"Internet",
"icon": "icon.png",
"mimes":["none"]

View File

@ -364,9 +364,9 @@
"name": "WVNC remote desktop",
"description": "https://raw.githubusercontent.com/lxsang/antosdk-apps/master/RemoteDesktop/README.md",
"category": "Internet",
"author": "",
"version": "0.1.6-a",
"dependencies": ["libwvnc@0.1.2-a"],"category":"Internet","icon":"icon.png","mimes":["none"],
"author": "Dany LE",
"version": "0.1.7-b",
"dependencies": [],"category":"Internet","icon":"icon.png","mimes":["none"],
"download": "https://raw.githubusercontent.com/lxsang/antosdk-apps/master/RemoteDesktop/build/release/RemoteDesktop.zip"
},
{