From 918878b0c5ad23a472c52833c7c1105979ec4cf8 Mon Sep 17 00:00:00 2001 From: Jovi Hsu Date: Sun, 14 Nov 2021 22:32:56 +0800 Subject: [PATCH] vnc raw encoding --- backend/src/agent/agent.rs | 8 +- backend/src/user/auth.rs | 2 +- frontend/Cargo.toml | 3 +- frontend/src/pages/page_remote.rs | 59 +++++- frontend/src/protocal/common.rs | 68 +++++++ frontend/src/protocal/vnc.rs | 290 ++++++++++++++++++++++++++---- 6 files changed, 386 insertions(+), 44 deletions(-) diff --git a/backend/src/agent/agent.rs b/backend/src/agent/agent.rs index c6ee786..d779b59 100644 --- a/backend/src/agent/agent.rs +++ b/backend/src/agent/agent.rs @@ -15,8 +15,8 @@ struct TcpCodec; impl Encoder for TcpCodec { type Error = io::Error; - fn encode(&mut self, item: Bytes, _dst: &mut BytesMut) -> Result<(), Self::Error> { - info!("encoding: {:?}", item); + fn encode(&mut self, _item: Bytes, _dst: &mut BytesMut) -> Result<(), Self::Error> { + // info!("encoding: {:?}", item); Ok(()) } } @@ -26,7 +26,7 @@ impl Decoder for TcpCodec { type Error = io::Error; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - info!("recv from server: {:?}", src); + // info!("recv from server: {:?}", src); if 0 == src.len() { return Ok(None); } @@ -98,7 +98,7 @@ impl StreamHandler> for Agent { fn handle(&mut self, msg: Result, ctx: &mut Context) { match msg { Ok(data) => { - info!("recv from server: {:?}", data); + // info!("recv from server: {:?}", data); if self.ws_addr.is_some() { ctx.address().do_send(AgentMsg::SendToClient(data)); } else { diff --git a/backend/src/user/auth.rs b/backend/src/user/auth.rs index 850691f..bfc29f1 100644 --- a/backend/src/user/auth.rs +++ b/backend/src/user/auth.rs @@ -51,7 +51,7 @@ impl Handler for Authenticator { fn handle(&mut self, msg: AuthMsg, _ctx: &mut Context) -> Self::Result { match msg { - AuthMsg::DoAuth(auth_info) => { + AuthMsg::DoAuth(_auth_info) => { // if auth_info.username == "admin" && auth_info.password == "admin" { // AuthResult::AuthSuccess // } else { diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index 8f678a4..8671d8a 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -20,7 +20,8 @@ crate-type = ["cdylib", "rlib"] wasm-bindgen = "^0.2" yew = "0.18" js-sys = "0.3.55" -web-sys = "0.3.55" +web-sys = {version="0.3.55", features=["HtmlCanvasElement", "CanvasRenderingContext2d", "ImageData"]} +gloo = "0.4.0" yew-router = "0.15" # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires diff --git a/frontend/src/pages/page_remote.rs b/frontend/src/pages/page_remote.rs index 46dfa6b..e0786af 100644 --- a/frontend/src/pages/page_remote.rs +++ b/frontend/src/pages/page_remote.rs @@ -1,4 +1,6 @@ use serde_json::{json, Value}; +use wasm_bindgen::{Clamped, JsValue}; +use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData}; use yew::{ format::Json, html, @@ -9,6 +11,8 @@ use yew::{ }, }; +use gloo::timers::callback::Interval; + use crate::{ components::{self, input::Input, ws::WebsocketMsg}, protocal::{common::*, vnc::VncHandler}, @@ -27,6 +31,9 @@ pub struct PageRemote { request_password: bool, username: String, password: String, + canvas: NodeRef, + canvas_ctx: Option, + interval: Option, } #[derive(Clone, PartialEq, Properties)] @@ -41,6 +48,7 @@ pub enum RemoteMsg { UpdateUsername(String), UpdatePassword(String), SendCredential, + RequireFrame(u8), } impl Component for PageRemote { @@ -60,6 +68,9 @@ impl Component for PageRemote { request_password: false, username: String::from(""), password: String::from(""), + canvas: NodeRef::default(), + canvas_ctx: None, + interval: None, } } @@ -140,6 +151,16 @@ impl Component for PageRemote { let _ = self.protocal_out_handler(out); true } + RemoteMsg::RequireFrame(incremental) => { + let out = self.handler.require_frame(incremental); + if self.interval.is_none() { + let link = self.link.clone(); + let tick = + Interval::new(250, move || link.send_message(RemoteMsg::RequireFrame(1))); + self.interval = Some(tick); + } + self.protocal_out_handler(out) + } } } @@ -167,6 +188,7 @@ impl Component for PageRemote { {self.button_connect_view()} + {self.error_msg.clone()} @@ -197,7 +219,42 @@ impl PageRemote { self.request_password = true; true } - _ => unimplemented!(), + ProtocalHandlerOutput::RenderCanvas(crs) => { + let canvas = self.canvas.cast::().unwrap(); + let ctx = match &self.canvas_ctx { + Some(ctx) => ctx, + None => { + let ctx = CanvasRenderingContext2d::from(JsValue::from( + canvas.get_context("2d").unwrap().unwrap(), + )); + self.canvas_ctx = Some(ctx); + self.canvas_ctx.as_ref().unwrap() + } + }; + + for cr in crs { + let data = ImageData::new_with_u8_clamped_array_and_sh( + Clamped(&cr.data), + cr.width as u32, + cr.height as u32, + ) + .unwrap(); + ConsoleService::log(&format!( + "renderring at ({}, {}), width {}, height {}", + cr.x, cr.y, cr.width, cr.height + )); + let _ = ctx.put_image_data(&data, cr.x as f64, cr.y as f64); + } + true + } + ProtocalHandlerOutput::SetCanvas(width, height) => { + let canvas = self.canvas.cast::().unwrap(); + canvas.set_width(width as u32); + canvas.set_height(height as u32); + self.link.send_message(RemoteMsg::RequireFrame(0)); + true + } + _ => false, } } diff --git a/frontend/src/protocal/common.rs b/frontend/src/protocal/common.rs index b505696..00f1ffb 100644 --- a/frontend/src/protocal/common.rs +++ b/frontend/src/protocal/common.rs @@ -1,10 +1,21 @@ use std::slice::Iter; + +pub struct CanvasData { + pub x: u16, + pub y: u16, + pub width: u16, + pub height: u16, + pub data: Vec, +} + pub enum ProtocalHandlerOutput { Ok, WsBuf(Vec), Err(String), RequireUsername, RequirePassword, + SetCanvas(u16, u16), + RenderCanvas(Vec), } pub struct ProtocalHandler @@ -29,12 +40,17 @@ where pub fn set_credential(&mut self, username: &str, password: &str) -> ProtocalHandlerOutput { self.inner.set_credential(username, password) } + + pub fn require_frame(&mut self, incremental: u8) -> ProtocalHandlerOutput { + self.inner.require_frame(incremental) + } } pub trait ProtocalImpl { fn new() -> Self; fn handle(&mut self, input: &[u8]) -> ProtocalHandlerOutput; fn set_credential(&mut self, username: &str, password: &str) -> ProtocalHandlerOutput; + fn require_frame(&mut self, incremental: u8) -> ProtocalHandlerOutput; } pub struct StreamReader<'a> { @@ -112,3 +128,55 @@ impl<'a> StreamReader<'a> { self.inner.next().is_none() } } + +pub struct StreamWriter<'a> { + inner: &'a mut Vec, +} + +impl<'a> StreamWriter<'a> { + pub fn new(buf: &'a mut Vec) -> Self { + Self { inner: buf } + } + + pub fn write_u8(&mut self, b: u8) { + self.inner.push(b); + } + + pub fn write_u16(&mut self, b: u16) { + self.inner.extend_from_slice(&b.to_be_bytes()); + } + + pub fn write_u32(&mut self, b: u32) { + self.inner.extend_from_slice(&b.to_be_bytes()); + } + + pub fn write_s8(&mut self, b: i8) { + self.write_u8(b as u8); + } + + pub fn write_s16(&mut self, b: i16) { + self.write_u16(b as u16); + } + + pub fn write_s32(&mut self, b: i32) { + self.write_u32(b as u32); + } + + pub fn write_string_with_len(&mut self, s: &str) { + self.inner.extend_from_slice(s.as_bytes()); + } + + pub fn write_string_l16(&mut self, s: &str) { + self.write_u16(s.len() as u16); + self.write_string_with_len(s); + } + + pub fn write_string_l32(&mut self, s: &str) { + self.write_u32(s.len() as u32); + self.write_string_with_len(s); + } + + pub fn write_slice(&mut self, s: &[u8]) { + self.inner.extend_from_slice(s); + } +} diff --git a/frontend/src/protocal/vnc.rs b/frontend/src/protocal/vnc.rs index a0c1f22..6321741 100644 --- a/frontend/src/protocal/vnc.rs +++ b/frontend/src/protocal/vnc.rs @@ -8,20 +8,6 @@ const VNC_RFB38: &[u8; 12] = b"RFB 003.008\n"; const VNC_VER_UNSUPPORTED: &str = "unsupported version"; const VNC_FAILED: &str = "Connection failed with unknow reason"; -#[derive(Debug, Clone, Copy)] -enum VncState { - Handshake, - Authentication, -} - -#[derive(Debug, Clone, Copy)] -enum VncVersion { - NONE, - VNC33, - VNC37, - VNC38, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] pub enum SecurityType { @@ -37,7 +23,7 @@ pub enum SecurityType { } pub struct VncHandler { - inner: Box, + inner: Box, } impl ProtocalImpl for VncHandler { @@ -75,12 +61,19 @@ impl ProtocalImpl for VncHandler { } self.inner.handle(&pass_bytes) } + + fn require_frame(&mut self, incremental: u8) -> ProtocalHandlerOutput { + self.inner.frame_require(incremental) + } } -trait VncStateMachine { +trait VncState { fn handle(&mut self, _input: &[u8]) -> ProtocalHandlerOutput; + fn frame_require(&self, _incremental: u8) -> ProtocalHandlerOutput { + ProtocalHandlerOutput::Err(VNC_FAILED.to_string()) + } fn done(&self) -> bool; - fn next(&self) -> Box; + fn next(&self) -> Box; } struct VncHandShake { @@ -93,7 +86,7 @@ impl Default for VncHandShake { } } -impl VncStateMachine for VncHandShake { +impl VncState for VncHandShake { fn handle(&mut self, rfbversion: &[u8]) -> ProtocalHandlerOutput { let support_version = match rfbversion { b"RFB 003.003\n" => Ok(VNC_RFB33), @@ -113,7 +106,7 @@ impl VncStateMachine for VncHandShake { self.done } - fn next(&self) -> Box { + fn next(&self) -> Box { Box::new(VncAuthentiacator::default()) } } @@ -136,7 +129,7 @@ impl Default for VncAuthentiacator { } } -impl VncStateMachine for VncAuthentiacator { +impl VncState for VncAuthentiacator { fn handle(&mut self, input: &[u8]) -> ProtocalHandlerOutput { if self.security_type == SecurityType::VncAuth { if self.wait_password { @@ -153,7 +146,7 @@ impl VncStateMachine for VncAuthentiacator { self.done } - fn next(&self) -> Box { + fn next(&self) -> Box { Box::new(VncDrawing::default()) } } @@ -213,6 +206,9 @@ struct VncDrawing { pf: PixelFormat, name: String, server_init: bool, + buffer: Vec, + rects: Vec, + num_rects_left: u16, } impl Default for VncDrawing { @@ -223,25 +219,50 @@ impl Default for VncDrawing { pf: PixelFormat::default(), name: "".to_string(), server_init: false, + buffer: Vec::with_capacity(50), + rects: Vec::new(), + num_rects_left: 0, } } } -impl VncStateMachine for VncDrawing { +impl VncState for VncDrawing { fn handle(&mut self, input: &[u8]) -> ProtocalHandlerOutput { if self.server_init { - // ProtocalHandlerOutput::WsBuf(self.get_framebuffer_update_request()) - self.handle_server_init(input) + let mut sr = StreamReader::new(input); + if self.num_rects_left > 0 { + // still in the previous update frame request + self.extract_rects(&mut sr); + self.render_rects() + } else { + let msg_type = sr.read_u8().unwrap(); + + match msg_type { + 0 => self.handle_framebuffer_update(&mut sr), + 1 => self.handle_set_colour_map(&mut sr), + 2 => self.handle_bell(&mut sr), + 3 => self.handle_server_cut_text(&mut sr), + _ => ProtocalHandlerOutput::Err(VNC_FAILED.to_string()), + } + } } else { self.handle_server_init(input) } } + fn frame_require(&self, incremental: u8) -> ProtocalHandlerOutput { + if self.num_rects_left > 0 { + ProtocalHandlerOutput::Ok + } else { + self.framebuffer_update_request(incremental) + } + } + fn done(&self) -> bool { false } - fn next(&self) -> Box { + fn next(&self) -> Box { Box::new(VncEnds) } } @@ -256,22 +277,196 @@ impl VncDrawing { // 4 CARD32 name-length // name-length CARD8 array name-string fn handle_server_init(&mut self, init: &[u8]) -> ProtocalHandlerOutput { - let mut sr = StreamReader::new(init); - self.width = sr.read_u16().unwrap(); - self.height = sr.read_u16().unwrap(); - let mut pfb: [u8; 16] = [0u8; 16]; - sr.extract_slice(16, &mut pfb); - // This pixel format will be used unless the client requests a different format using the SetPixelFormat message - self.pf = (&pfb).into(); - self.name = sr.read_string_l32().unwrap(); - ConsoleService::log(&format!("VNC: {}x{}", self.width, self.height)); - ProtocalHandlerOutput::Ok + self.buffer.extend_from_slice(init); + if self.buffer.len() > 24 { + let mut sr = StreamReader::new(init); + self.width = sr.read_u16().unwrap(); + self.height = sr.read_u16().unwrap(); + let mut pfb: [u8; 16] = [0u8; 16]; + sr.extract_slice(16, &mut pfb); + // This pixel format will be used unless the client requests a different format using the SetPixelFormat message + self.pf = (&pfb).into(); + self.name = sr.read_string_l32().unwrap(); + ConsoleService::log(&format!("VNC: {}x{}", self.width, self.height)); + self.server_init = true; + ProtocalHandlerOutput::SetCanvas(self.width, self.height) + } else { + ProtocalHandlerOutput::Ok + } + } + + // No. of bytes Type [Value] Description + // 1 CARD8 3 message-type + // 1 CARD8 incremental + // 2 CARD16 x-position + // 2 CARD16 y-position + // 2 CARD16 width + // 2 CARD16 height + fn framebuffer_update_request(&self, incremental: u8) -> ProtocalHandlerOutput { + // ConsoleService::log(&format!("VNC: framebuffer_update_request {}", incremental)); + let mut out: Vec = Vec::new(); + let mut sw = StreamWriter::new(&mut out); + sw.write_u8(3); + sw.write_u8(incremental); + sw.write_u16(0); + sw.write_u16(0); + sw.write_u16(self.width); + sw.write_u16(self.height); + ProtocalHandlerOutput::WsBuf(out) + } + + // No. of bytes Type [Value] Description + // 1 CARD8 0 message-type + // 1 padding + // 2 CARD16 number-of-rectangles + // This is followed by number-of-rectanglesrectangles of pixel data. + fn handle_framebuffer_update(&mut self, sr: &mut StreamReader) -> ProtocalHandlerOutput { + let _padding = sr.read_u8().unwrap(); + self.num_rects_left = sr.read_u16().unwrap(); + self.rects = Vec::with_capacity(self.num_rects_left as usize); + self.extract_rects(sr); + self.render_rects() + } + + fn handle_set_colour_map(&mut self, _sr: &mut StreamReader) -> ProtocalHandlerOutput { + unimplemented!() + } + + fn handle_bell(&mut self, _sr: &mut StreamReader) -> ProtocalHandlerOutput { + unimplemented!() + } + + fn handle_server_cut_text(&mut self, _sr: &mut StreamReader) -> ProtocalHandlerOutput { + unimplemented!() + } + + fn extract_rects(&mut self, sr: &mut StreamReader) { + while self.num_rects_left > 0 { + // we always keep the last rect in the vec + // and all the rects that has already been re-assembly should already been rendered + if self.rects.len() > 0 && self.rects.last().unwrap().left_data > 0 { + // which means that there is one rects that has not been re-assembly + let last_rect = self.rects.last_mut().unwrap(); + while let Some(v) = sr.read_u8() { + last_rect.encoding_data.push(v); + last_rect.left_data -= 1; + if last_rect.left_data == 0 { + // at the end of the rect + self.num_rects_left -= 1; + break; + } + } + ConsoleService::log(&format!( + "VNC read: {}, pending {}", + last_rect.encoding_data.len(), + last_rect.left_data + )); + if last_rect.left_data == 0 { + // there is still some data left + // start a new rect + // it must be handled in the else branch + continue; + } else { + // break the while loop + // render as much as we can + break; + } + } else { + // a brand new rects + let x = sr.read_u16().unwrap(); + let y = sr.read_u16().unwrap(); + let width = sr.read_u16().unwrap(); + let height = sr.read_u16().unwrap(); + let encoding_type = sr.read_u32().unwrap(); + match encoding_type { + 0 => { + let mut left_data = width as u32 * height as u32 * 4; + let mut encoding_data: Vec = Vec::with_capacity(left_data as usize); + while let Some(v) = sr.read_u8() { + // read as much as we can + encoding_data.push(v); + if encoding_data.len() == left_data as usize { + break; + } + } + left_data -= encoding_data.len() as u32; + if left_data == 0 { + self.num_rects_left -= 1; + } + // ConsoleService::log(&format!("VNC read new: {}", encoding_data.len())); + self.rects.push(VncRect { + x, + y, + width, + height, + encoding_data, + encoding_type, + left_data, + }); + // break the while loop + // render as much as we can + break; + } + _ => { + ConsoleService::log(&format!( + "VNC: unknown encoding type {}", + encoding_type + )); + ConsoleService::log(&format!( + "VNC: x:{}, y:{}, w:{}, h:{}", + x, y, width, height + )); + ConsoleService::log(&format!("VNC: left_data:{:x?}", sr.read_u32())); + unimplemented!() + } + } + } + } + } + + fn render_rects(&mut self) -> ProtocalHandlerOutput { + let mut out: Vec = Vec::new(); + if self.rects.len() > 1 || self.num_rects_left == 0 { + let drain_len = { + if self.num_rects_left != 0 { + self.rects.len() - 1 + } else { + self.rects.len() + } + }; + + ConsoleService::log(&format!("VNC render {} rects", drain_len)); + for x in self.rects.drain(0..drain_len) { + let mut data: Vec = Vec::with_capacity(x.encoding_data.len()); + for i in 0..x.width { + for j in 0..x.height { + let idx = (i as usize + j as usize * x.width as usize) * 4; + + let b = x.encoding_data[idx + 0]; + let g = x.encoding_data[idx + 1]; + let r = x.encoding_data[idx + 2]; + let a = x.encoding_data[idx + 3]; + + data.extend_from_slice(&[r, g, b, a]); + } + } + out.push(CanvasData { + x: x.x, + y: x.y, + width: x.width, + height: x.height, + data, + }); + } + } + + ProtocalHandlerOutput::RenderCanvas(out) } } struct VncEnds; -impl VncStateMachine for VncEnds { +impl VncState for VncEnds { fn handle(&mut self, _input: &[u8]) -> ProtocalHandlerOutput { ProtocalHandlerOutput::Err(VNC_FAILED.to_string()) } @@ -280,7 +475,7 @@ impl VncStateMachine for VncEnds { false } - fn next(&self) -> Box { + fn next(&self) -> Box { Box::new(VncEnds) } } @@ -397,3 +592,24 @@ impl Default for PixelFormat { } } } + +//Each rectangle consists of: +// 2 CARD16 x-position +// 2 CARD16 y-position +// 2 CARD16 width +// 2 CARD16 height +// 4 CARD32 encoding-type: +// 0 raw encoding +// 1 copy rectangle encoding +// 2 RRE encoding +// 4 CoRRE encoding +// 5 Hextile encoding +struct VncRect { + x: u16, + y: u16, + width: u16, + height: u16, + encoding_type: u32, + encoding_data: Vec, + left_data: u32, +}