vnc raw encoding

backend/src/agent/agent.rs

@@ -15,8 +15,8 @@ struct TcpCodec;
 impl Encoder<Bytes> for TcpCodec {
     type Error = io::Error;
 
-    fn encode(&mut self, item: Bytes, _dst: &mut BytesMut) -> Result<(), Self::Error> {
+    fn encode(&mut self, _item: Bytes, _dst: &mut BytesMut) -> Result<(), Self::Error> {
-        info!("encoding: {:?}", item);
+        // info!("encoding: {:?}", item);
         Ok(())
     }
 }
@@ -26,7 +26,7 @@ impl Decoder for TcpCodec {
     type Error = io::Error;
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        info!("recv from server: {:?}", src);
+        // info!("recv from server: {:?}", src);
         if 0 == src.len() {
             return Ok(None);
         }
@@ -98,7 +98,7 @@ impl StreamHandler<Result<Bytes, io::Error>> for Agent {
     fn handle(&mut self, msg: Result<Bytes, io::Error>, ctx: &mut Context<Self>) {
         match msg {
             Ok(data) => {
-                info!("recv from server: {:?}", data);
+                // info!("recv from server: {:?}", data);
                 if self.ws_addr.is_some() {
                     ctx.address().do_send(AgentMsg::SendToClient(data));
                 } else {

backend/src/user/auth.rs

@@ -51,7 +51,7 @@ impl Handler<AuthMsg> for Authenticator {
 
     fn handle(&mut self, msg: AuthMsg, _ctx: &mut Context<Self>) -> Self::Result {
         match msg {
-            AuthMsg::DoAuth(auth_info) => {
+            AuthMsg::DoAuth(_auth_info) => {
                 // if auth_info.username == "admin" && auth_info.password == "admin" {
                 //     AuthResult::AuthSuccess
                 // } else {

frontend/Cargo.toml

@@ -20,7 +20,8 @@ crate-type = ["cdylib", "rlib"]
 wasm-bindgen = "^0.2"
 yew = "0.18"
 js-sys = "0.3.55"
-web-sys = "0.3.55"
+web-sys = {version="0.3.55", features=["HtmlCanvasElement", "CanvasRenderingContext2d", "ImageData"]}
+gloo = "0.4.0"
 yew-router = "0.15"
 # The `console_error_panic_hook` crate provides better debugging of panics by
 # logging them with `console.error`. This is great for development, but requires

frontend/src/pages/page_remote.rs

@@ -1,4 +1,6 @@
 use serde_json::{json, Value};
+use wasm_bindgen::{Clamped, JsValue};
+use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData};
 use yew::{
     format::Json,
     html,
@@ -9,6 +11,8 @@ use yew::{
     },
 };
 
+use gloo::timers::callback::Interval;
+
 use crate::{
     components::{self, input::Input, ws::WebsocketMsg},
     protocal::{common::*, vnc::VncHandler},
@@ -27,6 +31,9 @@ pub struct PageRemote {
     request_password: bool,
     username: String,
     password: String,
+    canvas: NodeRef,
+    canvas_ctx: Option<CanvasRenderingContext2d>,
+    interval: Option<Interval>,
 }
 
 #[derive(Clone, PartialEq, Properties)]
@@ -41,6 +48,7 @@ pub enum RemoteMsg {
     UpdateUsername(String),
     UpdatePassword(String),
     SendCredential,
+    RequireFrame(u8),
 }
 
 impl Component for PageRemote {
@@ -60,6 +68,9 @@ impl Component for PageRemote {
             request_password: false,
             username: String::from(""),
             password: String::from(""),
+            canvas: NodeRef::default(),
+            canvas_ctx: None,
+            interval: None,
         }
     }
 
@@ -140,6 +151,16 @@ impl Component for PageRemote {
                 let _ = self.protocal_out_handler(out);
                 true
             }
+            RemoteMsg::RequireFrame(incremental) => {
+                let out = self.handler.require_frame(incremental);
+                if self.interval.is_none() {
+                    let link = self.link.clone();
+                    let tick =
+                        Interval::new(250, move || link.send_message(RemoteMsg::RequireFrame(1)));
+                    self.interval = Some(tick);
+                }
+                self.protocal_out_handler(out)
+            }
         }
     }
 
@@ -167,6 +188,7 @@ impl Component for PageRemote {
                         {self.button_connect_view()}
                         <components::ws::WebsocketCtx
                         weak_link=ws_link onrecv=recv_msg/>
+                        <canvas id="remote-canvas"  ref=self.canvas.clone() ></canvas>
                         {self.error_msg.clone()}
                     </div>
                 </>
@@ -197,7 +219,42 @@ impl PageRemote {
                 self.request_password = true;
                 true
             }
-            _ => unimplemented!(),
+            ProtocalHandlerOutput::RenderCanvas(crs) => {
+                let canvas = self.canvas.cast::<HtmlCanvasElement>().unwrap();
+                let ctx = match &self.canvas_ctx {
+                    Some(ctx) => ctx,
+                    None => {
+                        let ctx = CanvasRenderingContext2d::from(JsValue::from(
+                            canvas.get_context("2d").unwrap().unwrap(),
+                        ));
+                        self.canvas_ctx = Some(ctx);
+                        self.canvas_ctx.as_ref().unwrap()
+                    }
+                };
+
+                for cr in crs {
+                    let data = ImageData::new_with_u8_clamped_array_and_sh(
+                        Clamped(&cr.data),
+                        cr.width as u32,
+                        cr.height as u32,
+                    )
+                    .unwrap();
+                    ConsoleService::log(&format!(
+                        "renderring at ({}, {}), width {}, height {}",
+                        cr.x, cr.y, cr.width, cr.height
+                    ));
+                    let _ = ctx.put_image_data(&data, cr.x as f64, cr.y as f64);
+                }
+                true
+            }
+            ProtocalHandlerOutput::SetCanvas(width, height) => {
+                let canvas = self.canvas.cast::<HtmlCanvasElement>().unwrap();
+                canvas.set_width(width as u32);
+                canvas.set_height(height as u32);
+                self.link.send_message(RemoteMsg::RequireFrame(0));
+                true
+            }
+            _ => false,
         }
     }
 

frontend/src/protocal/common.rs

@@ -1,10 +1,21 @@
 use std::slice::Iter;
+
+pub struct CanvasData {
+    pub x: u16,
+    pub y: u16,
+    pub width: u16,
+    pub height: u16,
+    pub data: Vec<u8>,
+}
+
 pub enum ProtocalHandlerOutput {
     Ok,
     WsBuf(Vec<u8>),
     Err(String),
     RequireUsername,
     RequirePassword,
+    SetCanvas(u16, u16),
+    RenderCanvas(Vec<CanvasData>),
 }
 
 pub struct ProtocalHandler<T>
@@ -29,12 +40,17 @@ where
     pub fn set_credential(&mut self, username: &str, password: &str) -> ProtocalHandlerOutput {
         self.inner.set_credential(username, password)
     }
+
+    pub fn require_frame(&mut self, incremental: u8) -> ProtocalHandlerOutput {
+        self.inner.require_frame(incremental)
+    }
 }
 
 pub trait ProtocalImpl {
     fn new() -> Self;
     fn handle(&mut self, input: &[u8]) -> ProtocalHandlerOutput;
     fn set_credential(&mut self, username: &str, password: &str) -> ProtocalHandlerOutput;
+    fn require_frame(&mut self, incremental: u8) -> ProtocalHandlerOutput;
 }
 
 pub struct StreamReader<'a> {
@@ -112,3 +128,55 @@ impl<'a> StreamReader<'a> {
         self.inner.next().is_none()
     }
 }
+
+pub struct StreamWriter<'a> {
+    inner: &'a mut Vec<u8>,
+}
+
+impl<'a> StreamWriter<'a> {
+    pub fn new(buf: &'a mut Vec<u8>) -> Self {
+        Self { inner: buf }
+    }
+
+    pub fn write_u8(&mut self, b: u8) {
+        self.inner.push(b);
+    }
+
+    pub fn write_u16(&mut self, b: u16) {
+        self.inner.extend_from_slice(&b.to_be_bytes());
+    }
+
+    pub fn write_u32(&mut self, b: u32) {
+        self.inner.extend_from_slice(&b.to_be_bytes());
+    }
+
+    pub fn write_s8(&mut self, b: i8) {
+        self.write_u8(b as u8);
+    }
+
+    pub fn write_s16(&mut self, b: i16) {
+        self.write_u16(b as u16);
+    }
+
+    pub fn write_s32(&mut self, b: i32) {
+        self.write_u32(b as u32);
+    }
+
+    pub fn write_string_with_len(&mut self, s: &str) {
+        self.inner.extend_from_slice(s.as_bytes());
+    }
+
+    pub fn write_string_l16(&mut self, s: &str) {
+        self.write_u16(s.len() as u16);
+        self.write_string_with_len(s);
+    }
+
+    pub fn write_string_l32(&mut self, s: &str) {
+        self.write_u32(s.len() as u32);
+        self.write_string_with_len(s);
+    }
+
+    pub fn write_slice(&mut self, s: &[u8]) {
+        self.inner.extend_from_slice(s);
+    }
+}

frontend/src/protocal/vnc.rs

@@ -8,20 +8,6 @@ const VNC_RFB38: &[u8; 12] = b"RFB 003.008\n";
 const VNC_VER_UNSUPPORTED: &str = "unsupported version";
 const VNC_FAILED: &str = "Connection failed with unknow reason";
 
-#[derive(Debug, Clone, Copy)]
-enum VncState {
-    Handshake,
-    Authentication,
-}
-
-#[derive(Debug, Clone, Copy)]
-enum VncVersion {
-    NONE,
-    VNC33,
-    VNC37,
-    VNC38,
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 #[repr(u8)]
 pub enum SecurityType {
@@ -37,7 +23,7 @@ pub enum SecurityType {
 }
 
 pub struct VncHandler {
-    inner: Box<dyn VncStateMachine>,
+    inner: Box<dyn VncState>,
 }
 
 impl ProtocalImpl for VncHandler {
@@ -75,12 +61,19 @@ impl ProtocalImpl for VncHandler {
         }
         self.inner.handle(&pass_bytes)
     }
+
+    fn require_frame(&mut self, incremental: u8) -> ProtocalHandlerOutput {
+        self.inner.frame_require(incremental)
+    }
 }
 
-trait VncStateMachine {
+trait VncState {
     fn handle(&mut self, _input: &[u8]) -> ProtocalHandlerOutput;
+    fn frame_require(&self, _incremental: u8) -> ProtocalHandlerOutput {
+        ProtocalHandlerOutput::Err(VNC_FAILED.to_string())
+    }
     fn done(&self) -> bool;
-    fn next(&self) -> Box<dyn VncStateMachine>;
+    fn next(&self) -> Box<dyn VncState>;
 }
 
 struct VncHandShake {
@@ -93,7 +86,7 @@ impl Default for VncHandShake {
     }
 }
 
-impl VncStateMachine for VncHandShake {
+impl VncState for VncHandShake {
     fn handle(&mut self, rfbversion: &[u8]) -> ProtocalHandlerOutput {
         let support_version = match rfbversion {
             b"RFB 003.003\n" => Ok(VNC_RFB33),
@@ -113,7 +106,7 @@ impl VncStateMachine for VncHandShake {
         self.done
     }
 
-    fn next(&self) -> Box<dyn VncStateMachine> {
+    fn next(&self) -> Box<dyn VncState> {
         Box::new(VncAuthentiacator::default())
     }
 }
@@ -136,7 +129,7 @@ impl Default for VncAuthentiacator {
     }
 }
 
-impl VncStateMachine for VncAuthentiacator {
+impl VncState for VncAuthentiacator {
     fn handle(&mut self, input: &[u8]) -> ProtocalHandlerOutput {
         if self.security_type == SecurityType::VncAuth {
             if self.wait_password {
@@ -153,7 +146,7 @@ impl VncStateMachine for VncAuthentiacator {
         self.done
     }
 
-    fn next(&self) -> Box<dyn VncStateMachine> {
+    fn next(&self) -> Box<dyn VncState> {
         Box::new(VncDrawing::default())
     }
 }
@@ -213,6 +206,9 @@ struct VncDrawing {
     pf: PixelFormat,
     name: String,
     server_init: bool,
+    buffer: Vec<u8>,
+    rects: Vec<VncRect>,
+    num_rects_left: u16,
 }
 
 impl Default for VncDrawing {
@@ -223,25 +219,50 @@ impl Default for VncDrawing {
             pf: PixelFormat::default(),
             name: "".to_string(),
             server_init: false,
+            buffer: Vec::with_capacity(50),
+            rects: Vec::new(),
+            num_rects_left: 0,
         }
     }
 }
 
-impl VncStateMachine for VncDrawing {
+impl VncState for VncDrawing {
     fn handle(&mut self, input: &[u8]) -> ProtocalHandlerOutput {
         if self.server_init {
-            // ProtocalHandlerOutput::WsBuf(self.get_framebuffer_update_request())
+            let mut sr = StreamReader::new(input);
-            self.handle_server_init(input)
+            if self.num_rects_left > 0 {
+                // still in the previous update frame request
+                self.extract_rects(&mut sr);
+                self.render_rects()
+            } else {
+                let msg_type = sr.read_u8().unwrap();
+
+                match msg_type {
+                    0 => self.handle_framebuffer_update(&mut sr),
+                    1 => self.handle_set_colour_map(&mut sr),
+                    2 => self.handle_bell(&mut sr),
+                    3 => self.handle_server_cut_text(&mut sr),
+                    _ => ProtocalHandlerOutput::Err(VNC_FAILED.to_string()),
+                }
+            }
         } else {
             self.handle_server_init(input)
         }
     }
 
+    fn frame_require(&self, incremental: u8) -> ProtocalHandlerOutput {
+        if self.num_rects_left > 0 {
+            ProtocalHandlerOutput::Ok
+        } else {
+            self.framebuffer_update_request(incremental)
+        }
+    }
+
     fn done(&self) -> bool {
         false
     }
 
-    fn next(&self) -> Box<dyn VncStateMachine> {
+    fn next(&self) -> Box<dyn VncState> {
         Box::new(VncEnds)
     }
 }
@@ -256,6 +277,8 @@ impl VncDrawing {
     // 4            CARD32          name-length
     // name-length  CARD8           array name-string
     fn handle_server_init(&mut self, init: &[u8]) -> ProtocalHandlerOutput {
+        self.buffer.extend_from_slice(init);
+        if self.buffer.len() > 24 {
             let mut sr = StreamReader::new(init);
             self.width = sr.read_u16().unwrap();
             self.height = sr.read_u16().unwrap();
@@ -265,13 +288,185 @@ impl VncDrawing {
             self.pf = (&pfb).into();
             self.name = sr.read_string_l32().unwrap();
             ConsoleService::log(&format!("VNC: {}x{}", self.width, self.height));
+            self.server_init = true;
+            ProtocalHandlerOutput::SetCanvas(self.width, self.height)
+        } else {
             ProtocalHandlerOutput::Ok
         }
     }
 
+    // No. of bytes     Type    [Value]     Description
+    // 1                CARD8   3           message-type
+    // 1                CARD8               incremental
+    // 2                CARD16              x-position
+    // 2                CARD16              y-position
+    // 2                CARD16              width
+    // 2                CARD16              height
+    fn framebuffer_update_request(&self, incremental: u8) -> ProtocalHandlerOutput {
+        // ConsoleService::log(&format!("VNC: framebuffer_update_request {}", incremental));
+        let mut out: Vec<u8> = Vec::new();
+        let mut sw = StreamWriter::new(&mut out);
+        sw.write_u8(3);
+        sw.write_u8(incremental);
+        sw.write_u16(0);
+        sw.write_u16(0);
+        sw.write_u16(self.width);
+        sw.write_u16(self.height);
+        ProtocalHandlerOutput::WsBuf(out)
+    }
+
+    // No. of bytes     Type    [Value]     Description
+    // 1                CARD8   0           message-type
+    // 1                                    padding
+    // 2                CARD16              number-of-rectangles
+    // This is followed by number-of-rectanglesrectangles of pixel data.
+    fn handle_framebuffer_update(&mut self, sr: &mut StreamReader) -> ProtocalHandlerOutput {
+        let _padding = sr.read_u8().unwrap();
+        self.num_rects_left = sr.read_u16().unwrap();
+        self.rects = Vec::with_capacity(self.num_rects_left as usize);
+        self.extract_rects(sr);
+        self.render_rects()
+    }
+
+    fn handle_set_colour_map(&mut self, _sr: &mut StreamReader) -> ProtocalHandlerOutput {
+        unimplemented!()
+    }
+
+    fn handle_bell(&mut self, _sr: &mut StreamReader) -> ProtocalHandlerOutput {
+        unimplemented!()
+    }
+
+    fn handle_server_cut_text(&mut self, _sr: &mut StreamReader) -> ProtocalHandlerOutput {
+        unimplemented!()
+    }
+
+    fn extract_rects(&mut self, sr: &mut StreamReader) {
+        while self.num_rects_left > 0 {
+            // we always keep the last rect in the vec
+            // and all the rects that has already been re-assembly should already been rendered
+            if self.rects.len() > 0 && self.rects.last().unwrap().left_data > 0 {
+                // which means that there is one rects that has not been re-assembly
+                let last_rect = self.rects.last_mut().unwrap();
+                while let Some(v) = sr.read_u8() {
+                    last_rect.encoding_data.push(v);
+                    last_rect.left_data -= 1;
+                    if last_rect.left_data == 0 {
+                        // at the end of the rect
+                        self.num_rects_left -= 1;
+                        break;
+                    }
+                }
+                ConsoleService::log(&format!(
+                    "VNC read: {}, pending {}",
+                    last_rect.encoding_data.len(),
+                    last_rect.left_data
+                ));
+                if last_rect.left_data == 0 {
+                    // there is still some data left
+                    // start a new rect
+                    // it must be handled in the else branch
+                    continue;
+                } else {
+                    // break the while loop
+                    // render as much as we can
+                    break;
+                }
+            } else {
+                // a brand new rects
+                let x = sr.read_u16().unwrap();
+                let y = sr.read_u16().unwrap();
+                let width = sr.read_u16().unwrap();
+                let height = sr.read_u16().unwrap();
+                let encoding_type = sr.read_u32().unwrap();
+                match encoding_type {
+                    0 => {
+                        let mut left_data = width as u32 * height as u32 * 4;
+                        let mut encoding_data: Vec<u8> = Vec::with_capacity(left_data as usize);
+                        while let Some(v) = sr.read_u8() {
+                            // read as much as we can
+                            encoding_data.push(v);
+                            if encoding_data.len() == left_data as usize {
+                                break;
+                            }
+                        }
+                        left_data -= encoding_data.len() as u32;
+                        if left_data == 0 {
+                            self.num_rects_left -= 1;
+                        }
+                        // ConsoleService::log(&format!("VNC read new: {}", encoding_data.len()));
+                        self.rects.push(VncRect {
+                            x,
+                            y,
+                            width,
+                            height,
+                            encoding_data,
+                            encoding_type,
+                            left_data,
+                        });
+                        // break the while loop
+                        // render as much as we can
+                        break;
+                    }
+                    _ => {
+                        ConsoleService::log(&format!(
+                            "VNC: unknown encoding type {}",
+                            encoding_type
+                        ));
+                        ConsoleService::log(&format!(
+                            "VNC: x:{}, y:{}, w:{}, h:{}",
+                            x, y, width, height
+                        ));
+                        ConsoleService::log(&format!("VNC: left_data:{:x?}", sr.read_u32()));
+                        unimplemented!()
+                    }
+                }
+            }
+        }
+    }
+
+    fn render_rects(&mut self) -> ProtocalHandlerOutput {
+        let mut out: Vec<CanvasData> = Vec::new();
+        if self.rects.len() > 1 || self.num_rects_left == 0 {
+            let drain_len = {
+                if self.num_rects_left != 0 {
+                    self.rects.len() - 1
+                } else {
+                    self.rects.len()
+                }
+            };
+
+            ConsoleService::log(&format!("VNC render {} rects", drain_len));
+            for x in self.rects.drain(0..drain_len) {
+                let mut data: Vec<u8> = Vec::with_capacity(x.encoding_data.len());
+                for i in 0..x.width {
+                    for j in 0..x.height {
+                        let idx = (i as usize + j as usize * x.width as usize) * 4;
+
+                        let b = x.encoding_data[idx + 0];
+                        let g = x.encoding_data[idx + 1];
+                        let r = x.encoding_data[idx + 2];
+                        let a = x.encoding_data[idx + 3];
+
+                        data.extend_from_slice(&[r, g, b, a]);
+                    }
+                }
+                out.push(CanvasData {
+                    x: x.x,
+                    y: x.y,
+                    width: x.width,
+                    height: x.height,
+                    data,
+                });
+            }
+        }
+
+        ProtocalHandlerOutput::RenderCanvas(out)
+    }
+}
+
 struct VncEnds;
 
-impl VncStateMachine for VncEnds {
+impl VncState for VncEnds {
     fn handle(&mut self, _input: &[u8]) -> ProtocalHandlerOutput {
         ProtocalHandlerOutput::Err(VNC_FAILED.to_string())
     }
@@ -280,7 +475,7 @@ impl VncStateMachine for VncEnds {
         false
     }
 
-    fn next(&self) -> Box<dyn VncStateMachine> {
+    fn next(&self) -> Box<dyn VncState> {
         Box::new(VncEnds)
     }
 }
@@ -397,3 +592,24 @@ impl Default for PixelFormat {
         }
     }
 }
+
+//Each rectangle consists of:
+// 2                CARD16              x-position
+// 2                CARD16              y-position
+// 2                CARD16              width
+// 2                CARD16              height
+// 4                CARD32              encoding-type:
+//                          0           raw encoding
+//                          1           copy rectangle encoding
+//                          2           RRE encoding
+//                          4           CoRRE encoding
+//                          5           Hextile encoding
+struct VncRect {
+    x: u16,
+    y: u16,
+    width: u16,
+    height: u16,
+    encoding_type: u32,
+    encoding_data: Vec<u8>,
+    left_data: u32,
+}