Merge pull request #14 from HsuJv/dev

KeyEvent & PointerEvent support

backend/src/agent/agent.rs

@@ -111,6 +111,9 @@ impl StreamHandler<Result<Bytes, io::Error>> for Agent {
             }
             Err(err) => {
                 error!("error: {:?}", err);
+                if self.ws_addr.is_some() {
+                    self.ws_addr.as_ref().unwrap().do_send(ws::WsMsg::Close);
+                }
                 ctx.address().do_send(AgentMsg::Shutdown);
             }
         }

backend/src/agent/ws.rs

@@ -16,6 +16,7 @@ use super::agent::*;
 #[rtype(result = "()")]
 pub enum WsMsg {
     SendToClient(Bytes),
+    Close,
 }
 
 /// Define Websocket actor
@@ -41,6 +42,9 @@ impl Handler<WsMsg> for WsSession {
             WsMsg::SendToClient(data) => {
                 ctx.binary(data);
             }
+            WsMsg::Close => {
+                ctx.stop();
+            }
         };
     }
 }

backend/src/main.rs

@@ -44,7 +44,7 @@ impl Default for AppData {
 fn setup_logger() {
     let logger = femme::pretty::Logger::new();
     async_log::Logger::wrap(logger, || 12)
-        .start(log::LevelFilter::Trace)
+        .start(log::LevelFilter::Warn)
         .unwrap();
 }
 

frontend/src/app.rs

@@ -97,8 +97,8 @@ impl Component for App {
                     }
                 }
                 <footer class="footer">
-                    { "Powered by " }
+                    // { "Powered by " }
-                    <a href="https://yew.rs">{ "Yew" }</a>
+                    // <a href="https://yew.rs">{ "Yew" }</a>
                 </footer>
             </>
         }

frontend/src/lib.rs

@@ -5,8 +5,6 @@ mod protocal;
 mod utils;
 
 use wasm_bindgen::prelude::*;
-use wasm_bindgen::JsCast;
-use web_sys::{console, KeyboardEvent};
 
 #[cfg(feature = "wee_alloc")]
 #[global_allocator]
@@ -14,20 +12,6 @@ static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
 
 #[wasm_bindgen]
 pub fn run_app() -> Result<(), JsValue> {
-    let window = web_sys::window().unwrap();
-    let handler_submit = move |e: KeyboardEvent| {
-        e.stop_propagation();
-        console::log_1(&format!("{:?}", e).into())
-    };
-
-    let handler = Box::new(handler_submit) as Box<dyn FnMut(_)>;
-
-    let cb = Closure::wrap(handler);
-
-    window
-        .add_event_listener_with_callback("keydown", cb.as_ref().unchecked_ref())
-        .unwrap();
-    cb.forget();
     yew::start_app::<app::App>();
 
     Ok(())

frontend/src/pages/page_remote.rs

@@ -1,5 +1,5 @@
 use serde_json::{json, Value};
-use wasm_bindgen::{Clamped, JsValue};
+use wasm_bindgen::{prelude::Closure, Clamped, JsCast, JsValue};
 use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData};
 use yew::{
     format::Json,
@@ -15,7 +15,7 @@ use gloo::timers::callback::Interval;
 
 use crate::{
     components::{self, input::Input, ws::WebsocketMsg},
-    protocal::{common::*, vnc::VncHandler},
+    protocal::{common::*, vnc::vnc::VncHandler},
     utils::WeakComponentLink,
 };
 
@@ -155,7 +155,7 @@ impl Component for PageRemote {
                 if self.interval.is_none() {
                     let link = self.link.clone();
                     let tick =
-                        Interval::new(250, move || link.send_message(RemoteMsg::RequireFrame(1)));
+                        Interval::new(20, move || link.send_message(RemoteMsg::RequireFrame(1)));
                     self.interval = Some(tick);
                 }
                 self.protocal_out_handler()
@@ -256,6 +256,7 @@ impl PageRemote {
                         let canvas = self.canvas.cast::<HtmlCanvasElement>().unwrap();
                         canvas.set_width(width as u32);
                         canvas.set_height(height as u32);
+                        self.bind_mouse_and_key(&canvas);
                         self.link.send_message(RemoteMsg::RequireFrame(0));
                         let ctx = match &self.canvas_ctx {
                             Some(ctx) => ctx,
@@ -324,4 +325,104 @@ impl PageRemote {
             html! {}
         }
     }
+
+    fn bind_mouse_and_key(&mut self, canvas: &HtmlCanvasElement) {
+        let window = web_sys::window().unwrap();
+        let handler = self.handler.clone();
+        let key_down = move |e: KeyboardEvent| {
+            e.stop_propagation();
+            handler.key_press(e, true);
+        };
+
+        let handler = Box::new(key_down) as Box<dyn FnMut(_)>;
+
+        let cb = Closure::wrap(handler);
+
+        window
+            .add_event_listener_with_callback("keydown", cb.as_ref().unchecked_ref())
+            .unwrap();
+        cb.forget();
+
+        let handler = self.handler.clone();
+        let key_up = move |e: KeyboardEvent| {
+            e.stop_propagation();
+            handler.key_press(e, false);
+        };
+
+        let handler = Box::new(key_up) as Box<dyn FnMut(_)>;
+
+        let cb = Closure::wrap(handler);
+
+        window
+            .add_event_listener_with_callback("keyup", cb.as_ref().unchecked_ref())
+            .unwrap();
+        cb.forget();
+
+        // On a conventional mouse, buttons 1, 2, and 3 correspond to the left,
+        // middle, and right buttons on the mouse.  On a wheel mouse, each step
+        // of the wheel upwards is represented by a press and release of button
+        // 4, and each step downwards is represented by a press and release of
+        // button 5.
+
+        // to do:
+        // calculate relation position
+        let handler = self.handler.clone();
+        let mouse_move = move |e: MouseEvent| {
+            e.stop_propagation();
+            handler.mouse_event(e, MouseEventType::MouseMove);
+        };
+
+        let handler = Box::new(mouse_move) as Box<dyn FnMut(_)>;
+
+        let cb = Closure::wrap(handler);
+
+        canvas
+            .add_event_listener_with_callback("mousemove", cb.as_ref().unchecked_ref())
+            .unwrap();
+        cb.forget();
+
+        let handler = self.handler.clone();
+        let mouse_down = move |e: MouseEvent| {
+            e.stop_propagation();
+            handler.mouse_event(e, MouseEventType::MouseDown);
+        };
+
+        let handler = Box::new(mouse_down) as Box<dyn FnMut(_)>;
+
+        let cb = Closure::wrap(handler);
+
+        canvas
+            .add_event_listener_with_callback("mousedown", cb.as_ref().unchecked_ref())
+            .unwrap();
+        cb.forget();
+
+        let handler = self.handler.clone();
+        let mouse_up = move |e: MouseEvent| {
+            e.stop_propagation();
+            handler.mouse_event(e, MouseEventType::MouseUp);
+        };
+
+        let handler = Box::new(mouse_up) as Box<dyn FnMut(_)>;
+
+        let cb = Closure::wrap(handler);
+
+        canvas
+            .add_event_listener_with_callback("mouseup", cb.as_ref().unchecked_ref())
+            .unwrap();
+        cb.forget();
+
+        let get_context_menu = move |e: MouseEvent| {
+            e.prevent_default();
+            e.stop_propagation();
+        };
+
+        let handler = Box::new(get_context_menu) as Box<dyn FnMut(_)>;
+
+        let cb = Closure::wrap(handler);
+
+        canvas
+            .add_event_listener_with_callback("contextmenu", cb.as_ref().unchecked_ref())
+            .unwrap();
+        cb.forget();
+    }
 }

frontend/src/protocal/common.rs

@@ -1,3 +1,5 @@
+use std::{rc::Rc, sync::Mutex};
+
 pub struct CanvasData {
     pub x: u16,
     pub y: u16,
@@ -6,6 +8,12 @@ pub struct CanvasData {
     pub data: Vec<u8>,
 }
 
+pub enum MouseEventType {
+    MouseDown,
+    MouseUp,
+    MouseMove,
+}
+
 pub enum ProtocalHandlerOutput {
     WsBuf(Vec<u8>),
     Err(String),
@@ -20,7 +28,18 @@ pub struct ProtocalHandler<T>
 where
     T: ProtocalImpl,
 {
-    inner: T,
+    inner: Rc<Mutex<T>>,
+}
+
+impl<T> Clone for ProtocalHandler<T>
+where
+    T: ProtocalImpl,
+{
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
 }
 
 impl<T> ProtocalHandler<T>
@@ -28,36 +47,62 @@ where
     T: ProtocalImpl,
 {
     pub fn new() -> Self {
-        Self { inner: T::new() }
+        Self {
+            inner: Rc::new(Mutex::new(T::new())),
+        }
     }
 
-    pub fn do_input(&mut self, input: Vec<u8>) {
+    pub fn do_input(&self, input: Vec<u8>) {
-        self.inner.do_input(input);
+        self.inner.as_ref().lock().unwrap().do_input(input);
     }
 
-    pub fn get_output(&mut self) -> Vec<ProtocalHandlerOutput> {
+    pub fn get_output(&self) -> Vec<ProtocalHandlerOutput> {
-        self.inner.get_output()
+        self.inner.as_ref().lock().unwrap().get_output()
     }
 
-    pub fn set_credential(&mut self, username: &str, password: &str) {
+    pub fn set_credential(&self, username: &str, password: &str) {
-        self.inner.set_credential(username, password);
+        self.inner
+            .as_ref()
+            .lock()
+            .unwrap()
+            .set_credential(username, password);
     }
 
-    pub fn set_resolution(&mut self, width: u16, height: u16) {
+    pub fn set_resolution(&self, width: u16, height: u16) {
-        self.inner.set_resolution(width, height);
+        self.inner
+            .as_ref()
+            .lock()
+            .unwrap()
+            .set_resolution(width, height);
     }
 
-    pub fn require_frame(&mut self, incremental: u8) {
+    pub fn require_frame(&self, incremental: u8) {
-        self.inner.require_frame(incremental);
+        self.inner
+            .as_ref()
+            .lock()
+            .unwrap()
+            .require_frame(incremental);
+    }
+
+    pub fn key_press(&self, key: web_sys::KeyboardEvent, down: bool) {
+        self.inner.as_ref().lock().unwrap().key_press(key, down);
+    }
+
+    pub fn mouse_event(&self, mouse: web_sys::MouseEvent, et: MouseEventType) {
+        self.inner.as_ref().lock().unwrap().mouse_event(mouse, et);
     }
 }
 
 pub trait ProtocalImpl {
-    fn new() -> Self;
+    fn new() -> Self
+    where
+        Self: Sized;
     fn do_input(&mut self, input: Vec<u8>);
     fn get_output(&mut self) -> Vec<ProtocalHandlerOutput>;
     fn set_credential(&mut self, username: &str, password: &str);
     fn set_resolution(&mut self, width: u16, height: u16);
+    fn key_press(&mut self, key: web_sys::KeyboardEvent, down: bool);
+    fn mouse_event(&mut self, mouse: web_sys::MouseEvent, et: MouseEventType);
     fn require_frame(&mut self, incremental: u8);
 }
 

frontend/src/protocal/mod.rs

@@ -1,3 +1,2 @@
 pub mod common;
-mod des;
 pub mod vnc;

frontend/src/protocal/vnc/mod.rs

@@ -0,0 +1,4 @@
+mod des;
+pub mod vnc;
+mod x11cursor;
+mod x11keyboard;

frontend/src/protocal/vnc/vnc.rs

@@ -1,5 +1,6 @@
-use super::common::*;
+use crate::protocal::common::MouseEventType;
-use super::des;
+
+use super::{super::common::*, des, x11cursor::MouseUtils, x11keyboard};
 use yew::services::ConsoleService;
 
 const VNC_RFB33: &[u8; 12] = b"RFB 003.003\n";
@@ -35,6 +36,7 @@ pub enum VncEncoding {
     DesktopSizePseudo = -223,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum VncState {
     Init,
     Handshake,
@@ -58,6 +60,7 @@ pub struct VncHandler {
     security_type: SecurityType,
     challenge: [u8; 16],
     reader: StreamReader,
+    mouse: MouseUtils,
     require: usize,
     width: u16,
     height: u16,
@@ -87,6 +90,7 @@ impl ProtocalImpl for VncHandler {
             security_type: SecurityType::Invalid,
             challenge: [0; 16],
             reader: StreamReader::new(Vec::with_capacity(10)),
+            mouse: MouseUtils::new(),
             require: 12, // the handleshake message length
             width: 0,
             height: 0,
@@ -162,6 +166,26 @@ impl ProtocalImpl for VncHandler {
         // VNC client doen't support resolution change
     }
 
+    fn key_press(&mut self, key: web_sys::KeyboardEvent, down: bool) {
+        if self.state != VncState::Connected {
+            return;
+        }
+        let key = x11keyboard::KeyboardUtils::get_keysym(key);
+        if let ServerMessage::None = self.msg_handling {
+            self.send_key_event(key, down);
+        }
+    }
+
+    fn mouse_event(&mut self, mouse: web_sys::MouseEvent, et: MouseEventType) {
+        if self.state != VncState::Connected {
+            return;
+        }
+        let (x, y, mask) = self.mouse.get_mouse_sym(mouse, et);
+        if let ServerMessage::None = self.msg_handling {
+            self.send_pointer_event(x, y, mask);
+        }
+    }
+
     fn require_frame(&mut self, incremental: u8) {
         if 0 == incremental {
             // first frame
@@ -264,6 +288,46 @@ impl VncHandler {
         self.outbuf.extend_from_slice(&out);
     }
 
+    // +--------------+--------------+--------------+
+    // | No. of bytes | Type [Value] | Description  |
+    // +--------------+--------------+--------------+
+    // | 1            | U8 [4]       | message-type |
+    // | 1            | U8           | down-flag    |
+    // | 2            |              | padding      |
+    // | 4            | U32          | key          |
+    // +--------------+--------------+--------------+
+    fn send_key_event(&mut self, key: u32, down: bool) {
+        let mut out = Vec::with_capacity(10);
+        let mut sw = StreamWriter::new(&mut out);
+        sw.write_u8(4); // message-type
+        sw.write_u8(if down { 1 } else { 0 }); // down
+        sw.write_u16(0); // padding
+        sw.write_u32(key); // key
+
+        // ConsoleService::log(&format!("send key event {:x?} {:?}", key, down));
+        self.outbuf.extend_from_slice(&out);
+    }
+
+    // +--------------+--------------+--------------+
+    // | No. of bytes | Type [Value] | Description  |
+    // +--------------+--------------+--------------+
+    // | 1            | U8 [5]       | message-type |
+    // | 1            | U8           | button-mask  |
+    // | 2            | U16          | x-position   |
+    // | 2            | U16          | y-position   |
+    // +--------------+--------------+--------------+
+    fn send_pointer_event(&mut self, x: u16, y: u16, mask: u8) {
+        let mut out = Vec::with_capacity(10);
+        let mut sw = StreamWriter::new(&mut out);
+        sw.write_u8(5); // message-type
+        sw.write_u8(mask); // mask
+        sw.write_u16(x); // x
+        sw.write_u16(y); // y
+
+        ConsoleService::log(&format!("send mouse event {:x?} {:x?} {:#08b}", x, y, mask));
+        self.outbuf.extend_from_slice(&out);
+    }
+
     // No. of bytes     Type    [Value]     Description
     // 1                CARD8   3           message-type
     // 1                CARD8               incremental

frontend/src/protocal/vnc/x11cursor.rs

@@ -0,0 +1,38 @@
+use crate::protocal::common::MouseEventType;
+
+pub struct MouseUtils {
+    down: bool,
+    mask: u8,
+}
+
+impl MouseUtils {
+    pub fn new() -> Self {
+        Self {
+            down: false,
+            mask: 0,
+        }
+    }
+    pub fn get_mouse_sym(
+        &mut self,
+        event: web_sys::MouseEvent,
+        et: MouseEventType,
+    ) -> (u16, u16, u8) {
+        let x: u16 = event.client_x().try_into().unwrap_or(0);
+        let y: u16 = event.client_y().try_into().unwrap_or(0);
+        let mask: u8 = (event.button() << 1).try_into().unwrap_or(0);
+
+        match et {
+            MouseEventType::MouseDown => {
+                self.down = true;
+                self.mask = self.down as u8 | mask;
+            }
+            MouseEventType::MouseUp => {
+                self.down = false;
+                self.mask = self.down as u8 & (!mask);
+            }
+
+            MouseEventType::MouseMove => {}
+        }
+        (x, y, self.mask)
+    }
+}