vncviewer: support for back/forward mouse buttons

This commit contains work originally done by manny33:
  * #1711

This commit implements the pseudo-encoding ExtendedMouseButtons which
makes it possible to use the back/forward mouse buttons.

With this change, we have to keep track of the mouse button state
ourselves, as FLTK does not keep track of button states other than the
three standard buttons (LMB/MMB/RMB). Unfortunately, there is a bug in
FLTK where don't get enough information in certain edge cases to keep a
consistent state. There is a workaround in this commit for the three
standard buttons, but not for the back/forward buttons.

common/rfb/CConnection.cxx

@@ -835,6 +835,7 @@ void CConnection::updateEncodings()
   encodings.push_back(pseudoEncodingContinuousUpdates);
   encodings.push_back(pseudoEncodingFence);
   encodings.push_back(pseudoEncodingQEMUKeyEvent);
+  encodings.push_back(pseudoEncodingExtendedMouseButtons);
 
   if (Decoder::supported(preferredEncoding)) {
     encodings.push_back(preferredEncoding);

common/rfb/CMsgHandler.cxx

@@ -75,6 +75,11 @@ void CMsgHandler::endOfContinuousUpdates()
   server.supportsContinuousUpdates = true;
 }
 
+void CMsgHandler::supportsExtendedMouseButtons()
+{
+  server.supportsExtendedMouseButtons = true;
+}
+
 void CMsgHandler::supportsQEMUKeyEvent()
 {
   server.supportsQEMUKeyEvent = true;

common/rfb/CMsgHandler.h

@@ -57,6 +57,7 @@ namespace rfb {
     virtual void fence(uint32_t flags, unsigned len, const uint8_t data[]);
     virtual void endOfContinuousUpdates();
     virtual void supportsQEMUKeyEvent();
+    virtual void supportsExtendedMouseButtons();
     virtual void serverInit(int width, int height,
                             const PixelFormat& pf,
                             const char* name) = 0;

common/rfb/CMsgReader.cxx

@@ -202,6 +202,10 @@ bool CMsgReader::readMsg()
       handler->supportsQEMUKeyEvent();
       ret = true;
       break;
+    case pseudoEncodingExtendedMouseButtons:
+      handler->supportsExtendedMouseButtons();
+      ret = true;
+      break;
     default:
       ret = readRect(dataRect, rectEncoding);
       break;

common/rfb/CMsgWriter.cxx

@@ -176,15 +176,37 @@ void CMsgWriter::writeKeyEvent(uint32_t keysym, uint32_t keycode, bool down)
 void CMsgWriter::writePointerEvent(const Point& pos, uint16_t buttonMask)
 {
   Point p(pos);
+  bool extendedMouseButtons;
+
   if (p.x < 0) p.x = 0;
   if (p.y < 0) p.y = 0;
   if (p.x >= server->width()) p.x = server->width() - 1;
   if (p.y >= server->height()) p.y = server->height() - 1;
 
+  // Only send extended pointerEvent message when needed
+  extendedMouseButtons = buttonMask & 0x180;
+
   startMsg(msgTypePointerEvent);
-  os->writeU8(buttonMask);
-  os->writeU16(p.x);
-  os->writeU16(p.y);
+  if (server->supportsExtendedMouseButtons && extendedMouseButtons) {
+    int higherBits;
+    int lowerBits;
+
+    higherBits = (buttonMask  >> 7) & 0xff;
+    lowerBits = buttonMask & 0x7f;
+    lowerBits |= 0x80; // Set marker bit to 1
+
+    higherBits &= 0x03; // Clear reserved bits
+
+    os->writeU8(lowerBits);
+    os->writeU16(p.x);
+    os->writeU16(p.y);
+    os->writeU8(higherBits);
+  } else {
+    buttonMask &= 0x7f; // Set marker bit to 0
+    os->writeU8(buttonMask);
+    os->writeU16(p.x);
+    os->writeU16(p.y);
+  }
   endMsg();
 }
 

common/rfb/ServerParams.cxx

@@ -32,7 +32,7 @@ ServerParams::ServerParams()
   : majorVersion(0), minorVersion(0),
     supportsQEMUKeyEvent(false),
     supportsSetDesktopSize(false), supportsFence(false),
-    supportsContinuousUpdates(false),
+    supportsContinuousUpdates(false), supportsExtendedMouseButtons(false),
     width_(0), height_(0),
     ledState_(ledUnknown)
 {

common/rfb/ServerParams.h

@@ -79,6 +79,7 @@ namespace rfb {
     bool supportsSetDesktopSize;
     bool supportsFence;
     bool supportsContinuousUpdates;
+    bool supportsExtendedMouseButtons;
 
   private:
 

tests/unit/emulatemb.cxx

@@ -42,14 +42,14 @@ rfb::BoolParameter emulateMiddleButton("dummy_name", "dummy_desc", true);
 class TestClass : public EmulateMB
 {
 public:
-  void sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask) override;
+  void sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask) override;
 
-  struct PointerEventParams {rfb::Point pos; uint8_t mask; };
+  struct PointerEventParams {rfb::Point pos; uint16_t mask; };
 
   std::vector<PointerEventParams> results;
 };
 
-void TestClass::sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void TestClass::sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   PointerEventParams params;
   params.pos = pos;

vncviewer/EmulateMB.cxx

@@ -199,7 +199,7 @@ EmulateMB::EmulateMB()
 {
 }
 
-void EmulateMB::filterPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void EmulateMB::filterPointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   int btstate;
   int action1, action2;
@@ -280,7 +280,7 @@ void EmulateMB::filterPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
 void EmulateMB::handleTimeout(rfb::Timer *t)
 {
   int action1, action2;
-  uint8_t buttonMask;
+  uint16_t buttonMask;
 
   if (&timer != t)
     return;
@@ -312,7 +312,7 @@ void EmulateMB::handleTimeout(rfb::Timer *t)
   state = stateTab[state][4][2];
 }
 
-void EmulateMB::sendAction(const rfb::Point& pos, uint8_t buttonMask, int action)
+void EmulateMB::sendAction(const rfb::Point& pos, uint16_t buttonMask, int action)
 {
   assert(action != 0);
 
@@ -325,7 +325,7 @@ void EmulateMB::sendAction(const rfb::Point& pos, uint8_t buttonMask, int action
   sendPointerEvent(pos, buttonMask);
 }
 
-int EmulateMB::createButtonMask(uint8_t buttonMask)
+int EmulateMB::createButtonMask(uint16_t buttonMask)
 {
   // Unset left and right buttons in the mask
   buttonMask &= ~0x5;

vncviewer/EmulateMB.h

@@ -26,22 +26,22 @@ class EmulateMB : public rfb::Timer::Callback {
 public:
   EmulateMB();
 
-  void filterPointerEvent(const rfb::Point& pos, uint8_t buttonMask);
+  void filterPointerEvent(const rfb::Point& pos, uint16_t buttonMask);
 
 protected:
-  virtual void sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask)=0;
+  virtual void sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask)=0;
 
   void handleTimeout(rfb::Timer *t) override;
 
 private:
-  void sendAction(const rfb::Point& pos, uint8_t buttonMask, int action);
+  void sendAction(const rfb::Point& pos, uint16_t buttonMask, int action);
 
-  int createButtonMask(uint8_t buttonMask);
+  int createButtonMask(uint16_t buttonMask);
 
 private:
   int state;
-  uint8_t emulatedButtonMask;
-  uint8_t lastButtonMask;
+  uint16_t emulatedButtonMask;
+  uint16_t lastButtonMask;
   rfb::Point lastPos, origPos;
   rfb::Timer timer;
 };

vncviewer/Viewport.cxx

@@ -115,9 +115,10 @@ enum { ID_DISCONNECT, ID_FULLSCREEN, ID_MINIMIZE, ID_RESIZE,
 static const WORD SCAN_FAKE = 0xaa;
 #endif
 
+
 Viewport::Viewport(int w, int h, const rfb::PixelFormat& /*serverPF*/, CConn* cc_)
   : Fl_Widget(0, 0, w, h), cc(cc_), frameBuffer(nullptr),
-    lastPointerPos(0, 0), lastButtonMask(0),
+    lastPointerPos(0, 0), mouseButtonMask(0), lastButtonMask(0),
 #ifdef WIN32
     altGrArmed(false),
 #endif
@@ -559,7 +560,7 @@ void Viewport::resize(int x, int y, int w, int h)
 int Viewport::handle(int event)
 {
   std::string filtered;
-  int buttonMask, wheelMask;
+  int wheelMask;
   DownMap::const_iterator iter;
 
   switch (event) {
@@ -589,6 +590,7 @@ int Viewport::handle(int event)
 
   case FL_LEAVE:
     window()->cursor(FL_CURSOR_DEFAULT);
+    mouseButtonMask = 0;
     // We want a last move event to help trigger edge stuff
     handlePointerEvent(Point(Fl::event_x() - x(), Fl::event_y() - y()), 0);
     return 1;
@@ -598,13 +600,40 @@ int Viewport::handle(int event)
   case FL_DRAG:
   case FL_MOVE:
   case FL_MOUSEWHEEL:
-    buttonMask = 0;
+
+    // FIXME: FLTK is not consistent in how it sends FL_PUSH/FL_RELEASE
+    // events:
+    //   * https://github.com/fltk/fltk/issues/1076
+    //   * https://github.com/fltk/fltk/issues/1077.
+    // For LMB/MMB/RMB, we can get the button states consistently.
+    // We can't get the state for back/forward, and have to keep track
+    // of them ourselves. Unfortunately as FLTK has these
+    // inconsistencies, we could end up in a scenario where the
+    // mouseButtonMask state is out of sync with reality
+    // (for back/forward). There isn't much we can do until the bug is
+    // resolved in FLTK.
+    mouseButtonMask &= ~(0x07);
     if (Fl::event_button1())
-      buttonMask |= 1;
+        mouseButtonMask |= 1;
     if (Fl::event_button2())
-      buttonMask |= 2;
+        mouseButtonMask |= 2;
     if (Fl::event_button3())
-      buttonMask |= 4;
+        mouseButtonMask |= 4;
+
+    switch (event) {
+    case FL_PUSH:
+      if (Fl::event_button() == 8)
+        mouseButtonMask |= 128;
+      else if (Fl::event_button() == 9)
+        mouseButtonMask |= 256;
+      break;
+    case FL_RELEASE:
+      if (Fl::event_button() == 8)
+          mouseButtonMask &= ~(128);
+      else if (Fl::event_button() == 9)
+        mouseButtonMask &= ~(256);
+      break;
+    }
 
     if (event == FL_MOUSEWHEEL) {
       wheelMask = 0;
@@ -620,10 +649,11 @@ int Viewport::handle(int event)
       // A quick press of the wheel "button", followed by a immediate
       // release below
       handlePointerEvent(Point(Fl::event_x() - x(), Fl::event_y() - y()),
-                         buttonMask | wheelMask);
+                         mouseButtonMask | wheelMask);
     } 
 
-    handlePointerEvent(Point(Fl::event_x() - x(), Fl::event_y() - y()), buttonMask);
+    handlePointerEvent(Point(Fl::event_x() - x(), Fl::event_y() - y()),
+                       mouseButtonMask);
     return 1;
 
   case FL_FOCUS:
@@ -660,7 +690,7 @@ int Viewport::handle(int event)
   return Fl_Widget::handle(event);
 }
 
-void Viewport::sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void Viewport::sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   if (viewOnly)
       return;
@@ -790,7 +820,7 @@ void Viewport::flushPendingClipboard()
 }
 
 
-void Viewport::handlePointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void Viewport::handlePointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   filterPointerEvent(pos, buttonMask);
 }
@@ -937,6 +967,8 @@ int Viewport::handleSystemEvent(void *event, void *data)
       (msg->message == WM_RBUTTONUP) ||
       (msg->message == WM_MBUTTONDOWN) ||
       (msg->message == WM_MBUTTONUP) ||
+      (msg->message == WM_XBUTTONDOWN) ||
+      (msg->message == WM_XBUTTONUP) ||
       (msg->message == WM_MOUSEWHEEL) ||
       (msg->message == WM_MOUSEHWHEEL)) {
     // We can't get a mouse event in the middle of an AltGr sequence, so

vncviewer/Viewport.h

@@ -70,7 +70,7 @@ class Viewport : public Fl_Widget, public EmulateMB {
   int handle(int event) override;
 
 protected:
-  void sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask) override;
+  void sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask) override;
 
 private:
   bool hasFocus();
@@ -81,7 +81,7 @@ class Viewport : public Fl_Widget, public EmulateMB {
 
   void flushPendingClipboard();
 
-  void handlePointerEvent(const rfb::Point& pos, uint8_t buttonMask);
+  void handlePointerEvent(const rfb::Point& pos, uint16_t buttonMask);
   static void handlePointerTimeout(void *data);
 
   void resetKeyboard();
@@ -111,7 +111,8 @@ class Viewport : public Fl_Widget, public EmulateMB {
   PlatformPixelBuffer* frameBuffer;
 
   rfb::Point lastPointerPos;
-  uint8_t lastButtonMask;
+  uint16_t mouseButtonMask;
+  uint16_t lastButtonMask;
 
   typedef std::map<int, uint32_t> DownMap;
   DownMap downKeySym;

win/rfb_win32/SDisplay.cxx

@@ -312,7 +312,7 @@ void SDisplay::handleClipboardData(const char* data) {
 }
 
 
-void SDisplay::pointerEvent(const Point& pos, uint8_t buttonmask) {
+void SDisplay::pointerEvent(const Point& pos, uint16_t buttonmask) {
   if (pb->getRect().contains(pos)) {
     Point screenPos = pos.translate(screenRect.tl);
     // - Check that the SDesktop doesn't need restarting

win/rfb_win32/SDisplay.h

@@ -80,7 +80,7 @@ namespace rfb {
       void handleClipboardRequest() override;
       void handleClipboardAnnounce(bool available) override;
       void handleClipboardData(const char* data) override;
-      void pointerEvent(const Point& pos, uint8_t buttonmask) override;
+      void pointerEvent(const Point& pos, uint16_t buttonmask) override;
       void keyEvent(uint32_t keysym, uint32_t keycode, bool down) override;
 
       // -=- Clipboard events

win/rfb_win32/SInput.cxx

@@ -65,7 +65,7 @@ win32::SPointer::SPointer()
 }
 
 void
-win32::SPointer::pointerEvent(const Point& pos, uint8_t buttonmask)
+win32::SPointer::pointerEvent(const Point& pos, uint16_t buttonmask)
 {
   // - We are specifying absolute coordinates
   DWORD flags = MOUSEEVENTF_ABSOLUTE;

win/rfb_win32/SInput.h

@@ -44,7 +44,7 @@ namespace rfb {
       // - Create a pointer event at a the given coordinates, with the
       //   specified button state.  The event must be specified using
       //   Screen coordinates.
-      void pointerEvent(const Point& pos, uint8_t buttonmask);
+      void pointerEvent(const Point& pos, uint16_t buttonmask);
     protected:
       Point last_position;
       uint8_t last_buttonmask;