Merge pull request neutrinolabs#2879 from Nexarian/add_egfx_multimon_…

…common Initial rfx progressive integration
Nexarian · Dec 20, 2023 · 02d329e · 02d329e
2 parents 9fbe0ad + 1ee3c64
commit 02d329e
Show file tree

Hide file tree

Showing 11 changed files with 245 additions and 70 deletions.
diff --git a/common/Makefile.am b/common/Makefile.am
@@ -71,6 +71,6 @@ libcommon_la_SOURCES = \
   $(PIXMAN_SOURCES)
 
 libcommon_la_LIBADD = \
-  -lpthread \
+  -lpthread -lrt \
   $(OPENSSL_LIBS) \
   $(DLOPEN_LIBS)
diff --git a/common/ms-rdpbcgr.h b/common/ms-rdpbcgr.h
@@ -52,19 +52,29 @@
 
 /* TS_UD_HEADER: type ((2.2.1.3.1) */
 /* TODO: to be renamed */
-#define SEC_TAG_CLI_INFO               0xc001 /* CS_CORE? */
-#define SEC_TAG_CLI_CRYPT              0xc002 /* CS_SECURITY? */
-#define SEC_TAG_CLI_CHANNELS           0xc003 /* CS_CHANNELS? */
-#define SEC_TAG_CLI_4                  0xc004 /* CS_CLUSTER? */
-#define SEC_TAG_CLI_MONITOR            0xc005 /* CS_MONITOR */
-#define SEC_TAG_CLI_MONITOR_EX         0xc008 /* CS_MONITOR_EX */
+#define SEC_TAG_CLI_INFO       0xc001 /* CS_CORE? */
+#define SEC_TAG_CLI_CRYPT      0xc002 /* CS_SECURITY? */
+#define SEC_TAG_CLI_CHANNELS   0xc003 /* CS_CHANNELS? */
+#define SEC_TAG_CLI_4          0xc004 /* CS_CLUSTER? */
+#define SEC_TAG_CLI_MONITOR    0xc005 /* CS_MONITOR */
+#define SEC_TAG_CLI_MONITOR_EX 0xc008 /* CS_MONITOR_EX */
 
 /* Client Core Data: colorDepth, postBeta2ColorDepth (2.2.1.3.2) */
-#define RNS_UD_COLOR_4BPP              0xCA00
-#define RNS_UD_COLOR_8BPP              0xCA01
-#define RNS_UD_COLOR_16BPP_555         0xCA02
-#define RNS_UD_COLOR_16BPP_565         0xCA03
-#define RNS_UD_COLOR_24BPP             0xCA04
+#define RNS_UD_COLOR_4BPP      0xCA00
+#define RNS_UD_COLOR_8BPP      0xCA01
+#define RNS_UD_COLOR_16BPP_555 0xCA02
+#define RNS_UD_COLOR_16BPP_565 0xCA03
+#define RNS_UD_COLOR_24BPP     0xCA04
+
+/* Client Core Data: supportedColorDepths (2.2.1.3.2) */
+#define RNS_UD_24BPP_SUPPORT 0x0001
+#define RNS_UD_16BPP_SUPPORT 0x0002
+#define RNS_UD_15BPP_SUPPORT 0x0004
+#define RNS_UD_32BPP_SUPPORT 0x0008
+
+/* Client Core Data: earlyCapabilityFlags (2.2.1.3.2) */
+#define RNS_UD_CS_WANT_32BPP_SESSION         0x0002
+#define RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL 0x0100
 
 /* Client Core Data: connectionType  (2.2.1.3.2) */
 #define CONNECTION_TYPE_MODEM          0x01

diff --git a/common/xrdp_client_info.h b/common/xrdp_client_info.h
@@ -215,6 +215,14 @@ struct xrdp_client_info
     unsigned int session_physical_height; /* in mm */
 
     int large_pointer_support_flags;
+    int gfx;
+};
+
+enum xrdp_encoder_flags
+{
+    NONE                                   = 0,
+    ENCODE_COMPLETE                        = 1 << 0,
+    GFX_PROGRESSIVE_RFX                    = 1 << 1
 };
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */

diff --git a/common/xrdp_constants.h b/common/xrdp_constants.h
@@ -291,4 +291,50 @@
 #define XR_RDP_SCAN_LSHIFT 42
 #define XR_RDP_SCAN_ALT    56
 
+// Since we're not guaranteed to have pixman, copy these directives.
+#define XRDP_PIXMAN_TYPE_ARGB   2
+#define XRDP_PIXMAN_TYPE_ABGR   3
+#define XRDP_PIXMAN_FORMAT(bpp,type,a,r,g,b)    (((bpp) << 24) |  \
+        ((type) << 16) | \
+        ((a) << 12) |    \
+        ((r) << 8) |     \
+        ((g) << 4) |     \
+        ((b)))
+
+#define XRDP_a8b8g8r8 \
+    XRDP_PIXMAN_FORMAT(32, XRDP_PIXMAN_TYPE_ABGR, 8, 8, 8, 8)
+
+#define XRDP_a8r8g8b8 \
+    XRDP_PIXMAN_FORMAT(32, XRDP_PIXMAN_TYPE_ARGB, 8, 8, 8, 8)
+
+#define XRDP_r5g6b5 \
+    XRDP_PIXMAN_FORMAT(16, XRDP_PIXMAN_TYPE_ARGB, 0, 5, 6, 5)
+
+#define XRDP_a1r5g5b5 \
+    XRDP_PIXMAN_FORMAT(16, XRDP_PIXMAN_TYPE_ARGB, 1, 5, 5, 5)
+
+#define XRDP_r3g3b2 \
+    XRDP_PIXMAN_FORMAT(8, XRDP_PIXMAN_TYPE_ARGB, 0, 3, 3, 2)
+
+// The last used constant in pixman is 63, so use 64+
+#define XRDP_nv12 \
+    XRDP_PIXMAN_FORMAT(12, 64, 0, 0, 0, 0)
+
+#define XRDP_i420 \
+    XRDP_PIXMAN_FORMAT(12, 65, 0, 0, 0, 0)
+
+#define XRDP_nv12_709fr \
+    XRDP_PIXMAN_FORMAT(12, 66, 0, 0, 0, 0)
+
+#define XRDP_yuv444_709fr \
+    XRDP_PIXMAN_FORMAT(32, 67, 0, 0, 0, 0)
+
+// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegfx/8131c1bc-1af8-4907-a05a-f72f4581160f
+#define XRDP_yuv444_v1_stream_709fr \
+    XRDP_PIXMAN_FORMAT(32, 68, 0, 0, 0, 0)
+
+// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegfx/781406c3-5e24-4f2b-b6ff-42b76bf64f6d
+#define XRDP_yuv444_v2_stream_709fr \
+    XRDP_PIXMAN_FORMAT(32, 69, 0, 0, 0, 0)
+
 #endif
diff --git a/libxrdp/xrdp_sec.c b/libxrdp/xrdp_sec.c
@@ -2114,10 +2114,14 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s)
     in_uint16_le(s, supportedColorDepths);
     LOG_DEVEL(LOG_LEVEL_TRACE, "Received [MS-RDPBCGR] TS_UD_CS_CORE "
               "<Optional Field> supportedColorDepths %s",
-              supportedColorDepths == 0x0001 ? "RNS_UD_24BPP_SUPPORT" :
-              supportedColorDepths == 0x0002 ? "RNS_UD_16BPP_SUPPORT" :
-              supportedColorDepths == 0x0004 ? "RNS_UD_15BPP_SUPPORT" :
-              supportedColorDepths == 0x0008 ? "RNS_UD_32BPP_SUPPORT" :
+              supportedColorDepths == RNS_UD_24BPP_SUPPORT
+              ? "RNS_UD_24BPP_SUPPORT" :
+              supportedColorDepths == RNS_UD_16BPP_SUPPORT
+              ? "RNS_UD_16BPP_SUPPORT" :
+              supportedColorDepths == RNS_UD_15BPP_SUPPORT
+              ? "RNS_UD_15BPP_SUPPORT" :
+              supportedColorDepths == RNS_UD_32BPP_SUPPORT
+              ? "RNS_UD_32BPP_SUPPORT" :
               "unknown");
 
     if (!s_check_rem(s, 2))
@@ -2129,11 +2133,20 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s)
     LOG_DEVEL(LOG_LEVEL_TRACE, "Received [MS-RDPBCGR] TS_UD_CS_CORE "
               "<Optional Field> earlyCapabilityFlags 0x%4.4x",
               earlyCapabilityFlags);
-    if ((earlyCapabilityFlags & 0x0002) && (supportedColorDepths & 0x0008))
+    if ((earlyCapabilityFlags & RNS_UD_CS_WANT_32BPP_SESSION)
+            && (supportedColorDepths & RNS_UD_32BPP_SUPPORT))
     {
         client_info->bpp = 32;
     }
-
+    if (earlyCapabilityFlags & RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL)
+    {
+        LOG(LOG_LEVEL_INFO, "client supports gfx protocol");
+        self->rdp_layer->client_info.gfx = 1;
+    }
+    else
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client DOES NOT support gfx");
+    }
     if (!s_check_rem(s, 64))
     {
         return 0;

diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c
@@ -32,10 +32,17 @@
 #include "rfxcodec_encode.h"
 #endif
 
-
-
 #define XRDP_SURCMD_PREFIX_BYTES 256
 
+#ifdef XRDP_RFXCODEC
+/* LH3 LL3, HH3 HL3, HL2 LH2, LH1 HH2, HH1 HL1 todo check this */
+static const unsigned char g_rfx_quantization_values[] =
+{
+    0x66, 0x66, 0x77, 0x88, 0x98,
+    0x76, 0x77, 0x88, 0x98, 0xA9
+};
+#endif
+
 /*****************************************************************************/
 static int
 process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
@@ -77,16 +84,24 @@ xrdp_encoder_create(struct xrdp_mm *mm)
 
     client_info = mm->wm->client_info;
 
+    /* RemoteFX 7.1 requires LAN but GFX does not */
     if (client_info->mcs_connection_type != CONNECTION_TYPE_LAN)
     {
-        return 0;
+        if ((mm->egfx_flags & (XRDP_EGFX_H264 | XRDP_EGFX_RFX_PRO)) == 0)
+        {
+            return 0;
+        }
     }
     if (client_info->bpp < 24)
     {
         return 0;
     }
 
-    self = (struct xrdp_encoder *)g_malloc(sizeof(struct xrdp_encoder), 1);
+    self = g_new0(struct xrdp_encoder, 1);
+    if (self == NULL)
+    {
+        return NULL;
+    }
     self->mm = mm;
 
     if (client_info->jpeg_codec_id != 0)
@@ -96,12 +111,29 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->in_codec_mode = 1;
         self->codec_quality = client_info->jpeg_prop[0];
         client_info->capture_code = 0;
-        client_info->capture_format =
-            /* XRDP_a8b8g8r8 */
-            (32 << 24) | (3 << 16) | (8 << 12) | (8 << 8) | (8 << 4) | 8;
+        client_info->capture_format = XRDP_a8b8g8r8;
         self->process_enc = process_enc_jpg;
     }
 #ifdef XRDP_RFXCODEC
+    else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO)
+    {
+        LOG(LOG_LEVEL_INFO,
+            "xrdp_encoder_create: starting gfx rfx pro codec session");
+        self->in_codec_mode = 1;
+        client_info->capture_code = 2;
+        self->process_enc = process_enc_rfx;
+        self->gfx = 1;
+        self->quants = (const char *) g_rfx_quantization_values;
+        self->num_quants = 2;
+        self->quant_idx_y = 0;
+        self->quant_idx_u = 1;
+        self->quant_idx_v = 1;
+        self->codec_handle = rfxcodec_encode_create(
+                                 mm->wm->screen->width,
+                                 mm->wm->screen->height,
+                                 RFX_FORMAT_YUV,
+                                 RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
+    }
     else if (client_info->rfx_codec_id != 0)
     {
         LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session");
@@ -120,9 +152,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->codec_id = client_info->h264_codec_id;
         self->in_codec_mode = 1;
         client_info->capture_code = 3;
-        client_info->capture_format =
-            /* XRDP_nv12 */
-            (12 << 24) | (64 << 16) | (0 << 12) | (0 << 8) | (0 << 4) | 0;
+        client_info->capture_format = XRDP_nv12;
         self->process_enc = process_enc_h264;
     }
     else
@@ -131,7 +161,9 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         return 0;
     }
 
-    LOG_DEVEL(LOG_LEVEL_INFO, "init_xrdp_encoder: initializing encoder codec_id %d", self->codec_id);
+    LOG_DEVEL(LOG_LEVEL_INFO,
+              "init_xrdp_encoder: initializing encoder codec_id %d",
+              self->codec_id);
 
     /* setup required FIFOs */
     self->fifo_to_proc = fifo_create(xrdp_enc_data_destructor);
@@ -146,8 +178,17 @@ xrdp_encoder_create(struct xrdp_mm *mm)
     self->xrdp_encoder_event_processed = g_create_wait_obj(buf);
     g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid);
     self->xrdp_encoder_term = g_create_wait_obj(buf);
-    self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15;
-    self->frames_in_flight = client_info->max_unacknowledged_frame_count;
+    if (client_info->gfx)
+    {
+        // Magic numbers... Why?
+        self->frames_in_flight = 2;
+        self->max_compressed_bytes = 3145728;
+    }
+    else
+    {
+        self->frames_in_flight = client_info->max_unacknowledged_frame_count;
+        self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15;
+    }
     /* make sure frames_in_flight is at least 1 */
     self->frames_in_flight = MAX(self->frames_in_flight, 1);
 
@@ -354,9 +395,9 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                     tiles[index].y = y;
                     tiles[index].cx = cx;
                     tiles[index].cy = cy;
-                    tiles[index].quant_y = 0;
-                    tiles[index].quant_cb = 0;
-                    tiles[index].quant_cr = 0;
+                    tiles[index].quant_y = self->quant_idx_y;
+                    tiles[index].quant_cb = self->quant_idx_u;
+                    tiles[index].quant_cr = self->quant_idx_v;
                 }
 
                 count = enc->num_drects;
@@ -376,9 +417,11 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                 tiles_written = rfxcodec_encode(self->codec_handle,
                                                 out_data + XRDP_SURCMD_PREFIX_BYTES,
                                                 &out_data_bytes, enc->data,
-                                                enc->width, enc->height, enc->width * 4,
+                                                enc->width, enc->height,
+                                                enc->width * 4,
                                                 rfxrects, enc->num_drects,
-                                                tiles, tiles_left, 0, 0);
+                                                tiles, enc->num_crects,
+                                                self->quants, self->num_quants);
             }
         }
 
@@ -399,7 +442,11 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
         enc_done->enc = enc;
         enc_done->cx = self->mm->wm->screen->width;
         enc_done->cy = self->mm->wm->screen->height;
-
+        if (self->gfx)
+        {
+            enc_done->flags = (enum xrdp_encoder_flags)
+                              ((int)enc_done->flags | GFX_PROGRESSIVE_RFX);
+        }
         enc_done->continuation = all_tiles_written > 0;
         if (tiles_written > 0)
         {
@@ -429,7 +476,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 static int
 process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 {
-    LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_x264:");
+    LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_h264: dummy func");
     return 0;
 }
 

diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h
@@ -3,6 +3,7 @@
 #define _XRDP_ENCODER_H
 
 #include "arch.h"
+#include "xrdp_client_info.h"
 struct fifo;
 
 struct xrdp_enc_data;
@@ -27,6 +28,13 @@ struct xrdp_encoder
     int frame_id_server; /* last frame id received from Xorg */
     int frame_id_server_sent;
     int frames_in_flight;
+    int gfx;
+    int gfx_ack_off;
+    const char *quants;
+    int num_quants;
+    int quant_idx_y;
+    int quant_idx_u;
+    int quant_idx_v;
 };
 
 /* used when scheduling tasks in xrdp_encoder.c */
@@ -63,6 +71,7 @@ struct xrdp_enc_data_done
     int y;
     int cx;
     int cy;
+    enum xrdp_encoder_flags flags;
 };
 
 typedef struct xrdp_enc_data_done XRDP_ENC_DATA_DONE;

diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c
@@ -79,7 +79,13 @@ xrdp_mm_create(struct xrdp_wm *owner)
               self->wm->client_info->rfx_codec_id,
               self->wm->client_info->h264_codec_id);
 
-    self->encoder = xrdp_encoder_create(self);
+    if ((self->wm->client_info->gfx == 0) &&
+            ((self->wm->client_info->h264_codec_id != 0) ||
+             (self->wm->client_info->jpeg_codec_id != 0) ||
+             (self->wm->client_info->rfx_codec_id != 0)))
+    {
+        self->encoder = xrdp_encoder_create(self);
+    }
 
     return self;
 }