From 87d92c178fb09b6d5c9d0b48711bc41564dcdf19 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Wed, 20 Sep 2023 16:11:32 -0700 Subject: [PATCH] work on gfx --- xrdp/xrdp_encoder.c | 290 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 238 insertions(+), 52 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 78e00fea3f..e7ab763f4d 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -785,29 +785,194 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) /*****************************************************************************/ static struct stream * gfx_wiretosurface1(struct xrdp_encoder *self, - struct xrdp_egfx_bulk *bulk, struct stream *in_s) + struct xrdp_egfx_bulk *bulk, struct stream *in_s, + struct xrdp_enc_gfx_cmd *enc_gfx_cmd) { - int surface_id = 0; - int codec_id = 0; - int pixel_format = 0; - struct xrdp_egfx_rect dest_rect = { 0 }; - void *bitmap_data = 0; - int bitmap_data_length = 0; +#ifdef XRDP_X264 + int index; + int surface_id; + int codec_id; + int pixel_format; + int num_rects_d; + int num_rects_c; + struct stream *rv; + short left; + short top; + short width; + short height; + int bitmap_data_length; + int flags; + struct xrdp_egfx_rect *d_rects; + struct xrdp_egfx_rect *c_rects; + struct xrdp_egfx_rect d_rect; + int error; + struct stream ls; + struct stream *s; + int rcount; + struct xrdp_egfx_rect *rrects; if (self->codec_handle_gfx[1] == NULL) { return NULL; } - return xrdp_egfx_wire_to_surface1(bulk, surface_id, - codec_id, pixel_format, &dest_rect, - bitmap_data, bitmap_data_length); + s = &ls; + g_memset(s, 0, sizeof(struct stream)); + ls.size = self->max_compressed_bytes; + ls.data = g_new(char, ls.size); + if (ls.data == NULL) + { + return NULL; + } + ls.p = ls.data; + if (!s_check_rem(in_s, 15)) + { + g_free(s->data); + return NULL; + } + in_uint16_le(in_s, surface_id); + in_uint16_le(in_s, codec_id); + in_uint8(in_s, pixel_format); + in_uint32_le(in_s, flags); + in_uint16_le(in_s, num_rects_d); + if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) || + (!s_check_rem(in_s, num_rects_d * 8))) + { + g_free(s->data); + return NULL; + } + d_rects = g_new0(struct xrdp_egfx_rect, num_rects_d); + if (d_rects == NULL) + { + g_free(s->data); + return NULL; + } + for (index = 0; index < num_rects_d; index++) + { + in_uint16_le(in_s, left); + in_uint16_le(in_s, top); + in_uint16_le(in_s, width); + in_uint16_le(in_s, height); + g_writeln("rects_d index %d left %d top %d width %d height %d", + index, left, top, width, height); + d_rects[index].x1 = left; + d_rects[index].y1 = top; + d_rects[index].x2 = left + width; + d_rects[index].y2 = top + height; + + } + if (!s_check_rem(in_s, 2)) + { + g_free(s->data); + g_free(d_rects); + return NULL; + } + in_uint16_le(in_s, num_rects_c); + if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) || + (!s_check_rem(in_s, num_rects_c * 8))) + { + g_free(s->data); + g_free(d_rects); + return NULL; + } + c_rects = g_new0(struct xrdp_egfx_rect, num_rects_c); + if (c_rects == NULL) + { + g_free(s->data); + g_free(d_rects); + return NULL; + } + for (index = 0; index < num_rects_c; index++) + { + in_uint16_le(in_s, left); + in_uint16_le(in_s, top); + in_uint16_le(in_s, width); + in_uint16_le(in_s, height); + g_writeln("rects_c index %d left %d top %d width %d height %d", + index, left, top, width, height); + c_rects[index].x1 = left; + c_rects[index].y1 = top; + c_rects[index].x2 = left + width; + c_rects[index].y2 = top + height; + } + rcount = num_rects_d; + rrects = d_rects; + if (rcount > 15) + { + rcount = num_rects_c; + rrects = c_rects; + } + /* RFX_AVC420_METABLOCK */ + out_uint32_le(s, rcount); /* numRegionRects */ + for (index = 0; index < rcount; index++) + { + out_uint16_le(s, rrects[index].x1); + out_uint16_le(s, rrects[index].y1); + out_uint16_le(s, rrects[index].x2); + out_uint16_le(s, rrects[index].y2); + } + for (index = 0; index < rcount; index++) + { + out_uint8(s, 23); /* qp */ + out_uint8(s, 100); /* quality level 0..100 */ + } + + if (!s_check_rem(in_s, 8)) + { + g_free(s->data); + g_free(c_rects); + g_free(d_rects); + return NULL; + } + in_uint16_le(in_s, left); + in_uint16_le(in_s, top); + in_uint16_le(in_s, width); + in_uint16_le(in_s, height); + d_rect.x1 = left; + d_rect.y1 = top; + d_rect.x2 = left + width; + d_rect.y2 = top + height; + g_writeln("left %d top %d width %d height %d", left, top, width, height); + if (ENC_IS_BIT_SET(flags, 0)) + { + /* already compressed */ + out_uint8a(s, enc_gfx_cmd->data, enc_gfx_cmd->data_bytes); + } + else + { + bitmap_data_length = (int) ((s->data + s->size) - s->p); + error = xrdp_encoder_x264_encode(self->codec_handle_gfx[1], 0, + width, height, 0, + enc_gfx_cmd->data, + s->p, &bitmap_data_length); + if (error != 0) + { + g_writeln("xrdp_encoder_x264_encode failed error %d", error); + } + } + s_mark_end(s); + g_free(c_rects); + g_free(d_rects); + bitmap_data_length = (int) (s->end - s->data); + rv = xrdp_egfx_wire_to_surface1(bulk, surface_id, + codec_id, + pixel_format, &d_rect, + s->data, bitmap_data_length); + g_free(s->data); + return rv; +#else + (void)self; + (void)bulk; + (void)in_s; + (void)enc_gfx_cmd; + return NULL; +#endif } /*****************************************************************************/ static struct stream * gfx_wiretosurface2(struct xrdp_encoder *self, struct xrdp_egfx_bulk *bulk, struct stream *in_s, - char *screen_data) + struct xrdp_enc_gfx_cmd *enc_gfx_cmd) { #ifdef XRDP_RFXCODEC int index; @@ -817,6 +982,7 @@ gfx_wiretosurface2(struct xrdp_encoder *self, int pixel_format; int num_rects_d; int num_rects_c; + struct stream *rv; short left; short top; short width; @@ -825,12 +991,15 @@ gfx_wiretosurface2(struct xrdp_encoder *self, int bitmap_data_length; struct rfx_tile *tiles; struct rfx_rect *rfxrects; + int flags; + int tiles_written; + int do_free; if (self->codec_handle_gfx[0] == NULL) { return NULL; } - if (!s_check_rem(in_s, 11)) + if (!s_check_rem(in_s, 15)) { return NULL; } @@ -838,17 +1007,16 @@ gfx_wiretosurface2(struct xrdp_encoder *self, in_uint16_le(in_s, codec_id); in_uint32_le(in_s, codec_context_id); in_uint8(in_s, pixel_format); + in_uint32_le(in_s, flags); in_uint16_le(in_s, num_rects_d); - - rfxrects = g_new(struct rfx_rect, num_rects_d); - if (rfxrects == NULL) + if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) || + (!s_check_rem(in_s, num_rects_d * 8))) { return NULL; } - - if (!s_check_rem(in_s, num_rects_d * 8)) + rfxrects = g_new0(struct rfx_rect, num_rects_d); + if (rfxrects == NULL) { - g_free(rfxrects); return NULL; } for (index = 0; index < num_rects_d; index++) @@ -870,17 +1038,15 @@ gfx_wiretosurface2(struct xrdp_encoder *self, return NULL; } in_uint16_le(in_s, num_rects_c); - - tiles = g_new(struct rfx_tile, num_rects_c); - if (tiles == NULL) + if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) || + (!s_check_rem(in_s, num_rects_c * 8))) { g_free(rfxrects); return NULL; } - - if (!s_check_rem(in_s, num_rects_c * 8)) + tiles = g_new0(struct rfx_tile, num_rects_c); + if (tiles == NULL) { - g_free(tiles); g_free(rfxrects); return NULL; } @@ -890,7 +1056,8 @@ gfx_wiretosurface2(struct xrdp_encoder *self, in_uint16_le(in_s, top); in_uint16_le(in_s, width); in_uint16_le(in_s, height); - g_writeln("rects_c index %d left %d top %d width %d height %d", index, left, top, width, height); + g_writeln("rects_c index %d left %d top %d width %d height %d", + index, left, top, width, height); tiles[index].x = left; tiles[index].y = top; tiles[index].cx = width; @@ -910,39 +1077,58 @@ gfx_wiretosurface2(struct xrdp_encoder *self, in_uint16_le(in_s, width); in_uint16_le(in_s, height); g_writeln("left %d top %d width %d height %d", left, top, width, height); - - bitmap_data_length = 16 * 1024 * 1024; - bitmap_data = g_new(char, bitmap_data_length); - - int tiles_written; - tiles_written = rfxcodec_encode(self->codec_handle_gfx[0], - bitmap_data, - &bitmap_data_length, - screen_data, - width, height, - width * 4, - rfxrects, num_rects_d, - tiles, num_rects_c, - self->quants, self->num_quants); - g_writeln("tiles_written %d", tiles_written); - - g_free(bitmap_data); + do_free = 0; + if (ENC_IS_BIT_SET(flags, 0)) + { + /* already compressed */ + bitmap_data_length = enc_gfx_cmd->data_bytes; + bitmap_data = enc_gfx_cmd->data; + } + else + { + bitmap_data_length = self->max_compressed_bytes; + bitmap_data = g_new(char, bitmap_data_length); + if (bitmap_data == NULL) + { + g_free(tiles); + g_free(rfxrects); + return NULL; + } + do_free = 1; + tiles_written = rfxcodec_encode(self->codec_handle_gfx[0], + bitmap_data, + &bitmap_data_length, + enc_gfx_cmd->data, + width, height, + width * 4, + rfxrects, num_rects_d, + tiles, num_rects_c, + self->quants, self->num_quants); + g_writeln("tiles_written %d", tiles_written); + if (tiles_written < 1) + { + g_writeln("error tiles_written %d", tiles_written); + } + } g_free(tiles); g_free(rfxrects); - - return xrdp_egfx_wire_to_surface2(bulk, surface_id, - codec_id, codec_context_id, - pixel_format, - bitmap_data, bitmap_data_length); -} + rv = xrdp_egfx_wire_to_surface2(bulk, surface_id, + codec_id, codec_context_id, + pixel_format, + bitmap_data, bitmap_data_length); + if (do_free) + { + g_free(bitmap_data); + } + return rv; #else (void)self; (void)bulk; (void)in_s; - (void)screen_data; + (void)enc_gfx_cmd; return NULL; -} #endif +} /*****************************************************************************/ static struct stream * @@ -1143,10 +1329,10 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) switch (cmd_id) { case XR_RDPGFX_CMDID_WIRETOSURFACE_1: /* 0x0001 */ - s = gfx_wiretosurface1(self, bulk, &in_s); + s = gfx_wiretosurface1(self, bulk, &in_s, &(enc->u.gfx)); break; case XR_RDPGFX_CMDID_WIRETOSURFACE_2: /* 0x0002 */ - s = gfx_wiretosurface2(self, bulk, &in_s, enc->u.gfx.data); + s = gfx_wiretosurface2(self, bulk, &in_s, &(enc->u.gfx)); break; case XR_RDPGFX_CMDID_SOLIDFILL: /* 0x0004 */ s = gfx_solidfill(self, bulk, &in_s);