Skip to content

Commit

Permalink
work on gfx
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsorg71
jsorg71 committed Sep 20, 2023
1 parent 9b50c41 commit 87d92c1
Showing 1 changed file with 238 additions and 52 deletions.
290 changes: 238 additions & 52 deletions xrdp/xrdp_encoder.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -785,29 +785,194 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
/*****************************************************************************/
static struct stream *
gfx_wiretosurface1(struct xrdp_encoder *self,
struct xrdp_egfx_bulk *bulk, struct stream *in_s)
struct xrdp_egfx_bulk *bulk, struct stream *in_s,
struct xrdp_enc_gfx_cmd *enc_gfx_cmd)
{
int surface_id = 0;
int codec_id = 0;
int pixel_format = 0;
struct xrdp_egfx_rect dest_rect = { 0 };
void *bitmap_data = 0;
int bitmap_data_length = 0;
#ifdef XRDP_X264
int index;
int surface_id;
int codec_id;
int pixel_format;
int num_rects_d;
int num_rects_c;
struct stream *rv;
short left;
short top;
short width;
short height;
int bitmap_data_length;
int flags;
struct xrdp_egfx_rect *d_rects;
struct xrdp_egfx_rect *c_rects;
struct xrdp_egfx_rect d_rect;
int error;
struct stream ls;
struct stream *s;
int rcount;
struct xrdp_egfx_rect *rrects;

if (self->codec_handle_gfx[1] == NULL)
{
return NULL;
}
return xrdp_egfx_wire_to_surface1(bulk, surface_id,
codec_id, pixel_format, &dest_rect,
bitmap_data, bitmap_data_length);
s = &ls;
g_memset(s, 0, sizeof(struct stream));
ls.size = self->max_compressed_bytes;
ls.data = g_new(char, ls.size);
if (ls.data == NULL)
{
return NULL;
}
ls.p = ls.data;
if (!s_check_rem(in_s, 15))
{
g_free(s->data);
return NULL;
}
in_uint16_le(in_s, surface_id);
in_uint16_le(in_s, codec_id);
in_uint8(in_s, pixel_format);
in_uint32_le(in_s, flags);
in_uint16_le(in_s, num_rects_d);
if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) ||
(!s_check_rem(in_s, num_rects_d * 8)))
{
g_free(s->data);
return NULL;
}
d_rects = g_new0(struct xrdp_egfx_rect, num_rects_d);
if (d_rects == NULL)
{
g_free(s->data);
return NULL;
}
for (index = 0; index < num_rects_d; index++)
{
in_uint16_le(in_s, left);
in_uint16_le(in_s, top);
in_uint16_le(in_s, width);
in_uint16_le(in_s, height);
g_writeln("rects_d index %d left %d top %d width %d height %d",
index, left, top, width, height);
d_rects[index].x1 = left;
d_rects[index].y1 = top;
d_rects[index].x2 = left + width;
d_rects[index].y2 = top + height;

}
if (!s_check_rem(in_s, 2))
{
g_free(s->data);
g_free(d_rects);
return NULL;
}
in_uint16_le(in_s, num_rects_c);
if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) ||
(!s_check_rem(in_s, num_rects_c * 8)))
{
g_free(s->data);
g_free(d_rects);
return NULL;
}
c_rects = g_new0(struct xrdp_egfx_rect, num_rects_c);
if (c_rects == NULL)
{
g_free(s->data);
g_free(d_rects);
return NULL;
}
for (index = 0; index < num_rects_c; index++)
{
in_uint16_le(in_s, left);
in_uint16_le(in_s, top);
in_uint16_le(in_s, width);
in_uint16_le(in_s, height);
g_writeln("rects_c index %d left %d top %d width %d height %d",
index, left, top, width, height);
c_rects[index].x1 = left;
c_rects[index].y1 = top;
c_rects[index].x2 = left + width;
c_rects[index].y2 = top + height;
}
rcount = num_rects_d;
rrects = d_rects;
if (rcount > 15)
{
rcount = num_rects_c;
rrects = c_rects;
}
/* RFX_AVC420_METABLOCK */
out_uint32_le(s, rcount); /* numRegionRects */
for (index = 0; index < rcount; index++)
{
out_uint16_le(s, rrects[index].x1);
out_uint16_le(s, rrects[index].y1);
out_uint16_le(s, rrects[index].x2);
out_uint16_le(s, rrects[index].y2);
}
for (index = 0; index < rcount; index++)
{
out_uint8(s, 23); /* qp */
out_uint8(s, 100); /* quality level 0..100 */
}

if (!s_check_rem(in_s, 8))
{
g_free(s->data);
g_free(c_rects);
g_free(d_rects);
return NULL;
}
in_uint16_le(in_s, left);
in_uint16_le(in_s, top);
in_uint16_le(in_s, width);
in_uint16_le(in_s, height);
d_rect.x1 = left;
d_rect.y1 = top;
d_rect.x2 = left + width;
d_rect.y2 = top + height;
g_writeln("left %d top %d width %d height %d", left, top, width, height);
if (ENC_IS_BIT_SET(flags, 0))
{
/* already compressed */
out_uint8a(s, enc_gfx_cmd->data, enc_gfx_cmd->data_bytes);
}
else
{
bitmap_data_length = (int) ((s->data + s->size) - s->p);
error = xrdp_encoder_x264_encode(self->codec_handle_gfx[1], 0,
width, height, 0,
enc_gfx_cmd->data,
s->p, &bitmap_data_length);
if (error != 0)
{
g_writeln("xrdp_encoder_x264_encode failed error %d", error);
}
}
s_mark_end(s);
g_free(c_rects);
g_free(d_rects);
bitmap_data_length = (int) (s->end - s->data);
rv = xrdp_egfx_wire_to_surface1(bulk, surface_id,
codec_id,
pixel_format, &d_rect,
s->data, bitmap_data_length);
g_free(s->data);
return rv;
#else
(void)self;
(void)bulk;
(void)in_s;
(void)enc_gfx_cmd;
return NULL;
#endif
}

/*****************************************************************************/
static struct stream *
gfx_wiretosurface2(struct xrdp_encoder *self,
struct xrdp_egfx_bulk *bulk, struct stream *in_s,
char *screen_data)
struct xrdp_enc_gfx_cmd *enc_gfx_cmd)
{
#ifdef XRDP_RFXCODEC
int index;
Expand All @@ -817,6 +982,7 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
int pixel_format;
int num_rects_d;
int num_rects_c;
struct stream *rv;
short left;
short top;
short width;
Expand All @@ -825,30 +991,32 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
int bitmap_data_length;
struct rfx_tile *tiles;
struct rfx_rect *rfxrects;
int flags;
int tiles_written;
int do_free;

if (self->codec_handle_gfx[0] == NULL)
{
return NULL;
}
if (!s_check_rem(in_s, 11))
if (!s_check_rem(in_s, 15))
{
return NULL;
}
in_uint16_le(in_s, surface_id);
in_uint16_le(in_s, codec_id);
in_uint32_le(in_s, codec_context_id);
in_uint8(in_s, pixel_format);
in_uint32_le(in_s, flags);
in_uint16_le(in_s, num_rects_d);

rfxrects = g_new(struct rfx_rect, num_rects_d);
if (rfxrects == NULL)
if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) ||
(!s_check_rem(in_s, num_rects_d * 8)))
{
return NULL;
}

if (!s_check_rem(in_s, num_rects_d * 8))
rfxrects = g_new0(struct rfx_rect, num_rects_d);
if (rfxrects == NULL)
{
g_free(rfxrects);
return NULL;
}
for (index = 0; index < num_rects_d; index++)
Expand All @@ -870,17 +1038,15 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
return NULL;
}
in_uint16_le(in_s, num_rects_c);

tiles = g_new(struct rfx_tile, num_rects_c);
if (tiles == NULL)
if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) ||
(!s_check_rem(in_s, num_rects_c * 8)))
{
g_free(rfxrects);
return NULL;
}

if (!s_check_rem(in_s, num_rects_c * 8))
tiles = g_new0(struct rfx_tile, num_rects_c);
if (tiles == NULL)
{
g_free(tiles);
g_free(rfxrects);
return NULL;
}
Expand All @@ -890,7 +1056,8 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
in_uint16_le(in_s, top);
in_uint16_le(in_s, width);
in_uint16_le(in_s, height);
g_writeln("rects_c index %d left %d top %d width %d height %d", index, left, top, width, height);
g_writeln("rects_c index %d left %d top %d width %d height %d",
index, left, top, width, height);
tiles[index].x = left;
tiles[index].y = top;
tiles[index].cx = width;
Expand All @@ -910,39 +1077,58 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
in_uint16_le(in_s, width);
in_uint16_le(in_s, height);
g_writeln("left %d top %d width %d height %d", left, top, width, height);

bitmap_data_length = 16 * 1024 * 1024;
bitmap_data = g_new(char, bitmap_data_length);

int tiles_written;
tiles_written = rfxcodec_encode(self->codec_handle_gfx[0],
bitmap_data,
&bitmap_data_length,
screen_data,
width, height,
width * 4,
rfxrects, num_rects_d,
tiles, num_rects_c,
self->quants, self->num_quants);
g_writeln("tiles_written %d", tiles_written);

g_free(bitmap_data);
do_free = 0;
if (ENC_IS_BIT_SET(flags, 0))
{
/* already compressed */
bitmap_data_length = enc_gfx_cmd->data_bytes;
bitmap_data = enc_gfx_cmd->data;
}
else
{
bitmap_data_length = self->max_compressed_bytes;
bitmap_data = g_new(char, bitmap_data_length);
if (bitmap_data == NULL)
{
g_free(tiles);
g_free(rfxrects);
return NULL;
}
do_free = 1;
tiles_written = rfxcodec_encode(self->codec_handle_gfx[0],
bitmap_data,
&bitmap_data_length,
enc_gfx_cmd->data,
width, height,
width * 4,
rfxrects, num_rects_d,
tiles, num_rects_c,
self->quants, self->num_quants);
g_writeln("tiles_written %d", tiles_written);
if (tiles_written < 1)
{
g_writeln("error tiles_written %d", tiles_written);
}
}
g_free(tiles);
g_free(rfxrects);

return xrdp_egfx_wire_to_surface2(bulk, surface_id,
codec_id, codec_context_id,
pixel_format,
bitmap_data, bitmap_data_length);
}
rv = xrdp_egfx_wire_to_surface2(bulk, surface_id,
codec_id, codec_context_id,
pixel_format,
bitmap_data, bitmap_data_length);
if (do_free)
{
g_free(bitmap_data);
}
return rv;
#else
(void)self;
(void)bulk;
(void)in_s;
(void)screen_data;
(void)enc_gfx_cmd;
return NULL;
}
#endif
}

/*****************************************************************************/
static struct stream *
Expand Down Expand Up @@ -1143,10 +1329,10 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
switch (cmd_id)
{
case XR_RDPGFX_CMDID_WIRETOSURFACE_1: /* 0x0001 */
s = gfx_wiretosurface1(self, bulk, &in_s);
s = gfx_wiretosurface1(self, bulk, &in_s, &(enc->u.gfx));
break;
case XR_RDPGFX_CMDID_WIRETOSURFACE_2: /* 0x0002 */
s = gfx_wiretosurface2(self, bulk, &in_s, enc->u.gfx.data);
s = gfx_wiretosurface2(self, bulk, &in_s, &(enc->u.gfx));
break;
case XR_RDPGFX_CMDID_SOLIDFILL: /* 0x0004 */
s = gfx_solidfill(self, bulk, &in_s);
Expand Down

0 comments on commit 87d92c1

Please sign in to comment.