From 4efd280653f96d780f1ac3e60cc3fa21b2c35ebe Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Wed, 6 Nov 2024 22:53:40 -0800 Subject: [PATCH] dynamic resize working with accel_assist --- xrdp_accel_assist/xrdp_accel_assist.c | 89 ++++++++------------- xrdp_accel_assist/xrdp_accel_assist.h | 2 + xrdp_accel_assist/xrdp_accel_assist_nvenc.c | 12 ++- xrdp_accel_assist/xrdp_accel_assist_nvenc.h | 3 +- xrdp_accel_assist/xrdp_accel_assist_x11.c | 8 +- xrdp_accel_assist/xrdp_accel_assist_x11.h | 3 +- 6 files changed, 51 insertions(+), 66 deletions(-) diff --git a/xrdp_accel_assist/xrdp_accel_assist.c b/xrdp_accel_assist/xrdp_accel_assist.c index fb9be6325c..5c7b497978 100644 --- a/xrdp_accel_assist/xrdp_accel_assist.c +++ b/xrdp_accel_assist/xrdp_accel_assist.c @@ -43,14 +43,13 @@ struct xorgxrdp_info struct trans *xorg_trans; struct trans *xrdp_trans; struct source_info si; - int resizing; int shmem_fd_ret; int shmem_bytes_ret; + int idr_count; int pad0; }; static int g_display_num = 0; -int xrdp_invalidate = 0; /*****************************************************************************/ static int @@ -71,6 +70,7 @@ gfx_wiretosurface1(struct xorgxrdp_info *xi, struct stream *s) int height; int cdata_bytes; int rv; + int encoder_flags; char *flags_pointer; char *final_pointer; @@ -166,10 +166,17 @@ gfx_wiretosurface1(struct xorgxrdp_info *xi, struct stream *s) (void)top; cdata_bytes = GFX_MAP_SIZE; + encoder_flags = 0; + if (xi->idr_count > 0) + { + encoder_flags = XH_ENC_FLAGS_FORCEIDR; + xi->idr_count--; + } rv = xrdp_accel_assist_x11_encode_pixmap(0, 0, width, height, surface_id, num_rects_c, crects, - addr, &cdata_bytes); + addr, &cdata_bytes, + encoder_flags); LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface1: rv %d cdata_bytes %d", rv, cdata_bytes); @@ -248,7 +255,6 @@ xorg_process_message_62(struct xorgxrdp_info *xi, struct stream *s) } break; case 0x000E: /* XR_RDPGFX_CMDID_RESETGRAPHICS */ - xrdp_invalidate = 1; LOG(LOG_LEVEL_INFO, "xorg_process_message_62: " "XR_RDPGFX_CMDID_RESETGRAPHICS detected"); break; @@ -351,6 +357,7 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct stream *s) int cdata_bytes; int index; int recv_bytes; + int encoder_flags; enum encoder_result rv; struct xh_rect *crects; char *bmpdata; @@ -360,6 +367,7 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct stream *s) (void)twidth; (void)theight; + (void)frame_id; /* dirty pixels */ in_uint16_le(s, num_drects); @@ -389,26 +397,6 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct stream *s) in_uint16_le(s, theight); char *final_pointer = s->p; - if (xi->resizing == 3) - { - if (xrdp_invalidate > 0 && frame_id == 1) - { - // Let it through. We are no longer resizing. - LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing frame ID 1. Unblocking encoder. Invalidate is %d.", xrdp_invalidate); - xi->resizing = 0; - } - else - { - LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. Invalidate is %d", frame_id, xrdp_invalidate); - return 0; - } - } - - if (xi->resizing > 0) - { - return 0; - } - if (xi->shmem_fd_ret != -1) { LOG(LOG_LEVEL_ERROR, "xorg_process_message_64: xi->shmem_fd_ret " @@ -441,12 +429,18 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct stream *s) if ((bmpdata != NULL) && (flags & 1)) { cdata_bytes = 16 * 1024 * 1024; + encoder_flags = 0; + if (xi->idr_count > 0) + { + encoder_flags = XH_ENC_FLAGS_FORCEIDR; + xi->idr_count--; + } rv = xrdp_accel_assist_x11_encode_pixmap(left, top, width, height, (flags >> 28) & 0xF, num_crects, crects, bmpdata + 4, - &cdata_bytes); + &cdata_bytes, encoder_flags); if (rv == ENCODER_ERROR) { LOG(LOG_LEVEL_ERROR, "error %d", rv); @@ -550,10 +544,6 @@ xorg_process_message(struct xorgxrdp_info *xi, struct stream *s) s->p = phold + size; s->end = endhold; } - if (xi->resizing > 0) - { - return 0; - } } else if (type == 100) { @@ -568,10 +558,6 @@ xorg_process_message(struct xorgxrdp_info *xi, struct stream *s) case 1: LOG(LOG_LEVEL_DEBUG, "calling xrdp_accel_assist_x11_delete_all_pixmaps"); xrdp_accel_assist_x11_delete_all_pixmaps(); - if (xi->resizing == 1) - { - xi->resizing = 2; - } break; case 2: in_uint16_le(s, width); @@ -582,24 +568,22 @@ xorg_process_message(struct xorgxrdp_info *xi, struct stream *s) LOG(LOG_LEVEL_DEBUG, "calling xrdp_accel_assist_x11_create_pixmap"); xrdp_accel_assist_x11_create_pixmap(width, height, magic, con_id, mon_id); - if (xi->resizing == 2) - { - xi->resizing = 3; - } break; } s->p = phold + size; } + /* this will force I frames for 10 frames */ + xi->idr_count = 10; + LOG(LOG_LEVEL_INFO, "setting idr_count to %d", xi->idr_count); } s->p = s->data; - if (xi->shmem_fd_ret == -1) { - // Using system-v or no shared memory + /* no shared memory, ok */ ret = trans_write_copy_s(xi->xrdp_trans, s); return ret; } - // Using posix shared memory + /* using posix shared memory, got to pass fd on to xrdp */ ret = trans_force_write_s(xi->xrdp_trans, s); if (ret) { @@ -611,7 +595,8 @@ xorg_process_message(struct xorgxrdp_info *xi, struct stream *s) { return 1; } - return g_file_close(xi->shmem_fd_ret); + g_file_close(xi->shmem_fd_ret); + return 0; } /*****************************************************************************/ @@ -668,25 +653,23 @@ xrdp_process_message(struct xorgxrdp_info *xi, struct stream *s) in_uint32_le(s, len); in_uint16_le(s, msg_type1); - if (msg_type1 == 103) // client message + if (msg_type1 == 103) /* client message */ { in_uint32_le(s, msg_type2); - if (msg_type2 == 200) // invalidate + if (msg_type2 == 200) /* invalidate */ { LOG(LOG_LEVEL_DEBUG, "Invalidate found (len: %d, msg1: %d, msg2: %d)", len, msg_type1, msg_type2); - /* - 10 is an arbitrary number. Anecdotally, you need the first 10 frames to be key frames to make sure the client - receives at least one of them. - */ - xrdp_invalidate += 10; } - else if (msg_type2 == 300) // resize + else if (msg_type2 == 300) /* resize */ { - LOG(LOG_LEVEL_DEBUG, "Resize found (len: %d, msg1: %d, msg2: %d)", len, msg_type1, msg_type2); - xi->resizing = 1; + LOG(LOG_LEVEL_DEBUG, "Resize 300 found (len: %d, msg1: %d, msg2: %d)", len, msg_type1, msg_type2); + } + else if (msg_type2 == 302) /* resize */ + { + LOG(LOG_LEVEL_DEBUG, "Resize 302 found (len: %d, msg1: %d, msg2: %d)", len, msg_type1, msg_type2); } } - //Reset read pointer + /* Reset read pointer */ s->p = s->data; return trans_write_copy_s(xi->xorg_trans, s); } @@ -971,8 +954,6 @@ main(int argc, char **argv) xrdp_fd = g_atoi(g_getenv("XORGXRDP_XRDP_FD")); LOG(LOG_LEVEL_INFO, "xrdp_fd: %d", xrdp_fd); - xi.resizing = 0; - xi.xorg_trans = trans_create(TRANS_MODE_UNIX, 128 * 1024, 128 * 1024); xi.xorg_trans->sck = xorg_fd; xi.xorg_trans->status = TRANS_STATUS_UP; diff --git a/xrdp_accel_assist/xrdp_accel_assist.h b/xrdp_accel_assist/xrdp_accel_assist.h index 6f794feb5b..b04528420d 100644 --- a/xrdp_accel_assist/xrdp_accel_assist.h +++ b/xrdp_accel_assist/xrdp_accel_assist.h @@ -35,6 +35,8 @@ struct xh_rect short h; }; +#define XH_ENC_FLAGS_FORCEIDR (1 << 0) + enum encoder_result { INCREMENTAL_FRAME_ENCODED, /* P frame */ diff --git a/xrdp_accel_assist/xrdp_accel_assist_nvenc.c b/xrdp_accel_assist/xrdp_accel_assist_nvenc.c index 1f41b5b924..bd8e7de0f1 100644 --- a/xrdp_accel_assist/xrdp_accel_assist_nvenc.c +++ b/xrdp_accel_assist/xrdp_accel_assist_nvenc.c @@ -65,8 +65,6 @@ struct enc_info NV_ENC_REGISTERED_PTR registeredResource; }; -extern int xrdp_invalidate; - /*****************************************************************************/ int xrdp_accel_assist_nvenc_init(void) @@ -306,7 +304,8 @@ xrdp_accel_assist_nvenc_delete_encoder(struct enc_info *ei) /*****************************************************************************/ enum encoder_result xrdp_accel_assist_nvenc_encode(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes) + void *cdata, int *cdata_bytes, + int flags) { NV_ENC_PIC_PARAMS picParams; NV_ENC_LOCK_BITSTREAM lockBitstream; @@ -326,12 +325,11 @@ xrdp_accel_assist_nvenc_encode(struct enc_info *ei, int tex, picParams.inputTimeStamp = ei->frameCount; picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; picParams.encodePicFlags = NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; - if (xrdp_invalidate > 0 || ei->frameCount == 0) + if ((flags & XH_ENC_FLAGS_FORCEIDR) || (ei->frameCount < 1)) { picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; - LOG(LOG_LEVEL_INFO, "Forcing NVENC H264 IDR SPSPPS for frame id: %d," - "invalidate is: %d", ei->frameCount, xrdp_invalidate); - xrdp_invalidate = MAX(0, xrdp_invalidate - 1); + LOG(LOG_LEVEL_INFO, "Forcing NVENC H264 IDR SPSPPS for frame id: %d", + ei->frameCount); } nv_error = g_enc_funcs.nvEncEncodePicture(ei->enc, &picParams); rv = ENCODER_ERROR; diff --git a/xrdp_accel_assist/xrdp_accel_assist_nvenc.h b/xrdp_accel_assist/xrdp_accel_assist_nvenc.h index 75fc2ab2b9..23bc90c402 100644 --- a/xrdp_accel_assist/xrdp_accel_assist_nvenc.h +++ b/xrdp_accel_assist/xrdp_accel_assist_nvenc.h @@ -28,6 +28,7 @@ int xrdp_accel_assist_nvenc_delete_encoder(struct enc_info *ei); enum encoder_result xrdp_accel_assist_nvenc_encode(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes); + void *cdata, int *cdata_bytes, + int flags); #endif diff --git a/xrdp_accel_assist/xrdp_accel_assist_x11.c b/xrdp_accel_assist/xrdp_accel_assist_x11.c index 692b50885b..f1d728aa21 100644 --- a/xrdp_accel_assist/xrdp_accel_assist_x11.c +++ b/xrdp_accel_assist/xrdp_accel_assist_x11.c @@ -78,7 +78,8 @@ struct enc_funcs struct enc_info **ei); int (*destroy_enc)(struct enc_info *ei); enum encoder_result (*encode)(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes); + void *cdata, int *cdata_bytes, + int flags); }; static struct enc_funcs g_enc_funcs[] = @@ -834,7 +835,8 @@ enum encoder_result xrdp_accel_assist_x11_encode_pixmap(int left, int top, int width, int height, int mon_id, int num_crects, struct xh_rect *crects, - void *cdata, int *cdata_bytes) + void *cdata, int *cdata_bytes, + int flags) { struct mon_info *mi; struct shader_info *si; @@ -862,6 +864,6 @@ xrdp_accel_assist_x11_encode_pixmap(int left, int top, int width, int height, XFlush(g_display); /* encode */ rv = g_enc_funcs[g_enc].encode(mi->ei, mi->enc_texture, - cdata, cdata_bytes); + cdata, cdata_bytes, flags); return rv; } diff --git a/xrdp_accel_assist/xrdp_accel_assist_x11.h b/xrdp_accel_assist/xrdp_accel_assist_x11.h index dd95e608c2..6122979961 100644 --- a/xrdp_accel_assist/xrdp_accel_assist_x11.h +++ b/xrdp_accel_assist/xrdp_accel_assist_x11.h @@ -42,6 +42,7 @@ enum encoder_result xrdp_accel_assist_x11_encode_pixmap(int left, int top, int width, int height, int mon_id, int num_crects, struct xh_rect *crects, - void *cdata, int *cdata_bytes); + void *cdata, int *cdata_bytes, + int flags); #endif