diff --git a/packages/editor/src/components/post-featured-image/index.js b/packages/editor/src/components/post-featured-image/index.js
index 46a194f311a5e..acf366506cc41 100644
--- a/packages/editor/src/components/post-featured-image/index.js
+++ b/packages/editor/src/components/post-featured-image/index.js
@@ -18,7 +18,7 @@ import {
 	Notice,
 } from '@wordpress/components';
 import { isBlobURL } from '@wordpress/blob';
-import { useState, useRef, useEffect } from '@wordpress/element';
+import { useState, useRef } from '@wordpress/element';
 import { compose } from '@wordpress/compose';
 import { useSelect, withDispatch, withSelect } from '@wordpress/data';
 import {
@@ -102,17 +102,10 @@ function PostFeaturedImage( {
 	noticeOperations,
 	isRequestingFeaturedImageMedia,
 } ) {
-	const toggleRef = useRef();
+	const returnsFocusRef = useRef( false );
 	const [ isLoading, setIsLoading ] = useState( false );
 	const { getSettings } = useSelect( blockEditorStore );
 	const { mediaSourceUrl } = getMediaDetails( media, currentPostId );
-	const toggleFocusTimerRef = useRef();
-
-	useEffect( () => {
-		return () => {
-			clearTimeout( toggleFocusTimerRef.current );
-		};
-	}, [] );
 
 	function onDropFiles( filesList ) {
 		getSettings().mediaUpload( {
@@ -164,6 +157,13 @@ function PostFeaturedImage( {
 		);
 	}
 
+	function returnFocus( node ) {
+		if ( returnsFocusRef.current && node ) {
+			node.focus();
+			returnsFocusRef.current = false;
+		}
+	}
+
 	const isMissingMedia =
 		! isRequestingFeaturedImageMedia && !! featuredImageId && ! media;
 
@@ -203,7 +203,7 @@ function PostFeaturedImage( {
 								) : (
 									<Button
 										__next40pxDefaultSize
-										ref={ toggleRef }
+										ref={ returnFocus }
 										className={
 											! featuredImageId
 												? 'editor-post-featured-image__toggle'
@@ -276,12 +276,10 @@ function PostFeaturedImage( {
 											className="editor-post-featured-image__action"
 											onClick={ () => {
 												onRemoveImage();
-												// The toggle button is rendered conditionally, we need
-												// to wait it is rendered before moving focus to it.
-												toggleFocusTimerRef.current =
-													setTimeout( () => {
-														toggleRef.current?.focus();
-													} );
+												// Signal that the toggle button should be focused,
+												// when it is rendered. Can't focus it directly here
+												// because it's rendered conditionally.
+												returnsFocusRef.current = true;
 											} }
 											variant={
 												isMissingMedia