diff --git a/src/js/media/controllers/featured-image.js b/src/js/media/controllers/featured-image.js
index 5fd5797bd0..0f87ff659c 100644
--- a/src/js/media/controllers/featured-image.js
+++ b/src/js/media/controllers/featured-image.js
@@ -106,9 +106,7 @@ FeaturedImage = Library.extend(/** @lends wp.media.controller.FeaturedImage.prot
 	 */
 	updateSelection: function() {
 		var selection = this.get('selection'),
-			library = this.get('library'),
 			id = wp.media.view.settings.post.featuredImageId,
-			infiniteScrolling = wp.media.view.settings.infiniteScrolling,
 			attachment;
 
 		if ( '' !== id && -1 !== id ) {
@@ -117,10 +115,6 @@ FeaturedImage = Library.extend(/** @lends wp.media.controller.FeaturedImage.prot
 		}
 
 		selection.reset( attachment ? [ attachment ] : [] );
-
-		if ( ! infiniteScrolling && library.hasMore() ) {
-			library.more();
-		}
 	}
 });
 
diff --git a/src/js/media/controllers/replace-image.js b/src/js/media/controllers/replace-image.js
index c400d22b10..46dce6d1ba 100644
--- a/src/js/media/controllers/replace-image.js
+++ b/src/js/media/controllers/replace-image.js
@@ -90,24 +90,28 @@ ReplaceImage = Library.extend(/** @lends wp.media.controller.ReplaceImage.protot
 	 * @since 3.9.0
 	 */
 	activate: function() {
-		this.updateSelection();
+		this.frame.on( 'content:render:browse', this.updateSelection, this );
+
 		Library.prototype.activate.apply( this, arguments );
 	},
 
+	/**
+	 * @since 5.9.0
+	 */
+	deactivate: function() {
+		this.frame.off( 'content:render:browse', this.updateSelection, this );
+
+		Library.prototype.deactivate.apply( this, arguments );
+	},
+
 	/**
 	 * @since 3.9.0
 	 */
 	updateSelection: function() {
 		var selection = this.get('selection'),
-			library = this.get('library'),
-			attachment = this.image.attachment,
-			infiniteScrolling = wp.media.view.settings.infiniteScrolling;
+			attachment = this.image.attachment;
 
 		selection.reset( attachment ? [ attachment ] : [] );
-
-		if ( ! infiniteScrolling && library.getTotalAttachments() === 0 && library.hasMore() ) {
-			library.more();
-		}
 	}
 });