diff --git a/deps/media-playback/media-playback/decode.c b/deps/media-playback/media-playback/decode.c
index 1ab44f0e4..017c1beb7 100644
--- a/deps/media-playback/media-playback/decode.c
+++ b/deps/media-playback/media-playback/decode.c
@@ -279,18 +279,6 @@ bool mp_decode_next(struct mp_decode *d)
 			}
 		}
 
-		if (!d->audio && d->m->is_network && !d->got_first_keyframe) {
-			if (d->pkt.flags & AV_PKT_FLAG_KEY) {
-				d->got_first_keyframe = true;
-			} else {
-				av_packet_unref(&d->orig_pkt);
-				av_init_packet(&d->orig_pkt);
-				av_init_packet(&d->pkt);
-				d->packet_pending = false;
-				return true;
-			}
-		}
-
 		ret = decode_packet(d, &got_frame);
 
 		if (!got_frame && ret == 0) {
diff --git a/deps/media-playback/media-playback/media.c b/deps/media-playback/media-playback/media.c
index 78424d96e..dbb7f80fa 100644
--- a/deps/media-playback/media-playback/media.c
+++ b/deps/media-playback/media-playback/media.c
@@ -357,6 +357,13 @@ static void mp_media_next_video(mp_media_t *m, bool preload)
 	frame->height = f->height;
 	frame->flip = flip;
 
+	if (m->is_network && !d->got_first_keyframe) {
+		if (!f->key_frame)
+			return;
+
+		d->got_first_keyframe = true;
+	}
+
 	if (preload)
 		m->v_preload_cb(m->opaque, frame);
 	else