diff --git a/src/win.c b/src/win.c
index c77fcc2..7ec5ff3 100644
--- a/src/win.c
+++ b/src/win.c
@@ -707,6 +707,7 @@ static void win_set_shadow(session_t *ps, struct managed_win *w, bool shadow_new
 	// Mark the old extents as damaged if the shadow is removed
 	if (!w->shadow) {
 		add_damage(ps, &extents);
+		win_release_shadow(ps->backend_data, w);
 	}
 
 	pixman_region32_clear(&extents);