Improve transparency detection for blur

Previously, only the WM windows (window manager usually reparents the
application windows under its own window so it can manage them) are
checked for alpha channels. Some of the window managers (e.g. awesome)
acquire alpha channels for all of the WM windows, whether the underlying
application windows have alpha channel or not.

With this change, the application windows are also checked for alpha
channels. If a WM window has alpha but its child doesn't, only the WM
frame will be considered to be semi-transparent. Thus preventing some
unnecessary blurring.

Closes #191

Signed-off-by: Yuxuan Shui <yshuiv7@gmail.com>

src/backend/backend.c

@@ -164,21 +164,35 @@ void paint_all_new(session_t *ps, struct managed_win *t, bool ignore_damage) {
 		// Blur window background
 		bool win_transparent = ps->backend_data->ops->is_image_transparent(
 		    ps->backend_data, w->win_image);
-		bool frame_transparent = w->frame_opacity != 1;
+		auto real_win_mode = w->mode;
+		if (win_transparent && real_win_mode == WMODE_SOLID) {
+			// Compton core thought the window is opaque, but background thinks
+			// the window is transparent. Since the core doesn't have extra
+			// information to determine which part of the window is
+			// transparent, just assume the whole window is.
+			real_win_mode = WMODE_TRANS;
+		}
+
 		if (w->blur_background &&
-		    (win_transparent || (ps->o.blur_background_frame && frame_transparent))) {
+		    (ps->o.force_win_blend || real_win_mode == WMODE_TRANS ||
+		     (ps->o.blur_background_frame && real_win_mode == WMODE_FRAME_TRANS))) {
 			// Minimize the region we try to blur, if the window
 			// itself is not opaque, only the frame is.
 			// TODO resize blur region to fix black line artifact
-			if (!win_is_solid(ps, w)) {
+			if (real_win_mode == WMODE_TRANS || ps->o.force_win_blend) {
 				// We need to blur the bounding shape of the window
 				// (reg_paint = reg_bound \cap reg_damage)
 				ps->backend_data->ops->blur(ps->backend_data, w->opacity,
 				                            ps->backend_blur_context,
 				                            &reg_paint, &reg_visible);
-			} else if (frame_transparent && ps->o.blur_background_frame) {
+			} else {
 				// Window itself is solid, we only need to blur the frame
 				// region
+
+				// Readability assertions
+				assert(ps->o.blur_background_frame);
+				assert(real_win_mode == WMODE_FRAME_TRANS);
+
 				auto reg_blur = win_get_region_frame_local_by_val(w);
 				pixman_region32_translate(&reg_blur, w->g.x, w->g.y);
 				// make sure reg_blur \in reg_damage

src/compton.c

@@ -567,11 +567,11 @@ static struct managed_win *paint_preprocess(session_t *ps, bool *fade_running) {
 		// fading is enabled, and could create inconsistency when the wallpaper
 		// is not correctly set.
 		if (ps->o.unredir_if_possible && is_highest) {
-			if (win_is_solid(ps, w) &&
-			    (w->frame_opacity == 1 || !win_has_frame(w)) &&
-			    win_is_fullscreen(ps, w) && !w->unredir_if_possible_excluded)
+			if (w->mode == WMODE_SOLID && !ps->o.force_win_blend &&
+			    win_is_fullscreen(ps, w) && !w->unredir_if_possible_excluded) {
 				unredir_possible = true;
 			}
+		}
 
 		w->prev_trans = bottom;
 		bottom = w;

src/render.c

@@ -715,7 +715,7 @@ win_blur_background(session_t *ps, struct managed_win *w, xcb_render_picture_t t
 		// Minimize the region we try to blur, if the window itself is not
 		// opaque, only the frame is.
 		region_t reg_blur = win_get_bounding_shape_global_by_val(w);
-		if (win_is_solid(ps, w)) {
+		if (w->mode == WMODE_FRAME_TRANS && !ps->o.force_win_blend) {
 			region_t reg_noframe;
 			pixman_region32_init(&reg_noframe);
 			win_get_region_noframe_local(w, &reg_noframe);
@@ -895,8 +895,9 @@ void paint_all(session_t *ps, struct managed_win *t, bool ignore_damage) {
 			set_tgt_clip(ps, &reg_tmp);
 			// Blur window background
 			if (w->blur_background &&
-			    (!win_is_solid(ps, w) ||
-			     (ps->o.blur_background_frame && w->frame_opacity != 1)))
+			    (w->mode == WMODE_TRANS ||
+			     (ps->o.blur_background_frame && w->mode == WMODE_FRAME_TRANS) ||
+			     ps->o.force_win_blend))
 				win_blur_background(ps, w, ps->tgt_buffer.pict, &reg_tmp);
 
 			// Painting the window

src/win.c

@@ -417,13 +417,36 @@ bool win_has_alpha(const struct managed_win *w) {
 	       w->pictfmt->direct.alpha_mask;
 }
 
+bool win_client_has_alpha(const struct managed_win *w) {
+	return w->client_pictfmt && w->client_pictfmt->type == XCB_RENDER_PICT_TYPE_DIRECT &&
+	       w->client_pictfmt->direct.alpha_mask;
+}
+
 winmode_t win_calc_mode(const struct managed_win *w) {
-	if (win_has_alpha(w) || w->opacity < 1.0) {
+	if (w->opacity < 1.0) {
 		return WMODE_TRANS;
 	}
-	if (w->frame_opacity != 1.0) {
+	if (w->frame_opacity != 1.0 && win_has_frame(w)) {
 		return WMODE_FRAME_TRANS;
 	}
+
+	if (win_has_alpha(w)) {
+		// The WM window has alpha
+		if (win_client_has_alpha(w)) {
+			// The client window also has alpha, the entire window is
+			// transparent
+			return WMODE_TRANS;
+		}
+		if (win_has_frame(w)) {
+			// The client window doesn't have alpha, but we have a WM frame
+			// window, which has alpha.
+			return WMODE_FRAME_TRANS;
+		}
+		// Although the WM window has alpha, the frame window has 0 size, so
+		// consider the window solid
+	}
+
+	//log_trace("Window %#010x(%s) is solid", w->client_win, w->name);
 	return WMODE_SOLID;
 }
 
@@ -690,15 +713,14 @@ void win_determine_invert_color(session_t *ps, struct managed_win *w) {
 	win_set_invert_color(ps, w, invert_color_new);
 }
 
-void win_set_blur_background(session_t *ps, struct managed_win *w, bool blur_background_new) {
+static void win_set_blur_background(session_t *ps, struct managed_win *w, bool blur_background_new) {
 	if (w->blur_background == blur_background_new)
 		return;
 
 	w->blur_background = blur_background_new;
 
-	// Only consider window damaged if it's previously painted with background
-	// blurred
-	if (!win_is_solid(ps, w) || (ps->o.blur_background_frame && w->frame_opacity != 1))
+	// This damage might not be absolutely necessary (e.g. when the window is opaque),
+	// but blur_background changes should be rare, so this should be fine.
 	add_damage_from_win(ps, w);
 }
 
@@ -709,8 +731,8 @@ void win_determine_blur_background(session_t *ps, struct managed_win *w) {
 	if (w->a.map_state != XCB_MAP_STATE_VIEWABLE)
 		return;
 
-	bool blur_background_new = ps->o.blur_method &&
-	                           !c2_match(ps, w, ps->o.blur_background_blacklist, NULL);
+	bool blur_background_new =
+	    ps->o.blur_method && !c2_match(ps, w, ps->o.blur_background_blacklist, NULL);
 
 	win_set_blur_background(ps, w, blur_background_new);
 }
@@ -873,6 +895,16 @@ void win_mark_client(session_t *ps, struct managed_win *w, xcb_window_t client)
 
 	// Update window focus state
 	win_update_focused(ps, w);
+
+	auto r = xcb_get_window_attributes_reply(
+	    ps->c, xcb_get_window_attributes(ps->c, w->client_win), NULL);
+	if (!r) {
+		log_error("Failed to get client window attributes");
+		return;
+	}
+
+	w->client_pictfmt = x_get_pictform_for_visual(ps->c, r->visual);
+	free(r);
 }
 
 /**
@@ -1029,6 +1061,7 @@ struct win *fill_win(session_t *ps, struct win *w) {
 	    // Initialized in this function
 	    .a = {0},
 	    .pictfmt = NULL,
+	    .client_pictfmt = NULL,
 	    .widthb = 0,
 	    .heightb = 0,
 	    .shadow_dx = 0,
@@ -1140,6 +1173,7 @@ struct win *fill_win(session_t *ps, struct win *w) {
 	}
 
 	new->pictfmt = x_get_pictform_for_visual(ps->c, new->a.visual);
+	new->client_pictfmt = NULL;
 
 	list_replace(&w->stack_neighbour, &new->base.stack_neighbour);
 	struct win *replaced = NULL;
@@ -2127,13 +2161,6 @@ bool win_is_fullscreen(const session_t *ps, const struct managed_win *w) {
 	       (!w->bounding_shaped || w->rounded_corners);
 }
 
-/**
- * Check if a window will be painted solid.
- */
-bool win_is_solid(const session_t *ps, const struct managed_win *w) {
-	return WMODE_SOLID == w->mode && !ps->o.force_win_blend;
-}
-
 /**
  * Check if a window is really focused.
  */

src/win.h

@@ -180,8 +180,10 @@ struct managed_win {
 	xcb_get_geometry_reply_t g;
 	/// Xinerama screen this window is on.
 	int xinerama_scr;
-	/// Window visual pict format;
+	/// Window visual pict format
 	const xcb_render_pictforminfo_t *pictfmt;
+	/// Client window visual pict format
+	const xcb_render_pictforminfo_t *client_pictfmt;
 	/// Window painting mode.
 	winmode_t mode;
 	/// Whether the window has been damaged at least once.
@@ -343,7 +345,6 @@ void win_set_shadow(session_t *ps, struct managed_win *w, bool shadow_new);
 void win_determine_shadow(session_t *ps, struct managed_win *w);
 void win_set_invert_color(session_t *ps, struct managed_win *w, bool invert_color_new);
 void win_determine_invert_color(session_t *ps, struct managed_win *w);
-void win_set_blur_background(session_t *ps, struct managed_win *w, bool blur_background_new);
 void win_determine_blur_background(session_t *ps, struct managed_win *w);
 void win_on_wtype_change(session_t *ps, struct managed_win *w);
 void win_on_factor_change(session_t *ps, struct managed_win *w);
@@ -466,10 +467,6 @@ struct managed_win *find_toplevel2(session_t *ps, xcb_window_t wid);
  */
 bool attr_pure win_is_fullscreen(const session_t *ps, const struct managed_win *w);
 
-/**
- * Check if a window will be painted solid.
- */
-bool attr_pure win_is_solid(const session_t *ps, const struct managed_win *w);
 /**
  * Check if a window is really focused.
  */