backend: allow transparent windows to clip other windows

Transparent windows usually blends on top of other windows, this commit
adds an option to make transparent windows clip other windows like
non-transparent windows do.

Closes #265

Signed-off-by: Yuxuan Shui <yshuiv7@gmail.com>

man/picom.1.asciidoc

@@ -259,6 +259,9 @@ May also be one of the predefined kernels: `3x3box` (default), `5x5box`, `7x7box
 *--max-brightness*::
 	Dimming bright windows so their brightness doesn't exceed this set value. Brightness of a window is estimated by averaging all pixels in the window, so this could comes with a performance hit. Setting this to 1.0 disables this behaviour. Requires *--use-damage* to be disabled. (default: 1.0)
 
+*--transparent-clipping*::
+	Make transparent windows clip other windows like non-transparent windows do, instead of blending on top of them.
+
 FORMAT OF CONDITIONS
 --------------------
 Some options accept a condition string to match certain windows. A condition string is formed by one or more conditions, joined by logical operators.

src/backend/backend.c

@@ -134,9 +134,13 @@ void paint_all_new(session_t *ps, struct managed_win *t, bool ignore_damage) {
 	region_t reg_visible;
 	pixman_region32_init(&reg_visible);
 	pixman_region32_copy(&reg_visible, &ps->screen_reg);
-	if (t) {
+	if (t && !ps->o.transparent_clipping) {
 		// Calculate the region upon which the root window (wallpaper) is to be
 		// painted based on the ignore region of the lowest window, if available
+		//
+		// NOTE If transparent_clipping is enabled, transparent windows are
+		// included in the reg_ignore, but we still want to have the wallpaper
+		// beneath them, so we don't use reg_ignore for wallpaper in that case.
 		pixman_region32_subtract(&reg_visible, &reg_visible, t->reg_ignore);
 	}
 
@@ -171,6 +175,17 @@ void paint_all_new(session_t *ps, struct managed_win *t, bool ignore_damage) {
 		region_t reg_paint_in_bound;
 		pixman_region32_init(&reg_paint_in_bound);
 		pixman_region32_intersect(&reg_paint_in_bound, &reg_bound, &reg_paint);
+		if (ps->o.transparent_clipping) {
+			// <transparent-clipping-note>
+			// If transparent_clipping is enabled, we need to be SURE that
+			// things are not drawn inside reg_ignore, because otherwise they
+			// will appear underneath transparent windows.
+			// So here we have make sure reg_paint_in_bound \in reg_visible
+			// There are a few other places below where this is needed as
+			// well.
+			pixman_region32_intersect(&reg_paint_in_bound,
+			                          &reg_paint_in_bound, &reg_visible);
+		}
 
 		// Blur window background
 		// TODO since the background might change the content of the window (e.g.
@@ -216,6 +231,11 @@ void paint_all_new(session_t *ps, struct managed_win *t, bool ignore_damage) {
 				pixman_region32_translate(&reg_blur, w->g.x, w->g.y);
 				// make sure reg_blur \in reg_paint
 				pixman_region32_intersect(&reg_blur, &reg_blur, &reg_paint);
+				if (ps->o.transparent_clipping) {
+					// ref: <transparent-clipping-note>
+					pixman_region32_intersect(&reg_blur, &reg_blur,
+					                          &reg_visible);
+				}
 				ps->backend_data->ops->blur(ps->backend_data, blur_opacity,
 				                            ps->backend_blur_context,
 				                            &reg_blur, &reg_visible);
@@ -254,6 +274,12 @@ void paint_all_new(session_t *ps, struct managed_win *t, bool ignore_damage) {
 				    &ps->xinerama_scr_regs[w->xinerama_scr]);
 			}
 
+			if (ps->o.transparent_clipping) {
+				// ref: <transparent-clipping-note>
+				pixman_region32_intersect(&reg_shadow, &reg_shadow,
+				                          &reg_visible);
+			}
+
 			assert(w->shadow_image);
 			if (w->opacity == 1) {
 				ps->backend_data->ops->compose(

src/config.h

@@ -235,6 +235,10 @@ typedef struct options {
 
 	// Don't use EWMH to detect fullscreen applications
 	bool no_ewmh_fullscreen;
+
+	// Make transparent windows clip other windows, instead of blending on top of
+	// them
+	bool transparent_clipping;
 } options_t;
 
 extern const char *const BACKEND_STRS[NUM_BKEND + 1];

src/config_libconfig.c

@@ -364,6 +364,8 @@ char *parse_config_libconfig(options_t *opt, const char *config_file, bool *shad
 	lcfg_lookup_bool(&cfg, "detect-client-leader", &opt->detect_client_leader);
 	// --no-ewmh-fullscreen
 	lcfg_lookup_bool(&cfg, "no-ewmh-fullscreen", &opt->no_ewmh_fullscreen);
+	// --transparent-clipping
+	lcfg_lookup_bool(&cfg, "transparent-clipping", &opt->transparent_clipping);
 	// --shadow-exclude
 	parse_cfg_condlst(&cfg, &opt->shadow_blacklist, "shadow-exclude");
 	// --fade-exclude

src/options.c

@@ -321,7 +321,11 @@ static void usage(const char *argv0, int ret) {
 	    "\n"
 	    "--no-ewmh-fullscreen\n"
 	    "  Do not use EWMH to detect fullscreen windows. Reverts to checking\n"
-	    "  if a window is fullscreen based only on its size and coordinates.\n";
+	    "  if a window is fullscreen based only on its size and coordinates.\n"
+	    "\n"
+	    "--transparent-clipping\n"
+	    "  Make transparent windows clip other windows like non-transparent windows\n"
+	    "  do, instead of blending on top of them\n";
 	FILE *f = (ret ? stderr : stdout);
 	fprintf(f, usage_text, argv0);
 #undef WARNING_DISABLED
@@ -417,6 +421,7 @@ static const struct option longopts[] = {
     {"no-use-damage", no_argument, NULL, 324},
     {"no-vsync", no_argument, NULL, 325},
     {"max-brightness", required_argument, NULL, 326},
+    {"transparent-clipping", no_argument, NULL, 327},
     {"experimental-backends", no_argument, NULL, 733},
     {"monitor-repaint", no_argument, NULL, 800},
     {"diagnostics", no_argument, NULL, 801},
@@ -802,6 +807,7 @@ bool get_cfg(options_t *opt, int argc, char *const *argv, bool shadow_enable,
 		case 326:
 			opt->max_brightness = atof(optarg);
 			break;
+		P_CASEBOOL(327, transparent_clipping);
 
 		P_CASEBOOL(733, experimental_backends);
 		P_CASEBOOL(800, monitor_repaint);
@@ -835,6 +841,12 @@ bool get_cfg(options_t *opt, int argc, char *const *argv, bool shadow_enable,
 		return false;
 	}
 
+	if (opt->transparent_clipping && !opt->experimental_backends) {
+		log_error("Transparent clipping only works with the experimental "
+		          "backends");
+		return false;
+	}
+
 	// Range checking and option assignments
 	opt->fade_delta = max2(opt->fade_delta, 1);
 	opt->shadow_radius = max2(opt->shadow_radius, 0);

src/picom.c

@@ -549,10 +549,11 @@ static struct managed_win *paint_preprocess(session_t *ps, bool *fade_running) {
 			w->reg_ignore = rc_region_ref(last_reg_ignore);
 		}
 
-		// If the window is solid, we add the window region to the
+		// If the window is solid, or we enabled clipping for transparent windows,
-		// ignored region
+		// we add the window region to the ignored region
 		// Otherwise last_reg_ignore shouldn't change
-		if (w->mode != WMODE_TRANS && !ps->o.force_win_blend) {
+		if ((w->mode != WMODE_TRANS && !ps->o.force_win_blend) ||
+		    ps->o.transparent_clipping) {
 			// w->mode == WMODE_SOLID or WMODE_FRAME_TRANS
 			region_t *tmp = rc_region_new();
 			if (w->mode == WMODE_SOLID) {