Add debug mode

When enabled, the result will be redirected to a window, and compton
won't take over the screen.

Makes debugging easier.

Signed-off-by: Yuxuan Shui <yshuiv7@gmail.com>

src/backend/gl/glx.c

@@ -25,6 +25,7 @@
 #include "backend/gl/gl_common.h"
 #include "backend/gl/glx.h"
 #include "common.h"
+#include "compton.h"
 #include "compiler.h"
 #include "config.h"
 #include "log.h"
@@ -221,7 +222,7 @@ static backend_t *glx_init(session_t *ps) {
 
 	gd->display = ps->dpy;
 	gd->screen = ps->scr;
-	gd->target_win = ps->overlay != XCB_NONE ? ps->overlay : ps->root;
+	gd->target_win = session_get_target_window(ps);
 
 	XVisualInfo *pvis = NULL;
 
@@ -310,10 +311,7 @@ static backend_t *glx_init(session_t *ps) {
 	}
 
 	// Attach GLX context
-	GLXDrawable tgt = ps->overlay;
-	if (!tgt) {
-		tgt = ps->root;
-	}
+	GLXDrawable tgt = gd->target_win;
 	if (!glXMakeCurrent(ps->dpy, tgt, gd->ctx)) {
 		log_error("Failed to attach GLX context.");
 		goto end;

src/backend/xrender/xrender.c

@@ -14,6 +14,7 @@
 #include "backend/backend.h"
 #include "backend/backend_common.h"
 #include "common.h"
+#include "compton.h"
 #include "config.h"
 #include "kernel.h"
 #include "log.h"
@@ -532,18 +533,12 @@ backend_t *backend_xrender_init(session_t *ps) {
 	xd->black_pixel = solid_picture(ps->c, ps->root, true, 1, 0, 0, 0);
 	xd->white_pixel = solid_picture(ps->c, ps->root, true, 1, 1, 1, 1);
 
-	if (ps->overlay != XCB_NONE) {
-		xd->target = x_create_picture_with_visual_and_pixmap(
-		    ps->c, ps->vis, ps->overlay, 0, NULL);
-		xd->target_win = ps->overlay;
-	} else {
-		xcb_render_create_picture_value_list_t pa = {
-		    .subwindowmode = XCB_SUBWINDOW_MODE_INCLUDE_INFERIORS,
-		};
-		xd->target = x_create_picture_with_visual_and_pixmap(
-		    ps->c, ps->vis, ps->root, XCB_RENDER_CP_SUBWINDOW_MODE, &pa);
-		xd->target_win = ps->root;
-	}
+	xd->target_win = session_get_target_window(ps);
+	xcb_render_create_picture_value_list_t pa = {
+	    .subwindowmode = XCB_SUBWINDOW_MODE_INCLUDE_INFERIORS,
+	};
+	xd->target = x_create_picture_with_visual_and_pixmap(
+	    ps->c, ps->vis, xd->target_win, XCB_RENDER_CP_SUBWINDOW_MODE, &pa);
 
 	auto pictfmt = x_get_pictform_for_visual(ps->c, ps->vis);
 	if (!pictfmt) {

src/common.h

@@ -171,6 +171,8 @@ typedef struct session {
 	// Damage root_damage;
 	/// X Composite overlay window. Used if <code>--paint-on-overlay</code>.
 	xcb_window_t overlay;
+	/// The target window for debug mode
+	xcb_window_t debug_window;
 	/// Whether the root tile is filled by compton.
 	bool root_tile_fill;
 	/// Picture of the root window background.

src/compton.c

@@ -798,7 +798,7 @@ void configure_root(session_t *ps, int width, int height) {
 		} else {
 			if (!initialize_backend(ps)) {
 				log_fatal("Failed to re-initialize backend after root "
-						"change, aborting...");
+				          "change, aborting...");
 				ps->quit = true;
 				// TODO only event handlers should request ev_break,
 				// otherwise it's too hard to keep track of what can break
@@ -902,7 +902,9 @@ static bool register_cm(session_t *ps) {
 	// Unredirect the window if it's redirected, just in case
 	if (ps->redirected)
 		xcb_composite_unredirect_window(ps->c, ps->reg_win,
-		                                XCB_COMPOSITE_REDIRECT_MANUAL);
+		                                ps->o.debug_mode
+		                                    ? XCB_COMPOSITE_REDIRECT_AUTOMATIC
+		                                    : XCB_COMPOSITE_REDIRECT_MANUAL);
 
 	{
 		XClassHint *h = XAllocClassHint();
@@ -1094,6 +1096,45 @@ static bool init_overlay(session_t *ps) {
 	return true;
 }
 
+static bool init_debug_window(session_t *ps) {
+	xcb_colormap_t colormap = x_new_id(ps->c);
+	ps->debug_window = x_new_id(ps->c);
+
+	auto err = xcb_request_check(
+	    ps->c, xcb_create_colormap_checked(ps->c, XCB_COLORMAP_ALLOC_NONE, colormap,
+	                                       ps->root, ps->vis));
+	if (err) {
+		goto err_out;
+	}
+
+	err = xcb_request_check(
+	    ps->c, xcb_create_window_checked(ps->c, (uint8_t)ps->depth, ps->debug_window,
+	                                     ps->root, 0, 0, to_u16_checked(ps->root_width),
+	                                     to_u16_checked(ps->root_height), 0,
+	                                     XCB_WINDOW_CLASS_INPUT_OUTPUT, ps->vis,
+	                                     XCB_CW_COLORMAP, (uint32_t[]){colormap, 0}));
+	if (err) {
+		goto err_out;
+	}
+
+	err = xcb_request_check(ps->c, xcb_map_window(ps->c, ps->debug_window));
+	if (err) {
+		goto err_out;
+	}
+	return true;
+
+err_out:
+	free(err);
+	return false;
+}
+
+xcb_window_t session_get_target_window(session_t *ps) {
+	if (ps->o.debug_mode) {
+		return ps->debug_window;
+	}
+	return ps->overlay != XCB_NONE ? ps->overlay : ps->root;
+}
+
 /**
  * Redirect all windows.
  *
@@ -1109,7 +1150,9 @@ static bool redir_start(session_t *ps) {
 		xcb_map_window(ps->c, ps->overlay);
 	}
 
-	xcb_composite_redirect_subwindows(ps->c, ps->root, XCB_COMPOSITE_REDIRECT_MANUAL);
+	xcb_composite_redirect_subwindows(ps->c, ps->root,
+	                                  ps->o.debug_mode ? XCB_COMPOSITE_REDIRECT_AUTOMATIC
+	                                                   : XCB_COMPOSITE_REDIRECT_MANUAL);
 
 	x_sync(ps->c);
 
@@ -1155,7 +1198,9 @@ static void redir_stop(session_t *ps) {
 
 	destroy_backend(ps);
 
-	xcb_composite_unredirect_subwindows(ps->c, ps->root, XCB_COMPOSITE_REDIRECT_MANUAL);
+	xcb_composite_unredirect_subwindows(ps->c, ps->root,
+	                                    ps->o.debug_mode ? XCB_COMPOSITE_REDIRECT_AUTOMATIC
+	                                                     : XCB_COMPOSITE_REDIRECT_MANUAL);
 	// Unmap overlay window
 	if (ps->overlay)
 		xcb_unmap_window(ps->c, ps->overlay);
@@ -1791,8 +1836,14 @@ static session_t *session_init(int argc, char **argv, Display *dpy,
 
 	// Overlay must be initialized before double buffer, and before creation
 	// of OpenGL context.
-	if (!init_overlay(ps)) {
-		goto err;
+	if (!ps->o.debug_mode) {
+		if (!init_overlay(ps)) {
+			goto err;
+		}
+	} else {
+		if (!init_debug_window(ps)) {
+			goto err;
+		}
 	}
 
 	ps->drivers = detect_driver(ps->c, ps->backend_data, ps->root);

src/compton.h

@@ -59,6 +59,8 @@ void discard_ignore(session_t *ps, unsigned long sequence);
 
 void set_root_flags(session_t *ps, uint64_t flags);
 
+xcb_window_t session_get_target_window(session_t *);
+
 /**
  * Set a <code>switch_t</code> array of all unset wintypes to true.
  */

src/config.h

@@ -62,6 +62,8 @@ typedef struct options {
 	// === Debugging ===
 	bool monitor_repaint;
 	bool print_diagnostics;
+	/// Render to a separate window instead of taking over the screen
+	bool debug_mode;
 	// === General ===
 	/// Use the experimental new backends?
 	bool experimental_backends;

src/options.c

@@ -308,9 +308,14 @@ static void usage(int ret) {
 	    "--benchmark-wid window-id\n"
 	    "  Specify window ID to repaint in benchmark mode. If omitted or is 0,\n"
 	    "  the whole screen is repainted.\n"
+	    "\n"
 	    "--monitor-repaint\n"
 	    "  Highlight the updated area of the screen. For debugging the xrender\n"
-	    "  backend only.\n";
+	    "  backend only.\n"
+	    "\n"
+	    "--debug-mode\n"
+	    "  Render into a separate window, and don't take over the screen. Useful\n"
+	    "  when you want to attach a debugger to compton\n";
 	FILE *f = (ret ? stderr : stdout);
 	fputs(usage_text, f);
 #undef WARNING_DISABLED
@@ -406,6 +411,7 @@ static const struct option longopts[] = {
     {"experimental-backends", no_argument, NULL, 733},
     {"monitor-repaint", no_argument, NULL, 800},
     {"diagnostics", no_argument, NULL, 801},
+    {"debug-mode", no_argument, NULL, 802},
     // Must terminate with a NULL entry
     {NULL, 0, NULL, 0},
 };
@@ -777,6 +783,7 @@ void get_cfg(options_t *opt, int argc, char *const *argv, bool shadow_enable,
 		P_CASEBOOL(733, experimental_backends);
 		P_CASEBOOL(800, monitor_repaint);
 		case 801: opt->print_diagnostics = true; break;
+		P_CASEBOOL(802, debug_mode);
 		default: usage(1); break;
 #undef P_CASEBOOL
 		}