Preparation for dummy backends

Dummy backends are backends that doesn't render anything on to the
screen or the target window. But they are still be useful for debug
purposes, and can be useful for other things too.

Signed-off-by: Yuxuan Shui <yshuiv7@gmail.com>

src/backend/backend.h

@@ -71,7 +71,6 @@ struct kernel_blur_args {
 };
 
 struct backend_operations {
-
 	// ===========    Initialization    ===========
 
 	/// Initialize the backend, prepare for rendering to the target window.
@@ -83,7 +82,7 @@ struct backend_operations {
 	void (*deinit)(backend_t *backend_data) attr_nonnull(1);
 
 	/// Called when rendering will be stopped for an unknown amount of
-	/// time (e.g. screen is unredirected). Free some resources.
+	/// time (e.g. when screen is unredirected). Free some resources.
 	///
 	/// Optional, not yet used
 	void (*pause)(backend_t *backend_data, session_t *ps);
@@ -115,7 +114,8 @@ struct backend_operations {
 	// but we have to discard the result (because the result of blurring that area
 	// will be wrong). That's why we cannot render into the back buffer directly.
 	// After rendering is done, `present` is called to update a portion of the actual
-	// back buffer, then present it to the screen.
+	// back buffer, then present it to the target (or update the target directly,
+	// if not back buffered).
 
 	/// Called before when a new frame starts.
 	///
@@ -143,11 +143,13 @@ struct backend_operations {
 	    attr_nonnull(1, 3, 4, 5);
 
 	/// Update part of the back buffer with the rendering buffer, then present the
-	/// back buffer onto the screen.
+	/// back buffer onto the target window (if not back buffered, update part of the
+	/// target window directly).
 	///
-	/// @param region part of the screen that should be updated. if NULL, update the
-	///               whole screen
-	void (*present)(backend_t *backend_data, const region_t *region) attr_nonnull(1);
+	/// Optional, if NULL, indicates the backend doesn't have render output
+	///
+	/// @param region part of the target that should be updated
+	void (*present)(backend_t *backend_data, const region_t *region) attr_nonnull(1, 2);
 
 	/**
 	 * Bind a X pixmap to the backend's internal image data structure.
@@ -202,7 +204,7 @@ struct backend_operations {
 	 * @param reg_op       the clip region, define the part of the image to be
 	 *                     operated on.
 	 * @param reg_visible  define the part of the image that will eventually
-	 *                     be visible on screen. this is a hint to the backend
+	 *                     be visible on target. this is a hint to the backend
 	 *                     for optimization purposes.
 	 * @param args         extra arguments, operation specific
 	 * @return a new image data structure containing the result
@@ -231,8 +233,6 @@ struct backend_operations {
 	enum driver (*detect_driver)(backend_t *backend_data);
 };
 
-typedef backend_t *(*backend_init_fn)(session_t *ps)attr_nonnull(1);
-
 extern struct backend_operations *backend_list[];
 
 void paint_all_new(session_t *ps, struct managed_win *const t, bool ignore_damage)

src/compton.c

@@ -1147,6 +1147,15 @@ xcb_window_t session_get_target_window(session_t *ps) {
 	return ps->overlay != XCB_NONE ? ps->overlay : ps->root;
 }
 
+static inline uint8_t session_redirection_mode(session_t *ps) {
+	if (ps->o.debug_mode || !backend_list[ps->o.backend]->present) {
+		// If the backend is not rendering to the screen, or if the backend
+		// doesn't render anything, we don't need to take over the screen.
+		return XCB_COMPOSITE_REDIRECT_AUTOMATIC;
+	}
+	return XCB_COMPOSITE_REDIRECT_MANUAL;
+}
+
 /**
  * Redirect all windows.
  *
@@ -1162,9 +1171,7 @@ static bool redir_start(session_t *ps) {
 		xcb_map_window(ps->c, ps->overlay);
 	}
 
-	xcb_composite_redirect_subwindows(ps->c, ps->root,
-	                                  ps->o.debug_mode ? XCB_COMPOSITE_REDIRECT_AUTOMATIC
-	                                                   : XCB_COMPOSITE_REDIRECT_MANUAL);
+	xcb_composite_redirect_subwindows(ps->c, ps->root, session_redirection_mode(ps));
 
 	x_sync(ps->c);
 
@@ -1210,9 +1217,7 @@ static void redir_stop(session_t *ps) {
 
 	destroy_backend(ps);
 
-	xcb_composite_unredirect_subwindows(ps->c, ps->root,
-	                                    ps->o.debug_mode ? XCB_COMPOSITE_REDIRECT_AUTOMATIC
-	                                                     : XCB_COMPOSITE_REDIRECT_MANUAL);
+	xcb_composite_unredirect_subwindows(ps->c, ps->root, session_redirection_mode(ps));
 	// Unmap overlay window
 	if (ps->overlay)
 		xcb_unmap_window(ps->c, ps->overlay);
@@ -1845,19 +1850,24 @@ static session_t *session_init(int argc, char **argv, Display *dpy,
 	rebuild_screen_reg(ps);
 
 	// Create registration window
-	if (!ps->o.debug_mode && !register_cm(ps)) {
+	// If we are not taking over the screen, we don't need to register as a compositor
+	if (session_redirection_mode(ps) == XCB_COMPOSITE_REDIRECT_MANUAL && !register_cm(ps)) {
 		exit(1);
 	}
 
-	// Overlay must be initialized before double buffer, and before creation
-	// of OpenGL context.
-	if (!ps->o.debug_mode) {
-		if (!init_overlay(ps)) {
-			goto err;
-		}
-	} else {
-		if (!init_debug_window(ps)) {
-			goto err;
+	// Target window must be initialized before the backend
+	//
+	// backend_operations::present == NULL means this backend doesn't need a target
+	// window
+	if (backend_list[ps->o.backend]->present != NULL) {
+		if (!ps->o.debug_mode) {
+			if (!init_overlay(ps)) {
+				goto err;
+			}
+		} else {
+			if (!init_debug_window(ps)) {
+				goto err;
+			}
 		}
 	}