Bug fix #140: Possible fix for CPU usage with --unredir-possible-delay

- Possible fix for high CPU usage with a low --unredir-possible-delay.
  Thanks to Feltzer for reporting. (#140)

- Rewrite some parts to (hopefully) increase performance, especially
  with --unredir-if-possible.

- Known issue: With GTX 670 and x11-drivers/nvidia-drivers-325.15, and
  compton --unredir-if-possible --config /dev/null, if you send a
  SIGUSR1 to compton when a full-screen solid window is there, in which
  case compton either redirects then immediately unredirects the screen,
  or just don't redirect it altogether, X freezes after compton
  unredirects the screen. Requests sent by other X clients are not
  responded until compton is killed, which indicates the possibility of
  a bug in X.  Attaching to X process shows X is on ./os/waitFor.c.
  Backend does not matter. --paint-on-overlay fixes the issue somehow.
  compton-git-v0.1_beta1-5-g4600f43-2013-08-28 doesn't exhibit the
  issue, but it's probably timing-related.

src/compton.c

@@ -1102,9 +1102,16 @@ paint_preprocess(session_t *ps, win *list) {
     next = w->next;
     opacity_t opacity_old = w->opacity;
 
+    // Data expiration
+    {
+      // Remove built shadow if needed
+      if (w->flags & WFLAG_SIZE_CHANGE)
+        free_paint(ps, &w->shadow_paint);
+
       // Destroy reg_ignore on all windows if they should expire
       if (ps->reg_ignore_expire)
         free_region(ps, &w->reg_ignore);
+    }
 
     // Update window opacity target and dim state if asked
     if (WFLAG_OPCT_CHANGE & w->flags) {
@@ -1115,43 +1122,33 @@ paint_preprocess(session_t *ps, win *list) {
     // Run fading
     run_fade(ps, w, steps);
 
+    // Opacity will not change, from now on.
+
     // Give up if it's not damaged or invisible, or it's unmapped and its
-    // pixmap is gone (for example due to a ConfigureNotify)
+    // pixmap is gone (for example due to a ConfigureNotify), or when it's
+    // excluded
     if (!w->damaged
         || w->a.x + w->a.width < 1 || w->a.y + w->a.height < 1
         || w->a.x >= ps->root_width || w->a.y >= ps->root_height
-        || ((IsUnmapped == w->a.map_state || w->destroyed)
+        || ((IsUnmapped == w->a.map_state || w->destroyed) && !w->paint.pixmap)
-          && !w->paint.pixmap)) {
+        || get_alpha_pict_o(ps, w->opacity) == ps->alpha_picts[0]
+        || w->paint_excluded)
       to_paint = false;
-    }
 
-    to_paint = to_paint && !w->paint_excluded;
+    // to_paint will never change afterward
 
-    if (to_paint) {
+    // Determine mode as early as possible
-      // If opacity changes
+    if (to_paint && (!w->to_paint || w->opacity != opacity_old))
-      if (w->opacity != opacity_old) {
       win_determine_mode(ps, w);
-        add_damage_win(ps, w);
-      }
-
-      if (get_alpha_pict_o(ps, w->opacity) == ps->alpha_picts[0])
-        to_paint = false;
-    }
 
     if (to_paint) {
       // Fetch bounding region
-      if (!w->border_size) {
+      if (!w->border_size)
         w->border_size = border_size(ps, w, true);
-      }
 
       // Fetch window extents
-      if (!w->extents) {
+      if (!w->extents)
         w->extents = win_extents(ps, w);
-        // If w->extents does not exist, the previous add_damage_win()
-        // call when opacity changes has no effect, so redo it here.
-        if (w->opacity != opacity_old)
-          add_damage_win(ps, w);
-      }
 
       // Calculate frame_opacity
       {
@@ -1164,6 +1161,8 @@ paint_preprocess(session_t *ps, win *list) {
         else
           w->frame_opacity = 0.0;
 
+        // Destroy all reg_ignore above when frame opaque state changes on
+        // SOLID mode
         if (w->to_paint && WMODE_SOLID == mode_old
             && (0.0 == frame_opacity_old) != (0.0 == w->frame_opacity))
           ps->reg_ignore_expire = true;
@@ -1174,24 +1173,18 @@ paint_preprocess(session_t *ps, win *list) {
         w->shadow_opacity = ps->o.shadow_opacity * w->frame_opacity;
       else
         w->shadow_opacity = ps->o.shadow_opacity * get_opacity_percent(w);
-
-      // Rebuild shadow if necessary
-      if (w->flags & WFLAG_SIZE_CHANGE) {
-        free_paint(ps, &w->shadow_paint);
     }
 
-      if (w->shadow && !paint_isvalid(ps, &w->shadow_paint))
+    // Add window to damaged area if its painting status changes
-        win_build_shadow(ps, w, 1);
+    // or opacity changes
-    }
+    if (to_paint != w->to_paint || w->opacity != opacity_old)
+      add_damage_win(ps, w);
 
+    // Destroy all reg_ignore above when window mode changes
     if ((to_paint && WMODE_SOLID == w->mode)
         != (w->to_paint && WMODE_SOLID == mode_old))
       ps->reg_ignore_expire = true;
 
-    // Add window to damaged area if its painting status changes
-    if (to_paint != w->to_paint)
-      add_damage_win(ps, w);
-
     if (to_paint) {
       // Generate ignore region for painting to reduce GPU load
       if (ps->reg_ignore_expire || !w->to_paint) {
@@ -1264,6 +1257,10 @@ paint_preprocess(session_t *ps, win *list) {
   if (UNSET != ps->o.redirected_force)
     unredir_possible = !ps->o.redirected_force;
 
+  // If there's no window to paint, and the screen isn't redirected,
+  // don't redirect it.
+  if (ps->o.unredir_if_possible && is_highest && !ps->redirected)
+    unredir_possible = true;
   if (unredir_possible) {
     if (ps->redirected) {
       if (!ps->o.unredir_if_possible_delay || ps->tmout_unredir_hit)
@@ -1785,6 +1782,10 @@ paint_all(session_t *ps, XserverRegion region, XserverRegion region_real, win *t
   for (win *w = t; w; w = w->prev_trans) {
     // Painting shadow
     if (w->shadow) {
+      // Lazy shadow building
+      if (!paint_isvalid(ps, &w->shadow_paint))
+        win_build_shadow(ps, w, 1);
+
       // Shadow is to be painted based on the ignore region of current
       // window
       if (w->reg_ignore) {
@@ -2050,6 +2051,13 @@ wid_get_prop_wintype(session_t *ps, Window wid) {
 
 static void
 map_win(session_t *ps, Window id) {
+  // Unmap overlay window if it got mapped but we are currently not
+  // in redirected state.
+  if (ps->overlay && id == ps->overlay && !ps->redirected) {
+    XUnmapWindow(ps->dpy, ps->overlay);
+    XFlush(ps->dpy);
+  }
+
   win *w = find_win(ps, id);
 
   // Don't care about window mapping if it's an InputOnly window
@@ -2831,12 +2839,11 @@ add_win(session_t *ps, Window id, Window prev) {
   assert(IsViewable == map_state || IsUnmapped == map_state);
   new->a.map_state = IsUnmapped;
 
+  if (InputOutput == new->a.class) {
        // Get window picture format
-  if (InputOutput == new->a.class)
     new->pictfmt = XRenderFindVisualFormat(ps->dpy, new->a.visual);
 
        // Create Damage for window
-  if (InputOutput == new->a.class) {
        set_ignore_next(ps);
        new->damage = XDamageCreate(ps->dpy, id, XDamageReportNonEmpty);
   }
@@ -6097,6 +6104,11 @@ init_overlay(session_t *ps) {
     // Retrieve DamageNotify on root window if we are painting on an
     // overlay
     // root_damage = XDamageCreate(ps->dpy, root, XDamageReportNonEmpty);
+
+    // Unmap overlay, firstly. But this typically does not work because
+    // the window isn't created yet.
+    // XUnmapWindow(ps->dpy, ps->overlay);
+    // XFlush(ps->dpy);
   }
   else {
     fprintf(stderr, "Cannot get X Composite overlay window. Falling "
@@ -6371,6 +6383,9 @@ tmout_unredir_callback(session_t *ps, timeout_t *tmout) {
  */
 static bool
 mainloop(session_t *ps) {
+  // Don't miss timeouts even when we have a LOT of other events!
+  timeout_run(ps);
+
   // Process existing events
   // Sometimes poll() returns 1 but no events are actually read,
   // causing XNextEvent() to block, I have no idea what's wrong, so we
@@ -6444,8 +6459,6 @@ mainloop(session_t *ps) {
   free(ptv);
   ptv = NULL;
 
-  timeout_run(ps);
-
   return true;
 }
 
@@ -6732,6 +6745,7 @@ session_init(session_t *ps_old, int argc, char **argv) {
     | ExposureMask
     | StructureNotifyMask
     | PropertyChangeMask);
+  XFlush(ps->dpy);
 
   ps->root_width = DisplayWidth(ps->dpy, ps->scr);
   ps->root_height = DisplayHeight(ps->dpy, ps->scr);