Schedule groups-backends in a queue

When loading groups from an auth backend (for example LDAP), Gerrit is
subject to its latency. This might be a problem because a slow backend,
or a very large number of groups, or both, might cause the warm-cache
command to last longer than the allowed timeouts imposed by Gerrit (or a
load balancer that fronts it).

Schedule the loading of the cache asynchronously over a new dedicated
queue named: "Groups-Backend-Cache-Warmer".

This allows to terminate the SSH command in a timely manner, while
loading the cache in the background.

The status of the cache loading can be observed via the `show-queue`
command as well as the `queue/groups_backend_cache_warmer/*` metrics.

Currently the size of the queue is not configurable and it is set to 8
threads.

Change-Id: Iccbb1f05bcf04c540a68e1d70cd6be9078a50815

admin/warm-cache-1.0.groovy

diff --git a/admin/warm-cache-1.0.groovy b/admin/warm-cache-1.0.groovy
index 88a8148..c3c4892 100644
--- a/admin/warm-cache-1.0.groovy
+++ b/admin/warm-cache-1.0.groovy
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import com.google.common.flogger.FluentLogger
 import com.google.gerrit.common.data.GlobalCapability
 import com.google.gerrit.sshd.*
 import com.google.gerrit.extensions.annotations.*
@@ -20,9 +21,14 @@
 import com.google.gerrit.server.IdentifiedUser
 import com.google.gerrit.reviewdb.client.AccountGroup
 import com.google.inject.*
-import org.kohsuke.args4j.*
+import com.google.gerrit.server.git.WorkQueue
+import org.apache.sshd.server.Environment
+
+import java.util.concurrent.ExecutorService
 
 abstract class BaseSshCommand extends SshCommand {
+  protected static final FluentLogger logger = FluentLogger.forEnclosingClass()
+
 
   void println(String msg) {
     stdout.println msg
@@ -142,32 +148,70 @@
 @Export("groups-backends")
 @RequiresCapability(GlobalCapability.ADMINISTRATE_SERVER)
 class WarmGroupsBackendsCache extends WarmAccountsCache {
+  private static final THREAD_POOL_SIZE = 8
+  private static final QUEUE_NAME = "Groups-Backend-Cache-Warmer"
 
   @Inject
   IdentifiedUser.GenericFactory userFactory
 
-  public void run() {
-    println "Loading groups ..."
+  @Inject WorkQueue queues
+
+  private static class GroupsBackendsTask implements Runnable {
+    IdentifiedUser user
+
+    GroupsBackendsTask(IdentifiedUser identifiedUser) {
+      user = identifiedUser
+    }
+
+    @Override
+    void run() {
+      def threadStart = System.currentTimeMillis()
+      def groupsUUIDs = user.getEffectiveGroups()?.getKnownGroups()
+      def threadElapsed = (System.currentTimeMillis() - threadStart)
+      logger.atInfo().log("Loaded %d groups for account %d in %s millis", groupsUUIDs.size(), user.getAccountId().get(), threadElapsed)
+    }
+
+    @Override
+    String toString() {
+      return "Warmup backend groups [accountId: ${user.getAccountId().get()}]"
+    }
+  }
+
+  ExecutorService executorService
+
+  private ExecutorService executor() {
+    def existingExecutor = queues.getExecutor(QUEUE_NAME)
+    if(existingExecutor != null) {
+      return existingExecutor
+    }
+    return queues.createQueue(THREAD_POOL_SIZE, QUEUE_NAME, true);
+  }
+
+  @Override
+  void start(Environment env) throws IOException {
+    super.start(env)
+    executorService = executor()
+  }
+
+  void run() {
+    println "Scheduling backend groups loading ..."
     def start = System.currentTimeMillis()
 
-    def loaded = 0
-    def allGroupsUUIDs = new HashSet<AccountGroup.UUID>()
+    def scheduled = 0
     def lastDisplay = 0
 
     for (accountId in accounts.allIds()) {
-      def user = userFactory.create(accountId)
-      def groupsUUIDs = user?.getEffectiveGroups()?.getKnownGroups()
-      if (groupsUUIDs != null) { allGroupsUUIDs.addAll(groupsUUIDs) }
+      scheduled++
+      executorService.submit(new GroupsBackendsTask(userFactory.create(accountId)))
 
-      loaded = allGroupsUUIDs.size()
-      if (loaded.intdiv(1000) > lastDisplay) {
-        println "$loaded groups"
-        lastDisplay = loaded.intdiv(1000)
+      if (scheduled.intdiv(1000) > lastDisplay) {
+        println "Scheduled loading of groups for $scheduled accounts"
+        lastDisplay = scheduled.intdiv(1000)
       }
     }
 
-    def elapsed = (System.currentTimeMillis()-start)/1000
-    println "$loaded groups loaded in $elapsed secs"
+    def elapsed = (System.currentTimeMillis() - start) / 1000
+    println "Scheduled loading of groups for $scheduled accounts in $elapsed secs"
   }
 }