commands: Refactor the cfensureconf algorithm to allow concurrency & future ci mode

Non conservative change:
The 'xxx is a partial override without its master entry' warning message has
been completely removed.
M hgext3rd/confman/commands.py +46 -7
@@ 77,19 77,58 @@ def ensureconf(ui, repo, *args, **opts):
 
     """
     confman, repo = readconf(ui, repo, args, opts)
-    confman.fill_missing()
+
+    # a {section:track} of already checked-out repos
+    ready = {}
 
     snaps = confman.readsnapshot()
 
-    failedrepos = []
-    for section, secconf, managed in confman.iterrepos():
+    # phase 1 - checkout all the repos that may extend the configuration,
+    # update the configuration, and repeat until no new section appear
+    confcomplete = False
+    while not confcomplete:
+        need_checkout = []
+        for section in confman.filtered_sections():
+            conf = confman.confs[section]
+            if confman._check_parameters(section, True):
+                continue
+            rev = snaps.get(conf['layout']) or conf.get('track', 'default')
+            if ready.get(section) == rev:
+                continue
+            if any(s.startswith("expand") for s in conf):
+                need_checkout.append((section, rev, conf))
+
+        if not need_checkout:
+            confcomplete = True
+
+        for section, rev, conf in need_checkout:
+            try:
+                confman.checkout_section(
+                    section, snaps, opts.get('keep_descendant', False)
+                )
+            except Exception:
+                return 1
+            ready[section] = rev
+
+        confman._readconf()
+
+    # phase 2 - at this point, we know the configuration is stable, we can
+    # checkout all the remaining repos without reading it.
+    need_checkout = []
+    for section in confman.filtered_sections():
+        conf = confman.confs[section]
+        confman._check_parameters(section, False)
+        rev = snaps.get(conf['layout']) or conf.get('track', 'default')
+        if ready.get(section) == rev:
+            continue
+        need_checkout.append((section, rev, conf))
+
+    for section, rev, conf in need_checkout:
         try:
             confman.checkout_section(section, snaps, opts.get('keep_descendant', False))
         except Exception:
-            failedrepos.append(section)
-
-    if failedrepos:
-        return 1
+            return 1
+        ready[section] = rev
 
 
 # baseline

          
M hgext3rd/confman/configuration.py +10 -1
@@ 80,7 80,6 @@ class configurationmanager(object):
                 )
                 raise error.Abort(b'configuration error')
             else:
-                ui.warn('%s is a partial override without its master entry\n' % section)
                 return True
 
     def pathfromsection(self, section):

          
@@ 116,6 115,16 @@ class configurationmanager(object):
     def sections(self):
         return self.confs.sections()
 
+    def filtered_sections(self):
+        include = set(self.opts.get('include_conf') or ())
+        exclude = set(self.opts.get('exclude_conf') or ()) | self.failed
+        exactmatch = set(self.args) or ()
+
+        for section in self.confs.sections():
+            if not _filtersection(section, exactmatch, include, exclude):
+                continue
+            yield section
+
     def iterrepos(self, skipmissing=True):
         """Yield (section, conf, repository) for each managed
         repository. Repository may be None if it's missing. Skip

          
M tests/test-broadcast.t +1 -1
@@ 43,10 43,10 @@ Prepare configuration repo
   cloning managed1 from ../foo to $TESTTMP/conf/managed/managed1
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  managed1
   cloning managed2 from ../bar to $TESTTMP/conf/managed2
   updating to branch default
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  managed1
   managed2
 
 

          
M tests/test-clear.t +3 -3
@@ 81,9 81,6 @@ First cfensureconf
   cloning foo from $TESTTMP/foo to $TESTTMP/conf/foo
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning bar.quux from $TESTTMP/quux to $TESTTMP/conf/bar/quux
-  updating to branch default
-  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   foo
   foo repo has no `default` path, using configuration pulluri `$TESTTMP/foo` instead
   searching for changes

          
@@ 91,6 88,9 @@ First cfensureconf
   updating to stable
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   updated to 11aceb0f834624f90c1cac5cecc9b09fd28df45f/stable from 797f8866b5b935983bc197b3a18d3b3c71c0643a/default
+  cloning bar.quux from $TESTTMP/quux to $TESTTMP/conf/bar/quux
+  updating to branch default
+  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   bar.quux
   updating to quux-version-1.0
   0 files updated, 0 files merged, 2 files removed, 0 files unresolved

          
M tests/test-confman.t +13 -11
@@ 73,9 73,6 @@ First cfensureconf
   cloning foo from $TESTTMP/foo to $TESTTMP/conf/foo
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning bar.quux from $TESTTMP/quux to $TESTTMP/conf/bar/quux
-  updating to branch default
-  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   foo
   foo repo has no `default` path, using configuration pulluri `$TESTTMP/foo` instead
   searching for changes

          
@@ 83,6 80,9 @@ First cfensureconf
   updating to stable
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   updated to 11aceb0f834624f90c1cac5cecc9b09fd28df45f/stable from 797f8866b5b935983bc197b3a18d3b3c71c0643a/default
+  cloning bar.quux from $TESTTMP/quux to $TESTTMP/conf/bar/quux
+  updating to branch default
+  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   bar.quux
   updating to quux-version-1.0
   0 files updated, 0 files merged, 2 files removed, 0 files unresolved

          
@@ 420,13 420,15 @@ Don't choke on unreachable repo
   |- newcomer (default public) tip [no snapshot] [no baseline]
   bogosection repository at "$TESTTMP/conf/bogo" not found
   $ hg cfensure
+  newcomer
   cloning bogosection from /an/unlikely/path to $TESTTMP/conf/bogo
   b'repository /an/unlikely/path not found'
+  [1]
+  $ hg cfensure
   newcomer
-  $ hg cfensure
   cloning bogosection from /an/unlikely/path to $TESTTMP/conf/bogo
   b'repository /an/unlikely/path not found'
-  newcomer
+  [1]
 
 Check baseline command
   $ cd $TESTTMP/conf

          
@@ 529,20 531,20 @@ Let's try cfpush
   cloning foo from $TESTTMP/foo to $TESTTMP/clone/foo
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning bar.quux from $TESTTMP/quux to $TESTTMP/clone/bar/quux
-  updating to branch default
-  4 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning newcomer from $TESTTMP/newcomer to $TESTTMP/clone/newcomer
-  updating to branch default
-  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   foo
   updating to 11aceb0f834624f90c1cac5cecc9b09fd28df45f
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 11aceb0f834624f90c1cac5cecc9b09fd28df45f/stable from 894929fd5b7377a7fafdf46ad8a8d32ac9ea95d7/default
+  cloning bar.quux from $TESTTMP/quux to $TESTTMP/clone/bar/quux
+  updating to branch default
+  4 files updated, 0 files merged, 0 files removed, 0 files unresolved
   bar.quux
   updating to 039d61f282bba2568097f09aa7707de56ce27b8e
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   updated to quux-version-2.0/default from 00210b48a6d3b84a28e92071ccd070a4429021f4/default
+  cloning newcomer from $TESTTMP/newcomer to $TESTTMP/clone/newcomer
+  updating to branch default
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   newcomer
   $ hg cfs
   clone

          
M tests/test-ensureconf.t +1 -0
@@ 420,6 420,7 @@ Good uri map file with wrong target
   cloning main from $TESTTMP/main to $TESTTMP/app2/main
   clone: using 'http://perdu/main' instead of '$TESTTMP/main'
   <urlopen error [Errno -2] Name or service not known>
+  [1]
 
 Good uri map file with good target
   $ cd $TESTTMP

          
M tests/test-expand.t +36 -37
@@ 86,13 86,13 @@ Prepare configuration repos (also at 0.1
   cloning lib1 from $TESTTMP/lib1 to $TESTTMP/libs/lib1
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning lib2 from $TESTTMP/lib2 to $TESTTMP/libs/lib2
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   lib1
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning lib2 from $TESTTMP/lib2 to $TESTTMP/libs/lib2
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   lib2
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved

          
@@ 126,34 126,34 @@ Then, "framework"
   cloning libs from $TESTTMP/libs to $TESTTMP/framework/dependencies/libs
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning lib1 from $TESTTMP/lib1 to $TESTTMP/framework/lib1
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning lib2 from $TESTTMP/lib2 to $TESTTMP/framework/lib2
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning core from $TESTTMP/core to $TESTTMP/framework/core
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning tier1 from $TESTTMP/tier1 to $TESTTMP/framework/tier1
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   libs
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from */default (glob)
+  cloning lib1 from $TESTTMP/lib1 to $TESTTMP/framework/lib1
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   lib1
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning lib2 from $TESTTMP/lib2 to $TESTTMP/framework/lib2
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   lib2
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning core from $TESTTMP/core to $TESTTMP/framework/core
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   core
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning tier1 from $TESTTMP/tier1 to $TESTTMP/framework/tier1
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   tier1
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved

          
@@ 189,55 189,55 @@ Finally, "app"
   cloning framework from $TESTTMP/framework to $TESTTMP/app/dependencies/framework
   updating to branch default
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning libs from $TESTTMP/libs to $TESTTMP/app/dependencies/libs
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning lib1 from $TESTTMP/lib1 to $TESTTMP/app/lib1
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning lib2 from $TESTTMP/lib2 to $TESTTMP/app/lib2
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning core from $TESTTMP/core to $TESTTMP/app/core
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning tier1 from $TESTTMP/tier1 to $TESTTMP/app/tier1
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning main from $TESTTMP/main to $TESTTMP/app/main
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  cloning tier2 from $TESTTMP/tier2 to $TESTTMP/app/tier2
-  updating to branch default
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   framework
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from */default (glob)
+  cloning libs from $TESTTMP/libs to $TESTTMP/app/dependencies/libs
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   libs
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from */default (glob)
+  cloning lib1 from $TESTTMP/lib1 to $TESTTMP/app/lib1
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   lib1
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning lib2 from $TESTTMP/lib2 to $TESTTMP/app/lib2
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   lib2
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning core from $TESTTMP/core to $TESTTMP/app/core
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   core
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning tier1 from $TESTTMP/tier1 to $TESTTMP/app/tier1
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   tier1
   updating to 0.1.0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning main from $TESTTMP/main to $TESTTMP/app/main
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   main
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   updated to 0.1.0/default from 68f1e6a871f275e4ba08b2bce7db0a0edc33084e/default
+  cloning tier2 from $TESTTMP/tier2 to $TESTTMP/app/tier2
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   tier2
   updating to 0
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved

          
@@ 686,7 686,6 @@ section but only few parameters
   partial_override
   framework repository at "$TESTTMP/partial_override/dependencies/framework" not found
   libs repository at "$TESTTMP/partial_override/dependencies/libs" not found
-  lib1 is a partial override without its master entry
   $ hg cfensureconf > /dev/null
   framework repo has no `default` path, using configuration pulluri `$TESTTMP/framework` instead
   libs repo has no `default` path, using configuration pulluri `$TESTTMP/libs` instead

          
M tests/test-git.t +2 -1
@@ 83,10 83,10 @@ Ensureconf
   Cloning into '$TESTTMP/conf/foo'...
   done.
   cloning foo from $TESTTMP/foo to $TESTTMP/conf/foo
+  foo
   cloning bar.quux from $TESTTMP/quux to $TESTTMP/conf/bar/quux
   updating to branch default
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  foo
   bar.quux
   updating to quux-version-1.0
   0 files updated, 0 files merged, 2 files removed, 0 files unresolved

          
@@ 247,5 247,6 @@ Git clone failure
   fatal: repository 'invalid.git' does not exist
   cloning agitrepo from invalid.git to $TESTTMP/test-git-failure/target
   Command '['git', 'clone', 'invalid.git', '$TESTTMP/test-git-failure/target']' returned non-zero exit status 128.
+  [1]
   $ [ -e target ] ; echo $?
   1