305ef1b4bb16 draft — Alain Leufroy tip 10 months ago
limit number of async processes using `config.PRCESSES_NB`
3 files changed, 47 insertions(+), 17 deletions(-)

M lairucrem/config.py
M lairucrem/controler.py
M lairucrem/process.py
M lairucrem/config.py +13 -0
@@ 1,3 1,5 @@ 
+import asyncio
+import multiprocessing
 import re
 from pathlib import Path
 

          
@@ 514,12 516,23 @@ KEYBINDINGS_SETTERS = {
 
 KEYBINDINGS = []
 
+# Optimal: > 6
+PROCESSES_NB = multiprocessing.cpu_count() - 1
+
 
 def has_mouse_keybindings():
     return 'mouse' in KEYBINDINGS
 
 
+PROCESSES_SEMAPHORE = None
+
+
 # Load user defined configuration
 CONFIG_FILE_PATH = Path('~/.config/lairucrem/config.py').expanduser()
 if CONFIG_FILE_PATH.exists():
     exec(CONFIG_FILE_PATH.read_text())
+
+
+if PROCESSES_SEMAPHORE is None:
+    # Do not overwrite PROCESSES_SEMAPHORE set from the user's config file.
+    PROCESSES_SEMAPHORE = asyncio.Semaphore(PROCESSES_NB + 1)

          
M lairucrem/controler.py +26 -11
@@ 106,11 106,12 @@ class _hgwalker(ensurable, waitable, urw
     async def _wait(self):
         await super()._wait()
         first = True
-        async for widget in self:
-            if first:
-                self.clear(full=True)
-                first = False
-            self.append(widget)
+        async with config.PROCESSES_SEMAPHORE:
+            async for widget in self:
+                if first:
+                    self.clear(full=True)
+                    first = False
+                self.append(widget)
         if first:               # nothing fetched
             self.clear(full=True)
         urwid.emit_signal(self, 'completed')

          
@@ 629,6 630,19 @@ class patchlistwalker(_multiwalker):
         self._diff.filename = filename
         self._diffstat.filename = filename
 
+    async def _wait(self):
+        # Load task is specific order to ensure that prefered sections
+        # appears quickly (PROCESSES_SEMAPHORE may delay a few of
+        # them).
+        await asyncio.gather(
+            self._diff.wait(),
+            self._summary.wait(),
+            self._description.wait(),
+            self._diffstat.wait(),
+            self._summaryextra.wait(),
+        )
+        urwid.emit_signal(self, 'completed')
+
 
 class patchlistbox(
         mixin.filterable_listbox,

          
@@ 771,12 785,13 @@ class maincontroler(ensurable):
     async def _grep_revset(self, pattern):
         self._grep_pattern = pattern
         csets = defaultdict(set)
-        async for line in process.hg('grep', '--all', '-l', pattern):
-            try:
-                data = dict(parse_colored_line(line))
-            except ValueError:
-                pass
-            csets[data['grep.rev']].add(data['grep.filename'])
+        with HG_SEMAPHORE:
+            async for line in process.hg('grep', '--all', '-l', pattern):
+                try:
+                    data = dict(parse_colored_line(line))
+                except ValueError:
+                    pass
+                csets[data['grep.rev']].add(data['grep.filename'])
         revset = ' or '.join(csets)
         return f'({revset})'
 

          
M lairucrem/process.py +8 -6
@@ 31,6 31,7 @@ async def _wait_process(proc, cmd=''):
         msg = cmd + '\n\n' + msg
         raise OSError(proc.returncode, msg)
 
+
 class hg:
     """Async generator yielding mercurial command outputs.
 

          
@@ 191,9 192,10 @@ async def hg_plain(*cmd):
     """Execute an Hg command in plain mode and return overall results."""
     env = os.environ.copy()
     env['HGPLAIN'] = '1'
-    hgproc = hg(*cmd, env=env)
-    try:
-        return ''.join([line async for line in hgproc])
-    except asyncio.CancelledError:
-        await hgproc.kill()
-    return ''
+    async with config.PROCESSES_SEMAPHORE:
+        hgproc = hg(*cmd, env=env)
+        try:
+            return ''.join([line async for line in hgproc])
+        except asyncio.CancelledError:
+            await hgproc.kill()
+        return ''