Move parallelize to util.py.

This commit is contained in:
Nick Brassel 2023-09-29 12:35:12 +10:00
parent 6803275180
commit 0110a4f79d
No known key found for this signature in database
2 changed files with 28 additions and 18 deletions

View File

@ -4,12 +4,12 @@ import contextlib
import functools import functools
import fnmatch import fnmatch
import logging import logging
import multiprocessing
import re import re
from typing import List, Tuple from typing import List, Tuple
from dotty_dict import dotty from dotty_dict import dotty
from milc import cli from milc import cli
from qmk.util import parallelize
from qmk.info import keymap_json from qmk.info import keymap_json
import qmk.keyboard import qmk.keyboard
import qmk.keymap import qmk.keymap
@ -25,23 +25,6 @@ def _set_log_level(level):
return old return old
@contextlib.contextmanager
def parallelize(parallel):
if not parallel:
yield map
return
with contextlib.suppress(ImportError):
from mpire import WorkerPool
with WorkerPool() as pool:
yield functools.partial(pool.imap_unordered, progress_bar=True)
return
with multiprocessing.Pool() as pool:
yield pool.imap_unordered
return
@contextlib.contextmanager @contextlib.contextmanager
def ignore_logging(): def ignore_logging():
old = _set_log_level(logging.CRITICAL) old = _set_log_level(logging.CRITICAL)

27
lib/python/qmk/util.py Normal file
View File

@ -0,0 +1,27 @@
"""Utility functions.
"""
import contextlib
import functools
import multiprocessing
@contextlib.contextmanager
def parallelize(do_parallel):
"""Returns a function that can be used in place of a map() call.
Attempts to use `mpire`, falling back to `multiprocessing` if it's not
available. If parallelization is not requested, returns the original map()
function.
"""
if not do_parallel:
yield map
return
with contextlib.suppress(ImportError):
from mpire import WorkerPool
with WorkerPool() as pool:
yield functools.partial(pool.imap_unordered, progress_bar=True)
return
with multiprocessing.Pool() as pool:
yield pool.imap_unordered
return