【Python】concurrency best practice for HTTP (RESTful) API - 草稿 10%

太过爱你忘了你带给我的痛 2023-01-21 03:25 136阅读 0赞

# 【Python】concurrency best practice #

*-- for HTTP (RESTful) API*

### Overview ###

*  【Python】concurrency best practice
 *   *  concurrent.furture.ThreadPoolExecutor
     *  die thread
     *  reusable code - map
     *  Usage Examples - N/A
     *  Reference

--------------------

## concurrent.furture.ThreadPoolExecutor ##

N/A

## die thread ##

detail: N/A

"""die_threads.py"""
    
    from functools import wraps
    from logging import getLogger
    
    logger = getLogger(__name__)
    THREAD_POISON: bool = False
    
    
    def poisoning(msg):
      global THREAD_POISON
      logger.warning('\n%s: you got me', str(msg))
      THREAD_POISON = True
      logger.warning('done poisoning, THREAD_POISON = %s', str(THREAD_POISON))
    
    
    class DieFromPOISON(Exception):
      pass
    
    
    def die_from_poison(fn):
      @wraps(fn)
      def wrapper(*args, **kwargs):
        if THREAD_POISON:
          raise DieFromPOISON(*args, **kwargs)
        return fn(*args, **kwargs)
      return wrapper

## reusable code - map ##

detail: N/A

"""concurrency.py"""
    
    from logging import getLogger
    from concurrent.futures import ThreadPoolExecutor
    from concurrent.futures import as_completed
    
    from .die_threads import poisoning
    from .die_threads import die_from_poison
    
    logger = getLogger(__name__)
    DEFAULT_MAX_CUR = 5
    
    
    def concurrency(fn, *iterables, max_cur: int = DEFAULT_MAX_CUR) -> tuple:
      successes, failures = list(), list()
      fn = die_from_poison(fn)  # TODO: check has been wrappered or not
      with ThreadPoolExecutor(max_workers=max_cur) as executor:
        to_do_map = {
        executor.submit(fn, *args): i
                     for i, args in enumerate(zip(*iterables))}
        done_iter = as_completed(to_do_map)
        # TODO: feature: processing bar.
        try:
          for future in done_iter:
            try:
              successes.append((to_do_map[future], future.result()))
            except Exception as err:
              failures.append((to_do_map[future], err))
        except KeyboardInterrupt as e:
          poisoning(e)
          logger.warning('waiting started threads finish...')
      return successes, failures

## Usage Examples - N/A ##

## Reference ##

*  die thread - threado