Python asynchronous programming demo: process/thread pool
using multiprocessing
from multiprocessing import Pool
def a(num):
print num
if __name__ == "__main__":
pool = Pool(3)
for i in range(10):
pool.apply_async(a,args=(i,))
pool.close()
pool.join()
using concurrent
import concurrent
from concurrent import futures
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for item in number_list:
executor.submit(evaluate_item, item)
or
import concurrent
from concurrent import futures
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
for item in number_list:
executor.submit(evaluate_item, item)
Just like ThreadPoolExecutor, ProcessPoolExecutor is an executor that uses a thread pool to execute tasks in parallel. However, unlike ThreadPoolExecutor, ProcessPoolExecutor uses a multi-core processing module that allows us to not be limited by the GIL and greatly reduces execution time.
using asyncio
import asyncio
import datetime
import time
def function_1(end_time, loop):
print ("function_1 called")
if (loop.time() + 1.0) < end_time:
loop.call_later(1, function_2, end_time, loop)
else:
loop.stop()
def function_2(end_time, loop):
print ("function_2 called ")
if (loop.time() + 1.0) < end_time:
loop.call_later(1, function_3, end_time, loop)
else:
loop.stop()
def function_3(end_time, loop):
print ("function_3 called")
if (loop.time() + 1.0) < end_time:
loop.call_later(1, function_1, end_time, loop)
else:
loop.stop()
def function_4(end_time, loop):
print ("function_4 called")
if (loop.time() + 1.0) < end_time:
loop.call_later(1, function_4, end_time, loop)
else:
loop.stop()
loop = asyncio.get_event_loop()
end_loop = loop.time() + 9.0
loop.call_soon(function_1, end_loop, loop)
# loop.call_soon(function_4, end_loop, loop)
loop.run_forever()
loop.close()
Reference
https://python-parallel-programmning-cookbook.readthedocs.io/zh_CN/latest/chapter4/02_Using_the_concurrent.futures_Python_modules.html
Small world. Big idea!
- Welcome to visit the knowledge base of SRE and DevOps!
- License under CC BY-NC 4.0
- No personal information is collected
- Made with Material for MkDocs and generative AI tools
- Copyright issue feedback me#imzye.com, replace # with @
- Get latest SRE news and discuss on Discord Channel