试试这个：
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
#loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
结果：
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://example.org/birthday
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://www.example.com/
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
Process finished with exit code 0
您只能从SecondClass() 中删除loop.close()，因为这将阻止脚本运行所有任务...