主页 / 问题 / 1477775
Accepted
Red Fox
Asked:2022-12-15 12:29:32 +0000 UTC2022-12-15 12:29:32 +0000 UTC

如何加速 Avatar 解析器？

  • 772

我练习了多线程并决定制作一个头像解析器。但是数据收集的速度还有很多不足之处。我究竟做错了什么？或者也许在 asyncio 下重新制作它？

import threading
import requests
from bs4 import BeautifulSoup
import lxml

def getImage():
        #https://cspromogame.ru/avatars?page=1999
        links = set()
        for page in range(1,101): #2000
            url = "https://cspromogame.ru/avatars?page={page}"
            req = requests.get(url=url)
            soup = BeautifulSoup(req.text,"lxml")
            a = soup.findAll("a", class_="avatars__link")
            for link in a:
                link = link.get("href")
                alinks = link.split("/")[-1].replace(".jpg","")
                req2 = requests.get(link)
                out = open(f"Avatars/Картинка_{alinks}.jpg",'wb')
                out.write(req2.content)
                out.close()
                print("Обработано ", alinks)
                
threads  = []
for i in range(11):
    t = threading.Thread(target=getImage)
    t.start()

for th in threads:
    th.join()
python-3.x

1 个回答

  1. Best Answer

    实施方案（供审查）

    from bs4 import BeautifulSoup
import lxml
import asyncio
import aiohttp
import time

start_time = time.time()

async def main():

    async def get_one_page(page:int):
        url = f'https://cspromogame.ru/avatars?page={page}'
              
        async with session.get(url) as resp:
            text = await resp.text()
            print(f'Page: {page}')

            soup = BeautifulSoup(text,"lxml")
            a = soup.findAll("a", class_="avatars__link")
            for link in a:
                link = link.get("href")
                alinks = link.split("/")[-1].replace(".jpg","")

                async with session.get(link) as resp:
                    filename = f'Avatars/Pic_{alinks}.jpg'
                    chunk_size = 64 * 1024
                    with open(filename, 'wb') as fd:
                        async for chunk in resp.content.iter_chunked(chunk_size):
                            fd.write(chunk)
                    print(f'Done: {filename}')

    async with aiohttp.ClientSession() as session:
        pages = [ get_one_page(page) for page in range(1, 3)]
        await asyncio.gather(*pages)

asyncio.run(main())

print("--- %s seconds ---" % (time.time() - start_time))
    • 3

相关问题