主页 / 问题 / 863175
Accepted
XxX
Asked:2020-08-02 01:08:51 +0000 UTC2020-08-02 01:08:51 +0000 UTC

PyQt - QWebEnginePage

  • 772

有一个解析器：

class AppWindow(QtWidgets.QMainWindow):
    def __init__(self):
        QtWidgets.QMainWindow.__init__(self)
        #...
    def start(self):
        self.thread1 = parser()
        t = threading.Thread(target=self.thread1.run)
        t.daemon = True
        t.start()
    #...

class parser(QtCore.QThread):
    def __init__(self):
        super(parser, self).__init__()
        QtCore.QThread.__init__(self)
        #...
    def run(self):
        #...
        data = [...]
        queue = mp.Queue()
        for i in data:
            queue.put(i)
        processes = [mp.Process(target = parse_data, args=(queue)) for i in range(10)]
        for process in processes:
            process.daemon = True
            process.start()
        #...

def parse_data(data):
    for elem in data:
        def parse_elem(elem):
            #...
            html = Client(elem['url'])
            #...
        p = ThreadPool(processes = 1)
        try:
            p_result = p.apply_async(parse_elem, (elem))
            res = p_result.get(timeout=60) # поток создаётся только для того, чтобы установить таймаут на выполнение
            p.terminate()
            p.close()
        except:
            #...


class Client(QWebEnginePage): # source: https://stackoverflow.com/questions/8049520/web-scraping-javascript-page-with-python#answer-51341435
    def __init__(self, url):
        global app
        print(url)
        self.app = QApplication(sys.argv)
        QWebEnginePage.__init__(self)
        self.html = ""
        self.loadFinished.connect(self.on_load_finished)
        self.load(QUrl(url))
        self.app.exec_()

    def on_load_finished(self):
        self.html = self.toHtml(self.Callable)

    def Callable(self, data):
        self.html = data
        self.app.quit()

def main():
    mp.freeze_support()
    app = QtWidgets.QApplication(sys.argv)
    appwindow = AppWindow()
    appwindow.show()
    sys.exit(app.exec_())


if __name__ == "__main__": 
    main()

但是调用Client失败，返回错误：WARNING: QApplication was not created in the main() thread.. 还没有完全弄清楚它是如何工作的。它可以Client工作吗？

python

1 个回答

  1. Best Answer

    这就是我得到的。将 5 个文件保存到磁盘：

    from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QUrl
from PyQt5.QtWebEngineWidgets import QWebEnginePage


class Client(QWebEnginePage):
    def __init__(self, urls):
        self.app = QApplication([])

        super().__init__()

        self.response_list = []
        self.loadFinished.connect(self._on_load_finished)

        for url in urls:
            self.load(QUrl(url))
            self.app.exec_()

    def _on_load_finished(self):
        self.toHtml(self.callable)

    def callable(self, html_str):
        self.response_list.append(html_str)
        self.app.quit()


def go(urls):
    client = Client(urls)
    return client.response_list


if __name__ == '__main__':
    urls = [
        [
            'http://doc.qt.io/Qt-5/qwebenginepage.html',
            'https://yandex.ru/',
        ],
        [
            'http://doc.qt.io/Qt-5/qwebenginepage.html',
            'https://www.google.ru/',
        ],
        [
            'https://www.google.ru/',
        ]
    ]

    from multiprocessing import Pool
    with Pool() as p:
        results = p.map(go, urls)
        print(len(results))

    number = 1

    for result in results:
        print(len(result))

        for html in result:
            with open('result_{}.html'.format(number), 'w', encoding='utf-8') as f:
                f.write(html)

            number += 1

    PS起初我在一个进程中做了一个同步版本，我想建议它，但在澄清我需要在不同进程中加载​​几个页面后，我开始挖掘QApplication在另一个进程中启动。玩过之后，我用一个子进程做了一个例子，然后用了几个. 它奏效了，而且很棒。好吧，拿同步版本并在多个进程中运行它不再困难。

    • 1

相关问题