我用scrapy写了一个蜘蛛，它使用了CrawlerProcess并且运行良好。
下一步，我想要一个用于pyqt5的输入数据的GUI对话框，并在scrapy工作时设置一个微调器。这项任务是使用虚拟过程完成的，并且工作正常。
我知道多线程与CrawlerProcess不兼容，所以我为CrawlerRunner更改了它。无论如何，它失败了：
signal.signal(signal.SIGINT, self.sigInt)
File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread
我创建了一个重现错误的最小示例：
from PyQt5.QtWidgets import QApplication, QDialog, QTabWidget, QWidget, QGroupBox, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QRunnable, QMetaObject, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon
from scrapy.crawler import CrawlerRunner
from scrapy import Request, Spider, Item, Field
from twisted.internet import reactor
from waitingspinnerwidget import QtWaitingSpinner
class RequestRunnable(QRunnable):
def __init__(self, dialog):
super(RequestRunnable, self).__init__()
self.w = dialog
print('runnable __init__')
def run(self):
print('run')
crawler = CrawlerRunner({
'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'FEED_FORMAT': 'json',
'FEED_URI': 'output.json',
'DOWNLOAD_DELAY': 3,
'LOG_STDOUT': True,
'LOG_FILE': 'scrapy_output.txt',
'ROBOTSTXT_OBEY': False,
'RETRY_ENABLED': True,
'RETRY_HTTP_CODES': [500, 503, 504, 400, 404, 408],
'RETRY_TIMES': 5
})
# instantiate a spider
spider = CustomSpider()
d = crawler.crawl(spider)
d.addBoth(lambda _: reactor.stop()) # I think this is for manually stopping the reactor when we are done
reactor.run() #TODO gives error
QMetaObject.invokeMethod(self.w, "FinishedDownload", Qt.QueuedConnection)
class CustomSpider(Spider):
name = 'quotes_spider'
allowed_domains = ['quotes.toscrape.com']
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
quotes = response.xpath("//div[@class='quote']//span[@class='text']/text()").extract()
yield {'quotes': quotes}
class DownloadDataDialog(QDialog):
def __init__(self, parent=None):
super(DownloadDataDialog, self).__init__(parent)
self.spinner = QtWaitingSpinner(self, True, True, Qt.ApplicationModal)
tabWidget = QTabWidget(self)
tabWidget.addTab(MyTab(tabWidget), "MyTab")
mainLayout = QVBoxLayout()
mainLayout.addWidget(tabWidget)
self.setLayout(mainLayout)
self.setWindowTitle("Download option chain data from web")
class MyTab(QWidget):
def __init__(self, parent=None):
super(MyTab, self).__init__(parent)
dataGroup = QGroupBox('Data')
getButton = QPushButton('Download')
getButton.clicked.connect(self.download_data)
dataLayout = QVBoxLayout()
dataLayout.addWidget(getButton)
dataGroup.setLayout(dataLayout)
mainLayout = QVBoxLayout()
mainLayout.addWidget(dataGroup)
mainLayout.addStretch(1)
self.setLayout(mainLayout)
def download_data(self):
self.parentWidget().window().spinner.start()
runnable = RequestRunnable(self)
QThreadPool.globalInstance().start(runnable)
@pyqtSlot()
def FinishedDownload(self):
self.parentWidget().window().spinner.stop()
if __name__ == '__main__':
import sys
app = QApplication(sys.argv)
tabdialog = DownloadDataDialog()
tabdialog.show()
sys.exit(app.exec_())
要执行它，您需要waitingspinnerwidget.py：
# -*- coding: utf-8 -*-
"""
The MIT License (MIT)
Copyright (c) 2012-2014 Alexander Turkin
Copyright (c) 2014 William Hallatt
Copyright (c) 2015 Jacob Dawid
Copyright (c) 2016 Luca Weiss
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import math
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
class QtWaitingSpinner(QWidget):
def __init__(self, parent, centerOnParent=True, disableParentWhenSpinning=False, modality=Qt.NonModal):
super().__init__(parent, Qt.Dialog | Qt.FramelessWindowHint)
self._centerOnParent = centerOnParent
self._disableParentWhenSpinning = disableParentWhenSpinning
# WAS IN initialize()
self._color = QColor(Qt.black)
self._roundness = 100.0
self._minimumTrailOpacity = 3.14159265358979323846
self._trailFadePercentage = 80.0
self._revolutionsPerSecond = 1.57079632679489661923
self._numberOfLines = 20
self._lineLength = 10
self._lineWidth = 2
self._innerRadius = 10
self._currentCounter = 0
self._isSpinning = False
self._timer = QTimer(self)
self._timer.timeout.connect(self.rotate)
self.updateSize()
self.updateTimer()
self.hide()
# END initialize()
self.setWindowModality(modality)
self.setAttribute(Qt.WA_TranslucentBackground)
def paintEvent(self, QPaintEvent):
self.updatePosition()
painter = QPainter(self)
painter.fillRect(self.rect(), Qt.transparent)
painter.setRenderHint(QPainter.Antialiasing, True)
if self._currentCounter >= self._numberOfLines:
self._currentCounter = 0
painter.setPen(Qt.NoPen)
for i in range(0, self._numberOfLines):
painter.save()
painter.translate(self._innerRadius + self._lineLength, self._innerRadius + self._lineLength)
rotateAngle = float(360 * i) / float(self._numberOfLines)
painter.rotate(rotateAngle)
painter.translate(self._innerRadius, 0)
distance = self.lineCountDistanceFromPrimary(i, self._currentCounter, self._numberOfLines)
color = self.currentLineColor(distance, self._numberOfLines, self._trailFadePercentage,
self._minimumTrailOpacity, self._color)
painter.setBrush(color)
painter.drawRoundedRect(QRect(0, -self._lineWidth / 2, self._lineLength, self._lineWidth), self._roundness,
self._roundness, Qt.RelativeSize)
painter.restore()
def start(self):
self.updatePosition()
self._isSpinning = True
self.show()
if self.parentWidget and self._disableParentWhenSpinning:
self.parentWidget().setEnabled(False)
if not self._timer.isActive():
self._timer.start()
self._currentCounter = 0
def stop(self):
self._isSpinning = False
self.hide()
if self.parentWidget() and self._disableParentWhenSpinning:
self.parentWidget().setEnabled(True)
if self._timer.isActive():
self._timer.stop()
self._currentCounter = 0
def setNumberOfLines(self, lines):
self._numberOfLines = lines
self._currentCounter = 0
self.updateTimer()
def setLineLength(self, length):
self._lineLength = length
self.updateSize()
def setLineWidth(self, width):
self._lineWidth = width
self.updateSize()
def setInnerRadius(self, radius):
self._innerRadius = radius
self.updateSize()
def color(self):
return self._color
def roundness(self):
return self._roundness
def minimumTrailOpacity(self):
return self._minimumTrailOpacity
def trailFadePercentage(self):
return self._trailFadePercentage
def revolutionsPersSecond(self):
return self._revolutionsPerSecond
def numberOfLines(self):
return self._numberOfLines
def lineLength(self):
return self._lineLength
def lineWidth(self):
return self._lineWidth
def innerRadius(self):
return self._innerRadius
def isSpinning(self):
return self._isSpinning
def setRoundness(self, roundness):
self._roundness = max(0.0, min(100.0, roundness))
def setColor(self, color=Qt.black):
self._color = QColor(color)
def setRevolutionsPerSecond(self, revolutionsPerSecond):
self._revolutionsPerSecond = revolutionsPerSecond
self.updateTimer()
def setTrailFadePercentage(self, trail):
self._trailFadePercentage = trail
def setMinimumTrailOpacity(self, minimumTrailOpacity):
self._minimumTrailOpacity = minimumTrailOpacity
def rotate(self):
self._currentCounter += 1
if self._currentCounter >= self._numberOfLines:
self._currentCounter = 0
self.update()
def updateSize(self):
size = (self._innerRadius + self._lineLength) * 2
self.setFixedSize(size, size)
def updateTimer(self):
self._timer.setInterval(1000 / (self._numberOfLines * self._revolutionsPerSecond))
def updatePosition(self):
if self.parentWidget() and self._centerOnParent:
self.move(self.parentWidget().width() / 2 - self.width() / 2,
self.parentWidget().height() / 2 - self.height() / 2)
dialogCenter = self.mapToGlobal(self.rect().center())
parentWindowCenter = self.parentWidget().window().mapToGlobal(self.parentWidget().window().rect().center())
self.move(parentWindowCenter - dialogCenter)
parentRect = QRect(self.parentWidget().mapToGlobal(QPoint(0, 0)), self.parentWidget().size())
self.move(QStyle.alignedRect(Qt.LeftToRight, Qt.AlignCenter, self.size(), parentRect).topLeft())
def lineCountDistanceFromPrimary(self, current, primary, totalNrOfLines):
distance = primary - current
if distance < 0:
distance += totalNrOfLines
return distance
def currentLineColor(self, countDistance, totalNrOfLines, trailFadePerc, minOpacity, colorinput):
color = QColor(colorinput)
if countDistance == 0:
return color
minAlphaF = minOpacity / 100.0
distanceThreshold = int(math.ceil((totalNrOfLines - 1) * trailFadePerc / 100.0))
if countDistance > distanceThreshold:
color.setAlphaF(minAlphaF)
else:
alphaDiff = color.alphaF() - minAlphaF
gradient = alphaDiff / float(distanceThreshold + 1)
resultAlpha = color.alphaF() - gradient * countDistance
# If alpha is out of bounds, clip it.
resultAlpha = min(1.0, max(0.0, resultAlpha))
color.setAlphaF(resultAlpha)
return color
完全追溯：
Traceback (most recent call last):
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 428, in fireEvent
DeferredList(beforeResults).addCallback(self._continueFiring)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 322, in addCallback
callbackKeywords=kw)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 311, in addCallbacks
self._runCallbacks()
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
--- <exception caught here> ---
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 441, in _continueFiring
callable(*args, **kwargs)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1256, in _reallyStartRunning
self._handleSignals()
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\posixbase.py", line 295, in _handleSignals
_SignalReactorMixin._handleSignals(self)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1221, in _handleSignals
signal.signal(signal.SIGINT, self.sigInt)
File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread
知道发生了什么以及如何解决它？
不要使用线程，只需使用qt反应器进行扭曲，这样你的扭曲应用程序（scrapy）和你的qt5应用程序都可以在同一个事件循环中运行：
安装qt5reactor：
pip install qt5reactor
然后添加代码来安装reactor并运行它：
app = QApplication(sys.argv)
import qt5reactor
qt5reactor.install()
reactor.run()