我正在尝试运行脚本来废弃网站上的新闻文本。
似乎在主页重组后对报废有限制。
而且我一直收到这个错误，但如果我测试一个单独的网址就可以了。
有什么建议？
Traceback (most recent call last):
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\bigkinds_headers_v1.3_20170522.py", line 157, in <module>
result = fetch_news_detail(news['href'])
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\bigkinds_headers_v1.3_20170522.py", line 107, in fetch_news_detail
res = urlopen(url).read().decode('utf-8', errors='ignore')
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 532, in open
response = meth(req, response)
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 570, in error
return self._call_chain(*args)
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\park\AppData\Local\Programs\Python\Python36-32\lib\urllib\request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
这是我的代码
import urllib.request
from bs4 import BeautifulSoup
import time
import urllib.parse
import json
from urllib.request import urlopen
import random
def fetch_news_list(page, keyword,start,end):
result = []
url = "https://www.bigkinds.or.kr/news/newsResult.do"
headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'Accept-Encoding': 'none', 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive'
}
param = {
'pageInfo':'newsResult',
'login_chk':'null',
'LOGIN_SN':'null',
'LOGIN_NAME':'null',
'indexName':'news',
'keyword':keyword,
'byLine':'',
'searchScope':'1',
'searchFtr':'1',
'startDate':start,
'endDate':end,
'sortMethod':'date',
'contentLength':'100',
'providerCode':'',
'categoryCode':'',
'incidentCode':'',
'dateCode':'',
'highlighting':'',
'sessionUSID':'',
'sessionUUID':'test',
'listMode':'',
'categoryTab':'',
'newsId':'',
'filterProviderCode':'',
'filterCategoryCode':'',
'filterIncidentCode':'',
'filterDateCode':'',
'startNo':page,
'resultNumber':'100',
'topmenuoff':'',
'resultState':'',
'keywordJson':'{"searchDetailTxt1":keyword,"agreeDetailTxt1":"","needDetailTxt1":"","exceptDetailTxt1":"","o_id":"option1","startDate":start,"endDate":end,"providerNm":"","categoryNm":"","incidentCategoryNm":"","providerCode":"","categoryCode":"","incidentCategoryCode":"","searchFtr":"1","searchScope":"1","searchKeyword":"keyword"}',
'keywordFilterJson':'',
'totalCount':'',
'interval':'',
'quotationKeyword1':'',
'quotationKeyword2':'',
'quotationKeyword3':'',
'searchFromUseYN':'N',
'mainTodayPersonYn':'',
'period':'1year'
}
param = urllib.parse.urlencode(param).encode()
req = urllib.request.Request(url, param, headers)
sleepTime = random.randint(4,10)
time.sleep(sleepTime)
print(str(sleepTime) + ' seconds wait.')
try :
res = urllib.request.urlopen(req)
except URLError as e:
if hasattr(e, 'reason'):
print('We failed to reach a server.')
print('Reason: ', e.reason)
elif hasattr(e, 'code'):
print('The server couldn\'t fulfill the request.')
print('Error code: ', e.code)
html = res.read()
soup = BeautifulSoup(html, "html.parser")
div_tags = soup.find_all('div', class_='resTxt')
for cts in div_tags:
ids = cts.find('h3')['id'][5:31]
title = cts.find('h3',class_='list_newsId').get_text(strip=True)
href = 'https://www.bigkinds.or.kr/news/detailView.do?docId=' + ids + '&returnCnt=1'
sets = {
'title' : title,
'href' : href
}
result. append(sets)
return result
def fetch_news_detail(url):
result = {}
res = urlopen(url).read().decode('utf-8', errors='ignore')
responseJson = json.loads(res)
category = responseJson.get("detail").get("CATEGORY_MAIN")
date = responseJson.get("detail").get("DATE")
provider = responseJson.get("detail").get("PROVIDER")
content = responseJson.get("detail").get("CONTENT")
result = {
'category': category,
'date': date,
'provider' : provider,
'content': content
}
return result
keyword = input('(eg., 외국인 NOT(증시 OR 순매수 OR 증권 OR 코스피 OR 코스닥 OR 주식 OR 주가 OR 투타 OR KBO OR 야구 OR KBL OR 농구 OR 축구 OR 올림픽 OR K리그))\n input word: ')
start = input('(eg., 2017-01-01)\n input startday: ')
end = input('(eg., 2017-02-01)\n input endday: ')
page = 1
count = 1
flag = True
f = open('bigkinds.txt', 'w', encoding='utf-8')
while True:
if not flag:
break
news_list = fetch_news_list(page, keyword,start,end)
sleepTime = random.randint(3,8)
time.sleep(sleepTime)
print(str(sleepTime) + ' seconds wait.')
for news in news_list:
result = fetch_news_detail(news['href'])
result['title'] = news['title']
f.write('==' * 40 + '\n')
f.write('category: ' + result['category'] + '\n')
f.write('title: ' + result['title'] + '\n')
f.write('date: ' + result['date'] + '\n')
f.write('provider: ' + result['provider'] + '\n')
f.write('content: ' + result['content'] + '\n')
f.write('==' * 40 + '\n')
count += 1
if count >=5002:
flag = False
break
sleepTime = random.randint(2,10)
time.sleep(sleepTime)
print(str(sleepTime) + ' seconds wait.')
page += 1
f.close()
从Yelp中抓取数据时遇到过这种问题。很难绕过这些限制。我建议您尝试以下方法。