如何在循环中未得到响应的情况下继续获取网页?
for (node of NodeList) {
const url = node.getAttribute('href')
const res = await fetch(url) //Code below won't execute if no response
const html = await response.text()
const scraper = new DOMParser()
const doc = scraper.parseFromString(text, 'text/html')
alert('successfully parsed')
}
由于它是循环工作的并且一次发送非常多的请求,即使是网络上的一个小而短的问题也会破坏整个过程,我该怎么办?
我从不循环 AJAX,它会让服务器不堪重负,并且浏览器(如果在浏览器中)将没有时间更新界面。
我会尝试这样的事情
仅在成功或错误时再次调用该函数来解决异步性
const urls = [...NodeList].map(node => node.getAttribute('href'));
const docs = [];
const max = urls.length;
let cnt = 0;
const getHtml = () => {
if (cnt >= max) {
processAllDocs(); // here we have them all
return;
}
try {
const res = fetch(url) //Code below won't execute if no response
.then(response => {
if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}
return response.text()
})
.then(html => {
const scraper = new DOMParser()
const doc = scraper.parseFromString(text, 'text/html')
docs.push(doc)
cnt++
getHtml(); // get next - use setTimeout if you want to throttle
} catch (err) {
console.log(err.message);
if (err.message.includes("something really bad, please stop")) return; // stop all processing (server down or ten 500 errors in a row)
getHtml(); // get same - use setTimeout if you want to throttle
}
}
getHtml();