我想编写一个程序来ping至少10000个URL。我写了一个小程序,发现它的速度不如我预期。
Ping 100个网址需要3-4分钟。
有人有任何更好的建议吗?
private static Map<String, String> findUnreachableUrls(Set<String> urls) {
Map<String, String> badUrls = new TreeMap<>();
for (String url : urls) {
HttpURLConnection connection;
try {
connection = (HttpURLConnection) new URL(url).openConnection();
connection.setRequestMethod("HEAD");
connection.connect();
int responseCode = connection.getResponseCode();
if (responseCode != 200 || responseCode != 302) {
badUrls.put(url, Integer.toString(responseCode));
}
} catch (IOException e) {
badUrls.put(url, e.getMessage());
}
}
return badUrls;
}
您应该使用并行线程,例如5个线程对20个URL进行相同的处理,最后汇总结果。这样可以使结果更快。最简单的解决方案是使用Java 8 Streams并行处理URL。以下是相同的示例程序:
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
public class Main {
public static void main(String[] args) {
Set<String> urlSet = new HashSet<>();
//Populate your set with the Strings
findUnreachableUrls(urlSet);
}
private static Map<String, String> findUnreachableUrls(Set<String> urls) {
Map<String, String> badUrls = new TreeMap<>();
urls.parallelStream().forEach(
url->{
badUrls.put(url,checkUrl(url));
}
);
return badUrls;
}
private static String checkUrl(String url)
{
HttpURLConnection connection;
String returnCode="";
try {
connection = (HttpURLConnection) new URL(url).openConnection();
connection.setRequestMethod("HEAD");
connection.connect();
int responseCode = connection.getResponseCode();
if (responseCode != 200 || responseCode != 302) {
returnCode=responseCode+"";
}
}catch(IOException e)
{
returnCode=e.getMessage();
}
return returnCode;
}
}
正如我写的那样,我将以这种方式进行操作(未经测试)。也许如果只有几个主机,并且有很多网址,则应该将已分组的网址进一步分割。
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class TestURLs implements Function<String, Optional<TestURLs.Tuple>> {
public static final int TIMEOUT = 3000;
public class Tuple {
final String url;
final String error;
public Tuple(String url, String error) {
this.url = url;
this.error = error;
}
}
public static enum HostNamePortExtractor implements Function<String, String>{
INSTANCE;
@Override
public String apply(String url) {
try {
URL u = new URL(url);
return u.getHost() + u.getPort();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
@Override
public Optional<Tuple> apply(String url) {
HttpURLConnection connection;
try {
connection = (HttpURLConnection) new URL(url).openConnection();
connection.setRequestMethod("HEAD");
connection.setReadTimeout(TIMEOUT);
connection.setConnectTimeout(TIMEOUT);
connection.connect();
int responseCode = connection.getResponseCode();
// are you sure? I think you would have liked to write here "and" not or
//if (responseCode != 200 || responseCode != 302) {
if (responseCode != 200 && responseCode != 302) {
return Optional.of(new Tuple(url, Integer.toString(responseCode)));
}
} catch (IOException e) {
return Optional.of(new Tuple(url, e.getMessage()));
}
return Optional.empty();
}
public Map<String, String> process() {
List<String> URLs = new ArrayList<>(); // add urls here
// group by hostname+port
Map<String, List<String>> groupedUrls = URLs.stream().collect(Collectors.groupingBy(HostNamePortExtractor.INSTANCE));
Stream<Tuple> errors = groupedUrls.keySet().parallelStream()
// I am not fully sure, but hoping that the stream() will go to the same thread
.flatMap(host -> groupedUrls.get(host).stream())
// go to the server
.map(this::apply)
// if there was no error, filter out the optional.empties
.filter(o -> o.isPresent())
// get the Tuple with url and the error
.map(o -> o.get());
// make a map
return errors.collect(Collectors.toMap(t -> t.url, t -> t.error));
}
public static void main(String[] args) {
TestURLs testUrls = new TestURLs();
testUrls.process().entrySet().forEach(e -> {
System.out.println(e.getKey() + " error: " + e.getValue());
});
}
}