Java程序ping数千个URL

问题描述 投票:-1回答:2

我想编写一个程序来ping至少10000个URL。我写了一个小程序,发现它的速度不如我预期。

Ping 100个网址需要3-4分钟。

有人有任何更好的建议吗?

private static Map<String, String> findUnreachableUrls(Set<String> urls) {
        Map<String, String> badUrls = new TreeMap<>();
        for (String url : urls) {
            HttpURLConnection connection;
            try {
                connection = (HttpURLConnection) new URL(url).openConnection();
                connection.setRequestMethod("HEAD");
                connection.connect();
                int responseCode = connection.getResponseCode();
                if (responseCode != 200 || responseCode != 302) {
                    badUrls.put(url, Integer.toString(responseCode));
                }
            } catch (IOException e) {
                badUrls.put(url, e.getMessage());
            }

        }
        return badUrls;
    }
java httpclient
2个回答
2
投票

您应该使用并行线程,例如5个线程对20个URL进行相同的处理,最后汇总结果。这样可以使结果更快。最简单的解决方案是使用Java 8 Streams并行处理URL。以下是相同的示例程序:

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;

public class Main {

    public static void main(String[] args) {
        Set<String> urlSet = new HashSet<>();
        //Populate your set with the Strings
        findUnreachableUrls(urlSet);
    }
    private static Map<String, String> findUnreachableUrls(Set<String> urls) {
        Map<String, String> badUrls = new TreeMap<>();
        urls.parallelStream().forEach(
                url->{
                    badUrls.put(url,checkUrl(url));
                }
        );
        return badUrls;
    }

    private static String checkUrl(String url)
    {
        HttpURLConnection connection;
        String returnCode="";
        try {
            connection = (HttpURLConnection) new URL(url).openConnection();
            connection.setRequestMethod("HEAD");
            connection.connect();
            int responseCode = connection.getResponseCode();
            if (responseCode != 200 || responseCode != 302) {
                returnCode=responseCode+"";
            }
        }catch(IOException e)
        {
            returnCode=e.getMessage();
        }
        return returnCode;
    }

}

0
投票

正如我写的那样,我将以这种方式进行操作(未经测试)。也许如果只有几个主机,并且有很多网址,则应该将已分组的网址进一步分割。

 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

 public class TestURLs implements Function<String, Optional<TestURLs.Tuple>> {

public static final int TIMEOUT = 3000;

public class Tuple {
    final String url;
    final String error;

    public Tuple(String url, String error) {
        this.url = url;
        this.error = error;
    }
}

public static enum HostNamePortExtractor implements Function<String, String>{

    INSTANCE;

    @Override
    public String apply(String url) {
        try {
            URL u = new URL(url);
            return u.getHost() + u.getPort();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

@Override
public Optional<Tuple> apply(String url) {
    HttpURLConnection connection;
    try {
        connection = (HttpURLConnection) new URL(url).openConnection();
        connection.setRequestMethod("HEAD");
        connection.setReadTimeout(TIMEOUT);
        connection.setConnectTimeout(TIMEOUT);
        connection.connect();
        int responseCode = connection.getResponseCode();
        // are you sure? I think you would have liked to write here "and" not or
        //if (responseCode != 200 || responseCode != 302) {
        if (responseCode != 200 && responseCode != 302) {
            return Optional.of(new Tuple(url, Integer.toString(responseCode)));
        }
    } catch (IOException e) {
        return Optional.of(new Tuple(url, e.getMessage()));
    }
    return Optional.empty();
}

public Map<String, String> process() {
        List<String> URLs = new ArrayList<>(); // add urls here
        // group by hostname+port
        Map<String, List<String>> groupedUrls = URLs.stream().collect(Collectors.groupingBy(HostNamePortExtractor.INSTANCE));
        Stream<Tuple> errors = groupedUrls.keySet().parallelStream()
                // I am not fully sure, but hoping that the stream() will go to the same thread
                .flatMap(host -> groupedUrls.get(host).stream())
                // go to the server
                .map(this::apply)
                // if there was no error, filter out the optional.empties
                .filter(o -> o.isPresent())
                // get the Tuple with url and the error
                .map(o -> o.get()); 
                // make a map
        return errors.collect(Collectors.toMap(t -> t.url, t -> t.error));
    }

public static void main(String[] args) {
    TestURLs testUrls = new TestURLs();
    testUrls.process().entrySet().forEach(e -> {
        System.out.println(e.getKey() + " error: " + e.getValue());
    });
}

}

© www.soinside.com 2019 - 2024. All rights reserved.