Java: How to read the content of a webpage or url?

In this post, I am going to show multiple programs/ways to read the content of a webpage or url in Java.

 

Approach 1: Using built-in HttpURLConnection class.

 

ReadDataFromUrlDemo1.java

package com.sample.app;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class ReadDataFromUrlDemo1 {

	public static String getContentFromUrl(String urlToRead) throws Exception {

		URL url = new URL(urlToRead);
		HttpURLConnection httpUrlConnection = (HttpURLConnection) url.openConnection();
		httpUrlConnection.setRequestMethod("GET");

		StringBuilder result = new StringBuilder();

		try (InputStream inputStream = httpUrlConnection.getInputStream();
				InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
				BufferedReader reader = new BufferedReader(inputStreamReader)) {
			
			for (String line; (line = reader.readLine()) != null;) {
				result.append(line);
			}
			
		}
		return result.toString();
	}

	public static void main(String args[]) throws Exception {
		String urlToRead = "https://self-learning-java-tutorial.blogspot.com/2014/02/blog-post.html";
		String result = getContentFromUrl(urlToRead);
		System.out.println(result);
	}

}

 

Approach 2: Using Spring RestTemplate. One of the key advantages of Http client over HttpURLConnection is, it can handle URL redirects and proxy negotiations etc.,

 

ReadDataFromUrlDemo2.java

package com.sample.app;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;

public class ReadDataFromUrlDemo2 {

	private static final RestTemplate REST_TEMPLATE = new RestTemplate(getClientHttpRequestFactory());

	private static ClientHttpRequestFactory getClientHttpRequestFactory() {
		int timeout = 5000;
		RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout).setConnectionRequestTimeout(timeout)
				.setSocketTimeout(timeout).build();
		CloseableHttpClient client = HttpClientBuilder.create().setDefaultRequestConfig(config).build();
		return new HttpComponentsClientHttpRequestFactory(client);
	}

	public static String getContentFromUrl(String urlToRead) throws Exception {
		ResponseEntity<String> responseEntity = REST_TEMPLATE.exchange(urlToRead, HttpMethod.GET, null, String.class);
		return responseEntity.getBody();

	}

	public static void main(String args[]) throws Exception {
		String urlToRead = "https://self-learning-java-tutorial.blogspot.com/2018/08/spring-framework.html";
		String result = getContentFromUrl(urlToRead);
		System.out.println(result);
	}
}

Dependencies used

<dependencies>
	<dependency>
		<groupId>org.apache.httpcomponents</groupId>
		<artifactId>httpclient</artifactId>
		<version>4.5.10</version>
	</dependency>

	<dependency>
		<groupId>org.springframework</groupId>
		<artifactId>spring-web</artifactId>
		<version>5.3.11</version>
	</dependency>

</dependencies>

Approach 3: Using apache http client.

 

ReadDataFromUrlDemo3.java

package com.sample.app;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

public class ReadDataFromUrlDemo3 {

	public static String getContentFromUrl(String urlToRead) throws Exception {
		CloseableHttpClient httpclient = HttpClients.createDefault();
		HttpGet httpget = new HttpGet(urlToRead);
		CloseableHttpResponse response = httpclient.execute(httpget);
		HttpEntity entity = response.getEntity();

		StringBuilder result = new StringBuilder();

		try (InputStream inputStream = entity.getContent();
				InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
				BufferedReader reader = new BufferedReader(inputStreamReader)) {

			for (String line; (line = reader.readLine()) != null;) {
				result.append(line);
			}

		}
		return result.toString();

	}

	public static void main(String args[]) throws Exception {
		String urlToRead = "https://self-learning-java-tutorial.blogspot.com/2018/08/spring-framework.html";
		String result = getContentFromUrl(urlToRead);
		System.out.println(result);
	}
}

  

Previous                                                    Next                                                    Home

RestTemplate: Disable ssl certificate validation

Following snippet return RestTemplate instance which ignores SSL certificate validation.

public RestTemplate getRestTemplate() throws BaseException {

 try {
  TrustStrategy acceptingTrustStrategy = new TrustStrategy() {
   @Override
   public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
    return true;
   }
  };
  SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom()
    .loadTrustMaterial(null, acceptingTrustStrategy).build();
  SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext, new NoopHostnameVerifier());
  CloseableHttpClient httpClient = HttpClients.custom().setSSLSocketFactory(csf).build();
  HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory();
  requestFactory.setHttpClient(httpClient);
  RestTemplate restTemplate = new RestTemplate(requestFactory);
  return restTemplate;
 } catch (Exception e) {
  e.printStackTrace();
 }

}

Previous                                                    Next                                                    Home

RestTemplate: Configure Request Time out

private static final RestTemplate REST_TEMPLATE = new RestTemplate(getClientHttpRequestFactory());

private static ClientHttpRequestFactory getClientHttpRequestFactory() {
    int timeout = 5000;
    RequestConfig config = RequestConfig.custom()
      .setConnectTimeout(timeout)
      .setConnectionRequestTimeout(timeout)
      .setSocketTimeout(timeout)
      .build();
    CloseableHttpClient client = HttpClientBuilder
      .create()
      .setDefaultRequestConfig(config)
      .build();
    return new HttpComponentsClientHttpRequestFactory(client);
}

You can get complete working application from this link.

Previous                                                    Next                                                    Home

RestTemplate: Get only response headers

public static HttpHeaders getHeaders(String url) {
 if (url == null || url.isEmpty()) {
  throw new RuntimeException("url must not be null or empty");
 }

 return REST_TEMPLATE.headForHeaders(url);
}

You can get complete working application from this link.

Previous                                                    Next                                                    Home