Java: How to read the content of a webpage or url?

In this post, I am going to show multiple programs/ways to read the content of a webpage or url in Java.

 

Approach 1: Using built-in HttpURLConnection class.

 

ReadDataFromUrlDemo1.java

package com.sample.app;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class ReadDataFromUrlDemo1 {

	public static String getContentFromUrl(String urlToRead) throws Exception {

		URL url = new URL(urlToRead);
		HttpURLConnection httpUrlConnection = (HttpURLConnection) url.openConnection();
		httpUrlConnection.setRequestMethod("GET");

		StringBuilder result = new StringBuilder();

		try (InputStream inputStream = httpUrlConnection.getInputStream();
				InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
				BufferedReader reader = new BufferedReader(inputStreamReader)) {
			
			for (String line; (line = reader.readLine()) != null;) {
				result.append(line);
			}
			
		}
		return result.toString();
	}

	public static void main(String args[]) throws Exception {
		String urlToRead = "https://self-learning-java-tutorial.blogspot.com/2014/02/blog-post.html";
		String result = getContentFromUrl(urlToRead);
		System.out.println(result);
	}

}

 

Approach 2: Using Spring RestTemplate. One of the key advantages of Http client over HttpURLConnection is, it can handle URL redirects and proxy negotiations etc.,

 

ReadDataFromUrlDemo2.java

package com.sample.app;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;

public class ReadDataFromUrlDemo2 {

	private static final RestTemplate REST_TEMPLATE = new RestTemplate(getClientHttpRequestFactory());

	private static ClientHttpRequestFactory getClientHttpRequestFactory() {
		int timeout = 5000;
		RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout).setConnectionRequestTimeout(timeout)
				.setSocketTimeout(timeout).build();
		CloseableHttpClient client = HttpClientBuilder.create().setDefaultRequestConfig(config).build();
		return new HttpComponentsClientHttpRequestFactory(client);
	}

	public static String getContentFromUrl(String urlToRead) throws Exception {
		ResponseEntity<String> responseEntity = REST_TEMPLATE.exchange(urlToRead, HttpMethod.GET, null, String.class);
		return responseEntity.getBody();

	}

	public static void main(String args[]) throws Exception {
		String urlToRead = "https://self-learning-java-tutorial.blogspot.com/2018/08/spring-framework.html";
		String result = getContentFromUrl(urlToRead);
		System.out.println(result);
	}
}

Dependencies used

<dependencies>
	<dependency>
		<groupId>org.apache.httpcomponents</groupId>
		<artifactId>httpclient</artifactId>
		<version>4.5.10</version>
	</dependency>

	<dependency>
		<groupId>org.springframework</groupId>
		<artifactId>spring-web</artifactId>
		<version>5.3.11</version>
	</dependency>

</dependencies>

Approach 3: Using apache http client.

 

ReadDataFromUrlDemo3.java

package com.sample.app;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

public class ReadDataFromUrlDemo3 {

	public static String getContentFromUrl(String urlToRead) throws Exception {
		CloseableHttpClient httpclient = HttpClients.createDefault();
		HttpGet httpget = new HttpGet(urlToRead);
		CloseableHttpResponse response = httpclient.execute(httpget);
		HttpEntity entity = response.getEntity();

		StringBuilder result = new StringBuilder();

		try (InputStream inputStream = entity.getContent();
				InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
				BufferedReader reader = new BufferedReader(inputStreamReader)) {

			for (String line; (line = reader.readLine()) != null;) {
				result.append(line);
			}

		}
		return result.toString();

	}

	public static void main(String args[]) throws Exception {
		String urlToRead = "https://self-learning-java-tutorial.blogspot.com/2018/08/spring-framework.html";
		String result = getContentFromUrl(urlToRead);
		System.out.println(result);
	}
}

  

Previous                                                    Next                                                    Home

Apache HTTP Client: set timeout

Step 1: Define Request configuration.

RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
requestConfigBuilder.setConnectTimeout(3 * 1000);
requestConfigBuilder.setConnectionRequestTimeout(3 * 1000);
requestConfigBuilder.setSocketTimeout(3 * 1000);
RequestConfig requestConfig = requestConfigBuilder.build();

 

Step 2: Define an instance of HttpRequestBase and set the request configuration defined in step 1.

 

HttpRequestBase httpget = new HttpGet(url);
httpget.setConfig(requestConfig);

 

Step 3: Execute the request using http client.

CloseableHttpClient httpclient = HttpClients.createDefault();
CloseableHttpResponse httpResponse = httpclient.execute(httpget);

 

Step 4: Get the response as string.

public static String getResponseAsString(CloseableHttpResponse closeableHttpResponse)
		throws UnsupportedOperationException, IOException {

	HttpEntity httpEntity = closeableHttpResponse.getEntity();
	return getResponseAsString(httpEntity.getContent());

}

public static String getResponseAsString(InputStream is) throws IOException {
	try (InputStreamReader isReader = new InputStreamReader(is)) {
		BufferedReader reader = new BufferedReader(isReader);
		StringBuffer sb = new StringBuffer();
		String str;
		while ((str = reader.readLine()) != null) {
			sb.append(str);
		}

		return sb.toString();
	}

}

 

 

 

 

Previous                                                    Next                                                    Home

Spring RestTemplate + HttpClient configuration example

This is continuation to my previous post. Please go through my previous post and setup the application to test this.

Step 1: Create new maven project ‘springRestTemplate’.

Step 2: Update pom.xml with maven dependencies.

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>springRestTemplate</groupId>
  <artifactId>springRestTemplate</artifactId>
  <version>1</version>

  <parent>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-parent</artifactId>
    <version>2.1.6.RELEASE</version>
  </parent>

  <name>springbootApp</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>

    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-web</artifactId>
    </dependency>

    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <version>4.5.10</version>
    </dependency>


  </dependencies>
</project>

Step 3: Create com.sample.app.model package and define Employee class.

Employee.java

package com.sample.app.model;

public class Employee {

  private int id;
  private String firstName;
  private String lastName;

  public int getId() {
    return id;
  }

  public void setId(int id) {
    this.id = id;
  }

  public String getFirstName() {
    return firstName;
  }

  public void setFirstName(String firstName) {
    this.firstName = firstName;
  }

  public String getLastName() {
    return lastName;
  }

  public void setLastName(String lastName) {
    this.lastName = lastName;
  }

  @Override
  public String toString() {
    return "Employee [id=" + id + ", firstName=" + firstName + ", lastName=" + lastName + "]";
  }

}

Step 4: Create a package ‘com.sample.app.uti’ and define HttpClient class.

HttpClient.java

package com.sample.app.util;

import java.util.Map;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;

public class HttpClient {

  private static final RestTemplate REST_TEMPLATE = new RestTemplate(getClientHttpRequestFactory());
  
  private static ClientHttpRequestFactory getClientHttpRequestFactory() {
      int timeout = 5000;
      RequestConfig config = RequestConfig.custom()
        .setConnectTimeout(timeout)
        .setConnectionRequestTimeout(timeout)
        .setSocketTimeout(timeout)
        .build();
      CloseableHttpClient client = HttpClientBuilder
        .create()
        .setDefaultRequestConfig(config)
        .build();
      return new HttpComponentsClientHttpRequestFactory(client);
  }

  private static HttpHeaders getHeadersFromMap(Map<String, String> headers) {
    HttpHeaders httpHeaders = new HttpHeaders();

    if (headers == null || headers.isEmpty()) {
      return httpHeaders;
    }

    for (Map.Entry<String, String> entry : headers.entrySet()) {
      httpHeaders.set(entry.getKey(), entry.getValue());
    }

    return httpHeaders;
  }

  public static ResponseEntity<String> get(String url) {
    return get(url, null);
  }

  public static ResponseEntity<String> get(String url, Map<String, String> headers) {
    if (url == null || url.isEmpty()) {
      return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("URL must not be null");
    }

    HttpEntity<Object> httpEntity = new HttpEntity<>(headers);

    return REST_TEMPLATE.exchange(url, HttpMethod.GET, httpEntity, String.class);

  }

  public static ResponseEntity<String> put(String url, String json) {
    return put(url, null, json);
  }

  public static ResponseEntity<String> put(String url, Map<String, String> headers, String json) {
    if (url == null || url.isEmpty()) {
      return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("URL must not be null");
    }

    HttpHeaders httpHeaders = getHeadersFromMap(headers);

    HttpEntity<Object> httpEntity = new HttpEntity<Object>(json, httpHeaders);

    return REST_TEMPLATE.exchange(url, HttpMethod.PUT, httpEntity, String.class);

  }

  public static ResponseEntity<String> post(String url, String json) {
    return put(url, null, json);
  }

  public static ResponseEntity<String> post(String url, Map<String, String> headers, String json) {
    if (url == null || url.isEmpty()) {
      return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("URL must not be null");
    }

    HttpHeaders httpHeaders = getHeadersFromMap(headers);

    HttpEntity<Object> httpEntity = new HttpEntity<Object>(json, httpHeaders);

    return REST_TEMPLATE.exchange(url, HttpMethod.POST, httpEntity, String.class);

  }

  public static ResponseEntity<String> delete(String url) {
    return delete(url, null);
  }

  public static ResponseEntity<String> delete(String url, Map<String, String> headers) {
    if (url == null || url.isEmpty()) {
      return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("URL must not be null or empty");
    }

    HttpHeaders httpHeaders = getHeadersFromMap(headers);

    HttpEntity<Object> httpEntity = new HttpEntity<Object>(httpHeaders);

    return REST_TEMPLATE.exchange(url, HttpMethod.DELETE, httpEntity, String.class);

  }

  public static <T> T getResponse(String url, Class<T> clazz) {
    if (url == null || url.isEmpty()) {
      throw new RuntimeException("url must not be null or empty");
    }

    return REST_TEMPLATE.getForObject(url, clazz);
  }

  public static HttpHeaders getHeaders(String url) {
    if (url == null || url.isEmpty()) {
      throw new RuntimeException("url must not be null or empty");
    }

    return REST_TEMPLATE.headForHeaders(url);
  }

}

Step 5: Define App.java.

App.java

package com.sample.app;

import java.util.HashMap;
import java.util.Map;

import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;

import com.sample.app.model.Employee;
import com.sample.app.util.HttpClient;


public class App {

  private static void printAllEmployees() {
    ResponseEntity<String> response = HttpClient.get("http://localhost:8080/api/v1/employees/");
    System.out.println(response.getBody());
  }

  public static void main(String args[]) {

    String emp1 = "{\"id\":1,\"firstName\":\"Harini\",\"lastName\":\"Gurram\"}";
    Map<String, String> headers = new HashMap<>();
    headers.put("Accept", "application/json");
    headers.put("Content-Type", "application/json");

    ResponseEntity<String> response = HttpClient.put("http://localhost:8080/api/v1/employees/1", headers, emp1);
    System.out.println(response.getBody());
    printAllEmployees();

    String emp2 = "{\"firstName\":\"Jaideep\",\"lastName\":\"Geera\"}";
    response = HttpClient.post("http://localhost:8080/api/v1/employees/", headers, emp2);
    System.out.println(response.getBody());
    printAllEmployees();
    
    response = HttpClient.delete("http://localhost:8080/api/v1/employees/3");
    System.out.println(response.getBody());
    printAllEmployees();
    
    Employee emp3 = HttpClient.getResponse("http://localhost:8080/api/v1/employees/1", Employee.class);
    System.out.println(emp3);
    
    HttpHeaders responseHeaders = HttpClient.getHeaders("http://localhost:8080/api/v1/employees/");
    System.out.println(responseHeaders);
    
  }
}

Total project structure looks like below.

Run App.java, you can see the responses in console.

You can download the complete working application from this link.

Previous                                                    Next                                                    Home