Lucene: Boosting Queries

Sometimes we may want to boost queries on specific fields. For example, I want to rank more to the documents that are written by the author ‘Ram’.

 

Example

Query authorBoostQuery = new BoostQuery(authorQuery, 3f);
Query authorBoostQuery = new BoostQuery(authorQuery, 3f);

 

Now, you can combine multiple queries using BooleanQuery like below.

BooleanQuery boolQuery = BooleanQuery.Builder.class.newInstance()
.add(authorBoostQuery, BooleanClause.Occur.SHOULD).add(titleQuery, BooleanClause.Occur.SHOULD)
.build();

 

Find the below working application.

 

DocumentUtil.java

 

package com.sample.app.util;

import java.util.Arrays;
import java.util.List;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;

public class DocumentUtil {

	public static Document getDocument(String id, String title, String description, String blog, List<String> authors) {
		Document doc = new Document();
		doc.add(new StringField("id", id, Field.Store.YES));
		doc.add(new TextField("title", title, Field.Store.YES));
		doc.add(new TextField("description", description, Field.Store.NO));
		doc.add(new TextField("blog", blog, Field.Store.YES));

		for (String author : authors) {
			doc.add(new StringField("author", author, Field.Store.YES));
		}

		return doc;

	}

	public static List<Document> getDocuments() {
		Document doc1 = getDocument("1", "Java World",
				"The original independent resource for Java developers, architects, and managers.", " javaworld.com",
				Arrays.asList("Ram"));
		Document doc2 = getDocument("2", "Oracle Blogs | The Java Source",
				" Java powers more than 4.5 billion devices including 800 million computers and 1.5 billion cell phones. If you love Java, this is the blog you must follow.",
				"blogs.oracle.com/java", Arrays.asList("Ram"));
		Document doc3 = getDocument("3", "A Java geek Professionals and experts",
				"Nicolas Fränkel's blog. IT architect focusing on Java, Java EE, and their surrounding ecosystems. He is a trainer, book writer, speaker & blogger.",
				"blog.frankel.ch", Arrays.asList("John"));
		Document doc4 = getDocument("4", "Self Learning for Java Developers By Krishna (Java)", "Learn Java fundamentals and other java libraries",
				"self-learning-java-tutorial.blogspot.com", Arrays.asList("Krishna"));

		return Arrays.asList(doc1, doc2, doc3, doc4);

	}
}

 

App.java

package com.sample.app;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.util.QueryBuilder;

import com.sample.app.util.DocumentUtil;

public class App {

	private static void printQueryResults(Query query, Directory directory) throws IOException {
		try (IndexReader indexReader = DirectoryReader.open(directory)) {
			IndexSearcher indexSearcher = new IndexSearcher(indexReader);
			TopDocs docs = indexSearcher.search(query, 10);
			ScoreDoc[] hits = docs.scoreDocs;
			System.out.print("Total Hits: " + docs.totalHits);
			System.out.print("Results: ");
			for (int i = 0; i < hits.length; i++) {
				Document d = indexSearcher.doc(hits[i].doc);

				System.out.println("\nTitle: " + d.get("title"));
				System.out.println("Score : " + hits[i].score);

				IndexableField[] fields = d.getFields("author");
				for (IndexableField field : fields) {
					System.out.println("Author: " + field.stringValue());
				}

				System.out.println();
			}

			System.out.println("**************************************\n");
		}
	}

	public static void main(String args[]) throws IOException, InstantiationException, IllegalAccessException {

		Analyzer analyzer = new StandardAnalyzer();
		IndexWriterConfig indexWriterConfig1 = new IndexWriterConfig(analyzer);

		Directory directory = new MMapDirectory(new File("/Users/Shared/lucene").toPath(), NoLockFactory.INSTANCE);

		try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig1)) {

			List<Document> documents = DocumentUtil.getDocuments();

			indexWriter.addDocuments(documents);
			indexWriter.commit();

			QueryBuilder queryBuilder = new QueryBuilder(analyzer);
			Query authorQuery = new TermQuery(new Term("author", "Ram"));
			Query titleQuery = queryBuilder.createPhraseQuery("title", "Java");

			Query authorBoostQuery = new BoostQuery(authorQuery, 3f);
			Query titleBoostQuery = new BoostQuery(titleQuery, 123.5f);

			System.out.println("\nBoosting based on author");
			BooleanQuery boolQuery = BooleanQuery.Builder.class.newInstance()
					.add(authorBoostQuery, BooleanClause.Occur.SHOULD).add(titleQuery, BooleanClause.Occur.SHOULD)
					.build();

			printQueryResults(boolQuery, directory);

			System.out.println("\nBoosting based on title");
			boolQuery = BooleanQuery.Builder.class.newInstance().add(authorQuery, BooleanClause.Occur.SHOULD)
					.add(titleBoostQuery, BooleanClause.Occur.SHOULD).build();

			printQueryResults(boolQuery, directory);

		}

	}
}

 

Output

Boosting based on author
Total Hits: 4 hitsResults: 
Title: Java World
Score : 1.0093331
Author: Ram


Title: Oracle Blogs | The Java Source
Score : 0.9940433
Author: Ram


Title: Self Learning for Java Developers By Krishna (Java)
Score : 0.057394832
Author: Krishna


Title: A Java geek Professionals and experts
Score : 0.045246847
Author: John

**************************************


Boosting based on title
Total Hits: 4 hitsResults: 
Title: Java World
Score : 8.23543
Author: Ram


Title: Self Learning for Java Developers By Krishna (Java)
Score : 7.088262
Author: Krishna


Title: Oracle Blogs | The Java Source
Score : 6.347131
Author: Ram


Title: A Java geek Professionals and experts
Score : 5.5879855
Author: John

**************************************

 

 

 

  

Previous                                                    Next                                                    Home

Lucene: Check any document is deleted as part of delete query or not

'indexWriter.hasDeletions()' method return true if this index has deletions (including buffered deletions).

 

DocumentUtil.java

package com.sample.app.util;

import java.util.Arrays;
import java.util.List;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;

public class DocumentUtil {

	private static Document getDocument(String id, String title, String description, String blog) {
		Document doc = new Document();
		doc.add(new TextField("id", id, Field.Store.YES));
		doc.add(new TextField("title", title, Field.Store.YES));
		doc.add(new TextField("description", description, Field.Store.NO));
		doc.add(new TextField("blog", blog, Field.Store.YES));
		
		return doc;

	}

	public static List<Document> getDocuments() {
		Document doc1 = getDocument("1", "JavaWorld",
				"The original independent resource for Java developers, architects, and managers.", " javaworld.com");
		Document doc2 = getDocument("2", "Oracle Blogs | The Java Source",
				" Java powers more than 4.5 billion devices including 800 million computers and 1.5 billion cell phones. If you love Java, this is the blog you must follow.",
				"blogs.oracle.com/java");
		Document doc3 = getDocument("3", "A Java geek",
				"Nicolas Fränkel's blog. IT architect focusing on Java, Java EE, and their surrounding ecosystems. He is a trainer, book writer, speaker & blogger.",
				"blog.frankel.ch");
		Document doc4 = getDocument("4", "Self Learning Java", "Learn Java fundamentals and other java libraries",
				"self-learning-java-tutorial.blogspot.com");

		return Arrays.asList(doc1, doc2, doc3, doc4);

	}
}

 

App.java

package com.sample.app;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiBits;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.util.Bits;

import com.sample.app.util.DocumentUtil;

public class App {

	private static void printAllDocuments(Directory directory) throws IOException {

		try (IndexReader indexReader = DirectoryReader.open(directory)) {
			System.out.println("All Documents in Lucene Index");
			Bits liveDocs = MultiBits.getLiveDocs(indexReader);
			for (int i = 0; i < indexReader.maxDoc(); i++) {
				if (liveDocs != null && !liveDocs.get(i))
					continue;

				Document doc = indexReader.document(i);
				System.out.println(doc.get("id") + ", " + doc.get("title"));
			}

			System.out.println();
		}

	}

	public static void main(String args[]) throws IOException {

		Analyzer analyzer = new StandardAnalyzer();
		IndexWriterConfig indexWriterConfig1 = new IndexWriterConfig(analyzer);

		Directory directory = new MMapDirectory(new File("/Users/Shared/lucene").toPath(), NoLockFactory.INSTANCE);

		try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig1)) {

			List<Document> documents = DocumentUtil.getDocuments();

			System.out.println("Adding " + documents.size() + " documents to Lucene");
			indexWriter.addDocuments(documents);
			indexWriter.commit();

			printAllDocuments(directory);

			System.out.println("\nAbout to delete all documents");

			indexWriter.deleteDocuments(new Term("id", "2"));
			indexWriter.commit();

			System.out.println("Has Deletions : " + indexWriter.hasDeletions());

			printAllDocuments(directory);
		}

	}
}

 

Output

Adding 4 documents to Lucene
All Documents in Lucene Index
1, JavaWorld
2, Oracle Blogs | The Java Source
3, A Java geek
4, Self Learning Java


About to delete all documents
Has Deletions : true
All Documents in Lucene Index
1, JavaWorld
3, A Java geek
4, Self Learning Java

 

 

 

 

 

Previous                                                    Next                                                    Home

Lucene: Get total number of documents matches to given query

TopDocs class provides 'totalHits' property that return total number of documents that matches to given query.

 

Example

TopDocs topDocs = indexSearcher.search(query, maxHitsPerPage);
TotalHits totalHits = topDocs.totalHits;
System.out.println("Total Hits: " + totalHits.value);

Find the below working application.

 

DocumentUtil.java

package com.sample.app.util;

import java.util.Arrays;
import java.util.List;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;

public class DocumentUtil {

	private static Document getDocument(String id, String title, String description, String blog) {
		Document doc = new Document();
		doc.add(new TextField("id", id, Field.Store.YES));
		doc.add(new TextField("title", title, Field.Store.YES));
		doc.add(new TextField("description", description, Field.Store.NO));
		doc.add(new TextField("blog", blog, Field.Store.YES));
		
		return doc;

	}

	public static List<Document> getDocuments() {
		Document doc1 = getDocument("1", "JavaWorld",
				"The original independent resource for Java developers, architects, and managers.", " javaworld.com");
		Document doc2 = getDocument("2", "Oracle Blogs | The Java Source",
				" Java powers more than 4.5 billion devices including 800 million computers and 1.5 billion cell phones. If you love Java, this is the blog you must follow.",
				"blogs.oracle.com/java");
		Document doc3 = getDocument("3", "A Java geek",
				"Nicolas Fränkel's blog. IT architect focusing on Java, Java EE, and their surrounding ecosystems. He is a trainer, book writer, speaker & blogger.",
				"blog.frankel.ch");
		Document doc4 = getDocument("4", "Self Learning Java", "Learn Java fundamentals and other java libraries",
				"self-learning-java-tutorial.blogspot.com");

		return Arrays.asList(doc1, doc2, doc3, doc4);

	}
}

App.java

package com.sample.app;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.util.QueryBuilder;

import com.sample.app.util.DocumentUtil;

public class App {

	public static void main(String args[]) throws IOException {

		Analyzer analyzer = new StandardAnalyzer();
		IndexWriterConfig config = new IndexWriterConfig(analyzer);

		Directory directory = new MMapDirectory(new File("/Users/Shared/lucene").toPath(), NoLockFactory.INSTANCE);

		try (IndexWriter indexWriter = new IndexWriter(directory, config)) {

			List<Document> documents = DocumentUtil.getDocuments();

			indexWriter.addDocuments(documents);

		}

		QueryBuilder queryBuilder = new QueryBuilder(analyzer);
		Query query = queryBuilder.createMinShouldMatchQuery("description", "java", 0.2f);
		int maxHitsPerPage = 2;

		try (IndexReader indexReader = DirectoryReader.open(directory)) {
			IndexSearcher indexSearcher = new IndexSearcher(indexReader);

			TopDocs topDocs = indexSearcher.search(query, maxHitsPerPage);
			TotalHits totalHits = topDocs.totalHits;

			System.out.println("Total Hits: " + totalHits.value);

			ScoreDoc[] hits = topDocs.scoreDocs;
			System.out.println("Results: ");
			for (int i = 0; i < hits.length; i++) {
				Document document = indexSearcher.doc(hits[i].doc);
				System.out.println("Title: " + document.get("title"));
			}
		}

	}

}

Output

Total Hits: 4
Results: 
Title: Self Learning Java
Title: A Java geek

 

 

Previous                                                    Next                                                    Home

Mongo shell: why pretty method fails on findOne method

‘pretty()’ method runs on a cursor. Since findOne() method do not return a cursor, pretty() method will not work on it.

> db.audit.findOne()
{
      "_id" : ObjectId("60bb0156baf44d88348459c4"),
      "type" : "login",
      "modifiedTimestamp" : 1622908612185
}
> 
> db.audit.findOne().pretty()
uncaught exception: TypeError: db.audit.findOne(...).pretty is not a function :
@(shell):1:1

Since find() method return a cursor, you can use pretty() method with find() method.

> db.audit.find().pretty()
{
      "_id" : ObjectId("60bb0156baf44d88348459c4"),
      "type" : "login",
      "modifiedTimestamp" : 1622908612185
}
{
      "_id" : ObjectId("60bb9c348251689f64fb3ece"),
      "type" : "logout",
      "client" : "browser",
      "userId" : 234,
      "createdTimestamp" : 1622907956987
}
{
      "_id" : ObjectId("60bb9c418251689f64fb3ecf"),
      "type" : "login",
      "modifiedTimestamp" : 1622908670676
}
{
      "_id" : ObjectId("60bb9c598251689f64fb3ed0"),
      "type" : "reset_credentials",
      "client" : "mobile",
      "userId" : 31,
      "createdTimestamp" : 1622907993918
}
{
      "_id" : ObjectId("60bb9c6f8251689f64fb3ed1"),
      "type" : "login",
      "client" : "mobile",
      "userId" : 41,
      "createdTimestamp" : 1622908015960,
      "modifiedTimestamp" : 1622908909572
}

 

 

 

Previous                                                    Next                                                    Home

Mongo shell: Convert find query results to an array

‘find().toArray()’ method converts the find() query results to an array. Since it loads all the matched documents into the memory, use toArray() method with caution.

> db.countryCapitals.find().toArray()
[
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed2"),
        "country" : "Afghanistan",
        "city" : "Kabul"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed3"),
        "country" : "Albania",
        "city" : "Tirana"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed4"),
        "country" : "Algeria",
        "city" : "Alger"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed5"),
        "country" : "American Samoa",
        "city" : "Fagatogo"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed6"),
        "country" : "Andorra",
        "city" : "Andorra la Vella"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed7"),
        "country" : "Angola",
        "city" : "Luanda"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed8"),
        "country" : "Anguilla",
        "city" : "The Valley"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ed9"),
        "country" : "Antarctica",
        "city" : null
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3eda"),
        "country" : "Antigua and Barbuda",
        "city" : "Saint John's"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3edb"),
        "country" : "Argentina",
        "city" : "Buenos Aires"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3edc"),
        "country" : "Armenia",
        "city" : "Yerevan"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3edd"),
        "country" : "Aruba",
        "city" : "Oranjestad"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ede"),
        "country" : "Australia",
        "city" : "Canberra"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3edf"),
        "country" : "Austria",
        "city" : "Wien"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee0"),
        "country" : "Azerbaijan",
        "city" : "Baku"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee1"),
        "country" : "Bahamas",
        "city" : "Nassau"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee2"),
        "country" : "Bahrain",
        "city" : "al-Manama"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee3"),
        "country" : "Bangladesh",
        "city" : "Dhaka"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee4"),
        "country" : "Barbados",
        "city" : "Bridgetown"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee5"),
        "country" : "Belarus",
        "city" : "Minsk"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee6"),
        "country" : "Belgium",
        "city" : "Bruxelles [Brussel]"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee7"),
        "country" : "Belize",
        "city" : "Belmopan"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee8"),
        "country" : "Benin",
        "city" : "Porto-Novo"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3ee9"),
        "country" : "Bermuda",
        "city" : "Hamilton"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3eea"),
        "country" : "Bhutan",
        "city" : "Thimphu"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3eeb"),
        "country" : "Bolivia",
        "city" : "La Paz"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3eec"),
        "country" : "Bosnia and Herzegovina",
        "city" : "Sarajevo"
    },
    {
        "_id" : ObjectId("60bbadbe8251689f64fb3eed"),
        "country" : "Botswana",
        "city" : "Gaborone"
    }
]

 

 

 

Previous                                                    Next                                                    Home

Jq: Complex queries example

 

emps.json

emps.json 
[{
		"id": 1,
		"firstName": "Ram",
		"lastName": "Gurram",
		"age": 23
	},
	{
		"id": 2,
		"firstName": "Sailaja",
		"lastName": "PTR",
		"age": 31
	},
	{
		"id": 3,
		"firstName": "Venkat",
		"lastName": "IT",
		"age": 30
	},
	{
		"id": 4,
		"firstName": "Gopi",
		"lastName": "Battu",
		"age": 34
	}
]

 

a. Get all the employees whose firstName contains string ‘a’

$cat emps.json | jq '.[] | select((.firstName | contains("a")))'
{
  "id": 1,
  "firstName": "Ram",
  "lastName": "Gurram",
  "age": 23
}
{
  "id": 2,
  "firstName": "Sailaja",
  "lastName": "PTR",
  "age": 31
}
{
  "id": 3,
  "firstName": "Venkat",
  "lastName": "IT",
  "age": 30
}

 

b. Get all the employees whose firstName contains string ‘ai’

$cat emps.json | jq '.[] | select((.firstName | contains("ai")))'
{
  "id": 2,
  "firstName": "Sailaja",
  "lastName": "PTR",
  "age": 31
}

 

c. Get all the ids of employees whose name contains string ‘ai’

$cat emps.json | jq '.[] | select((.firstName | contains("ai"))) | .id'
2

 

d. Get all the employees firstName and lastName whose id is even number

$cat emps.json | jq '.[] | select((.id % 2 == 0)) | .firstName+","+.lastName+"," +(.id|tostring)'
"Sailaja,PTR,2"
"Gopi,Battu,4"

e. Get all the employees firstName and lastName whose id is odd number

$cat emps.json | jq '.[] | select((.id % 2 != 0)) | .firstName+","+.lastName+"," +(.id|tostring)'
"Ram,Gurram,1"
"Venkat,IT,3"

 

Previous                                                    Next                                                    Home