Skip to content

ZCU-PUB: fix slow /browse/subject (~5s→~0.2s) — backport #10057 metadata-browse json.facet/numBuckets #1328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vidiecan wants to merge 2 commits into
base: customer/zcu-pub
Choose a base branch
from
Open

ZCU-PUB: fix slow /browse/subject (~5s→~0.2s) — backport #10057 metadata-browse json.facet/numBuckets #1328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a93184a
Add limit, offset, and a new parameter to calculate the total entry c…
toniprieto Oct 26, 2024
084b1fc
Refactor browse entries facet query to use JSON facet query
toniprieto Mar 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 3 additions & 27 deletions dspace-api/src/main/java/org/dspace/browse/BrowseEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,9 +422,6 @@ private BrowseInfo browseByValue(BrowserScope bs)
}
}

// this is the total number of results in answer to the query
int total = getTotalResults(true);

// set the ordering field (there is only one option)
dao.setOrderField("sort_value");

Expand All @@ -444,6 +441,9 @@ private BrowseInfo browseByValue(BrowserScope bs)
dao.setOffset(offset);
dao.setLimit(scope.getResultsPerPage());

// this is the total number of results in answer to the query
int total = getTotalResults(true);

// Holder for the results
List<String[]> results = null;

Expand Down Expand Up @@ -680,33 +680,9 @@ private int getTotalResults(boolean distinct)
// tell the browse query whether we are distinct
dao.setDistinct(distinct);

// ensure that the select is set to "*"
String[] select = {"*"};
dao.setCountValues(select);

// FIXME: it would be nice to have a good way of doing this in the DAO
// now reset all of the fields that we don't want to have constraining
// our count, storing them locally to reinstate later
String focusField = dao.getJumpToField();
String focusValue = dao.getJumpToValue();
int limit = dao.getLimit();
int offset = dao.getOffset();

dao.setJumpToField(null);
dao.setJumpToValue(null);
dao.setLimit(-1);
dao.setOffset(-1);

// perform the query and get the result
int count = dao.doCountQuery();

// now put back the values we removed for this method
dao.setJumpToField(focusField);
dao.setJumpToValue(focusValue);
dao.setLimit(limit);
dao.setOffset(offset);
dao.setCountValues(null);

log.debug(LogHelper.getHeader(context, "get_total_results_return", "return=" + count));

return count;
Expand Down
65 changes: 32 additions & 33 deletions dspace-api/src/main/java/org/dspace/browse/SolrBrowseDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import java.util.Comparator;
import java.util.List;

import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.util.ClientUtils;
Expand All @@ -21,7 +23,6 @@
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.dspace.discovery.DiscoverFacetField;
import org.dspace.discovery.DiscoverQuery;
import org.dspace.discovery.DiscoverQuery.SORT_ORDER;
import org.dspace.discovery.DiscoverResult;
Expand All @@ -32,7 +33,6 @@
import org.dspace.discovery.SearchServiceException;
import org.dspace.discovery.SearchUtils;
import org.dspace.discovery.configuration.DiscoveryConfiguration;
import org.dspace.discovery.configuration.DiscoveryConfigurationParameters;
import org.dspace.discovery.indexobject.IndexableItem;
import org.dspace.services.factory.DSpaceServicesFactory;

Expand Down Expand Up @@ -179,19 +179,30 @@ private DiscoverResult getSolrResponse() throws BrowseException {
addLocationScopeFilter(query);
addDefaultFilterQueries(query);
if (distinct) {
DiscoverFacetField dff;
if (StringUtils.isNotBlank(startsWith)) {
dff = new DiscoverFacetField(facetField,
DiscoveryConfigurationParameters.TYPE_TEXT, -1,
DiscoveryConfigurationParameters.SORT.VALUE, startsWith);
// We use a json.facet query for metadata browsing because it allows us to limit the results
// while obtaining the total number of facet values with numBuckets:true and sort in reverse order
// Example of json.facet query:
// {"<fieldName>": {"type":"terms","field": "<fieldName>_filter", "limit":0, "offset":0,
// "sort":"index desc", "numBuckets":true, "prefix":"<startsWith>"}}
ObjectNode jsonFacet = JsonNodeFactory.instance.objectNode();
ObjectNode entriesFacet = JsonNodeFactory.instance.objectNode();
entriesFacet.put("type", "terms");
entriesFacet.put("field", facetField + "_filter");
entriesFacet.put("limit", limit);
entriesFacet.put("offset", offset);
entriesFacet.put("numBuckets", true);
if (ascending) {
entriesFacet.put("sort", "index");
} else {
dff = new DiscoverFacetField(facetField,
DiscoveryConfigurationParameters.TYPE_TEXT, -1,
DiscoveryConfigurationParameters.SORT.VALUE);
entriesFacet.put("sort", "index desc");
}
if (StringUtils.isNotBlank(startsWith)) {
// Add the prefix to the json facet query
entriesFacet.put("prefix", startsWith);
}
query.addFacetField(dff);
query.setFacetMinCount(1);
jsonFacet.set(facetField, entriesFacet);
query.setMaxResults(0);
query.addProperty("json.facet", jsonFacet.toString());
} else {
query.setMaxResults(limit/* > 0 ? limit : 20*/);
if (offset > 0) {
Expand Down Expand Up @@ -248,8 +259,7 @@ public int doCountQuery() throws BrowseException {
DiscoverResult resp = getSolrResponse();
int count = 0;
if (distinct) {
List<FacetResult> facetResults = resp.getFacetResult(facetField);
count = facetResults.size();
count = (int) resp.getTotalEntries();
} else {
// we need to cast to int to respect the BrowseDAO contract...
count = (int) resp.getTotalSearchResults();
Expand All @@ -266,26 +276,15 @@ public List doValueQuery() throws BrowseException {
DiscoverResult resp = getSolrResponse();
List<FacetResult> facet = resp.getFacetResult(facetField);
int count = doCountQuery();
int start = offset > 0 ? offset : 0;
int max = limit > 0 ? limit : count; //if negative, return everything
int max = facet.size();
List<String[]> result = new ArrayList<>();
if (ascending) {
for (int i = start; i < (start + max) && i < count; i++) {
FacetResult c = facet.get(i);
String freq = showFrequencies ? String.valueOf(c.getCount())
: "";
result.add(new String[] {c.getDisplayedValue(),
c.getAuthorityKey(), freq});
}
} else {
for (int i = count - start - 1; i >= count - (start + max)
&& i >= 0; i--) {
FacetResult c = facet.get(i);
String freq = showFrequencies ? String.valueOf(c.getCount())
: "";
result.add(new String[] {c.getDisplayedValue(),
c.getAuthorityKey(), freq});
}

for (int i = 0; i < max && i < count; i++) {
FacetResult c = facet.get(i);
String freq = showFrequencies ? String.valueOf(c.getCount())
: "";
result.add(new String[] {c.getDisplayedValue(),
c.getAuthorityKey(), freq});
}

return result;
Expand Down
11 changes: 11 additions & 0 deletions dspace-api/src/main/java/org/dspace/discovery/DiscoverResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ public class DiscoverResult {
private List<IndexableObject> indexableObjects;
private Map<String, List<FacetResult>> facetResults;

// Total count of facet entries calculated for a metadata browsing query
private long totalEntries;

/**
* A map that contains all the documents sougth after, the key is a string representation of the Indexable Object
*/
Expand Down Expand Up @@ -64,6 +67,14 @@ public void setTotalSearchResults(long totalSearchResults) {
this.totalSearchResults = totalSearchResults;
}

public long getTotalEntries() {
return totalEntries;
}

public void setTotalEntries(long totalEntries) {
this.totalEntries = totalEntries;
}

public int getStart() {
return start;
}
Expand Down
40 changes: 40 additions & 0 deletions dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.json.BucketBasedJsonFacet;
import org.apache.solr.client.solrj.response.json.BucketJsonFacet;
import org.apache.solr.client.solrj.response.json.NestableJsonFacet;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
Expand Down Expand Up @@ -1062,6 +1065,8 @@ protected DiscoverResult retrieveResult(Context context, DiscoverQuery query)
}
//Resolve our facet field values
resolveFacetFields(context, query, result, skipLoadingResponse, solrQueryResponse);
//Resolve our json facet field values used for metadata browsing
resolveJsonFacetFields(context, result, solrQueryResponse);
}
// If any stale entries are found in the current page of results,
// we remove those stale entries and rerun the same query again.
Expand All @@ -1087,7 +1092,42 @@ protected DiscoverResult retrieveResult(Context context, DiscoverQuery query)
return result;
}

/**
* Process the 'json.facet' response, which is currently only used for metadata browsing
*
* @param context context object
* @param result the result object to add the facet results to
* @param solrQueryResponse the solr query response
* @throws SQLException if database error
*/
private void resolveJsonFacetFields(Context context, DiscoverResult result, QueryResponse solrQueryResponse)
throws SQLException {

NestableJsonFacet response = solrQueryResponse.getJsonFacetingResponse();
if (response != null && response.getBucketBasedFacetNames() != null) {
for (String facetName : response.getBucketBasedFacetNames()) {
BucketBasedJsonFacet facet = response.getBucketBasedFacets(facetName);
if (facet != null) {
result.setTotalEntries(facet.getNumBucketsCount());
for (BucketJsonFacet bucket : facet.getBuckets()) {
String facetValue = bucket.getVal() != null ? bucket.getVal().toString() : "";
String field = facetName + "_filter";
String displayedValue = transformDisplayedValue(context, field, facetValue);
String authorityValue = transformAuthorityValue(context, field, facetValue);
String sortValue = transformSortValue(context, field, facetValue);
String filterValue = displayedValue;
if (StringUtils.isNotBlank(authorityValue)) {
filterValue = authorityValue;
}
result.addFacetResult(facetName,
new DiscoverResult.FacetResult(filterValue, displayedValue,
authorityValue, sortValue, bucket.getCount(),
DiscoveryConfigurationParameters.TYPE_TEXT));
}
}
}
}
}

private void resolveFacetFields(Context context, DiscoverQuery query, DiscoverResult result,
boolean skipLoadingResponse, QueryResponse solrQueryResponse) throws SQLException {
Expand Down
Loading