Compare commits

...
Author SHA1 Message Date
Ruslan 90ae597a66 dbeaver/pro#5630 fix images 2025-07-29 16:33:18 +02:00
Evgenia 885e2c16fe Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-29 15:16:42 +02:00
Wroud 6156675fb9 fix: do not use cache if result dataFormat changed 2025-07-29 20:06:59 +08:00
Evgenia 945802cb23 Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-29 13:20:53 +02:00
Wroud 74bea16b2f Merge branch 'dbeaver/pro#6401/fix/async-task-update-after-finish' into dbeaver/pro#5630-add-cache-for-queries 2025-07-29 19:16:38 +08:00
Evgenia 9a9445ca01 Merge branch 'devel' into dbeaver/pro#6401/fix/async-task-update-after-finish 2025-07-29 10:54:35 +02:00
mr-anton-t c7b2980a30 Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-28 17:54:07 +02:00
Ruslan 08f4e469e3 dbeaver/pro#5630 fix npe 2025-07-28 17:39:47 +02:00
Ruslan 97cee0127d dbeaver/pro#5630 fix npe 2025-07-28 15:31:58 +02:00
Wroud d03b00e30a dbeaver/pro#6401 fix: async task update after finish 2025-07-28 19:56:06 +08:00
mr-anton-t c200d5743a Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-28 10:53:38 +02:00
Daria Marutkina a1d3edcb82 Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-25 16:12:45 +02:00
Daria Marutkina f0004d112b Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-25 15:24:49 +02:00
Daria Marutkina 3c34ce5f71 Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-23 09:22:51 +02:00
Ruslan 52e2ee4a70 dbeaver/pro#5630 add soft reference 2025-07-21 13:33:31 +02:00
Ruslan 93972f8af8 dbeaver/pro#5630 cache sql queries 2025-07-16 19:22:20 +02:00
Ruslan 65a99475e8 dbeaver/pro#5630 cache sql queries 2025-07-16 19:10:18 +02:00
Ruslan aa489ccd31 Merge remote-tracking branch 'origin/dbeaver/pro#5630-add-cache-for-queries' into dbeaver/pro#5630-add-cache-for-queries 2025-07-16 15:06:02 +02:00
Ruslan d90a765cc7 dbeaver/pro#5630 cache sql queries 2025-07-16 15:05:25 +02:00
Wroud 550373f48e fix: check added constraints 2025-07-16 20:56:33 +08:00
Ruslan 636725681a Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-16 13:52:36 +02:00
Ruslan 14de42933e dbeaver/pro#5630 cache sql queries 2025-07-15 19:34:27 +02:00
Ruslan 18bc462ad0 Merge remote-tracking branch 'origin/dbeaver/pro#5630-add-cache-for-queries' into dbeaver/pro#5630-add-cache-for-queries 2025-07-15 14:09:56 +02:00
Alexey Potsetsuev 964cd76178 Merge branch 'devel' into dbeaver/pro#5630-add-cache-for-queries 2025-07-15 20:06:06 +08:00
Ruslan 9ba77d490a dbeaver/pro#5630 cache sql queries 2025-07-15 14:06:03 +02:00
Wroud 3dddd20dcb fix: grouping 2025-07-15 20:03:50 +08:00
Wroud 8c7c3a545d fix: don't cache refresh requests 2025-07-15 20:03:03 +08:00
Wroud 4068c556df feat: use cache for data queries 2025-07-15 19:30:34 +08:00
Ruslan 2e273e0d6c dbeaver/pro#5630 cache sql queries 2025-07-15 13:03:10 +02:00
Ruslan 6dee34b1a7 dbeaver/pro#5630 cache sql queries 2025-07-15 12:57:40 +02:00
Ruslan 298a4e3310 dbeaver/pro#5630 cache sql queries 2025-07-15 12:20:54 +02:00
Ruslan db210317a3 dbeaver/pro#5830 cache 2025-07-14 17:10:56 +02:00
Ruslan 76f7e8b9d7 dbeaver/pro#5630 Add cache for queries 2025-07-14 12:52:02 +02:00
21 changed files with 434 additions and 131 deletions
@@ -361,7 +361,8 @@ extend type Mutation {
resultId: ID,
filter: SQLDataFilter,
dataFormat: ResultDataFormat, # requested data format. May be ignored by server
readLogs: Boolean # added 23.2.1
readLogs: Boolean, # added 23.2.1
useCache: Boolean @since(version: "25.1.3")
): AsyncTaskInfo!
"Creates async task for reading data from the container node"
@@ -372,7 +373,8 @@ extend type Mutation {
containerNodePath: ID!,
resultId: ID,
filter: SQLDataFilter,
dataFormat: ResultDataFormat
dataFormat: ResultDataFormat,
useCache: Boolean @since(version: "25.1.3")
): AsyncTaskInfo!
"Returns transaction log info for the specified project, connection and context"
@@ -99,7 +99,9 @@ public interface DBWServiceSQL extends DBWService {
@Nullable WebSQLDataFilter filter,
@Nullable WebDataFormat dataFormat,
boolean readLogs,
@NotNull WebSession webSession) throws DBException;
@NotNull WebSession webSession,
@Nullable Boolean useCache
) throws DBException;
@WebAction
WebAsyncTaskInfo asyncReadDataFromContainer(
@@ -107,7 +109,9 @@ public interface DBWServiceSQL extends DBWService {
@NotNull String nodePath,
@Nullable String resultId,
@Nullable WebSQLDataFilter filter,
@Nullable WebDataFormat dataFormat) throws DBWebException;
@Nullable WebDataFormat dataFormat,
@Nullable Boolean useCache
) throws DBWebException;
/**
* Reads dynamic trace from provided database results.
@@ -26,6 +26,7 @@ import io.cloudbeaver.model.session.WebAsyncTaskProcessor;
import io.cloudbeaver.model.session.WebSession;
import io.cloudbeaver.model.session.WebSessionProvider;
import org.jkiss.code.NotNull;
import org.jkiss.code.Nullable;
import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.DBConstants;
@@ -49,6 +50,7 @@ import org.jkiss.dbeaver.model.websocket.event.WSTransactionalCountEvent;
import org.jkiss.dbeaver.utils.RuntimeUtils;
import org.jkiss.utils.CommonUtils;
import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException;
import java.time.Instant;
import java.time.ZoneId;
@@ -65,11 +67,13 @@ import java.util.concurrent.atomic.AtomicInteger;
public class WebSQLContextInfo implements WebSessionProvider {
private static final Log log = Log.getLog(WebSQLContextInfo.class);
private static final int MAX_RESULT_SET_SIZE = 300;
private final transient WebSQLProcessor processor;
private final String id;
private final String projectId;
private final Map<String, WebSQLResultsInfo> resultInfoMap = new HashMap<>();
private final Map<String, SoftReference<WebSQLQueryResults>> queryResultsMap = new HashMap<>();
private final AtomicInteger resultId = new AtomicInteger();
@@ -190,7 +194,26 @@ public class WebSQLContextInfo implements WebSessionProvider {
return resultsInfo;
}
@Nullable
public WebSQLQueryResults getQueryResults(@NotNull String resultId) throws DBWebException {
SoftReference<WebSQLQueryResults> reference = queryResultsMap.get(resultId);
return reference == null ? null : reference.get();
}
public void trySaveQueryResults(@NotNull WebSQLQueryResults[] resultsArray, int requestedLimit) {
if (resultsArray[0].getResultSet() == null) {
return;
}
boolean isSingleResult = resultsArray.length == 1;
int actualRowCount = isSingleResult ? resultsArray[0].getResultSet().getRows().length : 0;
if (isSingleResult && actualRowCount < Math.min(requestedLimit, MAX_RESULT_SET_SIZE)) {
queryResultsMap.put(resultsArray[0].getResultSet().getId(), new SoftReference<>(resultsArray[0]));
}
}
public boolean closeResult(@NotNull String resultId) {
queryResultsMap.remove(resultId);
return resultInfoMap.remove(resultId) != null;
}
@@ -199,6 +222,7 @@ public class WebSQLContextInfo implements WebSessionProvider {
void dispose() {
resultInfoMap.clear();
queryResultsMap.clear();
}
@Override
@@ -73,7 +73,6 @@ public class WebSQLProcessor implements WebSessionProvider {
private static final String FILE_ID = "fileId";
private static final String TEMP_FILE_FOLDER = "temp-sql-upload-files";
private final WebSession webSession;
private final WebConnectionInfo connection;
private final SQLSyntaxManager syntaxManager;
@@ -166,20 +165,25 @@ public class WebSQLProcessor implements WebSessionProvider {
@Nullable WebSQLDataFilter filter,
@Nullable WebDataFormat dataFormat,
@NotNull WebSession webSession,
boolean readLogs) throws DBWebException, DBCException {
boolean readLogs,
@Nullable Boolean useCache
) throws DBWebException, DBCException {
if (filter == null) {
// Use default filter
filter = new WebSQLDataFilter();
}
long startTime = System.currentTimeMillis();
WebSQLExecuteInfo executeInfo = new WebSQLExecuteInfo();
var dataContainer = new WebSQLQueryDataContainer(connection.getDataSource(), syntaxManager, sql);
DBCExecutionContext context = getExecutionContext(dataContainer);
try {
final DBDDataFilter dataFilter = filter.makeDataFilter((resultId == null ? null : contextInfo.getResults(resultId)));
if (trySetCachedResults(useCache, resultId, contextInfo, filter, executeInfo)) {
return executeInfo;
}
if (dataFilter.hasFilters()) {
sql = context.getDataSource().getSQLDialect().addFiltersToQuery(
monitor,
@@ -281,6 +285,7 @@ public class WebSQLProcessor implements WebSessionProvider {
if (executeInfo.getResults().length == 0) {
executeInfo.setStatusMessage("No Data");
} else {
contextInfo.trySaveQueryResults(executeInfo.getResults(), filter.getLimit());
executeInfo.setStatusMessage("Executed");
}
@@ -294,15 +299,22 @@ public class WebSQLProcessor implements WebSessionProvider {
@NotNull DBSDataContainer dataContainer,
@Nullable String resultId,
@NotNull WebSQLDataFilter filter,
@Nullable WebDataFormat dataFormat) throws DBException {
@Nullable WebDataFormat dataFormat,
@Nullable Boolean useCache
) throws DBException {
WebSQLExecuteInfo executeInfo = new WebSQLExecuteInfo();
DBCExecutionContext executionContext = getExecutionContext(dataContainer);
DBDDataFilter dataFilter = filter.makeDataFilter((resultId == null ? null : contextInfo.getResults(resultId)));
DBExecUtils.tryExecuteRecover(monitor, connection.getDataSource(), param -> {
if (trySetCachedResults(useCache, resultId, contextInfo, filter, executeInfo)) {
return;
}
try (DBCSession session = executionContext.openSession(monitor, resolveQueryPurpose(dataFilter), "Read data from container")) {
try (WebSQLQueryDataReceiver dataReceiver = new WebSQLQueryDataReceiver(contextInfo, dataContainer, dataFormat)) {
try (WebSQLQueryDataReceiver dataReceiver = new WebSQLQueryDataReceiver(contextInfo, dataContainer, dataFormat, webSession)) {
DBCStatistics statistics = dataContainer.readData(
new WebExecutionSource(dataContainer, executionContext, this),
session,
@@ -324,6 +336,7 @@ public class WebSQLProcessor implements WebSessionProvider {
if (resultSet != null && resultSet.getRows() != null && resultSet.getResultsInfo() != null) {
resultSet.getResultsInfo().setQueryText(statistics.getQueryText());
executeInfo.setStatusMessage(resultSet.getRows().length + " row(s) fetched");
contextInfo.trySaveQueryResults(executeInfo.getResults(), filter.getLimit());
}
} catch (DBException e) {
throw new InvocationTargetException(e);
@@ -432,7 +445,7 @@ public class WebSQLProcessor implements WebSessionProvider {
sendTransactionalEvent(contextInfo);
}
WebSQLQueryResultSet updatedResultSet = new WebSQLQueryResultSet();
WebSQLQueryResultSet updatedResultSet = new WebSQLQueryResultSet(webSession);
updatedResultSet.setResultsInfo(resultsInfo);
updatedResultSet.setColumns(resultsInfo.getAttributes());
@@ -448,6 +461,34 @@ public class WebSQLProcessor implements WebSessionProvider {
return result;
}
/**
* Checks cache and sets executeInfo if cached results are found.
* @return true if cached results were used, false otherwise
*/
private boolean trySetCachedResults(
@Nullable Boolean useCache,
@Nullable String resultId,
@NotNull WebSQLContextInfo contextInfo,
@NotNull WebSQLDataFilter filter,
@NotNull WebSQLExecuteInfo executeInfo
) {
if (CommonUtils.toBoolean(useCache)) {
WebSQLResultCacheService webSQLResultCacheService = new WebSQLResultCacheService();
WebSQLQueryResults filterCachedResults = webSQLResultCacheService.getCachedSQLQueryResults(
resultId,
contextInfo,
filter
);
if (filterCachedResults != null) {
executeInfo.setResults(new WebSQLQueryResults[] {filterCachedResults});
executeInfo.setDuration(0);
executeInfo.setStatusMessage("Cached result set");
return true;
}
}
return false;
}
private void sendTransactionalEvent(WebSQLContextInfo contextInfo) {
int count = QMUtils.getTransactionState(getExecutionContext()).getUpdateCount();
webSession.addSessionEvent(
@@ -509,7 +550,7 @@ public class WebSQLProcessor implements WebSessionProvider {
}
DBDDataFilter filter = new DBDDataFilter(constraints);
DBSDataContainer dataContainer = resultsInfo.getDataContainer();
WebRowDataReceiver dataReceiver = new WebRowDataReceiver(resultsInfo.getAttributes(), row.getData(), dataFormat);
RowDataReceiver dataReceiver = new RowDataReceiver(resultsInfo.getAttributes());
dataContainer.readData(
new AbstractExecutionSource(dataContainer, getExecutionContext(dataContainer), this),
session,
@@ -606,7 +647,7 @@ public class WebSQLProcessor implements WebSessionProvider {
DBDAttributeBinding[] allAttributes = resultsInfo.getAttributes();
DBDAttributeBinding[] keyAttributes = rowIdentifier.getAttributes().toArray(new DBDAttributeBinding[0]);
WebSQLQueryResultSet updatedResultSet = new WebSQLQueryResultSet();
WebSQLQueryResultSet updatedResultSet = new WebSQLQueryResultSet(webSession);
updatedResultSet.setResultsInfo(resultsInfo);
updatedResultSet.setColumns(resultsInfo.getAttributes());
@@ -813,7 +854,7 @@ public class WebSQLProcessor implements WebSessionProvider {
}
@Nullable
public Object convertInputCellValue(DBCSession session, DBDAttributeBinding updateAttribute, Object cellRawValue, boolean justGenerateScript) throws DBCException {
public Object convertInputCellValue(DBCSession session, DBDAttributeBinding updateAttribute, Object cellRawValue, boolean justGenerateScript) throws DBException {
cellRawValue = WebSQLUtils.makePlainCellValue(session, updateAttribute, cellRawValue);
Object realCellValue = cellRawValue;
// In some cases we already have final value here
@@ -1028,7 +1069,8 @@ public class WebSQLProcessor implements WebSessionProvider {
@NotNull WebSQLExecuteInfo executeInfo,
@NotNull WebSQLDataFilter webDataFilter,
@NotNull DBDDataFilter dataFilter,
@Nullable WebDataFormat dataFormat) throws DBException {
@Nullable WebDataFormat dataFormat
) throws DBException {
List<WebSQLQueryResults> resultList = new ArrayList<>();
int maxResultsCount = resolveMaxResultsCount(dataContainer.getDataSource());
@@ -1041,7 +1083,7 @@ public class WebSQLProcessor implements WebSessionProvider {
if (resultSet == null) {
break;
}
try (WebSQLQueryDataReceiver dataReceiver = new WebSQLQueryDataReceiver(contextInfo, dataContainer, dataFormat)) {
try (WebSQLQueryDataReceiver dataReceiver = new WebSQLQueryDataReceiver(contextInfo, dataContainer, dataFormat, webSession)) {
readResultSet(dbStat.getSession(), resultSet, webDataFilter, dataReceiver);
results.setResultSet(dataReceiver.getResultSet());
dataReceiver.getResultSet().getResultsInfo().setQueryText(resultSet.getSourceStatement().getQueryString());
@@ -1162,30 +1204,6 @@ public class WebSQLProcessor implements WebSessionProvider {
}
}
public class WebRowDataReceiver extends RowDataReceiver {
private final WebDataFormat dataFormat;
public WebRowDataReceiver(DBDAttributeBinding[] curAttributes, Object[] rowValues, WebDataFormat dataFormat) {
super(curAttributes);
this.rowValues = rowValues;
this.dataFormat = dataFormat;
}
@Override
protected void fetchRowValues(DBCSession session, DBCResultSet resultSet) throws DBCException {
for (int i = 0; i < curAttributes.length; i++) {
final DBDAttributeBinding attr = curAttributes[i];
DBDValueHandler valueHandler = attr.getValueHandler();
Object attrValue = valueHandler.fetchValueObject(session, resultSet, attr, i);
// Patch result rows (adapt to web format)
rowValues[i] = WebSQLUtils.makeWebCellValue(webSession, attr, attrValue, dataFormat);
}
}
}
///////////////////////////////////////////////////////
// Utils
private static int resolveMaxResultsCount(@Nullable DBPDataSource dataSource) {
@@ -44,17 +44,18 @@ class WebSQLQueryDataReceiver implements DBDDataReceiver {
private final WebSQLContextInfo contextInfo;
private final DBSDataContainer dataContainer;
private final WebDataFormat dataFormat;
private final WebSQLQueryResultSet webResultSet = new WebSQLQueryResultSet();
private final WebSQLQueryResultSet webResultSet;
private DBDAttributeBinding[] bindings;
private DBCTrace trace;
private List<WebSQLQueryResultSetRow> rows = new ArrayList<>();
private final Number rowLimit;
WebSQLQueryDataReceiver(WebSQLContextInfo contextInfo, DBSDataContainer dataContainer, WebDataFormat dataFormat) {
WebSQLQueryDataReceiver(WebSQLContextInfo contextInfo, DBSDataContainer dataContainer, WebDataFormat dataFormat, WebSession webSession) {
this.contextInfo = contextInfo;
this.dataContainer = dataContainer;
this.dataFormat = dataFormat;
this.webResultSet = new WebSQLQueryResultSet(webSession);
rowLimit = ServletAppUtils.getServletApplication()
.getAppConfiguration()
.getResourceQuota(WebSQLConstants.QUOTA_PROP_ROW_LIMIT);
@@ -146,14 +147,6 @@ class WebSQLQueryDataReceiver implements DBDDataReceiver {
}
}
// Convert row values
for (WebSQLQueryResultSetRow row : rows) {
for (int i = 0; i < bindings.length; i++) {
DBDAttributeBinding binding = bindings[i];
row.getData()[i] = WebSQLUtils.makeWebCellValue(webSession, binding, row.getData()[i], dataFormat);
}
}
webResultSet.setColumns(bindings);
webResultSet.setRows(List.of(rows.toArray(new WebSQLQueryResultSetRow[0])));
webResultSet.setHasChildrenCollection(resultSet instanceof DBDSubCollectionResultSet);
@@ -16,13 +16,16 @@
*/
package io.cloudbeaver.service.sql;
import io.cloudbeaver.model.session.WebSession;
import org.jkiss.code.NotNull;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.data.DBDAttributeBinding;
import org.jkiss.dbeaver.model.exec.DBCException;
import org.jkiss.dbeaver.model.exec.DBCExecutionContext;
import org.jkiss.dbeaver.model.exec.DBExecUtils;
import org.jkiss.dbeaver.model.meta.Property;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -45,8 +48,26 @@ public class WebSQLQueryResultSet {
private boolean hasDynamicTrace;
private boolean readOnly;
private String readOnlyStatus;
private WebSession webSession;
public WebSQLQueryResultSet() {
public WebSQLQueryResultSet(WebSession webSession) {
this.webSession = webSession;
}
public WebSQLQueryResultSet(WebSQLQueryResultSet webSQLQueryResultSet) {
this.columns = new WebSQLQueryResultColumn[webSQLQueryResultSet.columns.length];
System.arraycopy(webSQLQueryResultSet.columns, 0, this.columns, 0, webSQLQueryResultSet.columns.length);
this.rows = new ArrayList<>(webSQLQueryResultSet.rows);
this.hasMoreData = webSQLQueryResultSet.hasMoreData;
this.resultsInfo = webSQLQueryResultSet.resultsInfo;
this.singleEntity = webSQLQueryResultSet.singleEntity;
this.hasRowIdentifier = webSQLQueryResultSet.hasRowIdentifier;
this.hasChildrenCollection = webSQLQueryResultSet.hasChildrenCollection;
this.isSupportsDataFilter = webSQLQueryResultSet.isSupportsDataFilter;
this.hasDynamicTrace = webSQLQueryResultSet.hasDynamicTrace;
this.readOnly = webSQLQueryResultSet.readOnly;
this.readOnlyStatus = webSQLQueryResultSet.readOnlyStatus;
this.webSession = webSQLQueryResultSet.webSession;
}
@Property
@@ -71,15 +92,32 @@ public class WebSQLQueryResultSet {
this.columns = columns;
}
@Property
@Deprecated
/**
* Get original rows from result set.
*/
public Object[][] getRows() {
return rows.stream().map(WebSQLQueryResultSetRow::getData).toArray(x -> new Object[x][1]);
}
@Property
public List<WebSQLQueryResultSetRow> getRowsWithMetaData() {
return rows;
// Convert row values
List<WebSQLQueryResultSetRow> result = new ArrayList<>();
for (WebSQLQueryResultSetRow row : rows) {
result.add(new WebSQLQueryResultSetRow(row));
}
try {
for (WebSQLQueryResultSetRow row : result) {
for (int i = 0; i < columns.length && row.getData() != null && row.getData().length == columns.length; i++) {
DBDAttributeBinding binding = columns[i].getAttribute();
row.getData()[i] = WebSQLUtils.makeWebCellValue(webSession, binding, row.getData()[i], WebDataFormat.resultset);
}
}
} catch (DBCException e) {
log.warn("Error converting row values", e);
}
return result;
}
public void setRows(List<WebSQLQueryResultSetRow> rows) {
@@ -1,5 +1,6 @@
package io.cloudbeaver.service.sql;
import java.util.HashMap;
import java.util.Map;
public class WebSQLQueryResultSetRow {
@@ -13,6 +14,13 @@ public class WebSQLQueryResultSetRow {
this.metaData = metaData;
}
public WebSQLQueryResultSetRow(WebSQLQueryResultSetRow webSQLQueryResultSetRow) {
this.data = new Object[webSQLQueryResultSetRow.data.length];
//fixme it is probably a issue with mutable objects in data array
System.arraycopy(webSQLQueryResultSetRow.data, 0, this.data, 0, webSQLQueryResultSetRow.data.length);
this.metaData = webSQLQueryResultSetRow.metaData == null ? null : new HashMap<>(webSQLQueryResultSetRow.metaData);
}
public Object[] getData() {
return data;
}
@@ -46,6 +46,13 @@ public class WebSQLQueryResults {
this.dataFormat = dataFormat == null ? WebDataFormat.resultset : dataFormat;
}
public WebSQLQueryResults(WebSQLQueryResults webSQLQueryResults) {
this.webSession = webSQLQueryResults.webSession;
this.dataFormat = webSQLQueryResults.dataFormat;
this.updateRowCount = webSQLQueryResults.updateRowCount;
this.resultSet = webSQLQueryResults.resultSet == null ? null : new WebSQLQueryResultSet(webSQLQueryResults.resultSet);
}
@Property
public WebDataFormat getDataFormat() {
return dataFormat;
@@ -0,0 +1,106 @@
package io.cloudbeaver.service.sql;
import org.jkiss.code.NotNull;
import org.jkiss.code.Nullable;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.DBUtils;
import org.jkiss.dbeaver.model.data.DBDAttributeBinding;
import org.jkiss.dbeaver.model.data.DBDAttributeConstraint;
import org.jkiss.dbeaver.model.data.DBDDataFilter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
public class WebSQLResultCacheService {
private static final Log log = Log.getLog(WebSQLResultCacheService.class);
/**
* Retrieves cached results by result ID and applies sorting.
*
* @param resultId the ID of the cached results
* @param contextInfo the SQL context information
* @param webSQLDataFilter the web SQL data filter to apply
* @return filtered cached results or null if no results found for the given ID
*/
public WebSQLQueryResults getCachedSQLQueryResults(
@Nullable String resultId,
@NotNull WebSQLContextInfo contextInfo,
@NotNull WebSQLDataFilter webSQLDataFilter
) {
WebSQLQueryResults result;
try {
if (resultId == null) {
return null;
}
WebSQLQueryResults cachedResults = contextInfo.getQueryResults(resultId);
if (cachedResults == null) {
return null;
}
// Work with a copy to not change cached results
result = new WebSQLQueryResults(cachedResults);
DBDDataFilter dataFilter = webSQLDataFilter.makeDataFilter(contextInfo.getResults(resultId));
sortCachedResults(result, dataFilter);
} catch (Exception e) {
log.warn("Error retrieving cached results for ID '" + resultId + "'", e);
return null;
}
return result;
}
private void sortCachedResults(
@NotNull WebSQLQueryResults cachedResults,
@NotNull DBDDataFilter dataFilter
) {
WebSQLQueryResultSet resultSet = cachedResults.getResultSet();
if (resultSet == null) {
return;
}
Object[][] allRows = resultSet.getRows();
if (allRows.length == 0) {
return;
}
List<WebSQLQueryResultSetRow> allRowsList = new ArrayList<>();
for (Object[] row : allRows) {
allRowsList.add(new WebSQLQueryResultSetRow(row, null));
}
List<DBDAttributeConstraint> orderConstraints = dataFilter.getOrderConstraints();
if (!orderConstraints.isEmpty()) {
allRowsList.sort((r1, r2) -> compareRowsByOrder(r1, r2, orderConstraints));
}
resultSet.setRows(allRowsList);
resultSet.setHasMoreData(resultSet.isHasMoreData());
}
private int compareRowsByOrder(
@NotNull WebSQLQueryResultSetRow r1,
@NotNull WebSQLQueryResultSetRow r2,
@NotNull List<DBDAttributeConstraint> orderConstraints
) {
int result = 0;
for (DBDAttributeConstraint attributeConstraint : orderConstraints) {
int colIndex = attributeConstraint.getAttribute() != null ? attributeConstraint.getAttribute().getOrdinalPosition() : -1;
if (colIndex < 0) {
continue;
}
Object cell1 = r1.getData()[colIndex];
Object cell2 = r2.getData()[colIndex];
Comparator<Object> comparator = null;
if (attributeConstraint.getAttribute() != null && attributeConstraint.getAttribute() instanceof DBDAttributeBinding binding) {
comparator = binding.getValueHandler().getComparator();
}
if (comparator != null) {
result = comparator.compare(cell1, cell2);
} else {
result = DBUtils.compareDataValues(cell1, cell2);
}
if (attributeConstraint.isOrderDescending()) {
result = -result;
}
if (result != 0) {
break;
}
}
return result;
}
}
@@ -190,14 +190,16 @@ public class WebServiceBindingSQL extends WebServiceBindingBase<DBWServiceSQL>
getDataFilter(env),
getDataFormat(env),
CommonUtils.toBoolean(env.getArgument("readLogs")),
getWebSession(env)))
getWebSession(env),
env.getArgument("useCache")))
.dataFetcher("asyncReadDataFromContainer", env ->
getService(env).asyncReadDataFromContainer(
getSQLContext(env),
env.getArgument("containerNodePath"),
env.getArgument("resultId"),
getDataFilter(env),
getDataFormat(env)
getDataFormat(env),
env.getArgument("useCache")
))
.dataFetcher("asyncSqlExecuteResults", env ->
getService(env).asyncGetQueryResults(
@@ -476,8 +476,9 @@ public class WebServiceSQL implements DBWServiceSQL {
@Nullable WebSQLDataFilter filter,
@Nullable WebDataFormat dataFormat,
boolean readLogs,
@NotNull WebSession webSession)
{
@NotNull WebSession webSession,
Boolean useCache
) {
WebAsyncTaskProcessor<String> runnable = new WebAsyncTaskProcessor<>() {
@Override
public void run(DBRProgressMonitor monitor) throws InvocationTargetException {
@@ -485,7 +486,7 @@ public class WebServiceSQL implements DBWServiceSQL {
monitor.beginTask("Execute query", 1);
monitor.subTask("Process query " + sql);
WebSQLExecuteInfo executeResults = contextInfo.getProcessor().processQuery(
monitor, contextInfo, sql, resultId, filter, dataFormat, webSession, readLogs);
monitor, contextInfo, sql, resultId, filter, dataFormat, webSession, readLogs, useCache);
this.result = executeResults.getStatusMessage();
this.extendedResults = executeResults;
} catch (Throwable e) {
@@ -504,7 +505,9 @@ public class WebServiceSQL implements DBWServiceSQL {
@NotNull String nodePath,
@Nullable String resultId,
@Nullable WebSQLDataFilter filter,
@Nullable WebDataFormat dataFormat) throws DBWebException {
@Nullable WebDataFormat dataFormat,
@Nullable Boolean useCache
) throws DBWebException {
WebAsyncTaskProcessor<String> runnable = new WebAsyncTaskProcessor<String>() {
@Override
public void run(DBRProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
@@ -521,7 +524,8 @@ public class WebServiceSQL implements DBWServiceSQL {
dataContainer,
resultId,
filter != null ? filter : new WebSQLDataFilter(),
dataFormat);
dataFormat,
useCache);
this.result = executeResults.getStatusMessage();
this.extendedResults = executeResults;
} catch (Throwable e) {
@@ -1,6 +1,6 @@
/*
* CloudBeaver - Cloud Database Manager
* Copyright (C) 2020-2024 DBeaver Corp and others
* Copyright (C) 2020-2025 DBeaver Corp and others
*
* Licensed under the Apache License, Version 2.0.
* you may not use this file except in compliance with the License.
@@ -55,10 +55,10 @@ export class AsyncTask {
this.initPromise = null;
this.onStatusChange = new SyncExecutor();
this.innerPromise = new Promise((resolve, reject) => {
this.reject = reject;
this.resolve = resolve;
});
const innerPromise = Promise.withResolvers<AsyncTaskInfo>();
this.innerPromise = innerPromise.promise;
this.resolve = innerPromise.resolve;
this.reject = innerPromise.reject;
makeObservable<this, 'pending' | '_cancelled' | 'taskInfo' | 'updatingAsync'>(this, {
_cancelled: observable,
@@ -78,6 +78,15 @@ export class AsyncTask {
await this.initPromise;
}
/**
* Asynchronously updates the task information using the provided getter function.
* If the task is already finished, the update is skipped to avoid redundant status updates.
* This prevents unnecessary updates in scenarios where the task was already completed
* during the initial request, and a subsequent status event is received.
*
* @param getter - A function that retrieves the latest AsyncTaskInfo for this task.
* @returns A promise that resolves when the update is complete.
*/
async updateInfoAsync(getter: (task: AsyncTask) => Promise<AsyncTaskInfo>): Promise<void> {
const init = this.info === null;
@@ -91,6 +100,11 @@ export class AsyncTask {
return;
}
if (this.info?.running === false) {
// If the task is finished, we don't need to update it again
return;
}
this.updatingAsync = true;
try {
if (this._cancelled && init) {
@@ -1,22 +1,23 @@
mutation asyncReadDataFromContainer(
$projectId: ID!
$connectionId: ID!
$contextId: ID!
$containerNodePath: ID!
$resultId: ID
$filter: SQLDataFilter
$dataFormat: ResultDataFormat
$projectId: ID!
$connectionId: ID!
$contextId: ID!
$containerNodePath: ID!
$resultId: ID
$filter: SQLDataFilter
$dataFormat: ResultDataFormat
$useCache: Boolean
) {
taskInfo: asyncReadDataFromContainer(
projectId: $projectId
connectionId: $connectionId
contextId: $contextId
containerNodePath: $containerNodePath
resultId: $resultId
filter: $filter
dataFormat: $dataFormat
) {
...AsyncTaskInfo
}
taskInfo: asyncReadDataFromContainer(
projectId: $projectId
connectionId: $connectionId
contextId: $contextId
containerNodePath: $containerNodePath
resultId: $resultId
filter: $filter
dataFormat: $dataFormat
useCache: $useCache
) {
...AsyncTaskInfo
}
}
@@ -1,23 +1,25 @@
mutation asyncSqlExecuteQuery(
$projectId: ID!
$connectionId: ID!
$contextId: ID!
$query: String!
$resultId: ID
$filter: SQLDataFilter
$dataFormat: ResultDataFormat
$readLogs: Boolean
$projectId: ID!
$connectionId: ID!
$contextId: ID!
$query: String!
$resultId: ID
$filter: SQLDataFilter
$dataFormat: ResultDataFormat
$readLogs: Boolean
$useCache: Boolean
) {
taskInfo: asyncSqlExecuteQuery(
projectId: $projectId
connectionId: $connectionId
contextId: $contextId
sql: $query
resultId: $resultId
filter: $filter
dataFormat: $dataFormat
readLogs: $readLogs
) {
...AsyncTaskInfo
}
taskInfo: asyncSqlExecuteQuery(
projectId: $projectId
connectionId: $connectionId
contextId: $contextId
sql: $query
resultId: $resultId
filter: $filter
dataFormat: $dataFormat
readLogs: $readLogs
useCache: $useCache
) {
...AsyncTaskInfo
}
}
@@ -10,7 +10,6 @@ import { injectable } from '@cloudbeaver/core-di';
import { CommonDialogService, DialogueStateResult } from '@cloudbeaver/core-dialogs';
import { ClipboardService } from '@cloudbeaver/core-ui';
import { replaceMiddle } from '@cloudbeaver/core-utils';
import { isNotNullDefined } from '@dbeaver/js-helpers';
import { ACTION_DELETE, ActionService, MenuBaseItem, MenuService } from '@cloudbeaver/core-view';
import {
DATA_CONTEXT_DV_DDM,
@@ -69,12 +68,6 @@ export class DataGridContextMenuFilterService {
await model.request(() => {
constraints.setFilter(resultColumn.position, operator, filterValue);
});
const whereFilter = model.requestInfo.requestFilter || model.source.options?.whereFilter;
if (isNotNullDefined(whereFilter)) {
constraints.setWhereFilter(whereFilter);
}
}
register(): void {
@@ -1,6 +1,6 @@
/*
* CloudBeaver - Cloud Database Manager
* Copyright (C) 2020-2024 DBeaver Corp and others
* Copyright (C) 2020-2025 DBeaver Corp and others
*
* Licensed under the Apache License, Version 2.0.
* you may not use this file except in compliance with the License.
@@ -17,9 +17,13 @@ export interface IDataGroupingOptions extends IDataQueryOptions {
}
export class GroupingDataSource extends QueryDataSource<IDataGroupingOptions> {
override async request(prevResults: IDatabaseResultSet[]): Promise<IDatabaseResultSet[]> {
override async request(
prevResults: IDatabaseResultSet[],
prevOptions: Readonly<IDataGroupingOptions> | null,
refresh: boolean,
): Promise<IDatabaseResultSet[]> {
await this.generateQuery();
return await super.request(prevResults);
return await super.request(prevResults, prevOptions, refresh);
}
private async generateQuery(): Promise<void> {
@@ -68,11 +68,15 @@ export class ContainerDataSource extends ResultSetDataSource<IDataContainerOptio
await this.currentTask?.cancel();
}
async request(prevResults: IDatabaseResultSet[]): Promise<IDatabaseResultSet[]> {
async request(
prevResults: IDatabaseResultSet[],
prevOptions: Readonly<IDataContainerOptions> | null,
refresh: boolean,
): Promise<IDatabaseResultSet[]> {
const executionContext = await this.ensureContextCreated();
const context = executionContext.context!;
const limit = this.count;
const task = await this.getRequestTask(prevResults, context);
const task = await this.getRequestTask(prevResults, prevOptions, refresh, context);
this.currentTask = executionContext.run(
async () => {
@@ -88,11 +92,16 @@ export class ContainerDataSource extends ResultSetDataSource<IDataContainerOptio
try {
const response = await this.currentTask;
const whereFilter = response.filterText || '';
this.options!.whereFilter = whereFilter || this.options?.whereFilter || '';
//@ts-expect-error
this.prevOptions!.whereFilter = whereFilter || this.options?.whereFilter || '';
this.requestInfo = {
originalQuery: response.fullQuery || '',
requestDuration: response.duration || 0,
requestMessage: response.statusMessage || '',
requestFilter: response.filterText || '',
requestFilter: whereFilter,
source: null,
};
@@ -191,7 +200,12 @@ export class ContainerDataSource extends ResultSetDataSource<IDataContainerOptio
return prevResults;
}
protected getConfig(prevResults: IDatabaseResultSet[], context: IConnectionExecutionContextInfo) {
protected getConfig(
prevResults: IDatabaseResultSet[],
prevOptions: Readonly<IDataContainerOptions> | null,
refresh: boolean,
context: IConnectionExecutionContextInfo,
) {
const options = this.options;
if (!options) {
@@ -201,7 +215,11 @@ export class ContainerDataSource extends ResultSetDataSource<IDataContainerOptio
const offset = this.offset;
const limit = this.count;
const resultId = this.getPreviousResultId(prevResults, context);
let useServerCache = false;
if (resultId !== undefined && !refresh) {
useServerCache = this.canUseCachedResults(prevResults, prevOptions);
}
return {
projectId: context.projectId,
connectionId: context.connectionId,
@@ -215,12 +233,18 @@ export class ContainerDataSource extends ResultSetDataSource<IDataContainerOptio
where: options.whereFilter || undefined,
},
dataFormat: this.dataFormat,
useCache: useServerCache,
};
}
protected async getRequestTask(prevResults: IDatabaseResultSet[], context: IConnectionExecutionContextInfo): Promise<AsyncTask> {
protected async getRequestTask(
prevResults: IDatabaseResultSet[],
prevOptions: Readonly<IDataContainerOptions> | null,
refresh: boolean,
context: IConnectionExecutionContextInfo,
): Promise<AsyncTask> {
const task = this.asyncTaskInfoService.create(async () => {
const config = this.getConfig(prevResults, context);
const config = this.getConfig(prevResults, prevOptions, refresh, context);
const { taskInfo } = await this.graphQLService.sdk.asyncReadDataFromContainer(config);
return taskInfo;
});
@@ -302,7 +302,7 @@ export abstract class DatabaseDataSource<TOptions, TResult extends IDatabaseData
this.options = toJS(this.prevOptions);
}
return this.requestDataAction();
return this.requestDataAction(true);
});
}
@@ -345,12 +345,13 @@ export abstract class DatabaseDataSource<TOptions, TResult extends IDatabaseData
await this.cancel();
}
abstract request(prevResults: TResult[]): Promise<TResult[]>;
abstract request(prevResults: TResult[], prevOptions: Readonly<TOptions> | null, refresh: boolean): Promise<TResult[]>;
abstract save(prevResults: TResult[]): Promise<TResult[]>;
private async requestDataAction(): Promise<TResult[]> {
private async requestDataAction(refresh = false): Promise<TResult[]> {
const prevOptions = this.prevOptions;
this.prevOptions = toJS(this.options);
return this.request(this.results)
return this.request(this.results, prevOptions, refresh)
.finally(() => {
this.outdated = false;
})
@@ -7,7 +7,7 @@
*/
import { makeObservable, observable } from 'mobx';
import type { IConnectionExecutionContext, IConnectionExecutionContextInfo } from '@cloudbeaver/core-connections';
import { isConnectionInfoParamEqual, type IConnectionExecutionContext, type IConnectionExecutionContextInfo } from '@cloudbeaver/core-connections';
import type { IServiceProvider } from '@cloudbeaver/core-di';
import type { ITask } from '@cloudbeaver/core-executor';
import { AsyncTaskInfoService } from '@cloudbeaver/core-root';
@@ -17,7 +17,10 @@ import { DatabaseDataSource } from '../DatabaseDataModel/DatabaseDataSource.js';
import { type IDatabaseDataOptions } from '../DatabaseDataModel/IDatabaseDataOptions.js';
import type { IDatabaseResultSet } from '../DatabaseDataModel/IDatabaseResultSet.js';
export abstract class ResultSetDataSource<TOptions = IDatabaseDataOptions> extends DatabaseDataSource<TOptions, IDatabaseResultSet> {
export abstract class ResultSetDataSource<TOptions extends IDatabaseDataOptions = IDatabaseDataOptions> extends DatabaseDataSource<
TOptions,
IDatabaseResultSet
> {
executionContext: IConnectionExecutionContext | null;
totalCountRequestTask: ITask<number> | null;
private keepExecutionContextOnDispose: boolean;
@@ -138,6 +141,51 @@ export abstract class ResultSetDataSource<TOptions = IDatabaseDataOptions> exten
return resultId;
}
protected canUseCachedResults(prevResults: IDatabaseResultSet[], prevOptions: Readonly<TOptions> | null): boolean {
if (
!prevOptions ||
this.results.some(result => !result.loadedFully) ||
(this.options?.connectionKey === undefined && this.options?.connectionKey !== prevOptions.connectionKey) ||
(this.options?.connectionKey !== undefined &&
prevOptions.connectionKey !== undefined &&
!isConnectionInfoParamEqual(this.options.connectionKey, prevOptions.connectionKey)) ||
this.options?.catalog !== prevOptions.catalog ||
this.options?.schema !== prevOptions.schema ||
this.options?.readLogs !== prevOptions.readLogs ||
this.options?.whereFilter !== prevOptions.whereFilter ||
prevResults.some(result => result.dataFormat !== this.dataFormat)
) {
return false;
}
for (const constraint of prevOptions.constraints || []) {
if (constraint.criteria !== undefined || constraint.operator !== undefined) {
const currentConstraint = this.options?.constraints.find(c => c.attributePosition === constraint.attributePosition);
if (
!currentConstraint ||
currentConstraint.criteria !== constraint.criteria ||
currentConstraint.operator !== constraint.operator ||
currentConstraint.value !== constraint.value
) {
return false;
}
}
}
for (const constraint of this.options.constraints || []) {
if (constraint.criteria !== undefined || constraint.operator !== undefined) {
const prevConstraint = prevOptions.constraints.find(c => c.attributePosition === constraint.attributePosition);
if (!prevConstraint) {
return false;
}
}
}
return true;
}
private setTotalCount(resultIndex: number, count: number): this {
const result = this.getResult(resultIndex);
@@ -171,6 +219,6 @@ export abstract class ResultSetDataSource<TOptions = IDatabaseDataOptions> exten
}
}
export function isResultSetDataSource<T = IDatabaseDataOptions>(dataSource: any): dataSource is ResultSetDataSource<T> {
export function isResultSetDataSource<T extends IDatabaseDataOptions = IDatabaseDataOptions>(dataSource: any): dataSource is ResultSetDataSource<T> {
return dataSource instanceof ResultSetDataSource;
}
@@ -1,6 +1,6 @@
/*
* CloudBeaver - Cloud Database Manager
* Copyright (C) 2020-2024 DBeaver Corp and others
* Copyright (C) 2020-2025 DBeaver Corp and others
*
* Licensed under the Apache License, Version 2.0.
* you may not use this file except in compliance with the License.
@@ -10,7 +10,7 @@ import { type IDatabaseDataModel } from '../DatabaseDataModel/IDatabaseDataModel
import { type IDatabaseDataOptions } from '../DatabaseDataModel/IDatabaseDataOptions.js';
import { ResultSetDataSource } from './ResultSetDataSource.js';
export function isResultSetDataModel<T = IDatabaseDataOptions>(
export function isResultSetDataModel<T extends IDatabaseDataOptions = IDatabaseDataOptions>(
dataModel: IDatabaseDataModel<any> | undefined | null,
): dataModel is IDatabaseDataModel<ResultSetDataSource<T>> {
return dataModel instanceof DatabaseDataModel && dataModel.source instanceof ResultSetDataSource;
@@ -174,7 +174,7 @@ export class QueryDataSource<TOptions extends IDataQueryOptions = IDataQueryOpti
return this;
}
async request(prevResults: IDatabaseResultSet[]): Promise<IDatabaseResultSet[]> {
async request(prevResults: IDatabaseResultSet[], prevOptions: Readonly<TOptions> | null, refresh: boolean): Promise<IDatabaseResultSet[]> {
const options = this.options;
const executionContext = this.executionContext;
const executionContextInfo = this.executionContext?.context;
@@ -185,9 +185,13 @@ export class QueryDataSource<TOptions extends IDataQueryOptions = IDataQueryOpti
const limit = this.count;
let firstResultId: string | undefined;
let useServerCache = false;
if (this.requestInfo.query === this.options?.query) {
firstResultId = this.getPreviousResultId(prevResults, executionContextInfo);
if (firstResultId !== undefined && !refresh) {
useServerCache = this.canUseCachedResults(prevResults, prevOptions);
}
}
const task = this.asyncTaskInfoService.create(async () => {
@@ -205,6 +209,7 @@ export class QueryDataSource<TOptions extends IDataQueryOptions = IDataQueryOpti
},
dataFormat: this.dataFormat,
readLogs: options.readLogs,
useCache: useServerCache,
});
return taskInfo;
@@ -224,6 +229,11 @@ export class QueryDataSource<TOptions extends IDataQueryOptions = IDataQueryOpti
try {
const response = await this.currentTask;
const whereFilter = response.filterText || '';
this.options!.whereFilter = whereFilter || this.options?.whereFilter || '';
//@ts-expect-error
this.prevOptions!.whereFilter = whereFilter || this.options?.whereFilter || '';
const results = this.innerGetResults(executionContextInfo, response, limit);
this.clearError();