From 2107f4f968f271ed08c7474afda305b59e9f67d9 Mon Sep 17 00:00:00 2001 From: Alexander Skoblikov Date: Thu, 30 Mar 2023 15:33:38 +0200 Subject: [PATCH] =?UTF-8?q?CB-3164=20file=20based=20lock=20controller;=20l?= =?UTF-8?q?ock=20project=20when=20we=20try=20update=20d=E2=80=A6=20(#1564)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CB-3164 file based lock controller; lock project when we try update datasources --- .../io.cloudbeaver.model/META-INF/MANIFEST.MF | 1 + .../model/app/BaseWebApplication.java | 3 +- .../cloudbeaver/model/app/WebApplication.java | 3 +- .../rm/local/LocalResourceController.java | 121 +++++---- .../model/rm/lock/RMFileLockController.java | 243 ++++++++++++++++++ .../io/cloudbeaver/model/rm/lock/RMLock.java | 63 +++++ .../cloudbeaver/model/rm/lock/RMLockInfo.java | 110 ++++++++ .../io/cloudbeaver/server/CBApplication.java | 3 +- .../navigator/impl/WebServiceNavigator.java | 6 +- .../cloudbeaver/model/rm/lock/RMLockTest.java | 203 +++++++++++++++ .../model/rm/lock/TestLockController.java | 44 ++++ .../test/platform/CEServerTestSuite.java | 14 +- 12 files changed, 757 insertions(+), 57 deletions(-) create mode 100644 server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMFileLockController.java create mode 100644 server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLock.java create mode 100644 server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLockInfo.java create mode 100644 server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/RMLockTest.java create mode 100644 server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/TestLockController.java diff --git a/server/bundles/io.cloudbeaver.model/META-INF/MANIFEST.MF b/server/bundles/io.cloudbeaver.model/META-INF/MANIFEST.MF index 17322ce437..2fd0574fd9 100644 --- a/server/bundles/io.cloudbeaver.model/META-INF/MANIFEST.MF +++ b/server/bundles/io.cloudbeaver.model/META-INF/MANIFEST.MF @@ -27,6 +27,7 @@ Export-Package: io.cloudbeaver, io.cloudbeaver.model.app, io.cloudbeaver.model.rm, io.cloudbeaver.model.rm.local, + io.cloudbeaver.model.rm.lock, io.cloudbeaver.model.session, io.cloudbeaver.model.user, io.cloudbeaver.registry, diff --git a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/BaseWebApplication.java b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/BaseWebApplication.java index ea48f86724..339b7de564 100644 --- a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/BaseWebApplication.java +++ b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/BaseWebApplication.java @@ -66,7 +66,8 @@ public abstract class BaseWebApplication extends BaseApplicationImpl implements } @Override - public RMController createResourceController(@NotNull SMCredentialsProvider credentialsProvider, @NotNull DBPWorkspace workspace) { + public RMController createResourceController(@NotNull SMCredentialsProvider credentialsProvider, + @NotNull DBPWorkspace workspace) throws DBException { throw new IllegalStateException("Resource controller is not supported by " + getClass().getSimpleName()); } diff --git a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/WebApplication.java b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/WebApplication.java index e9ae96d153..db618b3149 100644 --- a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/WebApplication.java +++ b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/app/WebApplication.java @@ -60,7 +60,8 @@ public interface WebApplication extends DBPApplication { DBSSecretController getSecretController(@NotNull SMCredentialsProvider credentialsProvider) throws DBException; - RMController createResourceController(@NotNull SMCredentialsProvider credentialsProvider, @NotNull DBPWorkspace workspace); + RMController createResourceController(@NotNull SMCredentialsProvider credentialsProvider, + @NotNull DBPWorkspace workspace) throws DBException; String getServerURL(); diff --git a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/local/LocalResourceController.java b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/local/LocalResourceController.java index 80218b7988..487655d88b 100644 --- a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/local/LocalResourceController.java +++ b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/local/LocalResourceController.java @@ -20,6 +20,7 @@ import io.cloudbeaver.DBWConstants; import io.cloudbeaver.WebProjectImpl; import io.cloudbeaver.model.app.WebApplication; import io.cloudbeaver.model.rm.RMUtils; +import io.cloudbeaver.model.rm.lock.RMFileLockController; import io.cloudbeaver.service.security.SMUtils; import io.cloudbeaver.service.sql.WebSQLConstants; import io.cloudbeaver.utils.WebAppUtils; @@ -34,7 +35,6 @@ import org.jkiss.dbeaver.model.DBPDataSourceContainer; import org.jkiss.dbeaver.model.DBPDataSourceFolder; import org.jkiss.dbeaver.model.app.DBPDataSourceRegistry; import org.jkiss.dbeaver.model.app.DBPProject; -import org.jkiss.dbeaver.model.app.DBPResourceTypeDescriptor; import org.jkiss.dbeaver.model.app.DBPWorkspace; import org.jkiss.dbeaver.model.auth.SMCredentials; import org.jkiss.dbeaver.model.auth.SMCredentialsProvider; @@ -82,6 +82,7 @@ public class LocalResourceController implements RMController { private final Path sharedProjectsPath; private final String globalProjectName; private Supplier smControllerSupplier; + private final RMFileLockController lockController; private final Map projectRegistries = new LinkedHashMap<>(); @@ -92,13 +93,14 @@ public class LocalResourceController implements RMController { Path userProjectsPath, Path sharedProjectsPath, Supplier smControllerSupplier - ) { + ) throws DBException { this.workspace = workspace; this.credentialsProvider = credentialsProvider; this.rootPath = rootPath; this.userProjectsPath = userProjectsPath; this.sharedProjectsPath = sharedProjectsPath; this.smControllerSupplier = smControllerSupplier; + this.lockController = new RMFileLockController(WebAppUtils.getWebApplication()); this.globalProjectName = DBWorkbench.getPlatform().getApplication().getDefaultProjectName(); } @@ -287,16 +289,18 @@ public class LocalResourceController implements RMController { @Override public void deleteProject(@NotNull String projectId) throws DBException { - RMProject project = makeProjectFromId(projectId, false); - Path targetPath = getProjectPath(projectId); - if (!Files.exists(targetPath)) { - throw new DBException("Project '" + project.getName() + "' doesn't exists"); - } - try { - IOUtils.deleteDirectory(targetPath); - getSecurityController().deleteAllObjectPermissions(projectId, SMObjects.PROJECT); - } catch (IOException e) { - throw new DBException("Error deleting project '" + project.getName() + "'", e); + try (var projectLock = lockController.lockProject(projectId, "deleteProject")) { + RMProject project = makeProjectFromId(projectId, false); + Path targetPath = getProjectPath(projectId); + if (!Files.exists(targetPath)) { + throw new DBException("Project '" + project.getName() + "' doesn't exists"); + } + try { + IOUtils.deleteDirectory(targetPath); + getSecurityController().deleteAllObjectPermissions(projectId, SMObjects.PROJECT); + } catch (IOException e) { + throw new DBException("Error deleting project '" + project.getName() + "'", e); + } } } @@ -347,7 +351,9 @@ public class LocalResourceController implements RMController { @NotNull String configuration, @Nullable List dataSourceIds ) throws DBException { - updateProjectDataSources(projectId, configuration, dataSourceIds); + try (var projectLock = lockController.lockProject(projectId, "createDatasources")) { + updateProjectDataSources(projectId, configuration, dataSourceIds); + } } @Override @@ -368,32 +374,38 @@ public class LocalResourceController implements RMController { } @Override - public void deleteProjectDataSources(@NotNull String projectId, @NotNull String[] dataSourceIds) throws DBException { - final DBPProject project = getProjectMetadata(projectId, false); - final DBPDataSourceRegistry registry = project.getDataSourceRegistry(); + public void deleteProjectDataSources(@NotNull String projectId, + @NotNull String[] dataSourceIds) throws DBException { + try (var projectLock = lockController.lockProject(projectId, "deleteDatasources")) { + final DBPProject project = getProjectMetadata(projectId, false); + final DBPDataSourceRegistry registry = project.getDataSourceRegistry(); - for (String dataSourceId : dataSourceIds) { - final DBPDataSourceContainer dataSource = registry.getDataSource(dataSourceId); + for (String dataSourceId : dataSourceIds) { + final DBPDataSourceContainer dataSource = registry.getDataSource(dataSourceId); - if (dataSource != null) { - registry.removeDataSource(dataSource); - } else { - log.warn("Could not find datasource " + dataSourceId + " for deletion"); + if (dataSource != null) { + registry.removeDataSource(dataSource); + } else { + log.warn("Could not find datasource " + dataSourceId + " for deletion"); + } } + registry.checkForErrors(); } - registry.checkForErrors(); } @Override - public void createProjectDataSourceFolder(@NotNull String projectId, @NotNull String folderPath) throws DBException { - DBPProject project = getProjectMetadata(projectId, false); - DBPDataSourceRegistry registry = project.getDataSourceRegistry(); - var result = Path.of(folderPath); - var newName = result.getFileName().toString(); - var parent = result.getParent(); - var parentFolder = parent == null ? null : registry.getFolder(parent.toString().replace("\\", "/")); - DBPDataSourceFolder newFolder = registry.addFolder(parentFolder, newName); - registry.checkForErrors(); + public void createProjectDataSourceFolder(@NotNull String projectId, + @NotNull String folderPath) throws DBException { + try (var projectLock = lockController.lockProject(projectId, "createDatasourceFolder")) { + DBPProject project = getProjectMetadata(projectId, false); + DBPDataSourceRegistry registry = project.getDataSourceRegistry(); + var result = Path.of(folderPath); + var newName = result.getFileName().toString(); + var parent = result.getParent(); + var parentFolder = parent == null ? null : registry.getFolder(parent.toString().replace("\\", "/")); + DBPDataSourceFolder newFolder = registry.addFolder(parentFolder, newName); + registry.checkForErrors(); + } } @Override @@ -402,17 +414,19 @@ public class LocalResourceController implements RMController { @NotNull String[] folderPaths, boolean dropContents ) throws DBException { - DBPProject project = getProjectMetadata(projectId, false); - DBPDataSourceRegistry registry = project.getDataSourceRegistry(); - for (String folderPath : folderPaths) { - DBPDataSourceFolder folder = registry.getFolder(folderPath); - if (folder != null) { - registry.removeFolder(folder, dropContents); - } else { - log.warn("Can not find folder by path [" + folderPath + "] for deletion"); + try (var projectLock = lockController.lockProject(projectId, "createDatasourceFolder")) { + DBPProject project = getProjectMetadata(projectId, false); + DBPDataSourceRegistry registry = project.getDataSourceRegistry(); + for (String folderPath : folderPaths) { + DBPDataSourceFolder folder = registry.getFolder(folderPath); + if (folder != null) { + registry.removeFolder(folder, dropContents); + } else { + log.warn("Can not find folder by path [" + folderPath + "] for deletion"); + } } + registry.checkForErrors(); } - registry.checkForErrors(); } @Override @@ -421,10 +435,12 @@ public class LocalResourceController implements RMController { @NotNull String oldPath, @NotNull String newPath ) throws DBException { - DBPProject project = getProjectMetadata(projectId, false); - DBPDataSourceRegistry registry = project.getDataSourceRegistry(); - registry.moveFolder(oldPath, newPath); - registry.checkForErrors(); + try (var projectLock = lockController.lockProject(projectId, "createDatasourceFolder")) { + DBPProject project = getProjectMetadata(projectId, false); + DBPDataSourceRegistry registry = project.getDataSourceRegistry(); + registry.moveFolder(oldPath, newPath); + registry.checkForErrors(); + } } @NotNull @@ -692,9 +708,11 @@ public class LocalResourceController implements RMController { @NotNull String propertyName, @Nullable Object propertyValue ) throws DBException { - validateResourcePath(resourcePath); - getProjectMetadata(projectId, false).setResourceProperty(resourcePath, propertyName, propertyValue); - return DEFAULT_CHANGE_ID; + try (var projectLock = lockController.lockProject(projectId, "resourcePropertyUpdate")) { + validateResourcePath(resourcePath); + getProjectMetadata(projectId, false).setResourceProperty(resourcePath, propertyName, propertyValue); + return DEFAULT_CHANGE_ID; + } } private void validateResourcePath(String resourcePath) throws DBException { @@ -923,9 +941,12 @@ public class LocalResourceController implements RMController { ); } - public static Builder builder(SMCredentialsProvider credentialsProvider, DBPWorkspace workspace, Supplier smControllerSupplier) { + public static Builder builder(SMCredentialsProvider credentialsProvider, + DBPWorkspace workspace, + Supplier smControllerSupplier) { return new Builder(workspace, credentialsProvider, smControllerSupplier); } + public static final class Builder { private final SMCredentialsProvider credentialsProvider; private final Supplier smController; @@ -959,7 +980,7 @@ public class LocalResourceController implements RMController { return this; } - public LocalResourceController build() { + public LocalResourceController build() throws DBException { return new LocalResourceController(workspace, credentialsProvider, rootPath, userProjectsPath, sharedProjectsPath, smController); } } diff --git a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMFileLockController.java b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMFileLockController.java new file mode 100644 index 0000000000..7207edcb4f --- /dev/null +++ b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMFileLockController.java @@ -0,0 +1,243 @@ +/* + * DBeaver - Universal Database Manager + * Copyright (C) 2010-2023 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudbeaver.model.rm.lock; + +import com.google.gson.Gson; +import io.cloudbeaver.model.app.WebApplication; +import org.jkiss.code.Nullable; +import org.jkiss.dbeaver.DBException; +import org.jkiss.dbeaver.Log; +import org.jkiss.dbeaver.model.app.DBPWorkspace; + +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +/** + * File based resource locks + */ +public class RMFileLockController { + private static final Log log = Log.getLog(RMFileLockController.class); + private static final int DEFAULT_MAX_LOCK_TIME = 1 * 60 * 1000; // 1 min + private static final int CHECK_PERIOD = 10; + + private static final String LOCK_META_FOLDER = ".locks"; + private static final String LOCK_FILE_EXTENSION = ".lock"; + + private final Gson gson = new Gson(); + + private final Path lockFolderPath; + private final String applicationId; + private final int maxLockTime; + + + public RMFileLockController(WebApplication application) throws DBException { + this(application, DEFAULT_MAX_LOCK_TIME); + } + + // for tests + public RMFileLockController(WebApplication application, int maxLockTime) throws DBException { + this.lockFolderPath = application.getWorkspaceDirectory() + .resolve(DBPWorkspace.METADATA_FOLDER) + .resolve(LOCK_META_FOLDER); + this.applicationId = application.getApplicationInstanceId(); + this.maxLockTime = maxLockTime; + } + + /** + * Lock the project for the duration of any operation. + * Other threads/processes will also see this lock, and will wait for it to end + * or force intercept lock, if the operation will take too long and + * exceeds the maximum available locking time {@link #maxLockTime} or the lock is invalid {@link #awaitUnlock)}. + * + * @param projectId - project to be locked + * @param operationName - executed operation name + * @return - lock + */ + public RMLock lockProject(String projectId, String operationName) throws DBException { + synchronized (RMFileLockController.class) { + try { + createLockFolderIfNeeded(); + createProjectFolder(projectId); + Path projectLockFile = getProjectLockFilePath(projectId); + + RMLockInfo lockInfo = new RMLockInfo.Builder(projectId, UUID.randomUUID().toString()) + .setApplicationId(applicationId) + .setOperationName(operationName) + .setOperationStartTime(System.currentTimeMillis()) + .build(); + createLockFile(projectLockFile, lockInfo); + return new RMLock(projectLockFile); + } catch (Exception e) { + throw new DBException("Failed to lock project: " + projectId, e); + } + } + } + + /** + * Check that project locked + */ + public boolean isProjectLocked(String projectId) { + Path projectLockFilePath = getProjectLockFilePath(projectId); + return isLocked(projectLockFilePath); + } + + protected boolean isLocked(Path lockFilePath) { + return Files.exists(lockFilePath); + } + + private void createLockFile(Path projectLockFile, RMLockInfo lockInfo) throws DBException, InterruptedException { + boolean lockFileCreated = false; + while (!lockFileCreated) { + if (Files.exists(projectLockFile)) { + awaitUnlock(lockInfo.getProjectId(), projectLockFile); + } + try { + Files.createFile(projectLockFile); + lockFileCreated = true; + } catch (IOException e) { + if (Files.exists(projectLockFile)) { + log.info("Looks like file was locked by another rm instance at the same time"); + continue; + } else { + throw new DBException("Failed to create lock file: " + projectLockFile, e); + } + } + + try { + Files.write(projectLockFile, gson.toJson(lockInfo).getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + log.error("Failed to write lock info, unlock project: " + lockInfo.getProjectId()); + try { + Files.deleteIfExists(lockFolderPath); + } catch (IOException ex) { + throw new DBException("Failed to remove invalid lock file: " + projectLockFile, ex); + } + throw new DBException("Failed to lock project: " + lockInfo.getProjectId(), e); + } + } + + } + + private void createProjectFolder(String projectId) throws DBException { + Path projectLocksFolder = lockFolderPath.resolve(projectId); + if (Files.exists(projectLocksFolder)) { + return; + } + try { + Files.createDirectories(projectLocksFolder); + } catch (Exception e) { + if (Files.exists(projectLocksFolder)) { + // ignore, because file can be created by another server + } else { + throw new DBException("Failed to create project lock folder: " + projectId, e); + } + } + } + + protected void awaitUnlock(String projectId, Path projectLockFile) throws InterruptedException, DBException { + if (!isLocked(projectLockFile)) { + return; + } + awaitingUnlock(projectId, projectLockFile); + } + + protected void awaitingUnlock(String projectId, Path projectLockFile) throws DBException, InterruptedException { + log.info("Waiting for a file to be unlocked: " + projectLockFile); + RMLockInfo originalLockInfo = readLockInfo(projectId, projectLockFile); + boolean fileUnlocked = originalLockInfo == null; //lock can be removed at the moment when we try to read lock file info + int maxIterations = maxLockTime / CHECK_PERIOD; + int currentCheckCount = 0; + + while (!fileUnlocked) { + fileUnlocked = !isLocked(projectLockFile); + if (currentCheckCount >= maxIterations || fileUnlocked) { + break; + } + if (originalLockInfo != null & originalLockInfo.isBlank()) { + // possible in situation where the project has just been locked + // and the lock information has not yet been written + originalLockInfo = readLockInfo(projectId, projectLockFile); + } + currentCheckCount++; + Thread.sleep(CHECK_PERIOD); + } + if (fileUnlocked) { + return; + } + + RMLockInfo currentLockInfo = readLockInfo(projectId, projectLockFile); + if (currentLockInfo == null) { + // file unlocked now + return; + } + + //checking that this is not a new lock from another operation + if (originalLockInfo.getOperationId().equals(currentLockInfo.getOperationId())) { + forceUnlock(projectLockFile); + } else { + awaitUnlock(projectId, lockFolderPath); + } + } + + protected void forceUnlock(Path projectLockFile) { + // something went wrong and lock is invalid + log.warn("File has not been unlocked within the expected period, force unlock"); + try { + Files.deleteIfExists(projectLockFile); + } catch (IOException e) { + log.error(e); + } + } + + @Nullable + /** + @return + - null if lock not exist; + - empty lock info if the lock has just been created and the information has not yet been written; + - lock info + */ + private RMLockInfo readLockInfo(String projectId, Path projectLockFile) throws DBException { + if (Files.notExists(projectLockFile)) { + return null; + } + try (Reader reader = Files.newBufferedReader(projectLockFile, StandardCharsets.UTF_8)) { + return gson.fromJson(reader, RMLockInfo.class); + } catch (IOException e) { + if (!isLocked(projectLockFile)) { + return null; + } + log.warn("Failed to read lock file info, but lock file still exist: " + projectLockFile); + return RMLockInfo.emptyLock(projectId); + } + } + + private Path getProjectLockFilePath(String projectId) { + return lockFolderPath.resolve(projectId).resolve(projectId + LOCK_FILE_EXTENSION); + } + + private void createLockFolderIfNeeded() throws IOException { + synchronized (RMFileLockController.class) { + if (Files.notExists(lockFolderPath)) { + Files.createDirectories(lockFolderPath); + } + } + } +} diff --git a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLock.java b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLock.java new file mode 100644 index 0000000000..70afbd4630 --- /dev/null +++ b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLock.java @@ -0,0 +1,63 @@ +/* + * DBeaver - Universal Database Manager + * Copyright (C) 2010-2023 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudbeaver.model.rm.lock; + +import org.jkiss.dbeaver.Log; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Resource Manager resource lock + */ +public class RMLock implements AutoCloseable { + private static final Log log = Log.getLog(RMLock.class); + + private final Path lockFilePath; + + public RMLock(Path lockFilePath) { + this.lockFilePath = lockFilePath; + } + + /** + * Unlock resource and remove .lock file + */ + public void unlock() { + try { + Files.deleteIfExists(lockFilePath); + } catch (IOException e) { + log.error("Failed to unlock file: " + lockFilePath, e); + if (Files.exists(lockFilePath)) { + // file still locket, try to unlock again + unlock(); + } + } + } + + /** + * @return path to the lock file + */ + protected Path getLockFilePath() { + return lockFilePath; + } + + @Override + public void close() { + unlock(); + } +} diff --git a/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLockInfo.java b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLockInfo.java new file mode 100644 index 0000000000..1cec6bbfa0 --- /dev/null +++ b/server/bundles/io.cloudbeaver.model/src/io/cloudbeaver/model/rm/lock/RMLockInfo.java @@ -0,0 +1,110 @@ +/* + * DBeaver - Universal Database Manager + * Copyright (C) 2010-2023 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudbeaver.model.rm.lock; + +public class RMLockInfo { + private final String applicationId; + private final String projectId; + private final String operationId; + private final String operationName; + private final long operationStartTime; + + private RMLockInfo( + String applicationId, + String projectId, + String operationId, + String operationName, + long operationStartTime + ) { + this.applicationId = applicationId; + this.projectId = projectId; + this.operationId = operationId; + this.operationName = operationName; + this.operationStartTime = operationStartTime; + } + + static RMLockInfo emptyLock( + String projectId + ) { + return new RMLockInfo( + "", + projectId, + "", + "", + System.currentTimeMillis() + ); + } + + + public boolean isBlank() { + return operationId.isEmpty(); + } + + public String getApplicationId() { + return applicationId; + } + + public String getProjectId() { + return projectId; + } + + public String getOperationId() { + return operationId; + } + + public String getOperationName() { + return operationName; + } + + public long getOperationStartTime() { + return operationStartTime; + } + + public static final class Builder { + private String applicationId; + private final String projectId; + private final String operationId; + private String operationName; + private long operationStartTime; + + public Builder(String projectId, String operationId) { + this.projectId = projectId; + this.operationId = operationId; + } + + + public Builder setApplicationId(String applicationId) { + this.applicationId = applicationId; + return this; + } + + + public Builder setOperationName(String operationName) { + this.operationName = operationName; + return this; + } + + public Builder setOperationStartTime(long operationStartTime) { + this.operationStartTime = operationStartTime; + return this; + } + + public RMLockInfo build() { + return new RMLockInfo(applicationId, projectId, operationId, operationName, operationStartTime); + } + } +} diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBApplication.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBApplication.java index c9baba4e47..2331b6c60f 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBApplication.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/server/CBApplication.java @@ -519,7 +519,8 @@ public class CBApplication extends BaseWebApplication implements WebAuthApplicat } @Override - public RMController createResourceController(@NotNull SMCredentialsProvider credentialsProvider, @NotNull DBPWorkspace workspace) { + public RMController createResourceController(@NotNull SMCredentialsProvider credentialsProvider, + @NotNull DBPWorkspace workspace) throws DBException { return LocalResourceController.builder(credentialsProvider, workspace, this::getSecurityController).build(); } diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/navigator/impl/WebServiceNavigator.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/navigator/impl/WebServiceNavigator.java index 3046da892d..157389783d 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/navigator/impl/WebServiceNavigator.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/navigator/impl/WebServiceNavigator.java @@ -89,9 +89,9 @@ public class WebServiceNavigator implements DBWServiceNavigator { nodeChildren = DBNUtils.getNodeChildrenFiltered(monitor, rootNode, true); } else { DBNNode parentNode = navigatorModel.getNodeByPath(monitor, parentPath); - if (parentNode == null) { - throw new DBWebException("Node '" + parentPath + "' not found"); - } + if (parentNode == null) { + throw new DBWebException("Node '" + parentPath + "' not found"); + } if (!parentNode.hasChildren(false)) { return EMPTY_NODE_LIST; } diff --git a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/RMLockTest.java b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/RMLockTest.java new file mode 100644 index 0000000000..c04eac1c02 --- /dev/null +++ b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/RMLockTest.java @@ -0,0 +1,203 @@ +/* + * DBeaver - Universal Database Manager + * Copyright (C) 2010-2023 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudbeaver.model.rm.lock; + +import io.cloudbeaver.test.platform.CEServerTestSuite; +import org.jkiss.dbeaver.Log; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public class RMLockTest { + private static final Log log = Log.getLog(RMLockTest.class); + private final String project1 = "s_fakeProject1"; + private final String project2 = "s_fakeProject2"; + + + @Test + public void testProjectAccessUsingSeveralControllers() throws Throwable { + var lockController1 = new TestLockController(CEServerTestSuite.getTestApp(), 1); + + CountDownLatch thread1CDL = new CountDownLatch(1); + CountDownLatch thread2CDL = new CountDownLatch(1); + CountDownLatch globalCountDown = new CountDownLatch(2); + + AtomicBoolean isLockedByThread1 = new AtomicBoolean(false); + AtomicReference exceptionReference = new AtomicReference<>(); + var thread1 = new Thread(() -> { + try (var lock = lockController1.lockProject(project1, "testThatProjectLocked1")) { + isLockedByThread1.set(true); + thread2CDL.countDown(); + thread1CDL.await(1, TimeUnit.MINUTES); + } catch (Throwable e) { + log.error(e); + exceptionReference.set(e); + } finally { + isLockedByThread1.set(false); + globalCountDown.countDown(); + } + }); + thread1.start(); + + int atLeastWaitCalls = 1; + var lockController2 = Mockito.spy(new TestLockController(CEServerTestSuite.getTestApp(), 100)); + Mockito.doAnswer(new Answer() { + private int count = 0; + + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + count++; + if (count == atLeastWaitCalls) { + //unlock thread1 + thread1CDL.countDown(); + } + return invocationOnMock.callRealMethod(); + } + }).when(lockController2).awaitingUnlock(Mockito.any(), Mockito.any()); + var thread2 = new Thread(() -> { + try { + thread2CDL.await(1, TimeUnit.MINUTES); + Assert.assertTrue("Thread 1 not running", isLockedByThread1.get()); + Assert.assertTrue("Project not locked", lockController2.isProjectLocked(project1)); + try (var lock = lockController2.lockProject(project1, "testThatProjectLocked2")) { + Assert.assertFalse("Project locked by thread1, but thread2 has access to project", isLockedByThread1.get()); + //that we were really waiting for the file and the lock was not removed earlier + Mockito.verify(lockController2, Mockito.atLeast(atLeastWaitCalls)).awaitingUnlock(Mockito.any(), Mockito.any()); + } + } catch (Throwable e) { + log.error(e); + exceptionReference.set(e); + } finally { + globalCountDown.countDown(); + } + }); + thread2.start(); + globalCountDown.await(1, TimeUnit.MINUTES); + if (exceptionReference.get() != null) { + throw exceptionReference.get(); + } + Assert.assertFalse(lockController2.isProjectLocked(project1)); + } + + @Test + public void testAccessToDifferentProjects() throws Throwable { + var lockController1 = new TestLockController(CEServerTestSuite.getTestApp(), 1); + + CountDownLatch thread1CDL = new CountDownLatch(1); + CountDownLatch thread2CDL = new CountDownLatch(1); + CountDownLatch globalCountDown = new CountDownLatch(2); + + AtomicBoolean isLockedByThread1 = new AtomicBoolean(false); + AtomicBoolean isLockedByThread2 = new AtomicBoolean(false); + AtomicReference exceptionReference = new AtomicReference<>(); + var thread1 = new Thread(() -> { + try (var lock = lockController1.lockProject(project1, "testAccessToDifferentProjects1")) { + isLockedByThread1.set(true); + thread1CDL.await(1, TimeUnit.MINUTES); + Assert.assertTrue("Project2 not locked by thread2", isLockedByThread2.get()); + thread2CDL.countDown(); + } catch (Throwable e) { + log.error(e); + exceptionReference.set(e); + } finally { + isLockedByThread1.set(false); + globalCountDown.countDown(); + } + }); + thread1.start(); + + var lockController2 = new TestLockController(CEServerTestSuite.getTestApp(), 1); + var thread2 = new Thread(() -> { + try { + try (var lock = lockController2.lockProject(project2, "testAccessToDifferentProjects2")) { + Assert.assertTrue("Project1 not locket by thread1", isLockedByThread1.get()); + isLockedByThread2.set(true); + thread1CDL.countDown(); + thread2CDL.await(); + } + } catch (Throwable e) { + log.error(e); + exceptionReference.set(e); + } finally { + isLockedByThread2.set(false); + globalCountDown.countDown(); + } + }); + thread2.start(); + globalCountDown.await(1, TimeUnit.MINUTES); + if (exceptionReference.get() != null) { + throw exceptionReference.get(); + } + + Assert.assertFalse(lockController2.isProjectLocked(project1)); + Assert.assertFalse(lockController2.isProjectLocked(project2)); + } + + @Test + public void testForceUnlock() throws Throwable { + var lockController1 = new TestLockController(CEServerTestSuite.getTestApp(), 1); + + CountDownLatch thread1CDL = new CountDownLatch(1); + CountDownLatch thread2CDL = new CountDownLatch(1); + CountDownLatch globalCountDown = new CountDownLatch(2); + + AtomicBoolean isLockedByThread1 = new AtomicBoolean(false); + AtomicReference exceptionReference = new AtomicReference<>(); + var thread1 = new Thread(() -> { + try (var lock = lockController1.lockProject(project1, "testForceUnlock1")) { + isLockedByThread1.set(true); + thread1CDL.await(1, TimeUnit.MINUTES); + } catch (Throwable e) { + log.error(e); + exceptionReference.set(e); + } finally { + isLockedByThread1.set(false); + globalCountDown.countDown(); + } + }); + thread1.start(); + + var lockController2 = Mockito.spy(new TestLockController(CEServerTestSuite.getTestApp(), 100)); + var thread2 = new Thread(() -> { + try { + try (var lock = lockController2.lockProject(project1, "testForceUnlock2")) { + Assert.assertTrue("Project1 not locket by thread1", isLockedByThread1.get()); + Mockito.verify(lockController2, Mockito.atLeast(5)).isLocked(Mockito.any()); + thread1CDL.countDown(); + } + } catch (Throwable e) { + log.error(e); + exceptionReference.set(e); + } finally { + globalCountDown.countDown(); + } + }); + thread2.start(); + globalCountDown.await(1, TimeUnit.MINUTES); + if (exceptionReference.get() != null) { + throw exceptionReference.get(); + } + Assert.assertFalse(lockController2.isProjectLocked(project1)); + } +} diff --git a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/TestLockController.java b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/TestLockController.java new file mode 100644 index 0000000000..74dcc31098 --- /dev/null +++ b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/model/rm/lock/TestLockController.java @@ -0,0 +1,44 @@ +/* + * DBeaver - Universal Database Manager + * Copyright (C) 2010-2023 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudbeaver.model.rm.lock; + +import io.cloudbeaver.model.app.WebApplication; +import org.jkiss.dbeaver.DBException; + +import java.nio.file.Path; + +public class TestLockController extends RMFileLockController { + public TestLockController(WebApplication application) throws DBException { + super(application); + } + + public TestLockController(WebApplication application, int maxTimeout) throws DBException { + super(application, maxTimeout); + } + + //avoid mockito access method error + @Override + public void awaitingUnlock(String projectId, Path projectLockFile) throws InterruptedException, DBException { + super.awaitingUnlock(projectId, projectLockFile); + } + + //avoid mockito access method error + @Override + public boolean isLocked(Path lockFilePath) { + return super.isLocked(lockFilePath); + } +} diff --git a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java index 39445bffd7..9b9de9c26f 100644 --- a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java +++ b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java @@ -17,6 +17,7 @@ package io.cloudbeaver.test.platform; +import io.cloudbeaver.model.rm.lock.RMLockTest; import io.cloudbeaver.server.CBApplication; import io.cloudbeaver.utils.WebTestUtils; import org.junit.AfterClass; @@ -29,7 +30,14 @@ import java.net.http.HttpClient; import java.nio.file.Path; @RunWith(Suite.class) -@Suite.SuiteClasses({PlatformTest.class, AuthenticationTest.class, ResourceManagerTest.class }) +@Suite.SuiteClasses( + { + PlatformTest.class, + AuthenticationTest.class, + ResourceManagerTest.class, + RMLockTest.class + } +) public class CEServerTestSuite { public static final String GQL_API_URL = "http://localhost:18978/api/gql"; @@ -80,6 +88,10 @@ public class CEServerTestSuite { testApp.stop(); } + public static CBApplication getTestApp() { + return testApp; + } + public static HttpClient getClient() { return client; }