From bbbfde85efe507d0348f24326337ce5672726b70 Mon Sep 17 00:00:00 2001 From: Dave B Date: Tue, 18 May 2021 15:42:22 -0400 Subject: [PATCH 1/8] Open input file in read mode --- tools/extract/liftOver_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/extract/liftOver_wrapper.py b/tools/extract/liftOver_wrapper.py index 6a59ddc6822..3ceb1c7a0d2 100644 --- a/tools/extract/liftOver_wrapper.py +++ b/tools/extract/liftOver_wrapper.py @@ -25,7 +25,7 @@ def safe_bed_file(infile): https://lists.soe.ucsc.edu/pipermail/genome/2007-May/013561.html """ fix_pat = re.compile("^(track|browser)") - with tempfile.NamedTemporaryFile(mode='w', delete=False) as out_handle, open(infile, 'w') as in_handle: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as out_handle, open(infile, 'r') as in_handle: for line in in_handle: if fix_pat.match(line): line = "#" + line From 189c1b1b1a6e1778e0133d64e21ecd8615465c2c Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 20 May 2021 12:02:42 +0200 Subject: [PATCH 2/8] hide edit for users --- client/src/components/Libraries/LibrariesList.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Libraries/LibrariesList.vue b/client/src/components/Libraries/LibrariesList.vue index 7b540aa764a..f6487cea1ad 100644 --- a/client/src/components/Libraries/LibrariesList.vue +++ b/client/src/components/Libraries/LibrariesList.vue @@ -87,7 +87,7 @@ Undelete -
+
Date: Fri, 21 May 2021 14:51:24 -0400 Subject: [PATCH 3/8] Version 21.1.1 of tool-util (tag galaxy-tool-util-21.1.1). --- packages/tool_util/HISTORY.rst | 2 +- packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index f6fce4fb098..ff5680758c9 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -6,7 +6,7 @@ History .. to_doc --------------------- -21.1.1.dev0 +21.1.1 (2021-05-21) --------------------- diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index a9cc5a8b7af..785f40a84a4 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,4 +1,4 @@ -__version__ = '21.1.1.dev0' +__version__ = '21.1.1' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From 9a97d616f8d8c1eaf89de8d9a14397f8ddbf9bb1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 21 May 2021 14:52:02 -0400 Subject: [PATCH 4/8] Starting work on tool-util 21.1.2 --- packages/tool_util/HISTORY.rst | 6 ++++++ packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index ff5680758c9..053ba4ab90e 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -5,6 +5,12 @@ History .. to_doc +--------------------- +21.1.2.dev0 +--------------------- + + + --------------------- 21.1.1 (2021-05-21) --------------------- diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index 785f40a84a4..fb12eda4e1b 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,4 +1,4 @@ -__version__ = '21.1.1' +__version__ = '21.1.2.dev0' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From 6039c6e47234313bb3dccf79d901df473988b5cb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 21 May 2021 20:44:10 +0200 Subject: [PATCH 5/8] Fix isolated job home dir on pulsar --- lib/galaxy/jobs/runners/pulsar.py | 23 ++++++++++++----- test/integration/test_job_environments.py | 30 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 4813d5bbb77..eff37f03774 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -1003,6 +1003,8 @@ class PulsarComputeEnvironment(ComputeEnvironment): self._working_directory = remote_job_config["working_directory"] self._sep = remote_job_config["system_properties"]["separator"] self._tool_dir = remote_job_config["tools_directory"] + self._tmp_dir = remote_job_config.get('tmp_dir') + self._shared_home_dir = remote_job_config.get('shared_home_dir') version_path = self.local_path_config.version_path() new_version_path = self.path_mapper.remote_version_path_rewrite(version_path) if new_version_path: @@ -1092,14 +1094,23 @@ class PulsarComputeEnvironment(ComputeEnvironment): return self._tool_dir def home_directory(self): - # TODO: revisit and implement this, won't break anything working in the - # meantime. - return None + return self._target_to_directory(self.job_wrapper.home_target) def tmp_directory(self): - # TODO: revisit and implement this, won't break anything working in the - # meantime. - return None + return self._target_to_directory(self.job_wrapper.tmp_target) + + def _target_to_directory(self, target): + tmp_dir = self._tmp_dir + if target is None or (target == "job_tmp_if_explicit" and tmp_dir is None): + return None + elif target in ["job_tmp", "job_tmp_if_explicit"]: + return "$_GALAXY_JOB_TMP_DIR" + elif target == "shared_home": + return self._shared_home_dir + elif target == "job_home": + return "$_GALAXY_JOB_HOME_DIR" + else: + raise Exception(f"Unknown target type [{target}]") def galaxy_url(self): return self.job_wrapper.get_destination_configuration("galaxy_infrastructure_url") diff --git a/test/integration/test_job_environments.py b/test/integration/test_job_environments.py index c1e41ed0d04..1a0bdca4384 100644 --- a/test/integration/test_job_environments.py +++ b/test/integration/test_job_environments.py @@ -15,6 +15,7 @@ SIMPLE_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "simple_job_conf.xml") IO_INJECTION_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "io_injection_job_conf.yml") SETS_TMP_DIR_TO_TRUE_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "sets_tmp_dir_to_true_job_conf.xml") SETS_TMP_DIR_AS_EXPRESSION_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "sets_tmp_dir_expression_job_conf.xml") +EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_job_conf.xml") JobEnvironmentProperties = collections.namedtuple("JobEnvironmentProperties", [ "user_id", @@ -115,6 +116,35 @@ class DefaultJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTest assert job_env.home == os.path.join(job_directory, "home"), job_env.home +class EmbeddedPulsarDefaultJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase): + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.jobs_directory = tempfile.mkdtemp() + config["jobs_directory"] = cls.jobs_directory + config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE + + @skip_without_tool("job_environment_default") + def test_default_environment_1801(self): + job_env = self._run_and_get_environment_properties() + + euid = os.geteuid() + egid = os.getgid() + + assert job_env.user_id == str(euid), job_env.user_id + assert job_env.group_id == str(egid), job_env.group_id + assert 'pulsar_staging' in job_env.pwd + assert job_env.pwd.endswith("/working") + # job ran in embedded pulsar, not local job dir + assert self.jobs_directory not in job_env.pwd + + # Newer tools get isolated home directories in job_directory/home + job_directory = os.path.dirname(job_env.pwd) + assert os.path.abspath(job_env.home) == os.path.join(job_directory, "home"), job_env.home + + # Since job_conf doesn't set tmp_dir parameter - temp isn't in job_directory + assert not job_env.tmp.startswith(job_directory) + + class TmpDirToTrueJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase): @classmethod From 533f86287dd726e689f38634aa825946c213de09 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 23 May 2021 12:26:34 +0200 Subject: [PATCH 6/8] add GALAXY_MEMORY_MB* to passt hrough --- lib/galaxy/tool_util/deps/dependencies.py | 2 +- lib/galaxy/tool_util/parser/interface.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py index 2d362f8cbc1..760eac66239 100644 --- a/lib/galaxy/tool_util/deps/dependencies.py +++ b/lib/galaxy/tool_util/deps/dependencies.py @@ -42,7 +42,7 @@ class ToolInfo: def __init__(self, container_descriptions=None, requirements=None, requires_galaxy_python_environment=False, env_pass_through=None, guest_ports=None, tool_id=None, tool_version=None, profile=-1): if env_pass_through is None: - env_pass_through = ["GALAXY_SLOTS"] + env_pass_through = ["GALAXY_SLOTS", "GALAXY_MEMORY_MB", "GALAXY_MEMORY_MB_PER_SLOT"] if container_descriptions is None: container_descriptions = [] if requirements is None: diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 8c549f3aa4a..96c974dd2d2 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -110,7 +110,8 @@ class ToolSource(metaclass=ABCMeta): return ["TMPDIR", "TMP", "TEMP"] def parse_docker_env_pass_through(self): - return ["GALAXY_SLOTS", "HOME", "_GALAXY_JOB_HOME_DIR", "_GALAXY_JOB_TMP_DIR"] + self.parse_tmp_directory_vars() + return ["GALAXY_SLOTS", "GALAXY_MEMORY_MB", "GALAXY_MEMORY_MB_PER_SLOT", "HOME", + "_GALAXY_JOB_HOME_DIR", "_GALAXY_JOB_TMP_DIR"] + self.parse_tmp_directory_vars() @abstractmethod def parse_interpreter(self): From fa6a91b1a378fee7ae2435d73523bd4539b38d1c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 24 May 2021 18:52:41 +0200 Subject: [PATCH 7/8] Bump up pulsar-galaxy-lib --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 4eb1b2c89a6..15cbbc5001e 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.5 +pulsar-galaxy-lib==0.14.6 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index e4a6769996d..9b18b2f30ae 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.5 +pulsar-galaxy-lib==0.14.6 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" From ea143935aa07ecd5a4c2b6eb370a145aace90322 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 25 May 2021 19:30:49 +0100 Subject: [PATCH 8/8] Add/drop index together with `job_id` column Also fix typo in old migration. --- .../migrate/versions/0123_add_workflow_request_tables.py | 2 +- .../model/migrate/versions/0173_add_job_id_to_dataset.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py b/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py index 9120e04b510..2c470ec18a6 100644 --- a/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py +++ b/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py @@ -95,7 +95,7 @@ def upgrade(migrate_engine): WorkflowUUID_column = Column("uuid", UUIDType, nullable=True) add_column(History_column, "workflow_invocation", metadata) add_column(State_column, "workflow_invocation", metadata) - add_column(SchedulerId_column, "workflow_invocation", metadata, index_nane="id_workflow_invocation_scheduler") + add_column(SchedulerId_column, "workflow_invocation", metadata, index_name="id_workflow_invocation_scheduler") add_column(HandlerId_column, "workflow_invocation", metadata, index_name="id_workflow_invocation_handler") add_column(WorkflowUUID_column, "workflow_invocation", metadata) diff --git a/lib/galaxy/model/migrate/versions/0173_add_job_id_to_dataset.py b/lib/galaxy/model/migrate/versions/0173_add_job_id_to_dataset.py index 1333758bc6d..50d0958be99 100644 --- a/lib/galaxy/model/migrate/versions/0173_add_job_id_to_dataset.py +++ b/lib/galaxy/model/migrate/versions/0173_add_job_id_to_dataset.py @@ -14,9 +14,7 @@ from sqlalchemy import ( from galaxy.model.migrate.versions.util import ( add_column, - add_index, drop_column, - drop_index, ) log = logging.getLogger(__name__) @@ -28,9 +26,8 @@ def upgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - job_id_column = Column('job_id', Integer, ForeignKey('job.id')) - add_column(job_id_column, 'dataset', metadata) - add_index('ix_dataset_job_id', 'dataset', 'job_id', metadata) + job_id_column = Column('job_id', Integer, ForeignKey('job.id'), index=True) + add_column(job_id_column, 'dataset', metadata, index_name='ix_dataset_job_id') def downgrade(migrate_engine): @@ -38,4 +35,3 @@ def downgrade(migrate_engine): metadata.reflect() drop_column('job_id', 'dataset', metadata) - drop_index('ix_dataset_job_id', 'dataset', 'job_id', metadata)