diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index a3cfe3653e5..45e263a3eed 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -34,22 +34,18 @@ def timenow(): def pruneLD(plinktasks=[], cd='./', vclbase=[]): """ """ - fd, plog = tempfile.mkstemp() - os.close(fd) - alog = [] - alog.append('## Rgenetics: http://rgenetics.org Galaxy Tools rgQC.py Plink pruneLD runner\n') - for task in plinktasks: # each is a list - vcl = vclbase + task - with open(plog, 'w') as sto: - subprocess.check_call(vcl, stdout=sto, stderr=sto, cwd=cd) - try: - lplog = open(plog).readlines() - lplog = [elem for elem in lplog if elem.find('Pruning SNP') == -1] - alog += lplog - alog.append('\n') - os.unlink(plog) # no longer needed - except Exception: - alog.append('### {} Strange - no std out from plink when running command line\n{}\n'.format(timenow(), ' '.join(vcl))) + alog = ['## Rgenetics: http://rgenetics.org Galaxy Tools rgQC.py Plink pruneLD runner\n'] + with tempfile.NamedTemporaryFile(mode='r+') as plog: + for task in plinktasks: # each is a list + vcl = vclbase + task + subprocess.check_call(vcl, stdout=plog, stderr=plog, cwd=cd) + try: + plog.seek(0) + lplog = [elem for elem in plog.readlines() if elem.find('Pruning SNP') == -1] + alog += lplog + alog.append('\n') + except Exception: + alog.append('### {} Strange - no std out from plink when running command line\n{}\n'.format(timenow(), ' '.join(vcl))) return alog diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 7e5452613a2..0eb3c2b8f67 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -50,8 +50,8 @@ def stream_url_to_file(path, file_sources=None): prefix = "url_paste" if file_sources and file_sources.looks_like_uri(path): file_source_path = file_sources.get_file_source_path(path) - _, temp_name = tempfile.mkstemp(prefix=prefix) - os.close(_) + with tempfile.NamedTemporaryFile(prefix=prefix, delete=False) as temp: + temp_name = temp.name file_source_path.file_source.realize_to(file_source_path.path, temp_name) return temp_name else: @@ -118,11 +118,10 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", Converts in place a file from universal line endings to Posix line endings. """ - fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) i = 0 NEWLINE_BYTE = 10 CR_BYTE = 13 - with open(fd, mode="wb") as fp, open(fname, mode="rb") as fi: + with tempfile.NamedTemporaryFile(mode='wb', prefix=tmp_prefix, dir=tmp_dir, delete=False) as fp, open(fname, mode='rb') as fi: last_char = None block = fi.read(block_size) last_block = b"" @@ -144,11 +143,11 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", i += 1 fp.write(b"\n") if in_place: - shutil.move(temp_name, fname) + shutil.move(fp.name, fname) # Return number of lines in file. return (i, None) else: - return (i, temp_name) + return (i, fp.name) def convert_newlines_sep2tabs(fname, in_place=True, patt=br"[^\S\n]+", tmp_dir=None, tmp_prefix="gxupload"): @@ -678,21 +677,20 @@ def handle_compressed_file( keep_compressed = getattr(datatype, 'compressed', False) # don't waste time decompressing if we sniff invalid contents if is_compressed and is_valid and auto_decompress and not keep_compressed: - fd, uncompressed = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) - compressed_file = DECOMPRESSION_FUNCTIONS[compressed_type](filename) - # TODO: it'd be ideal to convert to posix newlines and space-to-tab here as well - while True: - try: - chunk = compressed_file.read(CHUNK_SIZE) - except OSError as e: - os.close(fd) - os.remove(uncompressed) - compressed_file.close() - raise OSError('Problem uncompressing {} data, please try retrieving the data uncompressed: {}'.format(compressed_type, util.unicodify(e))) - if not chunk: - break - os.write(fd, chunk) - os.close(fd) + with tempfile.NamedTemporaryFile(prefix=tmp_prefix, dir=tmp_dir, delete=False) as uncompressed: + compressed_file = DECOMPRESSION_FUNCTIONS[compressed_type](filename) + # TODO: it'd be ideal to convert to posix newlines and space-to-tab here as well + while True: + try: + chunk = compressed_file.read(CHUNK_SIZE) + except OSError as e: + os.remove(uncompressed.name) + compressed_file.close() + raise OSError('Problem uncompressing {} data, please try retrieving the data uncompressed: {}'.format(compressed_type, util.unicodify(e))) + if not chunk: + break + uncompressed.write(chunk) + uncompressed = uncompressed.name compressed_file.close() if in_place: # Replace the compressed file with the uncompressed file diff --git a/lib/galaxy/datatypes/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py index 78df4cfc059..9de26d9e9a4 100644 --- a/lib/galaxy/datatypes/util/maf_utilities.py +++ b/lib/galaxy/datatypes/util/maf_utilities.py @@ -353,12 +353,11 @@ def build_maf_index_species_chromosomes(filename, index_species=None): # builds and returns ( index, index_filename ) for specified maf_file def build_maf_index(maf_file, species=None): - indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species) + indexes, *_ = build_maf_index_species_chromosomes(maf_file, species) if indexes is not None: - fd, index_filename = tempfile.mkstemp() - with os.fdopen(fd, 'w') as out: - indexes.write(out) - return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename) + with tempfile.NamedTemporaryFile(mode='w', delete=False) as index: + indexes.write(index) + return (bx.align.maf.Indexed(maf_file, index_filename=index.name, keep_open=True, parse_e_rows=False), index.name) return (None, None) diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py index eace788a900..4731f14e1a3 100644 --- a/lib/galaxy/metadata/__init__.py +++ b/lib/galaxy/metadata/__init__.py @@ -373,11 +373,9 @@ class JobExternalOutputMetadataWrapper(MetadataCollectionStrategy): assert not use_bin if include_command: # return command required to build - fd, fp = tempfile.mkstemp(suffix='.py', dir=tmp_dir, prefix="set_metadata_") - metadata_script_file = abspath(fp) - with os.fdopen(fd, 'w') as f: - f.write(SET_METADATA_SCRIPT) - return 'python "{}" {}'.format(metadata_path_on_compute(metadata_script_file), args) + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', dir=tmp_dir, prefix="set_metadata_", delete=False) as temp: + temp.write(SET_METADATA_SCRIPT) + return 'python "{}" {}'.format(metadata_path_on_compute(temp.name), args) else: # return args to galaxy_ext.metadata.set_metadata required to build return args diff --git a/lib/galaxy/tool_util/deps/conda_util.py b/lib/galaxy/tool_util/deps/conda_util.py index b6ddde2fe06..573cc43f5f8 100644 --- a/lib/galaxy/tool_util/deps/conda_util.py +++ b/lib/galaxy/tool_util/deps/conda_util.py @@ -428,8 +428,8 @@ def hash_conda_packages(conda_packages, conda_target=None): # shell makes sense for planemo, in Galaxy this should just execute # these commands as Python def install_conda(conda_context, force_conda_build=False): - f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install") - os.close(f) + with tempfile.NamedTemporaryFile(suffix=".sh", prefix="conda_install", delete=False) as temp: + script_path = temp.name download_cmd = commands.download_command(conda_link(), to=script_path) install_cmd = ['bash', script_path, '-b', '-p', conda_context.conda_prefix] package_targets = [ diff --git a/lib/galaxy/tool_util/locations/__init__.py b/lib/galaxy/tool_util/locations/__init__.py index d89e5b402ce..b3e0d9534e5 100644 --- a/lib/galaxy/tool_util/locations/__init__.py +++ b/lib/galaxy/tool_util/locations/__init__.py @@ -1,4 +1,3 @@ -import os import tempfile from abc import ( ABCMeta, @@ -20,6 +19,5 @@ class ToolLocationResolver(metaclass=ABCMeta): def _temp_path(self, uri_like): """Create an abstraction for this so we can configure and cache later.""" - handle, filename = tempfile.mkstemp(suffix=uri_like.split("/")[-1]) - os.close(handle) - return filename + with tempfile.NamedTemporaryFile(suffix=uri_like.split("/")[-1], delete=False) as temp: + return temp.name diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 65ea371acaa..1d814cf7017 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -176,27 +176,25 @@ def verify( def make_temp_fname(fname=None): """Safe temp name - preserve the file extension for tools that interpret it.""" suffix = os.path.split(fname)[-1] # ignore full path - fd, temp_prefix = tempfile.mkstemp(prefix='tmp', suffix=suffix) - os.close(fd) - return temp_prefix + with tempfile.NamedTemporaryFile(prefix='tmp', suffix=suffix, delete=False) as temp: + return temp.name def _bam_to_sam(local_name, temp_name): temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_') - fd, temp_temp = tempfile.mkstemp(suffix='.sam', prefix='history_bam_converted_to_sam_') - os.close(fd) - try: - pysam.view('-h', '-o%s' % temp_local.name, local_name) - except Exception as e: - msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e) - raise Exception(msg) - try: - pysam.view('-h', '-o%s' % temp_temp, temp_name) - except Exception as e: - msg = "Converting history BAM to SAM failed: %s" % unicodify(e) - raise Exception(msg) + with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp: + try: + pysam.view('-h', '-o%s' % temp_local.name, local_name) + except Exception as e: + msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e) + raise Exception(msg) + try: + pysam.view('-h', '-o%s' % temp.name, temp_name) + except Exception as e: + msg = "Converting history BAM to SAM failed: %s" % unicodify(e) + raise Exception(msg) os.remove(temp_name) - return temp_local, temp_temp + return temp_local, temp.name def _verify_checksum(data, checksum_type, expected_checksum_value): diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 45f21a22b6b..0f6bc035eae 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -518,8 +518,8 @@ class ToolEvaluator: directory = os.path.join(self.local_working_directory, "working") config_filename = os.path.join(directory, filename) else: - fd, config_filename = tempfile.mkstemp(dir=directory) - os.close(fd) + with tempfile.NamedTemporaryFile(dir=directory, delete=False) as temp: + config_filename = temp.name self.__write_workdir_file(config_filename, config_text, param_dict, is_template=is_template) self.__register_extra_file(name, config_filename) config_filenames.append(config_filename) @@ -542,8 +542,8 @@ class ToolEvaluator: is_template = False else: is_template = True - fd, config_filename = tempfile.mkstemp(dir=directory) - os.close(fd) + with tempfile.NamedTemporaryFile(dir=directory, delete=False) as temp: + config_filename = temp.name self.__write_workdir_file(config_filename, environment_variable_template, param_dict, is_template=is_template, strip=environment_variable_def.get("strip", False)) config_file_basename = os.path.basename(config_filename) # environment setup in job file template happens before `cd $working_directory` @@ -571,17 +571,15 @@ class ToolEvaluator: directory = self.local_working_directory command = self.tool.command if self.tool.profile < 16.04 and command and "$param_file" in command: - fd, param_filename = tempfile.mkstemp(dir=directory) - os.close(fd) - with open(param_filename, "w") as f: + with tempfile.NamedTemporaryFile(mode='w', dir=directory, delete=False) as param: for key, value in param_dict.items(): # parameters can be strings or lists of strings, coerce to list if not isinstance(value, list): value = [value] for elem in value: - f.write(f'{key}={elem}\n') - self.__register_extra_file('param_file', param_filename) - return param_filename + param.write(f'{key}={elem}\n') + self.__register_extra_file('param_file', param.name) + return param.name else: return None diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 093391c8184..c018d57b177 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -8,11 +8,9 @@ import os import random import re import sys -import tempfile from contextlib import contextmanager from json import loads -import packaging.version import pysam from bx.bbi.bigbed_file import BigBedFile from bx.bbi.bigwig_file import BigWigFile @@ -30,7 +28,6 @@ from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and # pysam 0.16.0.1 emits logs containing the word 'Error', this can confuse the stdout/stderr checkers. # Can be be removed once https://github.com/pysam-developers/pysam/issues/939 is resolved. pysam.set_verbosity(0) -PYSAM_INDEX_SYMLINK_NECESSARY = packaging.version.parse(pysam.__version__) <= packaging.version.parse('0.13.0') def float_nan(n): @@ -327,17 +324,9 @@ class TabixDataProvider(GenomeDataProvider, FilterableMixin): def open_data_file(self): # We create a symlink to the index file. This is # required until https://github.com/pysam-developers/pysam/pull/586 is merged. - if PYSAM_INDEX_SYMLINK_NECESSARY: - fd, index_path = tempfile.mkstemp(suffix='.tbi') - os.close(fd) - os.unlink(index_path) - os.symlink(self.converted_dataset.file_name, index_path) - else: - index_path = self.converted_dataset.file_name + index_path = self.converted_dataset.file_name with pysam.TabixFile(self.dependencies['bgzip'].file_name, index=index_path) as f: yield f - if PYSAM_INDEX_SYMLINK_NECESSARY: - os.unlink(index_path) def get_iterator(self, data_file, chrom, start, end, **kwargs): # chrom must be a string, start/end integers. diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 1f22df35e2f..78fdc33c65c 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -130,24 +130,23 @@ def get_upload_point(repository, **kwd): def handle_bz2(repository, uploaded_file_name): - fd, uncompressed = tempfile.mkstemp(prefix='repo_%d_upload_bunzip2_' % repository.id, - dir=os.path.dirname(uploaded_file_name), - text=False) - bzipped_file = bz2.BZ2File(uploaded_file_name, 'rb') - while 1: - try: - chunk = bzipped_file.read(basic_util.CHUNK_SIZE) - except OSError: - os.close(fd) - os.remove(uncompressed) - log.exception('Problem uncompressing bz2 data "%s"', uploaded_file_name) - return - if not chunk: - break - os.write(fd, chunk) - os.close(fd) - bzipped_file.close() - shutil.move(uncompressed, uploaded_file_name) + with tempfile.NamedTemporaryFile( + mode='wb', + prefix=f'repo_{repository.id}_upload_bunzip2_', + dir=os.path.dirname(uploaded_file_name), + delete=False, + ) as uncompressed, bz2.BZ2File(uploaded_file_name, 'rb') as bzipped_file: + while 1: + try: + chunk = bzipped_file.read(basic_util.CHUNK_SIZE) + except OSError: + os.remove(uncompressed.name) + log.exception(f'Problem uncompressing bz2 data "{uploaded_file_name}"') + return + if not chunk: + break + uncompressed.write(chunk) + shutil.move(uncompressed.name, uploaded_file_name) def handle_directory_changes(app, host, username, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, @@ -230,24 +229,23 @@ def handle_directory_changes(app, host, username, repository, full_path, filenam def handle_gzip(repository, uploaded_file_name): - fd, uncompressed = tempfile.mkstemp(prefix='repo_%d_upload_gunzip_' % repository.id, - dir=os.path.dirname(uploaded_file_name), - text=False) - gzipped_file = gzip.GzipFile(uploaded_file_name, 'rb') - while 1: - try: - chunk = gzipped_file.read(basic_util.CHUNK_SIZE) - except OSError: - os.close(fd) - os.remove(uncompressed) - log.exception('Problem uncompressing gz data "%s"', uploaded_file_name) - return - if not chunk: - break - os.write(fd, chunk) - os.close(fd) - gzipped_file.close() - shutil.move(uncompressed, uploaded_file_name) + with tempfile.NamedTemporaryFile( + mode='wb', + prefix=f'repo_{repository.id}_upload_gunzip_', + dir=os.path.dirname(uploaded_file_name), + delete=False + ) as uncompressed, gzip.GzipFile(uploaded_file_name, 'rb') as gzipped_file: + while 1: + try: + chunk = gzipped_file.read(basic_util.CHUNK_SIZE) + except OSError: + os.remove(uncompressed.name) + log.exception(f'Problem uncompressing gz data "{uploaded_file_name}"') + return + if not chunk: + break + uncompressed.write(chunk) + shutil.move(uncompressed.name, uploaded_file_name) def uncompress(repository, uploaded_file_name, uploaded_file_filename, isgzip=False, isbz2=False): diff --git a/lib/tool_shed/webapp/controllers/upload.py b/lib/tool_shed/webapp/controllers/upload.py index 5cdc1f5fac7..80035cb24b2 100644 --- a/lib/tool_shed/webapp/controllers/upload.py +++ b/lib/tool_shed/webapp/controllers/upload.py @@ -79,13 +79,12 @@ class UploadController(BaseUIController): status = 'error' uploaded_file = None if valid_url: - fd, uploaded_file_name = tempfile.mkstemp() - os.close(fd) - uploaded_file = open(uploaded_file_name, 'wb') - for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): - if chunk: - uploaded_file.write(chunk) - uploaded_file.flush() + with tempfile.NamedTemporaryFile(mode='wb', delete=False) as uploaded_file: + uploaded_file_name = uploaded_file.name + for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): + if chunk: + uploaded_file.write(chunk) + uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 elif file_data not in ('', None): diff --git a/test/functional/tools/directory_uri_copy_to.py b/test/functional/tools/directory_uri_copy_to.py index fd2b2d9426c..487a8a6c1f6 100644 --- a/test/functional/tools/directory_uri_copy_to.py +++ b/test/functional/tools/directory_uri_copy_to.py @@ -28,10 +28,9 @@ def main(argv=None): target_uri = directory_uri + "/helloworld" file_source_path = file_sources.get_file_source_path(target_uri) file_source = file_source_path.file_source - fd, temp_name = tempfile.mkstemp() - with open(fd, 'w') as f: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: f.write('hello world!\n') - file_source.write_from(file_source_path.path, temp_name) + file_source.write_from(file_source_path.path, f.name) def _parser(): diff --git a/test/integration/test_work_queue_put_failure.py b/test/integration/test_work_queue_put_failure.py index 09f5927d578..7e97d14fefe 100644 --- a/test/integration/test_work_queue_put_failure.py +++ b/test/integration/test_work_queue_put_failure.py @@ -26,10 +26,9 @@ class WorkQueuePutFailureTestCase(integration_util.IntegrationTestCase): @classmethod def handle_galaxy_config_kwds(cls, config, ): # config["jobs_directory"] = cls.jobs_directory - fd, path = tempfile.mkstemp(suffix='job_conf.yml') - with open(path, 'w') as job_conf: + with tempfile.NamedTemporaryFile(mode='w', suffix='job_conf.yml', delete=False) as job_conf: job_conf.write(job_conf_yaml) - config["job_config_file"] = path + config["job_config_file"] = job_conf.name # Disable tool dependency resolution. config["tool_dependency_dir"] = "none" diff --git a/test/integration/test_workflow_handler_configuration.py b/test/integration/test_workflow_handler_configuration.py index 3b59278f2e1..4b59d625841 100644 --- a/test/integration/test_workflow_handler_configuration.py +++ b/test/integration/test_workflow_handler_configuration.py @@ -81,13 +81,11 @@ steps: def config_file(template, assign_with=''): - fd, path = tempfile.mkstemp(suffix=".xml", prefix="workflow_handler_config_") - os.close(fd) - with open(path, 'w') as config: + with tempfile.NamedTemporaryFile(mode='w', suffix=".xml", prefix="workflow_handler_config_", delete=False) as config: if assign_with: assign_with = f'assign_with="{assign_with}"' config.write(template.substitute(assign_with=assign_with)) - return path + return config.name class BaseWorkflowHandlerConfigurationTestCase(integration_util.IntegrationTestCase): diff --git a/test/unit/data/datatypes/util.py b/test/unit/data/datatypes/util.py index 47d5be832b9..9c13b98e3be 100644 --- a/test/unit/data/datatypes/util.py +++ b/test/unit/data/datatypes/util.py @@ -24,14 +24,10 @@ def get_dataset(filename, index_attr='bam_index', dataset_id=1, has_data=True): @contextmanager def get_tmp_path(should_exist=False, suffix=''): - _, path = tempfile.mkstemp(suffix=suffix) - if not should_exist: - os.remove(path) - yield path - try: - os.remove(path) - except Exception: - pass + with tempfile.NamedTemporaryFile(suffix=suffix) as temp: + if not should_exist: + os.remove(temp.name) + yield temp.name @contextmanager diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 18d892137ee..fd5dfb41fe8 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -62,11 +62,9 @@ def user_context_fixture(user_ftp_dir=None): def assert_realizes_as(file_sources, uri, expected, user_context=None): file_source_path = file_sources.get_file_source_path(uri) - _, temp_name = tempfile.mkstemp() - file_source_path.file_source.realize_to(file_source_path.path, temp_name, user_context=user_context) - try: - with open(temp_name) as f: - realized_contents = f.read() + with tempfile.NamedTemporaryFile(mode='r') as temp: + file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) + realized_contents = temp.read() if realized_contents != expected: message = "Expected to realize contents at [{}] as [{}], instead found [{}]".format( uri, @@ -74,14 +72,10 @@ def assert_realizes_as(file_sources, uri, expected, user_context=None): realized_contents, ) raise AssertionError(message) - finally: - os.remove(temp_name) - return temp_name def write_from(file_sources, uri, content, user_context=None): file_source_path = file_sources.get_file_source_path(uri) - fd, temp_name = tempfile.mkstemp() - with open(fd, 'w') as f: + with tempfile.NamedTemporaryFile(mode='w') as f: f.write(content) - file_source_path.file_source.write_from(file_source_path.path, temp_name, user_context=user_context) + file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context) diff --git a/test/unit/queue_worker/conftest.py b/test/unit/queue_worker/conftest.py index 2e867eec1a4..725724463d2 100644 --- a/test/unit/queue_worker/conftest.py +++ b/test/unit/queue_worker/conftest.py @@ -22,10 +22,8 @@ def create_base_test(connection, amqp_type, amqp_connection=None): @pytest.fixture() def sqlite_connection(request): - fd, path = tempfile.mkstemp() - os.close(fd) - request.addfinalizer(lambda: os.remove(path)) - return 'sqlite:////%s' % path + with tempfile.NamedTemporaryFile() as temp: + yield f'sqlite:////{temp.name}' @pytest.fixture() diff --git a/test/unit/test_verify.py b/test/unit/test_verify.py index 2416f310737..e51011ca604 100644 --- a/test/unit/test_verify.py +++ b/test/unit/test_verify.py @@ -1,5 +1,4 @@ import collections -import os import tempfile import pytest @@ -24,10 +23,9 @@ TestFile = collections.namedtuple('TestFile', 'value path') def test_file_list(): files = [] for b, ext in [(F1, '.txt'), (F2, '.txt'), (F3, '.pdf'), (F4, '.txt'), (MULTILINE_MATCH, '.txt')]: - fd, path = tempfile.mkstemp(suffix=ext) - with os.fdopen(fd, 'wb') as out: + with tempfile.NamedTemporaryFile(mode='wb', suffix=ext, delete=False) as out: out.write(b) - files.append(TestFile(b, path)) + files.append(TestFile(b, out.name)) return files diff --git a/test/unit/tools/test_history_imp_exp.py b/test/unit/tools/test_history_imp_exp.py index 13d92fb70ce..070db91aca6 100644 --- a/test/unit/tools/test_history_imp_exp.py +++ b/test/unit/tools/test_history_imp_exp.py @@ -1,11 +1,8 @@ import json import os import tarfile +import tempfile from shutil import rmtree -from tempfile import ( - mkdtemp, - mkstemp, -) from galaxy import model from galaxy.exceptions import MalformedContents @@ -53,31 +50,32 @@ def _mock_app(store_by="id"): def _run_jihaw_cleanup_check_secure(history_archive, msg): malformed = False try: - app, _ = _run_jihaw_cleanup(history_archive.arc_directory) + _run_jihaw_cleanup(history_archive.arc_directory) except MalformedContents: malformed = True assert malformed def test_create_archive(): - tempdir = mkdtemp() - _, out_file = mkstemp() - dataset = os.path.join(tempdir, 'datasets/Pasted_Entry_1.txt') - history_attrs_file = os.path.join(tempdir, 'history_attrs.txt') - datasets_attrs_file = os.path.join(tempdir, 'datasets_attrs.txt') - jobs_attrs_file = os.path.join(tempdir, 'jobs_attrs.txt') - os.makedirs(os.path.join(tempdir, 'datasets')) - with open(dataset, 'w') as out: - out.write('Hello\n') - with open(history_attrs_file, 'w') as out: - out.write(HISTORY_ATTRS) - with open(datasets_attrs_file, 'w') as out: - out.write(DATASETS_ATTRS_EXPORT.format(file_name=dataset)) - with open(jobs_attrs_file, 'w') as out: - out.write(JOBS_ATTRS) - create_archive(tempdir, out_file, gzip=True) - with tarfile.open(out_file) as t: - assert sorted(t.getnames()) == sorted(['datasets', 'datasets/Pasted_Entry_1.txt', 'history_attrs.txt', 'datasets_attrs.txt', 'jobs_attrs.txt']), t.getnames() + tempdir = tempfile.mkdtemp() + with tempfile.NamedTemporaryFile() as temp: + out_file = temp.name + dataset = os.path.join(tempdir, 'datasets/Pasted_Entry_1.txt') + history_attrs_file = os.path.join(tempdir, 'history_attrs.txt') + datasets_attrs_file = os.path.join(tempdir, 'datasets_attrs.txt') + jobs_attrs_file = os.path.join(tempdir, 'jobs_attrs.txt') + os.makedirs(os.path.join(tempdir, 'datasets')) + with open(dataset, 'w') as out: + out.write('Hello\n') + with open(history_attrs_file, 'w') as out: + out.write(HISTORY_ATTRS) + with open(datasets_attrs_file, 'w') as out: + out.write(DATASETS_ATTRS_EXPORT.format(file_name=dataset)) + with open(jobs_attrs_file, 'w') as out: + out.write(JOBS_ATTRS) + create_archive(tempdir, out_file, gzip=True) + with tarfile.open(out_file) as t: + assert sorted(t.getnames()) == sorted(['datasets', 'datasets/Pasted_Entry_1.txt', 'history_attrs.txt', 'datasets_attrs.txt', 'jobs_attrs.txt']), t.getnames() def test_history_import_symlink(): @@ -620,7 +618,7 @@ def _setup_history_for_export(history_name): def _import_export(app, h, dest_export=None): if dest_export is None: - dest_parent = mkdtemp() + dest_parent = tempfile.mkdtemp() dest_export = os.path.join(dest_parent, "moo.tgz") dataset = model.Dataset(id=100) @@ -681,7 +679,7 @@ def test_import_1901_default(): def import_archive(archive_path, app=None): - dest_parent = mkdtemp() + dest_parent = tempfile.mkdtemp() dest_dir = os.path.join(dest_parent, 'dest') options = Dummy() @@ -715,7 +713,7 @@ def test_history_import_relpath_in_archive(): """ Ensure that a history import archive cannot reference a relative path outside the archive """ - dest_parent = mkdtemp() + dest_parent = tempfile.mkdtemp() with HistoryArchive(arcname_prefix='../insecure') as history_archive: history_archive.write_metafiles() @@ -728,7 +726,7 @@ def test_history_import_abspath_in_archive(): """ Ensure that a history import archive cannot reference a absolute path outside the archive """ - dest_parent = mkdtemp() + dest_parent = tempfile.mkdtemp() arcname_prefix = os.path.abspath(os.path.join(dest_parent, 'insecure')) with HistoryArchive(arcname_prefix=arcname_prefix) as history_archive: @@ -740,7 +738,7 @@ def test_history_import_abspath_in_archive(): class HistoryArchive: def __init__(self, arcname_prefix=None): - self.temp_directory = mkdtemp() + self.temp_directory = tempfile.mkdtemp() self.arc_directory = os.path.join(self.temp_directory, 'archive') self.arcname_prefix = arcname_prefix self.tar_file_path = os.path.join(self.temp_directory, 'archive.tar.gz') diff --git a/test/unit/util/test_utils.py b/test/unit/util/test_utils.py index 6248817f0c7..08ffd0fc04a 100644 --- a/test/unit/util/test_utils.py +++ b/test/unit/util/test_utils.py @@ -1,5 +1,4 @@ import errno -import os import tempfile import pytest @@ -76,9 +75,8 @@ def test_xml_to_string_pretty(): def test_parse_xml_enoent(): - fd, path = tempfile.mkstemp() - os.close(fd) - os.remove(path) + with tempfile.NamedTemporaryFile() as temp: + path = temp.name with pytest.raises(IOError) as excinfo: util.parse_xml(path) assert excinfo.value.errno == errno.ENOENT diff --git a/tools/extract/liftOver_wrapper.py b/tools/extract/liftOver_wrapper.py index b025742dd20..6a59ddc6822 100644 --- a/tools/extract/liftOver_wrapper.py +++ b/tools/extract/liftOver_wrapper.py @@ -25,14 +25,12 @@ def safe_bed_file(infile): https://lists.soe.ucsc.edu/pipermail/genome/2007-May/013561.html """ fix_pat = re.compile("^(track|browser)") - fd, fname = tempfile.mkstemp() - os.close(fd) - with open(infile) as in_handle, open(fname, 'w') as out_handle: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as out_handle, open(infile, 'w') as in_handle: for line in in_handle: if fix_pat.match(line): line = "#" + line out_handle.write(line) - return fname + return out_handle.name if len(sys.argv) < 9: