diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 14bd2a7b539..c79d1f9c8c6 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -221,6 +221,11 @@ class InvalidFileFormatError(MessageException): err_code = error_codes_by_name['INVALID_FILE_FORMAT'] +class ReferenceDataError(MessageException): + status_code = 500 + err_code = error_codes_by_name['REFERENCE_DATA_ERROR'] + + # non-web exceptions class ContainerCLIError(Exception): diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 629d799ea5d..4a746065df1 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -159,6 +159,11 @@ "code": 500005, "message": "File format not supported for this operation." }, + { + "name": "REFERENCE_DATA_ERROR", + "code": 500006, + "message": "Reference data required for program execution failed to load." + }, { "name": "NOT_IMPLEMENTED", "code": 501001, diff --git a/lib/galaxy/managers/genomes.py b/lib/galaxy/managers/genomes.py new file mode 100644 index 00000000000..f0970982756 --- /dev/null +++ b/lib/galaxy/managers/genomes.py @@ -0,0 +1,74 @@ +from typing import ( + Any, + List, +) + +from galaxy import model as m +from galaxy.exceptions import ( + ReferenceDataError, + RequestParameterInvalidException, +) +from galaxy.managers.context import ProvidesUserContext +from galaxy.structured_app import StructuredApp + + +class GenomesManager: + + def __init__(self, app: StructuredApp): + self._app = app + self.genomes = app.genomes + + def get_dbkeys(self, user: m.User, chrom_info: bool) -> List[List[str]]: + return self.genomes.get_dbkeys(user, chrom_info) + + def get_genome( + self, + trans: ProvidesUserContext, + id: str, + num: int, + chrom: str, + low: int, + high: int, + reference: bool + ) -> Any: + if reference: + region = self.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high) + return {'dataset_type': 'refseq', 'data': region.sequence} + else: + return self.genomes.chroms(trans, dbkey=id, num=num, chrom=chrom, low=low) + + def get_sequence( + self, + trans: ProvidesUserContext, + id: str, + chrom: str, + low: int, + high: int + ) -> Any: + region = self.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high) + return region.sequence + + def get_indexes(self, id: str, index_type: str) -> Any: + index_extensions = {'fasta_indexes': '.fai'} + if index_type not in index_extensions: + raise RequestParameterInvalidException(f'Invalid index type: {index_type}') + + tbl_entries = self._app.tool_data_tables.data_tables[index_type].data + ext = index_extensions[index_type] + index_filename = self._get_index_filename(id, tbl_entries, ext, index_type) + try: + with open(index_filename, mode='r') as f: + return f.read() + except OSError: + raise ReferenceDataError(f'Failed to load index file for {id}') + + def _get_index_filename(self, id, tbl_entries, ext, index_type): + try: + paths = [x[-1] for x in tbl_entries if id in x] + file_name = paths.pop() + except TypeError: + raise ReferenceDataError('Data tables not found for {index_type}') + except IndexError: + raise ReferenceDataError('Data tables not found for {index_type} for {id}') + else: + return f"{file_name}{ext}" diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index ed1b8cd30d1..8c8e265cb39 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -6,6 +6,10 @@ from json import loads from bx.seq.twobit import TwoBitFile +from galaxy.exceptions import ( + ObjectNotFound, + ReferenceDataError, +) from galaxy.util.bunch import Bunch log = logging.getLogger(__name__) @@ -86,6 +90,9 @@ class Genome: """ Returns representation of self as a dictionary. """ + # if there's no len_file, there's nothing to return + if not self.len_file: + raise ReferenceDataError(f'len_file not set for {self.key}') def check_int(s): if s.isdigit(): @@ -117,69 +124,70 @@ class Genome: # (b) whether there are previous, next chroms; # (c) index of start chrom. # - len_file_enumerate = enumerate(open(self.len_file)) - chroms = {} - prev_chroms = False - start_index = 0 - if chrom: - # Use starting chrom to start list. - found = False - count = 0 - for line_num, line in len_file_enumerate: - if line.startswith("#"): - continue - name, len = line.split("\t") - if found: - chroms[name] = int(len) - count += 1 - elif name == chrom: - # Found starting chrom. - chroms[name] = int(len) - count += 1 - found = True - start_index = line_num - if line_num != 0: - prev_chroms = True - if count >= num: - break - else: - # Use low to start list. - high = low + int(num) - prev_chroms = (low != 0) - start_index = low + with open(self.len_file) as f: + len_file_enumerate = enumerate(f) + chroms = {} + prev_chroms = False + start_index = 0 + if chrom: + # Use starting chrom to start list. + found = False + count = 0 + for line_num, line in len_file_enumerate: + if line.startswith("#"): + continue + name, len = line.split("\t") + if found: + chroms[name] = int(len) + count += 1 + elif name == chrom: + # Found starting chrom. + chroms[name] = int(len) + count += 1 + found = True + start_index = line_num + if line_num != 0: + prev_chroms = True + if count >= num: + break + else: + # Use low to start list. + high = low + int(num) + prev_chroms = (low != 0) + start_index = low - # Read chrom data from len file. - for line_num, line in len_file_enumerate: - if line_num < low: - continue - if line_num >= high: - break - if line.startswith("#"): - continue - # LEN files have format: - # - fields = line.split("\t") - chroms[fields[0]] = int(fields[1]) + # Read chrom data from len file. + for line_num, line in len_file_enumerate: + if line_num < low: + continue + if line_num >= high: + break + if line.startswith("#"): + continue + # LEN files have format: + # + fields = line.split("\t") + chroms[fields[0]] = int(fields[1]) - # Set flag to indicate whether there are more chroms after list. - next_chroms = False - try: - next(len_file_enumerate) - next_chroms = True - except StopIteration: - # No more chroms to read. - pass + # Set flag to indicate whether there are more chroms after list. + next_chroms = False + try: + next(len_file_enumerate) + next_chroms = True + except StopIteration: + # No more chroms to read. + pass - to_sort = [{'chrom': chrm, 'len': length} for chrm, length in chroms.items()] - to_sort.sort(key=lambda _: split_by_number(_['chrom'])) - return { - 'id': self.key, - 'reference': self.twobit_file is not None, - 'chrom_info': to_sort, - 'prev_chroms': prev_chroms, - 'next_chroms': next_chroms, - 'start_index': start_index - } + to_sort = [{'chrom': chrm, 'len': length} for chrm, length in chroms.items()] + to_sort.sort(key=lambda _: split_by_number(_['chrom'])) + return { + 'id': self.key, + 'reference': self.twobit_file is not None, + 'chrom_info': to_sort, + 'prev_chroms': prev_chroms, + 'next_chroms': next_chroms, + 'start_index': start_index + } class Genomes: @@ -208,13 +216,15 @@ class Genomes: if twobit_table is None: # Add genome data (twobit files) to genomes, directly from twobit.loc try: - for line in open(os.path.join(self.app.config.tool_data_path, "twobit.loc")): - if line.startswith("#"): - continue - val = line.split() - if len(val) == 2: - key, path = val - twobit_fields[key] = path + twobit_path = os.path.join(self.app.config.tool_data_path, "twobit.loc") + with open(twobit_path) as f: + for line in f: + if line.startswith("#"): + continue + val = line.split() + if len(val) == 2: + key, path = val + twobit_fields[key] = path except OSError: # Thrown if twobit.loc does not exist. log.exception("Error reading twobit.loc") @@ -246,18 +256,15 @@ class Genomes: rval = self.genomes[dbkey] return rval - def get_dbkeys(self, trans, chrom_info=False, **kwd): + def get_dbkeys(self, user, chrom_info=False): """ Returns all known dbkeys. If chrom_info is True, only dbkeys with chromosome lengths are returned. """ self.check_and_reload() dbkeys = [] # Add user's custom keys to dbkeys. - user_keys_dict = {} - user = trans.get_user() - if user: - if 'dbkeys' in user.preferences: - user_keys_dict = loads(user.preferences['dbkeys']) + if user and 'dbkeys' in user.preferences: + user_keys_dict = loads(user.preferences['dbkeys']) dbkeys.extend([(attributes['name'], key) for key, attributes in user_keys_dict.items()]) # Add app keys to dbkeys. @@ -325,14 +332,10 @@ class Genomes: elif dbkey in self.genomes: genome = self.genomes[dbkey] - # Set up return value or log exception if genome not found for key. - rval = None - if genome: - rval = genome.to_dict(num=num, chrom=chrom, low=low) - else: - log.exception('genome not found for key %s', dbkey) + if not genome: + raise ObjectNotFound(f'genome not found for key {dbkey}') - return rval + return genome.to_dict(num=num, chrom=chrom, low=low) def has_reference_data(self, dbkey, dbkey_owner=None): """ @@ -369,7 +372,7 @@ class Genomes: dbkey_user = trans.user if not self.has_reference_data(dbkey, dbkey_user): - return None + raise ReferenceDataError(f"No reference data for {dbkey}") # # Get twobit file with reference data. @@ -389,11 +392,15 @@ class Genomes: twobit_dataset = fasta_dataset.get_converted_dataset(trans, 'twobit') twobit_file_name = twobit_dataset.file_name + return self._get_reference_data(twobit_file_name, chrom, low, high) + + def _get_reference_data(twobit_file_name, chrom, low, high): # Read and return reference data. try: - twobit = TwoBitFile(open(twobit_file_name, 'rb')) - if chrom in twobit: - seq_data = twobit[chrom].get(int(low), int(high)) - return GenomeRegion(chrom=chrom, start=low, end=high, sequence=seq_data) - except OSError: - return None + with open(twobit_file_name, 'rb') as f: + twobit = TwoBitFile(f) + if chrom in twobit: + seq_data = twobit[chrom].get(int(low), int(high)) + return GenomeRegion(chrom=chrom, start=low, end=high, sequence=seq_data) + except OSError as e: + raise e() diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 4665a075b70..02e2669397a 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -77,7 +77,6 @@ def get_user(galaxy_session: Optional[model.GalaxySession] = Depends(get_session def get_trans(app: UniverseApplication = Depends(get_app), user: Optional[User] = Depends(get_user), galaxy_session: Optional[model.GalaxySession] = Depends(get_session), ) -> SessionRequestContext: - app.model.session.expunge_all() return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session) diff --git a/lib/galaxy/webapps/galaxy/api/genomes.py b/lib/galaxy/webapps/galaxy/api/genomes.py index ec8e06e7659..28f738824e4 100644 --- a/lib/galaxy/webapps/galaxy/api/genomes.py +++ b/lib/galaxy/webapps/galaxy/api/genomes.py @@ -1,50 +1,196 @@ -from galaxy import web +from typing import ( + Any, + List, +) + +from fastapi import ( + Depends, + Path, + Query, +) +from fastapi.responses import Response +from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter as APIRouter + +from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.genomes import GenomesManager +from galaxy.structured_app import StructuredApp +from galaxy.web import ( + expose_api_anonymous, + expose_api_raw_anonymous, +) from galaxy.web.framework.helpers import is_true from galaxy.webapps.base.controller import BaseAPIController +from . import ( + get_app, + get_trans, +) + +router = APIRouter(tags=['genomes']) + +IdPathParam: str = Path( + ..., + title='Genome ID', + description='Genome ID' +) + +ChromInfoQueryParam: bool = Query( + None, + title='ChromInfo', + description='If true, return genome keys with chromosome lengths' +) + +NumQueryParam: int = Query( + None, + title='Number', + description='Limits size of returned data', +) + +ChromQueryParam: Any = Query( + None, + title='Chrom', + description='Limits size of returned data', +) + +LowQueryParam: int = Query( + None, + title='Low', + description='Limits size of returned data', +) + +HighQueryParam: int = Query( + None, + title='High', + description='Limits size of returned data', +) + +FormatQueryParam: str = Query( + None, + title='Format', + description='Format' +) + +ReferenceQueryParam: bool = Query( + None, + title='Reference', + description='If true, return reference data' +) + +IndexTypeQueryParam: str = Query( + 'fasta_indexes', # currently this is the only supported index type + title='Index type', + description='Index type' +) + + +def get_genomes_manager(app: StructuredApp = Depends(get_app)) -> GenomesManager: + return GenomesManager(app) def get_id(base, format): if format: return f"{base}.{format}" - else: - return base + return base + + +@cbv(router) +class FastAPIGenomes: + manager: GenomesManager = Depends(get_genomes_manager) + + @router.get( + '/api/genomes', + summary='Return a list of installed genomes', + response_description='Installed genomes' + ) + def index( + self, + trans: ProvidesUserContext = Depends(get_trans), + chrom_info: bool = ChromInfoQueryParam + ) -> List[List[str]]: + return self.manager.get_dbkeys(trans.user, chrom_info) + + @router.get( + '/api/genomes/{id}', + summary='Return information about build ', + response_description='Information about genome build ' + ) + def show( + self, + trans: ProvidesUserContext = Depends(get_trans), + id: str = IdPathParam, + reference: bool = ReferenceQueryParam, + num: int = NumQueryParam, + chrom: str = ChromQueryParam, + low: int = LowQueryParam, + high: int = HighQueryParam, + format: str = FormatQueryParam, + ) -> Any: + id = get_id(id, format) + return self.manager.get_genome(trans, id, num, chrom, low, high, reference) + + @router.get( + '/api/genomes/{id}/indexes', + summary='Return all available indexes for a genome id for provided type', + response_description='Indexes for a genome id for provided type' + ) + def indexes( + self, + id: str = IdPathParam, + type: str = IndexTypeQueryParam, + format: str = FormatQueryParam, + ) -> Any: + id = get_id(id, format) + rval = self.manager.get_indexes(id, type) + return Response(rval) + + @router.get( + '/api/genomes/{id}/sequences', + summary='Return raw sequence data', + response_description='Raw sequence data' + ) + def sequences( + self, + trans: ProvidesUserContext = Depends(get_trans), + id: str = IdPathParam, + reference: bool = ReferenceQueryParam, + chrom: str = ChromQueryParam, + low: int = LowQueryParam, + high: int = HighQueryParam, + format: str = FormatQueryParam, + ) -> Any: + id = get_id(id, format) + rval = self.manager.get_sequence(trans, id, chrom, low, high) + return Response(rval) class GenomesController(BaseAPIController): """ RESTful controller for interactions with genome data. """ + def __init__(self, app: StructuredApp): + super().__init__(app) + self.manager = GenomesManager(app) - @web.legacy_expose_api_anonymous + @expose_api_anonymous def index(self, trans, **kwd): """ GET /api/genomes: returns a list of installed genomes """ + chrom_info = kwd.get('chrom_info') + return self.manager.get_dbkeys(trans.user, chrom_info) - return self.app.genomes.get_dbkeys(trans, **kwd) - - @web.json + @expose_api_anonymous def show(self, trans, id, num=None, chrom=None, low=None, high=None, **kwd): """ GET /api/genomes/{id} Returns information about build """ - - # Process kwds. - id = get_id(id, kwd.get('format', None)) + id = get_id(id, kwd.get('format')) reference = is_true(kwd.get('reference', False)) + return self.manager.get_genome(trans, id, num, chrom, low, high, reference) - # Return info. - rval = None - if reference: - region = self.app.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high) - rval = {'dataset_type': 'refseq', 'data': region.sequence} - else: - rval = self.app.genomes.chroms(trans, dbkey=id, num=num, chrom=chrom, low=low) - return rval - - @web.legacy_expose_api_raw_anonymous + @expose_api_raw_anonymous def indexes(self, trans, id, **kwd): """ GET /api/genomes/{id}/indexes?type={table name} @@ -52,26 +198,17 @@ class GenomesController(BaseAPIController): Returns all available indexes for a genome id for type={table name} For instance, /api/genomes/hg19/indexes?type=fasta_indexes """ - index_extensions = {'fasta_indexes': '.fai'} - id = get_id(id, kwd.get('format', None)) - index_type = kwd.get('type', None) + id = get_id(id, kwd.get('format')) + index_type = kwd.get('type') + return self.manager.get_indexes(id, index_type) - tbl_entries = self.app.tool_data_tables.data_tables[index_type].data - index_file_name = [x[-1] for x in tbl_entries if id in x].pop() - - if_open = open(index_file_name + index_extensions[index_type], mode='r') - return if_open.read() - - @web.legacy_expose_api_raw_anonymous - def sequences(self, trans, id, num=None, chrom=None, low=None, high=None, **kwd): + @expose_api_raw_anonymous + def sequences(self, trans, id, chrom=None, low=None, high=None, **kwd): """ GET /api/genomes/{id}/sequences This is a wrapper for accepting sequence requests that want a raw return, not json """ - id = get_id(id, kwd.get('format', None)) - reference = is_true(kwd.get('reference', False)) - assert reference - region = self.app.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high) - return region.sequence + id = get_id(id, kwd.get('format')) + return self.manager.get_sequence(trans, id, chrom, low, high) diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index e4a1056a560..ea8e4857a69 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -541,6 +541,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization id = kwd.get('id') if not id: return self.message_exception(trans, 'No visualization id received for editing.') + trans_user = trans.get_user() v = self.get_visualization(trans, id, check_ownership=True) if trans.request.method == 'GET': if v.slug is None: @@ -562,7 +563,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization 'type': 'select', 'optional': True, 'value': v.dbkey, - 'options': trans.app.genomes.get_dbkeys(trans, chrom_info=True), + 'options': trans.app.genomes.get_dbkeys(trans_user, chrom_info=True), 'help': 'Parameter to associate your visualization with a database key.' }, { 'name': 'annotation', @@ -590,7 +591,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization v.dbkey = v_dbkey if v_annotation: v_annotation = sanitize_html(v_annotation) - self.add_item_annotation(trans.sa_session, trans.get_user(), v, v_annotation) + self.add_item_annotation(trans.sa_session, trans_user, v, v_annotation) trans.sa_session.add(v) trans.sa_session.flush() return {'message': 'Attributes of \'%s\' successfully saved.' % v.title, 'status': 'success'} diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index d5bd24f7130..d46204354d1 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -24,6 +24,10 @@ api_tags_metadata = [ "name": "datatypes", "description": "Operations with supported data types.", }, + { + "name": "genomes", + "description": "Operations with genome data.", + }, { "name": "licenses", "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).", diff --git a/test/integration/test_genomes.py b/test/integration/test_genomes.py new file mode 100644 index 00000000000..50556ebee0b --- /dev/null +++ b/test/integration/test_genomes.py @@ -0,0 +1,111 @@ +import os +import tempfile +from unittest.mock import patch + +from galaxy.exceptions import ( + ObjectNotFound, + ReferenceDataError, +) +from galaxy_test.driver import integration_util + +BUILDS_DATA = ( + '?\tunspecified (?)', + 'hg_test\tdescription of hg_test', + 'hg_test_nolen\tdescription of hg_test_nolen', +) + +LEN_DATA = ( + 'chr1\t248956422', + 'chr2\t242193529', + 'chr3\t198295559', +) + + +def get_key(has_len_file=True): + pos = 1 if has_len_file else 2 + return BUILDS_DATA[pos].split('\t')[0] + + +class GenomesTestCase(integration_util.IntegrationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + genomes_dir = cls.temp_config_dir("test_genomes") + os.makedirs(genomes_dir) + cls._setup_builds_file(config, genomes_dir) + cls._setup_len_file(config, genomes_dir) + + @classmethod + def _setup_builds_file(cls, config, genomes_dir): + """Create builds file + set config option.""" + builds_file_path = os.path.join(genomes_dir, 'builds.txt') + config['builds_file_path'] = builds_file_path + with open(builds_file_path, 'w') as f: + f.write('\n'.join(BUILDS_DATA)) + + @classmethod + def _setup_len_file(cls, config, genomes_dir): + """Create len file + set config option.""" + config['len_file_path'] = genomes_dir # the config option is a dir + key = get_key() + len_file_path = os.path.join(genomes_dir, f'{key}.len') + with open(len_file_path, 'w') as f: + f.write('\n'.join(LEN_DATA)) + + def test_index(self): + response = self._get('genomes') + self._assert_status_code_is(response, 200) + rval = response.json() + expected_data = [item.split('\t')[::-1] for item in BUILDS_DATA] + assert rval == expected_data + + def test_show_valid(self): + key = get_key() + response = self._get(f'genomes/{key}') + self._assert_status_code_is(response, 200) + rval = response.json() + assert rval['id'] == key + assert len(rval['chrom_info']) == len(LEN_DATA) + + def test_show_valid_no_refdata(self): + key = get_key(has_len_file=False) + response = self._get(f'genomes/{key}') + self._assert_status_code_is(response, 500) + assert response.json()['err_code'] == ReferenceDataError.err_code.code + + def test_show_invalid(self): + response = self._get('genomes/invalid') + self._assert_status_code_is(response, 404) + assert response.json()['err_code'] == ObjectNotFound.err_code.code + + def test_sequences(self): + + class RefDataMock: + sequence = 'test-value' + + key = get_key() + with patch.object(self._app.genomes, 'has_reference_data', return_value=True), \ + patch.object(self._app.genomes, '_get_reference_data', return_value=RefDataMock()): + response = self._get(f'genomes/{key}/sequences') + self._assert_status_code_is(response, 200) + assert response.content == bytes(RefDataMock.sequence, 'utf-8') + + def test_sequences_no_data(self): + key = get_key() + with patch.object(self._app.genomes, 'has_reference_data', return_value=False): + response = self._get(f'genomes/{key}/sequences') + self._assert_status_code_is(response, 500) + assert response.json()['err_code'] == ReferenceDataError.err_code.code + + def test_indexes(self): + mock_key, mock_content, index_type, suffix = 'mykey', 'mydata', 'fasta_indexes', '.fai' + # write some data to a tempfile + with tempfile.NamedTemporaryFile(dir=self._tempdir, suffix=suffix, mode="w", delete=False) as tf: + tf.write(mock_content) + # make a mock containing the path to the tempfile + tmpfile_path = tf.name[:-len(suffix)] # chop off the extention + mock_data = [[mock_key, tmpfile_path]] + with patch.object(self._app.tool_data_tables.data_tables[index_type], 'data', new=mock_data): + response = self._get(f'genomes/{mock_key}/indexes?type={index_type}') + self._assert_status_code_is(response, 200) + assert response.content == bytes(mock_content, 'utf-8')