Merge pull request #11241 from ic4f/dev_fastapi_genomes

Migrate api/genomes to fastapi
This commit is contained in:
Sergey Golitsynskiy
2021-02-11 20:04:34 -05:00
committed by GitHub
9 changed files with 469 additions and 126 deletions
+5
View File
@@ -221,6 +221,11 @@ class InvalidFileFormatError(MessageException):
err_code = error_codes_by_name['INVALID_FILE_FORMAT']
class ReferenceDataError(MessageException):
status_code = 500
err_code = error_codes_by_name['REFERENCE_DATA_ERROR']
# non-web exceptions
class ContainerCLIError(Exception):
+5
View File
@@ -159,6 +159,11 @@
"code": 500005,
"message": "File format not supported for this operation."
},
{
"name": "REFERENCE_DATA_ERROR",
"code": 500006,
"message": "Reference data required for program execution failed to load."
},
{
"name": "NOT_IMPLEMENTED",
"code": 501001,
+74
View File
@@ -0,0 +1,74 @@
from typing import (
Any,
List,
)
from galaxy import model as m
from galaxy.exceptions import (
ReferenceDataError,
RequestParameterInvalidException,
)
from galaxy.managers.context import ProvidesUserContext
from galaxy.structured_app import StructuredApp
class GenomesManager:
def __init__(self, app: StructuredApp):
self._app = app
self.genomes = app.genomes
def get_dbkeys(self, user: m.User, chrom_info: bool) -> List[List[str]]:
return self.genomes.get_dbkeys(user, chrom_info)
def get_genome(
self,
trans: ProvidesUserContext,
id: str,
num: int,
chrom: str,
low: int,
high: int,
reference: bool
) -> Any:
if reference:
region = self.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high)
return {'dataset_type': 'refseq', 'data': region.sequence}
else:
return self.genomes.chroms(trans, dbkey=id, num=num, chrom=chrom, low=low)
def get_sequence(
self,
trans: ProvidesUserContext,
id: str,
chrom: str,
low: int,
high: int
) -> Any:
region = self.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high)
return region.sequence
def get_indexes(self, id: str, index_type: str) -> Any:
index_extensions = {'fasta_indexes': '.fai'}
if index_type not in index_extensions:
raise RequestParameterInvalidException(f'Invalid index type: {index_type}')
tbl_entries = self._app.tool_data_tables.data_tables[index_type].data
ext = index_extensions[index_type]
index_filename = self._get_index_filename(id, tbl_entries, ext, index_type)
try:
with open(index_filename, mode='r') as f:
return f.read()
except OSError:
raise ReferenceDataError(f'Failed to load index file for {id}')
def _get_index_filename(self, id, tbl_entries, ext, index_type):
try:
paths = [x[-1] for x in tbl_entries if id in x]
file_name = paths.pop()
except TypeError:
raise ReferenceDataError('Data tables not found for {index_type}')
except IndexError:
raise ReferenceDataError('Data tables not found for {index_type} for {id}')
else:
return f"{file_name}{ext}"
+94 -87
View File
@@ -6,6 +6,10 @@ from json import loads
from bx.seq.twobit import TwoBitFile
from galaxy.exceptions import (
ObjectNotFound,
ReferenceDataError,
)
from galaxy.util.bunch import Bunch
log = logging.getLogger(__name__)
@@ -86,6 +90,9 @@ class Genome:
"""
Returns representation of self as a dictionary.
"""
# if there's no len_file, there's nothing to return
if not self.len_file:
raise ReferenceDataError(f'len_file not set for {self.key}')
def check_int(s):
if s.isdigit():
@@ -117,69 +124,70 @@ class Genome:
# (b) whether there are previous, next chroms;
# (c) index of start chrom.
#
len_file_enumerate = enumerate(open(self.len_file))
chroms = {}
prev_chroms = False
start_index = 0
if chrom:
# Use starting chrom to start list.
found = False
count = 0
for line_num, line in len_file_enumerate:
if line.startswith("#"):
continue
name, len = line.split("\t")
if found:
chroms[name] = int(len)
count += 1
elif name == chrom:
# Found starting chrom.
chroms[name] = int(len)
count += 1
found = True
start_index = line_num
if line_num != 0:
prev_chroms = True
if count >= num:
break
else:
# Use low to start list.
high = low + int(num)
prev_chroms = (low != 0)
start_index = low
with open(self.len_file) as f:
len_file_enumerate = enumerate(f)
chroms = {}
prev_chroms = False
start_index = 0
if chrom:
# Use starting chrom to start list.
found = False
count = 0
for line_num, line in len_file_enumerate:
if line.startswith("#"):
continue
name, len = line.split("\t")
if found:
chroms[name] = int(len)
count += 1
elif name == chrom:
# Found starting chrom.
chroms[name] = int(len)
count += 1
found = True
start_index = line_num
if line_num != 0:
prev_chroms = True
if count >= num:
break
else:
# Use low to start list.
high = low + int(num)
prev_chroms = (low != 0)
start_index = low
# Read chrom data from len file.
for line_num, line in len_file_enumerate:
if line_num < low:
continue
if line_num >= high:
break
if line.startswith("#"):
continue
# LEN files have format:
# <chrom_name><tab><chrom_length>
fields = line.split("\t")
chroms[fields[0]] = int(fields[1])
# Read chrom data from len file.
for line_num, line in len_file_enumerate:
if line_num < low:
continue
if line_num >= high:
break
if line.startswith("#"):
continue
# LEN files have format:
# <chrom_name><tab><chrom_length>
fields = line.split("\t")
chroms[fields[0]] = int(fields[1])
# Set flag to indicate whether there are more chroms after list.
next_chroms = False
try:
next(len_file_enumerate)
next_chroms = True
except StopIteration:
# No more chroms to read.
pass
# Set flag to indicate whether there are more chroms after list.
next_chroms = False
try:
next(len_file_enumerate)
next_chroms = True
except StopIteration:
# No more chroms to read.
pass
to_sort = [{'chrom': chrm, 'len': length} for chrm, length in chroms.items()]
to_sort.sort(key=lambda _: split_by_number(_['chrom']))
return {
'id': self.key,
'reference': self.twobit_file is not None,
'chrom_info': to_sort,
'prev_chroms': prev_chroms,
'next_chroms': next_chroms,
'start_index': start_index
}
to_sort = [{'chrom': chrm, 'len': length} for chrm, length in chroms.items()]
to_sort.sort(key=lambda _: split_by_number(_['chrom']))
return {
'id': self.key,
'reference': self.twobit_file is not None,
'chrom_info': to_sort,
'prev_chroms': prev_chroms,
'next_chroms': next_chroms,
'start_index': start_index
}
class Genomes:
@@ -208,13 +216,15 @@ class Genomes:
if twobit_table is None:
# Add genome data (twobit files) to genomes, directly from twobit.loc
try:
for line in open(os.path.join(self.app.config.tool_data_path, "twobit.loc")):
if line.startswith("#"):
continue
val = line.split()
if len(val) == 2:
key, path = val
twobit_fields[key] = path
twobit_path = os.path.join(self.app.config.tool_data_path, "twobit.loc")
with open(twobit_path) as f:
for line in f:
if line.startswith("#"):
continue
val = line.split()
if len(val) == 2:
key, path = val
twobit_fields[key] = path
except OSError:
# Thrown if twobit.loc does not exist.
log.exception("Error reading twobit.loc")
@@ -246,18 +256,15 @@ class Genomes:
rval = self.genomes[dbkey]
return rval
def get_dbkeys(self, trans, chrom_info=False, **kwd):
def get_dbkeys(self, user, chrom_info=False):
""" Returns all known dbkeys. If chrom_info is True, only dbkeys with
chromosome lengths are returned. """
self.check_and_reload()
dbkeys = []
# Add user's custom keys to dbkeys.
user_keys_dict = {}
user = trans.get_user()
if user:
if 'dbkeys' in user.preferences:
user_keys_dict = loads(user.preferences['dbkeys'])
if user and 'dbkeys' in user.preferences:
user_keys_dict = loads(user.preferences['dbkeys'])
dbkeys.extend([(attributes['name'], key) for key, attributes in user_keys_dict.items()])
# Add app keys to dbkeys.
@@ -325,14 +332,10 @@ class Genomes:
elif dbkey in self.genomes:
genome = self.genomes[dbkey]
# Set up return value or log exception if genome not found for key.
rval = None
if genome:
rval = genome.to_dict(num=num, chrom=chrom, low=low)
else:
log.exception('genome not found for key %s', dbkey)
if not genome:
raise ObjectNotFound(f'genome not found for key {dbkey}')
return rval
return genome.to_dict(num=num, chrom=chrom, low=low)
def has_reference_data(self, dbkey, dbkey_owner=None):
"""
@@ -369,7 +372,7 @@ class Genomes:
dbkey_user = trans.user
if not self.has_reference_data(dbkey, dbkey_user):
return None
raise ReferenceDataError(f"No reference data for {dbkey}")
#
# Get twobit file with reference data.
@@ -389,11 +392,15 @@ class Genomes:
twobit_dataset = fasta_dataset.get_converted_dataset(trans, 'twobit')
twobit_file_name = twobit_dataset.file_name
return self._get_reference_data(twobit_file_name, chrom, low, high)
def _get_reference_data(twobit_file_name, chrom, low, high):
# Read and return reference data.
try:
twobit = TwoBitFile(open(twobit_file_name, 'rb'))
if chrom in twobit:
seq_data = twobit[chrom].get(int(low), int(high))
return GenomeRegion(chrom=chrom, start=low, end=high, sequence=seq_data)
except OSError:
return None
with open(twobit_file_name, 'rb') as f:
twobit = TwoBitFile(f)
if chrom in twobit:
seq_data = twobit[chrom].get(int(low), int(high))
return GenomeRegion(chrom=chrom, start=low, end=high, sequence=seq_data)
except OSError as e:
raise e()
@@ -77,7 +77,6 @@ def get_user(galaxy_session: Optional[model.GalaxySession] = Depends(get_session
def get_trans(app: UniverseApplication = Depends(get_app), user: Optional[User] = Depends(get_user),
galaxy_session: Optional[model.GalaxySession] = Depends(get_session),
) -> SessionRequestContext:
app.model.session.expunge_all()
return SessionRequestContext(app=app, user=user, galaxy_session=galaxy_session)
+173 -36
View File
@@ -1,50 +1,196 @@
from galaxy import web
from typing import (
Any,
List,
)
from fastapi import (
Depends,
Path,
Query,
)
from fastapi.responses import Response
from fastapi_utils.cbv import cbv
from fastapi_utils.inferring_router import InferringRouter as APIRouter
from galaxy.managers.context import ProvidesUserContext
from galaxy.managers.genomes import GenomesManager
from galaxy.structured_app import StructuredApp
from galaxy.web import (
expose_api_anonymous,
expose_api_raw_anonymous,
)
from galaxy.web.framework.helpers import is_true
from galaxy.webapps.base.controller import BaseAPIController
from . import (
get_app,
get_trans,
)
router = APIRouter(tags=['genomes'])
IdPathParam: str = Path(
...,
title='Genome ID',
description='Genome ID'
)
ChromInfoQueryParam: bool = Query(
None,
title='ChromInfo',
description='If true, return genome keys with chromosome lengths'
)
NumQueryParam: int = Query(
None,
title='Number',
description='Limits size of returned data',
)
ChromQueryParam: Any = Query(
None,
title='Chrom',
description='Limits size of returned data',
)
LowQueryParam: int = Query(
None,
title='Low',
description='Limits size of returned data',
)
HighQueryParam: int = Query(
None,
title='High',
description='Limits size of returned data',
)
FormatQueryParam: str = Query(
None,
title='Format',
description='Format'
)
ReferenceQueryParam: bool = Query(
None,
title='Reference',
description='If true, return reference data'
)
IndexTypeQueryParam: str = Query(
'fasta_indexes', # currently this is the only supported index type
title='Index type',
description='Index type'
)
def get_genomes_manager(app: StructuredApp = Depends(get_app)) -> GenomesManager:
return GenomesManager(app)
def get_id(base, format):
if format:
return f"{base}.{format}"
else:
return base
return base
@cbv(router)
class FastAPIGenomes:
manager: GenomesManager = Depends(get_genomes_manager)
@router.get(
'/api/genomes',
summary='Return a list of installed genomes',
response_description='Installed genomes'
)
def index(
self,
trans: ProvidesUserContext = Depends(get_trans),
chrom_info: bool = ChromInfoQueryParam
) -> List[List[str]]:
return self.manager.get_dbkeys(trans.user, chrom_info)
@router.get(
'/api/genomes/{id}',
summary='Return information about build <id>',
response_description='Information about genome build <id>'
)
def show(
self,
trans: ProvidesUserContext = Depends(get_trans),
id: str = IdPathParam,
reference: bool = ReferenceQueryParam,
num: int = NumQueryParam,
chrom: str = ChromQueryParam,
low: int = LowQueryParam,
high: int = HighQueryParam,
format: str = FormatQueryParam,
) -> Any:
id = get_id(id, format)
return self.manager.get_genome(trans, id, num, chrom, low, high, reference)
@router.get(
'/api/genomes/{id}/indexes',
summary='Return all available indexes for a genome id for provided type',
response_description='Indexes for a genome id for provided type'
)
def indexes(
self,
id: str = IdPathParam,
type: str = IndexTypeQueryParam,
format: str = FormatQueryParam,
) -> Any:
id = get_id(id, format)
rval = self.manager.get_indexes(id, type)
return Response(rval)
@router.get(
'/api/genomes/{id}/sequences',
summary='Return raw sequence data',
response_description='Raw sequence data'
)
def sequences(
self,
trans: ProvidesUserContext = Depends(get_trans),
id: str = IdPathParam,
reference: bool = ReferenceQueryParam,
chrom: str = ChromQueryParam,
low: int = LowQueryParam,
high: int = HighQueryParam,
format: str = FormatQueryParam,
) -> Any:
id = get_id(id, format)
rval = self.manager.get_sequence(trans, id, chrom, low, high)
return Response(rval)
class GenomesController(BaseAPIController):
"""
RESTful controller for interactions with genome data.
"""
def __init__(self, app: StructuredApp):
super().__init__(app)
self.manager = GenomesManager(app)
@web.legacy_expose_api_anonymous
@expose_api_anonymous
def index(self, trans, **kwd):
"""
GET /api/genomes: returns a list of installed genomes
"""
chrom_info = kwd.get('chrom_info')
return self.manager.get_dbkeys(trans.user, chrom_info)
return self.app.genomes.get_dbkeys(trans, **kwd)
@web.json
@expose_api_anonymous
def show(self, trans, id, num=None, chrom=None, low=None, high=None, **kwd):
"""
GET /api/genomes/{id}
Returns information about build <id>
"""
# Process kwds.
id = get_id(id, kwd.get('format', None))
id = get_id(id, kwd.get('format'))
reference = is_true(kwd.get('reference', False))
return self.manager.get_genome(trans, id, num, chrom, low, high, reference)
# Return info.
rval = None
if reference:
region = self.app.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high)
rval = {'dataset_type': 'refseq', 'data': region.sequence}
else:
rval = self.app.genomes.chroms(trans, dbkey=id, num=num, chrom=chrom, low=low)
return rval
@web.legacy_expose_api_raw_anonymous
@expose_api_raw_anonymous
def indexes(self, trans, id, **kwd):
"""
GET /api/genomes/{id}/indexes?type={table name}
@@ -52,26 +198,17 @@ class GenomesController(BaseAPIController):
Returns all available indexes for a genome id for type={table name}
For instance, /api/genomes/hg19/indexes?type=fasta_indexes
"""
index_extensions = {'fasta_indexes': '.fai'}
id = get_id(id, kwd.get('format', None))
index_type = kwd.get('type', None)
id = get_id(id, kwd.get('format'))
index_type = kwd.get('type')
return self.manager.get_indexes(id, index_type)
tbl_entries = self.app.tool_data_tables.data_tables[index_type].data
index_file_name = [x[-1] for x in tbl_entries if id in x].pop()
if_open = open(index_file_name + index_extensions[index_type], mode='r')
return if_open.read()
@web.legacy_expose_api_raw_anonymous
def sequences(self, trans, id, num=None, chrom=None, low=None, high=None, **kwd):
@expose_api_raw_anonymous
def sequences(self, trans, id, chrom=None, low=None, high=None, **kwd):
"""
GET /api/genomes/{id}/sequences
This is a wrapper for accepting sequence requests that
want a raw return, not json
"""
id = get_id(id, kwd.get('format', None))
reference = is_true(kwd.get('reference', False))
assert reference
region = self.app.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high)
return region.sequence
id = get_id(id, kwd.get('format'))
return self.manager.get_sequence(trans, id, chrom, low, high)
@@ -541,6 +541,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
id = kwd.get('id')
if not id:
return self.message_exception(trans, 'No visualization id received for editing.')
trans_user = trans.get_user()
v = self.get_visualization(trans, id, check_ownership=True)
if trans.request.method == 'GET':
if v.slug is None:
@@ -562,7 +563,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
'type': 'select',
'optional': True,
'value': v.dbkey,
'options': trans.app.genomes.get_dbkeys(trans, chrom_info=True),
'options': trans.app.genomes.get_dbkeys(trans_user, chrom_info=True),
'help': 'Parameter to associate your visualization with a database key.'
}, {
'name': 'annotation',
@@ -590,7 +591,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
v.dbkey = v_dbkey
if v_annotation:
v_annotation = sanitize_html(v_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), v, v_annotation)
self.add_item_annotation(trans.sa_session, trans_user, v, v_annotation)
trans.sa_session.add(v)
trans.sa_session.flush()
return {'message': 'Attributes of \'%s\' successfully saved.' % v.title, 'status': 'success'}
+4
View File
@@ -24,6 +24,10 @@ api_tags_metadata = [
"name": "datatypes",
"description": "Operations with supported data types.",
},
{
"name": "genomes",
"description": "Operations with genome data.",
},
{
"name": "licenses",
"description": "Operations with [SPDX licenses](https://spdx.org/licenses/).",
+111
View File
@@ -0,0 +1,111 @@
import os
import tempfile
from unittest.mock import patch
from galaxy.exceptions import (
ObjectNotFound,
ReferenceDataError,
)
from galaxy_test.driver import integration_util
BUILDS_DATA = (
'?\tunspecified (?)',
'hg_test\tdescription of hg_test',
'hg_test_nolen\tdescription of hg_test_nolen',
)
LEN_DATA = (
'chr1\t248956422',
'chr2\t242193529',
'chr3\t198295559',
)
def get_key(has_len_file=True):
pos = 1 if has_len_file else 2
return BUILDS_DATA[pos].split('\t')[0]
class GenomesTestCase(integration_util.IntegrationTestCase):
@classmethod
def handle_galaxy_config_kwds(cls, config):
genomes_dir = cls.temp_config_dir("test_genomes")
os.makedirs(genomes_dir)
cls._setup_builds_file(config, genomes_dir)
cls._setup_len_file(config, genomes_dir)
@classmethod
def _setup_builds_file(cls, config, genomes_dir):
"""Create builds file + set config option."""
builds_file_path = os.path.join(genomes_dir, 'builds.txt')
config['builds_file_path'] = builds_file_path
with open(builds_file_path, 'w') as f:
f.write('\n'.join(BUILDS_DATA))
@classmethod
def _setup_len_file(cls, config, genomes_dir):
"""Create len file + set config option."""
config['len_file_path'] = genomes_dir # the config option is a dir
key = get_key()
len_file_path = os.path.join(genomes_dir, f'{key}.len')
with open(len_file_path, 'w') as f:
f.write('\n'.join(LEN_DATA))
def test_index(self):
response = self._get('genomes')
self._assert_status_code_is(response, 200)
rval = response.json()
expected_data = [item.split('\t')[::-1] for item in BUILDS_DATA]
assert rval == expected_data
def test_show_valid(self):
key = get_key()
response = self._get(f'genomes/{key}')
self._assert_status_code_is(response, 200)
rval = response.json()
assert rval['id'] == key
assert len(rval['chrom_info']) == len(LEN_DATA)
def test_show_valid_no_refdata(self):
key = get_key(has_len_file=False)
response = self._get(f'genomes/{key}')
self._assert_status_code_is(response, 500)
assert response.json()['err_code'] == ReferenceDataError.err_code.code
def test_show_invalid(self):
response = self._get('genomes/invalid')
self._assert_status_code_is(response, 404)
assert response.json()['err_code'] == ObjectNotFound.err_code.code
def test_sequences(self):
class RefDataMock:
sequence = 'test-value'
key = get_key()
with patch.object(self._app.genomes, 'has_reference_data', return_value=True), \
patch.object(self._app.genomes, '_get_reference_data', return_value=RefDataMock()):
response = self._get(f'genomes/{key}/sequences')
self._assert_status_code_is(response, 200)
assert response.content == bytes(RefDataMock.sequence, 'utf-8')
def test_sequences_no_data(self):
key = get_key()
with patch.object(self._app.genomes, 'has_reference_data', return_value=False):
response = self._get(f'genomes/{key}/sequences')
self._assert_status_code_is(response, 500)
assert response.json()['err_code'] == ReferenceDataError.err_code.code
def test_indexes(self):
mock_key, mock_content, index_type, suffix = 'mykey', 'mydata', 'fasta_indexes', '.fai'
# write some data to a tempfile
with tempfile.NamedTemporaryFile(dir=self._tempdir, suffix=suffix, mode="w", delete=False) as tf:
tf.write(mock_content)
# make a mock containing the path to the tempfile
tmpfile_path = tf.name[:-len(suffix)] # chop off the extention
mock_data = [[mock_key, tmpfile_path]]
with patch.object(self._app.tool_data_tables.data_tables[index_type], 'data', new=mock_data):
response = self._get(f'genomes/{mock_key}/indexes?type={index_type}')
self._assert_status_code_is(response, 200)
assert response.content == bytes(mock_content, 'utf-8')