- added a new table 'sample_dataset' to store sample datasets & their info when they are transfered from the sequencer
- the datasets transfer page now uses a grid to facilitate bulk renaming
- bulk renaming possible to fix the problem with the way SOLiD generates datasets
- the remote file browser is now independent of a specific sample, the user may select any sample when transferring datasets from the sequencer
This commit is contained in:
Ramkrishna Chakrabarty
2010-07-29 10:55:08 -04:00
parent 457a08eb5b
commit c925dc1f2c
14 changed files with 653 additions and 242 deletions
+17 -8
View File
@@ -1584,7 +1584,7 @@ class Sample( object ):
COMPLETE = 'Complete',
ERROR = 'Error')
def __init__(self, name=None, desc=None, request=None, form_values=None,
bar_code=None, library=None, folder=None, dataset_files=None):
bar_code=None, library=None, folder=None):
self.name = name
self.desc = desc
self.request = request
@@ -1592,27 +1592,26 @@ class Sample( object ):
self.bar_code = bar_code
self.library = library
self.folder = folder
self.dataset_files = dataset_files
def current_state(self):
if self.events:
return self.events[0].state
return None
def untransferred_dataset_files(self):
count = 0
for df in self.dataset_files:
if df['status'] == self.transfer_status.NOT_STARTED:
for df in self.datasets:
if df.status == self.transfer_status.NOT_STARTED:
count = count + 1
return count
def inprogress_dataset_files(self):
count = 0
for df in self.dataset_files:
if df['status'] not in [self.transfer_status.NOT_STARTED, self.transfer_status.COMPLETE]:
for df in self.datasets:
if df.status not in [self.transfer_status.NOT_STARTED, self.transfer_status.COMPLETE]:
count = count + 1
return count
def transferred_dataset_files(self):
count = 0
for df in self.dataset_files:
if df['status'] == self.transfer_status.COMPLETE:
for df in self.datasets:
if df.status == self.transfer_status.COMPLETE:
count = count + 1
return count
def dataset_size(self, filepath):
@@ -1639,6 +1638,16 @@ class SampleEvent( object ):
self.state = sample_state
self.comment = comment
class SampleDataset( object ):
def __init__(self, sample=None, name=None, file_path=None,
status=None, error_msg=None, size=None):
self.sample = sample
self.name = name
self.file_path = file_path
self.status = status
self.error_msg = error_msg
self.size = size
class UserAddress( object ):
def __init__(self, user=None, desc=None, name=None, institution=None,
address=None, city=None, state=None, postal_code=None,
+16 -1
View File
@@ -574,7 +574,6 @@ Sample.table = Table('sample', metadata,
Column( "bar_code", TrimmedString( 255 ), index=True ),
Column( "library_id", Integer, ForeignKey( "library.id" ), index=True ),
Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ),
Column( "dataset_files", JSONType() ),
Column( "deleted", Boolean, index=True, default=False ) )
SampleState.table = Table('sample_state', metadata,
@@ -593,6 +592,17 @@ SampleEvent.table = Table('sample_event', metadata,
Column( "sample_state_id", Integer, ForeignKey( "sample_state.id" ), index=True ),
Column( "comment", TEXT ) )
SampleDataset.table = Table('sample_dataset', metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "sample_id", Integer, ForeignKey( "sample.id" ), index=True ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "file_path", TrimmedString( 255 ), nullable=False ),
Column( "status", TrimmedString( 255 ), nullable=False ),
Column( "error_msg", TEXT ),
Column( "size", TrimmedString( 255 ) ) )
Page.table = Table( "page", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
@@ -792,6 +802,8 @@ assign_mapper( context, Sample, Sample.table,
properties=dict(
events=relation( SampleEvent, backref="sample",
order_by=desc(SampleEvent.table.c.update_time) ),
datasets=relation( SampleDataset, backref="sample",
order_by=desc(SampleDataset.table.c.update_time) ),
values=relation( FormValues,
primaryjoin=( Sample.table.c.form_values_id == FormValues.table.c.id ) ),
request=relation( Request,
@@ -865,6 +877,9 @@ assign_mapper( context, SampleEvent, SampleEvent.table,
assign_mapper( context, SampleState, SampleState.table,
properties=None )
assign_mapper( context, SampleDataset, SampleDataset.table,
properties=None )
assign_mapper( context, UserAddress, UserAddress.table,
properties=dict(
user=relation( User,
+305 -143
View File
@@ -146,7 +146,7 @@ class RequestsGrid( grids.Grid ):
#
# ---- Request Type Gridr ------------------------------------------------------
# ---- Request Type Grid ------------------------------------------------------
#
class RequestTypeGrid( grids.Grid ):
# Custom column types
@@ -207,6 +207,57 @@ class RequestTypeGrid( grids.Grid ):
action='create_request_type' ) )
]
# ---- Data Transfer Grid ------------------------------------------------------
#
class DataTransferGrid( grids.Grid ):
# Custom column types
class NameColumn( grids.TextColumn ):
def get_value(self, trans, grid, sample_dataset):
return sample_dataset.name
class SizeColumn( grids.TextColumn ):
def get_value(self, trans, grid, sample_dataset):
return sample_dataset.size
class StatusColumn( grids.TextColumn ):
def get_value(self, trans, grid, sample_dataset):
return sample_dataset.status
# Grid definition
title = "Sample Datasets"
template = "admin/requests/datasets_grid.mako"
model_class = model.SampleDataset
default_sort_key = "-create_time"
num_rows_per_page = 50
preserve_state = True
use_paging = True
#default_filter = dict( deleted="False" )
columns = [
NameColumn( "Name",
#key="name",
model_class=model.SampleDataset,
link=( lambda item: dict( operation="view", id=item.id ) ),
attach_popup=True,
filterable="advanced" ),
SizeColumn( "Size",
#key='size',
model_class=model.SampleDataset,
filterable="advanced" ),
StatusColumn( "Status",
#key='status',
model_class=model.SampleDataset,
filterable="advanced" ),
]
columns.append( grids.MulticolFilterColumn( "Search",
cols_to_filter=[ columns[0] ],
key="free-text-search",
visible=False,
filterable="standard" ) )
operations = [
grids.GridOperation( "Start Transfer", allow_multiple=True, condition=( lambda item: item.status in [model.Sample.transfer_status.NOT_STARTED] ) ),
grids.GridOperation( "Rename", allow_multiple=True, allow_popup=False, condition=( lambda item: item.status in [model.Sample.transfer_status.NOT_STARTED] ) ),
grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: item.status in [model.Sample.transfer_status.NOT_STARTED] ) ),
]
def apply_query_filter( self, trans, query, **kwd ):
return query.filter_by( sample_id=kwd['sample_id'] )
#
# ---- Request Controller ------------------------------------------------------
#
@@ -214,6 +265,7 @@ class RequestTypeGrid( grids.Grid ):
class RequestsAdmin( BaseController ):
request_grid = RequestsGrid()
requesttype_grid = RequestTypeGrid()
datatx_grid = DataTransferGrid()
@web.expose
@@ -280,11 +332,11 @@ class RequestsAdmin( BaseController ):
# Avoid caching
trans.response.headers['Pragma'] = 'no-cache'
trans.response.headers['Expires'] = '0'
sample = trans.sa_session.query( self.app.model.Sample ).get( int(id) )
datatx_info = sample.request.type.datatx_info
request = trans.sa_session.query( self.app.model.Request ).get( int(id) )
datatx_info = request.type.datatx_info
cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( datatx_info['username'],
datatx_info['host'],
folder_path )
datatx_info['host'],
folder_path )
output = pexpect.run(cmd, events={'.ssword:*': datatx_info['password']+'\r\n',
pexpect.TIMEOUT:print_ticks},
timeout=10)
@@ -297,8 +349,8 @@ class RequestsAdmin( BaseController ):
# Avoid caching
trans.response.headers['Pragma'] = 'no-cache'
trans.response.headers['Expires'] = '0'
sample = trans.sa_session.query( self.app.model.Sample ).get( int(id) )
return self.__get_files(trans, sample, folder_path)
request = trans.sa_session.query( self.app.model.Request ).get( int(id) )
return self.__get_files(trans, request.type, folder_path)
def __reject_request(self, trans, **kwd):
try:
@@ -522,12 +574,120 @@ class RequestsAdmin( BaseController ):
#
# Data transfer from sequencer
#
@web.expose
@web.require_admin
def manage_datasets( self, trans, **kwd ):
if 'operation' in kwd:
operation = kwd['operation'].lower()
if not kwd.get( 'id', None ):
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='list',
status='error',
message="Invalid sample dataset ID") )
if operation == "view":
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(kwd['id']) )
return trans.fill_template( '/admin/requests/dataset.mako',
sample=sample_dataset.sample,
sample_dataset=sample_dataset)
def __get_files(self, trans, sample, folder_path):
elif operation == "delete":
id_list = util.listify( kwd['id'] )
for id in id_list:
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) )
sample_id = sample_dataset.sample_id
trans.sa_session.delete( sample_dataset )
trans.sa_session.flush()
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='manage_datasets',
sample_id=sample_id,
status='done',
message="%i dataset(s) have been removed." % len(id_list)) )
elif operation == "rename":
id_list = util.listify( kwd['id'] )
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id_list[0]) )
return trans.fill_template( '/admin/requests/rename_datasets.mako',
sample=sample_dataset.sample,
id_list=id_list )
elif operation == "start transfer":
id_list = util.listify( kwd['id'] )
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id_list[0]) )
self.__start_datatx(trans, sample_dataset.sample, id_list)
# Render the grid view
try:
sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] )
except:
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='list',
status='error',
message="Invalid sample ID" ) )
self.datatx_grid.title = 'Datasets of Sample "%s"' % sample.name
self.datatx_grid.global_actions = [
grids.GridAction( "Refresh",
dict( controller='requests_admin',
action='manage_datasets',
sample_id=sample.id) ),
grids.GridAction( "Select Datasets",
dict(controller='requests_admin',
action='get_data',
request_id=sample.request.id,
folder_path=sample.request.type.datatx_info['data_dir'],
sample_id=sample.id,
show_page=True)),
grids.GridAction( 'Data Library "%s"' % sample.library.name,
dict(controller='library_common',
action='browse_library',
cntrller='library_admin',
id=trans.security.encode_id( sample.library.id))),
grids.GridAction( "Browse this request",
dict( controller='requests_admin',
action='list',
operation='show',
id=trans.security.encode_id(sample.request.id)))]
return self.datatx_grid( trans, **kwd )
@web.expose
@web.require_admin
def rename_datasets( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
try:
sample = trans.sa_session.query( trans.app.model.Sample ).get( trans.security.decode_id(kwd['sample_id']))
except:
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='list',
status='error',
message="Invalid sample ID" ) )
if params.get( 'save_button', False ):
id_list = util.listify( kwd['id_list'] )
for id in id_list:
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) )
prepend = util.restore_text( params.get( 'prepend_%i' % sample_dataset.id, '' ) )
name = util.restore_text( params.get( 'name_%i' % sample_dataset.id, sample_dataset.name ) )
if prepend == 'None':
sample_dataset.name = name
else:
sample_dataset.name = prepend+'_'+name
trans.sa_session.add( sample_dataset )
trans.sa_session.flush()
return trans.fill_template( '/admin/requests/rename_datasets.mako',
sample=sample, id_list=id_list,
message='Changes saved successfully.',
status='done' )
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='manage_datasets',
sample_id=sample.id) )
def __get_files(self, trans, request_type, folder_path):
'''
This method retrieves the filenames to be transfer from the remote host.
'''
datatx_info = sample.request.type.datatx_info
datatx_info = request_type.datatx_info
if not datatx_info['host'] or not datatx_info['username'] or not datatx_info['password']:
message = "Error in sequencer login information."
return trans.response.send_redirect( web.url_for( controller='requests_common',
@@ -555,123 +715,152 @@ class RequestsAdmin( BaseController ):
return output.splitlines()
def __get_files_in_dir(self, trans, sample, folder_path):
tmpfiles = self.__get_files(trans, sample, folder_path)
for tf in tmpfiles:
if tf[-1] == os.sep:
self.__get_files_in_dir(trans, sample, os.path.join(folder_path, tf))
# def __get_files_in_dir(self, trans, sample, folder_path):
# tmpfiles = self.__get_files(trans, sample, folder_path)
# for tf in tmpfiles:
# if tf[-1] == os.sep:
# self.__get_files_in_dir(trans, sample, os.path.join(folder_path, tf))
# else:
# sample.dataset_files.append([os.path.join(folder_path, tf),
# sample.transfer_status.NOT_STARTED])
# trans.sa_session.add( sample )
# trans.sa_session.flush()
# return
def __samples_selectbox(self, trans, request, sample_id=None):
samples_selectbox = SelectField('sample_id')
for i, s in enumerate(request.samples):
if str(s.id) == sample_id:
samples_selectbox.add_option(s.name, s.id, selected=True)
else:
sample.dataset_files.append([os.path.join(folder_path, tf),
sample.transfer_status.NOT_STARTED])
trans.sa_session.add( sample )
trans.sa_session.flush()
return
samples_selectbox.add_option(s.name, s.id)
return samples_selectbox
@web.expose
@web.require_admin
def get_data(self, trans, **kwd):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
try:
sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] )
request = trans.sa_session.query( trans.app.model.Request ).get( kwd['request_id'] )
except:
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='list',
status='error',
message="Invalid sample ID" ) )
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
folder_path = util.restore_text( params.get( 'folder_path',
sample.request.type.datatx_info['data_dir'] ) )
message="Invalid request ID" ) )
files_list = util.listify( params.get( 'files_list', '' ) )
if params.get( 'start_transfer_button', False ) == 'True':
return self.__start_datatx(trans, sample)
folder_path = util.restore_text( params.get( 'folder_path',
request.type.datatx_info['data_dir'] ) )
sbox = self.__samples_selectbox(trans, request, kwd.get('sample_id', None))
if not folder_path:
return trans.fill_template( '/requests/common/get_data.mako',
cntrller='requests_admin',
sample=sample, files=[],
dataset_files=sample.dataset_files,
return trans.fill_template( '/admin/requests/get_data.mako',
cntrller='requests_admin', request=request,
samples_selectbox=sbox, files=[],
folder_path=folder_path )
if folder_path[-1] != os.sep:
folder_path = folder_path+os.sep
if params.get( 'browse_button', False ):
if params.get( 'show_page', False ):
if kwd.get('sample_id', None):
sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] )
if sample.datasets:
folder_path = os.path.dirname(sample.datasets[-1].file_path)
return trans.fill_template( '/admin/requests/get_data.mako',
cntrller='requests_admin', request=request,
samples_selectbox=sbox, files=[],
folder_path=folder_path,
status=status, message=message )
elif params.get( 'browse_button', False ):
# get the filenames from the remote host
files = self.__get_files(trans, sample, folder_path)
files = self.__get_files(trans, request.type, folder_path)
if folder_path[-1] != os.sep:
folder_path += os.sep
return trans.fill_template( '/requests/common/get_data.mako',
cntrller='requests_admin',
sample=sample, files=files,
dataset_files=sample.dataset_files,
folder_path=folder_path )
return trans.fill_template( '/admin/requests/get_data.mako',
cntrller='requests_admin', request=request,
samples_selectbox=sbox, files=files,
folder_path=folder_path,
status=status, message=message )
elif params.get( 'folder_up', False ):
if folder_path[-1] == os.sep:
folder_path = os.path.dirname(folder_path[:-1])
# get the filenames from the remote host
files = self.__get_files(trans, sample, folder_path)
files = self.__get_files(trans, request.type, folder_path)
if folder_path[-1] != os.sep:
folder_path += os.sep
return trans.fill_template( '/requests/common/get_data.mako',
cntrller='requests_admin',
sample=sample, files=files,
dataset_files=sample.dataset_files,
folder_path=folder_path )
return trans.fill_template( '/admin/requests/get_data.mako',
cntrller='requests_admin',request=request,
samples_selectbox=sbox, files=files,
folder_path=folder_path,
status=status, message=message )
elif params.get( 'open_folder', False ):
if len(files_list) == 1:
folder_path = os.path.join(folder_path, files_list[0])
# get the filenames from the remote host
files = self.__get_files(trans, sample, folder_path)
files = self.__get_files(trans, request.type, folder_path)
if folder_path[-1] != os.sep:
folder_path += os.sep
return trans.fill_template( '/requests/common/get_data.mako',
cntrller='requests_admin',
sample=sample, files=files,
dataset_files=sample.dataset_files,
folder_path=folder_path )
elif params.get( 'remove_dataset_button', False ):
# get the filenames from the remote host
files = self.__get_files(trans, sample, folder_path)
dataset_index = int(params.get( 'dataset_index', 0 ))
del sample.dataset_files[dataset_index]
trans.sa_session.add( sample )
trans.sa_session.flush()
return trans.fill_template( '/requests/common/get_data.mako',
cntrller='requests_admin',
sample=sample, files=files,
dataset_files=sample.dataset_files,
folder_path=folder_path)
elif params.get( 'select_files_button', False ):
folder_files = []
if len(files_list):
for f in files_list:
filepath = os.path.join(folder_path, f)
if f[-1] == os.sep:
# the selected item is a folder so transfer all the
# folder contents
# FIXME
#self.__get_files_in_dir(trans, sample, filepath)
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='get_data',
sample_id=sample.id,
folder_path=folder_path,
open_folder=True))
else:
sample.dataset_files.append(dict(filepath=filepath,
status=sample.transfer_status.NOT_STARTED,
name=self.__dataset_name(sample, filepath.split('/')[-1]),
error_msg='',
size=sample.dataset_size(filepath)))
trans.sa_session.add( sample )
trans.sa_session.flush()
return trans.fill_template( '/admin/requests/get_data.mako',
cntrller='requests_admin', request=request,
samples_selectbox=sbox, files=files,
folder_path=folder_path,
status=status, message=message )
elif params.get( 'select_show_datasets_button', False ):
sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] )
retval = self.__save_sample_datasets(trans, sample, files_list, folder_path)
if retval: message='The dataset(s) %s have been selected for sample <b>%s</b>' %(str(retval)[1:-1].replace("'", ""), sample.name)
else: message = None
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='manage_datasets',
sample_id=sample.id,
status='done',
message=message) )
elif params.get( 'select_more_button', False ):
sample = trans.sa_session.query( trans.app.model.Sample ).get( kwd['sample_id'] )
retval = self.__save_sample_datasets(trans, sample, files_list, folder_path)
if retval: message='The dataset(s) %s have been selected for sample <b>%s</b>' %(str(retval)[1:-1].replace("'", ""), sample.name)
else: message = None
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='get_data',
sample_id=sample.id,
request_id=sample.request.id,
folder_path=folder_path,
open_folder=True))
return trans.response.send_redirect( web.url_for( controller='requests_common',
cntrller='requests_admin' ,
action='show_datatx_page',
sample_id=trans.security.encode_id(sample.id),
folder_path=folder_path))
sample_id=sample.id,
open_folder=True,
status='done',
message=message))
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='get_data',
request_id=sample.request.id,
folder_path=folder_path,
show_page=True))
def __save_sample_datasets(self, trans, sample, files_list, folder_path):
files = []
if len(files_list):
for f in files_list:
filepath = os.path.join(folder_path, f)
if f[-1] == os.sep:
# the selected item is a folder so transfer all the
# folder contents
# FIXME
#self.__get_files_in_dir(trans, sample, filepath)
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='get_data',
request=sample.request,
folder_path=folder_path,
open_folder=True))
else:
sample_dataset = trans.app.model.SampleDataset( sample=sample,
file_path=filepath,
status=sample.transfer_status.NOT_STARTED,
name=self.__dataset_name(sample, filepath.split('/')[-1]),
error_msg='',
size=sample.dataset_size(filepath))
trans.sa_session.add( sample_dataset )
trans.sa_session.flush()
files.append(str(sample_dataset.name))
return files
def __dataset_name(self, sample, filepath):
name = filepath.split('/')[-1]
@@ -735,7 +924,7 @@ class RequestsAdmin( BaseController ):
trans.sa_session.flush()
return datatx_user
def __send_message(self, trans, datatx_info, sample):
def __send_message(self, trans, datatx_info, sample, id_list):
'''
This method creates the xml message and sends it to the rabbitmq server
'''
@@ -752,20 +941,20 @@ class RequestsAdmin( BaseController ):
</data_transfer>'''
dataset_xml = \
'''<dataset>
<index>%(INDEX)s</index>
<dataset_id>%(ID)s</dataset_id>
<name>%(NAME)s</name>
<file>%(FILE)s</file>
</dataset>'''
datasets = ''
for index, dataset in enumerate(sample.dataset_files):
if dataset['status'] == sample.transfer_status.NOT_STARTED:
datasets = datasets + dataset_xml % dict(INDEX=str(index),
NAME=dataset['name'],
FILE=dataset['filepath'])
sample.dataset_files[index]['status'] = sample.transfer_status.IN_QUEUE
trans.sa_session.add( sample )
trans.sa_session.flush()
for id in id_list:
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) )
if sample_dataset.status == sample.transfer_status.NOT_STARTED:
datasets = datasets + dataset_xml % dict(ID=str(sample_dataset.id),
NAME=sample_dataset.name,
FILE=sample_dataset.file_path)
sample_dataset.status = sample.transfer_status.IN_QUEUE
trans.sa_session.add( sample_dataset )
trans.sa_session.flush()
data = xml % dict(DATA_HOST=datatx_info['host'],
DATA_USER=datatx_info['username'],
DATA_PASSWORD=datatx_info['password'],
@@ -790,7 +979,7 @@ class RequestsAdmin( BaseController ):
chan.close()
conn.close()
def __start_datatx(self, trans, sample):
def __start_datatx(self, trans, sample, id_list):
# data transfer user
datatx_user = self.__setup_datatx_user(trans, sample.library, sample.folder)
# validate sequecer information
@@ -799,46 +988,19 @@ class RequestsAdmin( BaseController ):
not datatx_info['username'] or \
not datatx_info['password']:
message = "Error in sequencer login information."
return trans.response.send_redirect( web.url_for( controller='requests_common',
cntrller='requests_admin' ,
action='show_datatx_page',
sample_id=trans.security.encode_id(sample.id),
status='error',
message=message))
self.__send_message(trans, datatx_info, sample)
return trans.response.send_redirect( web.url_for( controller='requests_common',
cntrller='requests_admin' ,
action='show_datatx_page',
sample_id=trans.security.encode_id(sample.id),
folder_path=datatx_info['data_dir']))
@web.expose
@web.require_admin
def dataset_details( self, trans, **kwd ):
try:
sample = trans.sa_session.query( trans.app.model.Sample ).get( trans.security.decode_id(kwd['sample_id']) )
except:
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='list',
action='manage_datasets',
sample_id=sample.id,
status='error',
message="Invalid sample ID" ) )
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
dataset_index = int( params.get( 'dataset_index', '' ) )
if params.get('save', '') == 'Save':
sample.dataset_files[dataset_index]['name'] = util.restore_text( params.get( 'name',
sample.dataset_files[dataset_index]['name'] ) )
trans.sa_session.add( sample )
trans.sa_session.flush()
status = 'done'
message = 'Saved the changes made to the dataset.'
return trans.fill_template( '/admin/requests/dataset.mako',
sample=sample,
dataset_index=dataset_index,
message=message,
status=status)
message=message) )
self.__send_message(trans, datatx_info, sample, id_list)
message="%i dataset(s) have been queued for transfer from the sequencer. Click on <b>Refresh</b> button above to get the latest transfer status." % len(id_list)
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='manage_datasets',
sample_id=sample.id,
status='done',
message=message) )
#
##
#### Request Type Stuff ###################################################
##
+11 -9
View File
@@ -34,7 +34,7 @@ class RequestsCommon( BaseController ):
if sample.current_state().name != state:
rval[id] = {
"state": sample.current_state().name,
"datasets": len(sample.dataset_files),
"datasets": len(sample.datasets),
"html_state": unicode( trans.fill_template( "requests/common/sample_state.mako", sample=sample, cntrller=cntrller ), 'utf-8' ),
"html_datasets": unicode( trans.fill_template( "requests/common/sample_datasets.mako", trans=trans, sample=sample, cntrller=cntrller ), 'utf-8' )
}
@@ -518,7 +518,6 @@ class RequestsCommon( BaseController ):
barcode=s.bar_code,
library=s.library,
folder=s.folder,
dataset_files=s.dataset_files,
field_values=s.values.content,
lib_widget=lib_widget,
folder_widget=folder_widget))
@@ -530,7 +529,6 @@ class RequestsCommon( BaseController ):
barcode='',
library=None,
folder=None,
dataset_files=[],
field_values=['' for field in request.type.sample_form.fields],
lib_widget=lib_widget,
folder_widget=folder_widget))
@@ -792,8 +790,7 @@ class RequestsCommon( BaseController ):
request, form_values,
current_samples[sample_index]['barcode'],
current_samples[sample_index]['library'],
current_samples[sample_index]['folder'],
dataset_files=[])
current_samples[sample_index]['folder'])
trans.sa_session.add( s )
trans.sa_session.flush()
@@ -1078,12 +1075,17 @@ class RequestsCommon( BaseController ):
operation='show',
status='error',
message="Set a data library and folder for <b>%s</b> to transfer dataset(s)." % sample.name,
id=trans.security.encode_id(sample.request.id) ) )
id=trans.security.encode_id(sample.request.id) ) )
if cntrller == 'requests_admin':
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='manage_datasets',
sample_id=sample.id) )
if params.get( 'folder_path', '' ):
folder_path = util.restore_text( params.get( 'folder_path', '' ) )
else:
if len(sample.dataset_files):
folder_path = os.path.dirname(sample.dataset_files[-1]['filepath'][:-1])
if len(sample.datasets):
folder_path = os.path.dirname(sample.datasets[-1]['filepath'][:-1])
else:
folder_path = util.restore_text( sample.request.type.datatx_info.get('data_dir', '') )
if folder_path and folder_path[-1] != os.sep:
@@ -1094,6 +1096,6 @@ class RequestsCommon( BaseController ):
message = 'The sequencer login information is incomplete. Click on the <b>Sequencer information</b> to add login details.'
return trans.fill_template( '/requests/common/get_data.mako',
cntrller=cntrller, sample=sample,
dataset_files=sample.dataset_files,
dataset_files=sample.datasets,
message=message, status=status, files=[],
folder_path=folder_path )
@@ -70,12 +70,12 @@ class DataTransfer(object):
self.config_id_secret = config_id_secret
count=0
while True:
index = self.get_value_index(self.dom, 'index', count)
dataset_id = self.get_value_index(self.dom, 'dataset_id', count)
file = self.get_value_index(self.dom, 'file', count)
name = self.get_value_index(self.dom, 'name', count)
if file:
self.dataset_files.append(dict(name=name,
index=int(index),
dataset_id=int(dataset_id),
file=file))
else:
break
@@ -149,7 +149,7 @@ class DataTransfer(object):
def print_ticks(d):
pass
for i, df in enumerate(self.dataset_files):
self.update_status(Sample.transfer_status.TRANSFERRING, df['index'])
self.update_status(Sample.transfer_status.TRANSFERRING, df['dataset_id'])
try:
cmd = "scp %s@%s:'%s' '%s/%s'" % ( self.username,
self.host,
@@ -168,7 +168,7 @@ class DataTransfer(object):
raise Exception(msg)
except Exception, e:
msg = traceback.format_exc()
self.update_status('Error', df['index'], msg)
self.update_status('Error', df['dataset_id'], msg)
def add_to_library(self):
@@ -189,29 +189,17 @@ class DataTransfer(object):
log.debug(e)
self.error_and_exit(str(e))
def update_status(self, status, dataset_index='All', msg=''):
def update_status(self, status, dataset_id='All', msg=''):
'''
Update the data transfer status for this dataset in the database
'''
try:
log.debug('Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str(dataset_index), str(self.sample_id) ) )
df = from_json_string(self.galaxydb.get_sample_dataset_files(self.sample_id))
if dataset_index == 'All':
log.debug('Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str(dataset_id), str(self.sample_id) ) )
if dataset_id == 'All':
for dataset in self.dataset_files:
df[dataset['index']]['status'] = status
if status == 'Error':
df[dataset['index']]['error_msg'] = msg
else:
df[dataset['index']]['error_msg'] = ''
self.galaxydb.set_sample_dataset_status(dataset['dataset_id'], status, msg)
else:
df[dataset_index]['status'] = status
if status == 'Error':
df[dataset_index]['error_msg'] = msg
else:
df[dataset_index]['error_msg'] = ''
self.galaxydb.set_sample_dataset_files(self.sample_id, to_json_string(df))
self.galaxydb.set_sample_dataset_status(dataset_id, status, msg)
log.debug('done.')
except:
log.error(traceback.format_exc())
@@ -229,12 +217,12 @@ class DataTransfer(object):
rc = rc + node.data
return rc
def get_value_index(self, dom, tag_name, index):
def get_value_index(self, dom, tag_name, dataset_id):
'''
This method extracts the tag value from the xml message
'''
try:
nodelist = dom.getElementsByTagName(tag_name)[index].childNodes
nodelist = dom.getElementsByTagName(tag_name)[dataset_id].childNodes
except:
return None
rc = ""
@@ -28,11 +28,12 @@ class GalaxyDbInterface(object):
def __init__(self, dbstr):
self.dbstr = dbstr
self.db_engine = create_engine(self.dbstr)
self.db_engine.echo = False
self.db_engine.echo = True
self.metadata = MetaData(self.db_engine)
self.session = sessionmaker(bind=self.db_engine)
self.event_table = Table('sample_event', self.metadata, autoload=True )
self.sample_table = Table('sample', self.metadata, autoload=True )
self.sample_dataset_table = Table('sample_dataset', self.metadata, autoload=True )
self.request_table = Table('request', self.metadata, autoload=True )
self.request_event_table = Table('request_event', self.metadata, autoload=True )
self.state_table = Table('sample_state', self.metadata, autoload=True )
@@ -105,7 +106,7 @@ class GalaxyDbInterface(object):
create_time=datetime.utcnow(),
sample_id=sample_id,
sample_state_id=int(new_state_id),
comment='bar code scanner')
comment='Update by barcode scan')
# if all the samples for this request are in the final state
# then change the request state to 'Complete'
result = select(columns=[self.sample_table.c.id],
@@ -126,23 +127,22 @@ class GalaxyDbInterface(object):
request_id=self.request_id,
state=request_state,
comment='All samples of this request have finished processing.')
def get_sample_dataset_files(self, sample_id):
subsubquery = select(columns=[self.sample_table.c.dataset_files],
whereclause=self.sample_table.c.id==sample_id)
return subsubquery.execute().fetchall()[0][0]
def set_sample_dataset_files(self, sample_id, value):
u = self.sample_table.update(whereclause=self.sample_table.c.id==sample_id)
u.execute(dataset_files=value)
def set_sample_dataset_status(self, id, new_status, msg=None):
u = self.sample_dataset_table.update(whereclause=self.sample_dataset_table.c.id==int(id))
u.execute(status=new_status)
if new_status == 'Error':
u.execute(error_msg=msg)
else:
u.execute(error_msg='')
return
if __name__ == '__main__':
print '''This file should not be run directly. To start the Galaxy AMQP Listener:
%sh run_galaxy_listener.sh'''
dbstr = 'postgres://postgres:postgres@localhost/galaxy_uft'
dbstr = 'postgres://postgres:postgres@localhost/g2'
parser = optparse.OptionParser()
parser.add_option('-n', '--name', help='name of the sample field', dest='name', \
+8 -24
View File
@@ -12,60 +12,44 @@
<ul class="manage-table-actions">
<li>
<a class="action-button" href="${h.url_for( controller='requests_common', action='show_datatx_page', cntrller='requests_admin', sample_id=trans.security.encode_id(sample.id) )}">
<span>Dataset transfer page</span></a>
<span>Browse datasets</span></a>
</li>
</ul>
<div class="toolForm">
<div class="toolFormTitle">Dataset details</div>
<div class="toolFormTitle">Dataset Information</div>
<div class="toolFormBody">
<form name="dataset_details" action="${h.url_for( controller='requests_admin', action='dataset_details', save_changes=True, sample_id=trans.security.encode_id(sample.id), dataset_index=dataset_index )}" method="post" >
<%
dataset = sample.dataset_files[dataset_index]
%>
<div class="form-row">
<label>Name:</label>
<div style="float: left; width: 250px; margin-right: 10px;">
%if dataset['status'] in [sample.transfer_status.IN_QUEUE, sample.transfer_status.NOT_STARTED]:
<input type="text" name="name" value="${dataset['name']}" size="60"/>
%else:
${dataset['name']}
%endif
${sample_dataset.name}
</div>
<div style="clear: both"></div>
</div>
<div class="form-row">
<label>File on the Sequencer:</label>
<div style="float: left; width: 250px; margin-right: 10px;">
${dataset['filepath']}
##<input type="text" name="filepath" value="${dataset['filepath']}" size="100" readonly/>
${sample_dataset.file_path}
</div>
<div style="clear: both"></div>
</div>
<div class="form-row">
<label>Size:</label>
<div style="float: left; width: 250px; margin-right: 10px;">
${dataset.get('size', 'Unknown')}
${sample_dataset.size}
</div>
<div style="clear: both"></div>
</div>
<div class="form-row">
<label>Transfer status:</label>
<div style="float: left; width: 250px; margin-right: 10px;">
${dataset['status']}
${sample_dataset.status}
<br/>
%if dataset['status'] == sample.transfer_status.ERROR:
${dataset['error_msg']}
%if sample_dataset.status == sample.transfer_status.ERROR:
${sample_dataset.error_msg}
%endif
</div>
<div style="clear: both"></div>
</div>
%if dataset['status'] in [sample.transfer_status.IN_QUEUE, sample.transfer_status.NOT_STARTED]:
<div class="form-row">
<input type="submit" name="save" value="Save"/>
</div>
%endif
</form>
</div>
</div>
@@ -0,0 +1,32 @@
<%def name="custom_javascripts()">
<script type="text/javascript">
$("#select-dataset-action-button").bind( "click", function(e) {
alert('afdhblvi')
$.ajax({
url: "${h.url_for( controller='requests_admin', action='remote_file_browser' )}",
data: {id: 6},
error: function() { alert( "Couldn't create new browser" ) },
success: function(form_html) {
show_modal("Select file", form_html, {
"Cancel": function() { window.location = "${h.url_for( controller='requests_admin', action='list' )}"; },
"Continue": function() { $(document).trigger("convert_dbkeys"); continue_fn(); }
});
$("#new-title").focus();
replace_big_select_inputs();
}
});
}
</script>
</%def>
<%def name="javascripts()">
${h.js( "galaxy.base", "galaxy.panels", "json2", "jquery", "jquery.event.drag", "jquery.autocomplete", "jquery.mousewheel", "trackster", "ui.core", "ui.sortable" )}
${self.custom_javascripts()}
${parent.javascripts()}
</%def>
<%inherit file="/grid_base.mako"/>
+144
View File
@@ -0,0 +1,144 @@
<%inherit file="/base.mako"/>
<%namespace file="/message.mako" import="render_msg" />
<script type="text/javascript">
$(document).ready(function(){
//hide the all of the element with class msg_body
$(".msg_body").hide();
//toggle the componenet with class msg_body
$(".msg_head").click(function(){
$(this).next(".msg_body").slideToggle(450);
});
});
</script>
<script type="text/javascript">
function display_file_details(request_id, folder_path)
{
var w = document.get_data.files_list.selectedIndex;
var selected_value = document.get_data.files_list.options[w].value;
var cell = $("#file_details");
if(selected_value.charAt(selected_value.length-1) != '/')
{
// Make ajax call
$.ajax( {
type: "POST",
url: "${h.url_for( controller='requests_admin', action='get_file_details' )}",
dataType: "json",
data: { id: request_id, folder_path: document.get_data.folder_path.value+selected_value },
success : function ( data ) {
cell.html( '<label>'+data+'</label>' )
}
});
}
else
{
cell.html( '' )
}
}
</script>
<script type="text/javascript">
function open_folder1(request_id, folder_path)
{
var w = document.get_data.files_list.selectedIndex;
var selected_value = document.get_data.files_list.options[w].value;
var cell = $("#file_details");
if(selected_value.charAt(selected_value.length-1) == '/')
{
document.get_data.folder_path.value = document.get_data.folder_path.value+selected_value
// Make ajax call
$.ajax( {
type: "POST",
url: "${h.url_for( controller='requests_admin', action='open_folder' )}",
dataType: "json",
data: { id: request_id, folder_path: document.get_data.folder_path.value },
success : function ( data ) {
document.get_data.files_list.options.length = 0
for(i=0; i<data.length; i++)
{
var newOpt = new Option(data[i], data[i]);
document.get_data.files_list.options[i] = newOpt;
}
//cell.html( '<label>'+data+'</label>' )
}
});
}
else
{
cell.html( '' )
}
}
</script>
<style type="text/css">
.msg_head {
padding: 0px 0px;
cursor: pointer;
}
}
</style>
%if message:
${render_msg( message, status )}
%endif
<br/>
<br/>
<ul class="manage-table-actions">
<li>
<a class="action-button" href="${h.url_for( controller='requests_admin', action='manage_request_types', operation='view', id=trans.security.encode_id(request.type.id) )}">
<span>Sequencer information</span></a>
</li>
<li>
<a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='show', id=trans.security.encode_id(request.id) )}">
<span>Browse this request</span></a>
</li>
</ul>
<form name="get_data" id="get_data" action="${h.url_for( controller='requests_admin', cntrller=cntrller, action='get_data', request_id=request.id)}" method="post" >
<div class="toolFormTitle">Select files for transfer</div>
<div class="toolForm">
<div class="form-row">
<label>Sample:</label>
${samples_selectbox.get_html()}
<div class="toolParamHelp" style="clear: both;">
Select the sample with which you want to associate the dataset(s)
</div>
<br/>
<label>Folder path on the sequencer:</label>
<input type="text" name="folder_path" value="${folder_path}" size="100"/>
<input type="submit" name="browse_button" value="List contents"/>
##<input type="submit" name="open_folder" value="Open folder"/>
<input type="submit" name="folder_up" value="Up"/>
</div>
<div class="form-row">
<select name="files_list" id="files_list" style="max-width: 60%; width: 98%; height: 150px; font-size: 100%;" ondblclick="open_folder1(${request.id}, '${folder_path}')" onChange="display_file_details(${request.id}, '${folder_path}')" multiple>
%for index, f in enumerate(files):
<option value="${f}">${f}</option>
%endfor
</select>
<br/>
<div id="file_details" class="toolParamHelp" style="clear: both;">
</div>
</div>
<div class="form-row">
<!-- <div class="toolParamHelp" style="clear: both;">
After selecting dataset(s), be sure to click on the <b>Start transfer</b> button.
Once the transfer is complete the dataset(s) will show up on this page.
</div>-->
<input type="submit" name="select_show_datasets_button" value="Select & show datasets"/>
<input type="submit" name="select_more_button" value="Select more"/>
</div>
</div>
</form>
@@ -0,0 +1,66 @@
<%inherit file="/base.mako"/>
<%namespace file="/message.mako" import="render_msg" />
%if message:
${render_msg( message, status )}
%endif
<h3>Rename datasets for Sample "${sample.name}"</h3>
<br/>
${render_msg('A dataset can be renamed only if its status is "Not Started"', 'ok')}
<ul class="manage-table-actions">
<li>
<a class="action-button" href="${h.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample.id )}">
<span>Browse datasets</span></a>
</li>
<li>
<a class="action-button" href="${h.url_for( controller='requests_admin', action='list', operation='show', id=trans.security.encode_id(sample.request.id) )}">
<span>Browse this request</span></a>
</li>
</ul>
<form name="rename_datasets" id="rename_datasets" action="${h.url_for( controller='requests_admin', action='rename_datasets', id_list=id_list, sample_id=trans.security.encode_id(sample.id))}" method="post" >
<table class="grid">
<thead>
<tr>
<th>Prepend directory name</th>
<th>Name</th>
<th>Path on Sequencer</th>
</tr>
<thead>
<tbody>
%for id in id_list:
<%
sample_dataset = trans.sa_session.query( trans.app.model.SampleDataset ).get( trans.security.decode_id(id) )
import os
id = trans.security.decode_id(id)
%>
%if sample_dataset.status == trans.app.model.Sample.transfer_status.NOT_STARTED:
<tr>
<td>
<select name="prepend_${sample_dataset.id}" last_selected_value="2">
<option value="None" selected></option>
%for option_index, option in enumerate(sample_dataset.file_path.split(os.sep)[:-1]):
%if option.strip():
<option value="${option}">${option}</option>
%endif
%endfor
</select>
</td>
<td>
<input type="text" name="name_${sample_dataset.id}" value="${sample_dataset.name}" size="100"/>
</td>
<td>
${sample_dataset.file_path}
</td>
</tr>
%endif
%endfor
</tbody>
</table>
<br/>
<div class="form-row">
<input type="submit" name="save_button" value="Save"/>
<input type="submit" name="cancel_button" value="Close"/>
</div>
</form>
+2
View File
@@ -79,6 +79,8 @@
class_plural = "Libraries"
elif a_class == model.HistoryDatasetAssociation:
class_plural = "Datasets"
elif a_class == model.SampleDataset:
class_plural = "Sample Datasets"
elif a_class == model.FormDefinitionCurrent:
class_plural = "Forms"
else:
+1 -2
View File
@@ -94,8 +94,7 @@ $(document).ready(function(){
</style>
<h2>Data transfer from Sequencer</h2>
<h3>Sample "${sample.name}" of Request "${sample.request.name}"</h3>
<h2>Datasets of Sample "${sample.name}"</h2>
<br/>
<br/>
@@ -1,5 +1,5 @@
<%def name="render_sample_datasets( cntrller, sample )">
<a href="${h.url_for(controller='requests_common', cntrller=cntrller, action='show_datatx_page', sample_id=trans.security.encode_id(sample.id))}">${sample.transferred_dataset_files()}/${len(sample.dataset_files)}</a>
<a href="${h.url_for(controller='requests_common', cntrller=cntrller, action='show_datatx_page', sample_id=trans.security.encode_id(sample.id))}">${sample.transferred_dataset_files()}/${len(sample.datasets)}</a>
</%def>
+26 -18
View File
@@ -147,30 +147,29 @@ $(document).ready(function(){
</div>
<ul class="manage-table-actions">
%if request.unsubmitted() and request.samples:
<li>
<li><a class="action-button" id="seqreq-${request.id}-popup" class="menubutton">Sequencing Request Actions</a></li>
<div popupmenu="seqreq-${request.id}-popup">
%if request.unsubmitted() and request.samples:
<a class="action-button" confirm="More samples cannot be added to this request once it is submitted. Click OK to submit." href="${h.url_for( controller=cntrller, action='list', operation='Submit', id=trans.security.encode_id(request.id) )}">
<span>Submit request</span></a>
</li>
%endif
%if cntrller == 'requests_admin' and trans.user_is_admin():
%if request.submitted():
<li>
<a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='reject', id=trans.security.encode_id(request.id))}">
<span>Reject request</span></a>
</li>
<span>Submit</span></a>
%endif
%endif
<li>
<a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='Edit', id=trans.security.encode_id(request.id))}">
<span>Edit</span></a>
<a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='events', id=trans.security.encode_id(request.id) )}">
<span>History</span></a>
</li>
%if cntrller == 'requests_admin' and trans.user_is_admin():
%if request.submitted():
<a class="action-button" href="${h.url_for( controller=cntrller, action='list', operation='reject', id=trans.security.encode_id(request.id))}">
<span>Reject</span></a>
<a class="action-button" href="${h.url_for( controller='requests_admin', action='get_data', show_page=True, request_id=request.id)}">
<span>Select dataset(s) to transfer</span></a>
%endif
%endif
</div>
<li>
<a class="action-button" href="${h.url_for( controller=cntrller, action='list')}">
<span>Browse requests</span></a>
</li>
</ul>
@@ -396,12 +395,21 @@ $(document).ready(function(){
<td>${info['name']}</td>
<td>${info['barcode']}</td>
%if sample.request.unsubmitted():
<td>Unsubmitted</td>
##<td>Unsubmitted</td>
<td>
<div id="history-name-container">
<div id="history-name" class="tooltip editable-text" title="Click to rename history">Unsubmitted</div>
</div>
</td>
%else:
<td id="sampleState-${sample.id}">${render_sample_state( cntrller, sample )}</td>
%endif
%if info['library']:
<td><a href="${h.url_for( controller='library_common', action='browse_library', cntrller='library', id=trans.security.encode_id( info['library'].id ) )}">${info['library'].name}</a></td>
%if cntrller == 'requests':
<td><a href="${h.url_for( controller='library_common', action='browse_library', cntrller='library', id=trans.security.encode_id( info['library'].id ) )}">${info['library'].name}</a></td>
%elif cntrller == 'requests_admin':
<td><a href="${h.url_for( controller='library_common', action='browse_library', cntrller='library_admin', id=trans.security.encode_id( info['library'].id ) )}">${info['library'].name}</a></td>
%endif
%else:
<td></td>
%endif