opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-28 02:58:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix the remaining 1334 flake8 errors in scripts/ . Close #391.

Browse Source
This commit is contained in:
Nicola Soranzo
2016-02-18 15:45:15 +00:00
parent f5277a654a
commit 93348e7199
55 changed files with 1513 additions and 1439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/flake8_blacklist.txt
-9
View File
@@ -4,12 +4,3 @@ database/
doc/patch.py
doc/source/conf.py
lib/galaxy/util/jstree.py
scripts/api/
scripts/data_libraries/
scripts/loc_files/
scripts/microbes/
scripts/others/
scripts/scramble/
scripts/tool_shed/
scripts/tools/
scripts/transfer.py
scripts/api/common.py
+17 -10
View File
@@ -3,7 +3,6 @@ Common methods used by the API sample scripts.
"""
import json
import logging
import os
import sys
import urllib2
@@ -11,6 +10,7 @@ from Crypto.Cipher import Blowfish
log = logging.getLogger( __name__ )
def make_url( api_key, url, args=None ):
"""
Adds the API Key to the URL if it's not already there.
@@ -24,6 +24,7 @@ def make_url( api_key, url, args=None ):
args.insert( 0, ( 'key', api_key ) )
return url + argsep + '&'.join( [ '='.join( t ) for t in args ] )
def get( api_key, url ):
"""
Do the actual GET.
@@ -35,29 +36,32 @@ def get( api_key, url ):
print "URL did not return JSON data: %s" % e
sys.exit(1)
def post( api_key, url, data ):
"""
Do the actual POST.
"""
url = make_url( api_key, url )
req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) )
req = urllib2.Request( url, headers={ 'Content-Type': 'application/json' }, data=json.dumps( data ) )
return json.loads( urllib2.urlopen( req ).read() )
def put( api_key, url, data ):
"""
Do the actual PUT
"""
url = make_url( api_key, url )
req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ))
req = urllib2.Request( url, headers={ 'Content-Type': 'application/json' }, data=json.dumps( data ))
req.get_method = lambda: 'PUT'
return json.loads( urllib2.urlopen( req ).read() )
def __del( api_key, url, data ):
"""
Do the actual DELETE
"""
url = make_url( api_key, url )
req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ))
req = urllib2.Request( url, headers={ 'Content-Type': 'application/json' }, data=json.dumps( data ))
req.get_method = lambda: 'DELETE'
return json.loads( urllib2.urlopen( req ).read() )
@@ -70,7 +74,7 @@ def display( api_key, url, return_formatted=True ):
r = get( api_key, url )
except urllib2.HTTPError, e:
print e
print e.read( 1024 ) # Only return the first 1K of errors.
print e.read( 1024 ) # Only return the first 1K of errors.
sys.exit( 1 )
if type( r ) == unicode:
print 'error: %s' % r
@@ -85,7 +89,7 @@ def display( api_key, url, return_formatted=True ):
# All collection members should have a name in the response.
# url is optional
if 'url' in i:
print '#%d: %s' % (n+1, i.pop( 'url' ) )
print '#%d: %s' % (n + 1, i.pop( 'url' ) )
if 'name' in i:
print ' name: %s' % i.pop( 'name' )
for k, v in i.items():
@@ -103,6 +107,7 @@ def display( api_key, url, return_formatted=True ):
else:
print 'response is unknown type: %s' % type( r )
def submit( api_key, url, data, return_formatted=True ):
"""
Sends an API POST request and acts as a generic formatter for the JSON response.
@@ -116,7 +121,7 @@ def submit( api_key, url, data, return_formatted=True ):
print e.read( 1024 )
sys.exit( 1 )
else:
return 'Error. '+ str( e.read( 1024 ) )
return 'Error. ' + str( e.read( 1024 ) )
if not return_formatted:
return r
print 'Response'
@@ -139,6 +144,7 @@ def submit( api_key, url, data, return_formatted=True ):
else:
print r
def update( api_key, url, data, return_formatted=True ):
"""
Sends an API PUT request and acts as a generic formatter for the JSON response.
@@ -152,13 +158,14 @@ def update( api_key, url, data, return_formatted=True ):
print e.read( 1024 )
sys.exit( 1 )
else:
return 'Error. '+ str( e.read( 1024 ) )
return 'Error. ' + str( e.read( 1024 ) )
if not return_formatted:
return r
print 'Response'
print '--------'
print r
def delete( api_key, url, data, return_formatted=True ):
"""
Sends an API DELETE request and acts as a generic formatter for the JSON response.
@@ -172,13 +179,14 @@ def delete( api_key, url, data, return_formatted=True ):
print e.read( 1024 )
sys.exit( 1 )
else:
return 'Error. '+ str( e.read( 1024 ) )
return 'Error. ' + str( e.read( 1024 ) )
if not return_formatted:
return r
print 'Response'
print '--------'
print r
def encode_id( config_id_secret, obj_id ):
"""
utility method to encode ID's
@@ -190,4 +198,3 @@ def encode_id( config_id_secret, obj_id ):
s = ( "!" * ( 8 - len(s) % 8 ) ) + s
# Encrypt
return id_cipher.encrypt( s ).encode( 'hex' )
scripts/api/copy_hda_to_library_folder.py
+4 -4
View File
@@ -1,13 +1,14 @@
#!/usr/bin/env python
usage = "USAGE: copy_hda_to_library_folder.py <base url> <api key> <hda id> <library id> <folder id> [ message ]"
import os, sys
import os
import sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
# -----------------------------------------------------------------------------
def copy_hda_to_library_folder( base_url, key, hda_id, library_id, folder_id, message='' ):
url = 'http://%s/api/libraries/%s/contents' %( base_url, library_id )
url = 'http://%s/api/libraries/%s/contents' % ( base_url, library_id )
payload = {
'folder_id' : folder_id,
'create_type' : 'file',
@@ -19,7 +20,6 @@ def copy_hda_to_library_folder( base_url, key, hda_id, library_id, folder_id, me
return submit( key, url, payload )
# -----------------------------------------------------------------------------
if __name__ == '__main__':
num_args = len( sys.argv )
if num_args < 6:
scripts/api/create.py
+3 -1
View File
@@ -4,9 +4,11 @@ Generic POST/create script
usage: create.py key url [key=value ...]
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
data = {}
scripts/api/data_manager_example_execute.py
+26 -24
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python
##Dan Blankenberg
##Very simple example of using the API to run Data Managers
##Script makes the naive assumption that dbkey==sequence id, which in many cases is not true nor desired
##*** This script is not recommended for use as-is on a production server ***
# Dan Blankenberg
# Very simple example of using the API to run Data Managers
# Script makes the naive assumption that dbkey==sequence id, which in many cases is not true nor desired
# *** This script is not recommended for use as-is on a production server ***
import os
import sys
@@ -19,6 +19,7 @@ FETCH_GENOME_TOOL_ID = 'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manage
BUILD_INDEX_TOOLS_ID = [ 'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_bwa_index_builder/bwa_index_builder_data_manager/0.0.1',
'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_bwa_index_builder/bwa_color_space_index_builder_data_manager/0.0.1' ]
def run_tool( tool_id, history_id, params, api_key, galaxy_url, wait=True, sleep_time=None, **kwargs ):
sleep_time = sleep_time or DEFAULT_SLEEP_TIME
tools_url = urlparse.urljoin( galaxy_url, 'api/tools' )
@@ -40,60 +41,61 @@ def run_tool( tool_id, history_id, params, api_key, galaxy_url, wait=True, sleep
outputs.pop( 0 )
if wait and outputs:
time.sleep( sleep_time )
return rval
def get_dataset_state( hda_id, api_key, galaxy_url ):
datasets_url = urlparse.urljoin( galaxy_url, 'api/datasets/%s' % hda_id )
dataset_info = get( api_key, datasets_url )
return dataset_info['state']
def dataset_is_terminal( hda_id, api_key, galaxy_url ):
dataset_state = get_dataset_state( hda_id, api_key, galaxy_url )
return dataset_state in [ 'ok', 'error' ]
if __name__ == '__main__':
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option( '-k', '--key', dest='api_key', action='store', type="string", default=None, help='API Key.' )
parser.add_option( '-u', '--url', dest='base_url', action='store', type="string", default='http://localhost:8080', help='Base URL of Galaxy Server' )
parser.add_option( '-d', '--dbkey', dest='dbkeys', action='append', type="string", default=[], help='List of dbkeys to download and Index' )
parser.add_option( '-s', '--sleep_time', dest='sleep_time', action='store', type="int", default=DEFAULT_SLEEP_TIME, help='How long to sleep between check loops' )
(options, args) = parser.parse_args()
#check options
# check options
assert options.api_key is not None, ValueError( 'You must specify an API key.' )
assert options.dbkeys, ValueError( 'You must specify at least one dbkey to use.' )
#check user is admin
configuration_options = get( options.api_key, urlparse.urljoin( options.base_url, 'api/configuration' ) )
if 'library_import_dir' not in configuration_options: #hack to check if is admin user
# check user is admin
configuration_options = get( options.api_key, urlparse.urljoin( options.base_url, 'api/configuration' ) )
if 'library_import_dir' not in configuration_options: # hack to check if is admin user
print "Warning: Data Managers are only available to admin users. The API Key provided does not appear to belong to an admin user. Will attempt to run anyway."
#Fetch Genomes
# Fetch Genomes
dbkeys = {}
for dbkey in options.dbkeys:
if dbkey not in dbkeys:
dbkeys[ dbkey ] = run_tool( FETCH_GENOME_TOOL_ID, None, { 'dbkey':dbkey, 'reference_source|reference_source_selector': 'ucsc', 'reference_source|requested_dbkey': dbkey }, options.api_key, options.base_url, wait=False )
dbkeys[ dbkey ] = run_tool( FETCH_GENOME_TOOL_ID, None, { 'dbkey': dbkey, 'reference_source|reference_source_selector': 'ucsc', 'reference_source|requested_dbkey': dbkey }, options.api_key, options.base_url, wait=False )
else:
"dbkey (%s) was specified more than once, skipping additional specification." % ( dbkey )
print 'Genomes Queued for downloading.'
#Start indexers
# Start indexers
indexing_tools = []
while dbkeys:
for dbkey, value in dbkeys.items():
if dataset_is_terminal( value['outputs'][0]['id'], options.api_key, options.base_url ):
del dbkeys[ dbkey ]
for tool_id in BUILD_INDEX_TOOLS_ID:
indexing_tools.append( run_tool( tool_id, None, { 'all_fasta_source':dbkey }, options.api_key, options.base_url, wait=False ) )
indexing_tools.append( run_tool( tool_id, None, { 'all_fasta_source': dbkey }, options.api_key, options.base_url, wait=False ) )
if dbkeys:
time.sleep( options.sleep_time )
print 'All genomes downloaded and indexers now queued.'
#Wait for indexers to finish
# Wait for indexers to finish
while indexing_tools:
for i, indexing_tool_value in enumerate( indexing_tools ):
if dataset_is_terminal( indexing_tool_value['outputs'][0]['id'], options.api_key, options.base_url ):
@@ -102,5 +104,5 @@ if __name__ == '__main__':
break
if indexing_tools:
time.sleep( options.sleep_time )
print 'All indexers have been run, please check results.'
scripts/api/delete.py
+3 -1
View File
@@ -4,9 +4,11 @@ Generic DELETE/delete script
usage: delete.py key url
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import delete
data = {}
scripts/api/display.py
+4 -1
View File
@@ -1,7 +1,10 @@
#!/usr/bin/env python
import os
import sys
import urllib2
import os, sys, urllib2
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import display
try:
scripts/api/example_watch_folder.py
+9 -7
View File
@@ -14,9 +14,12 @@ import os
import shutil
import sys
import time
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit, display
def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
# Find/Create data library with the above name. Assume we're putting datasets in the root folder '/'
libs = display(api_key, api_url + 'libraries', return_formatted=False)
@@ -25,14 +28,14 @@ def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
if library['name'] == data_library:
library_id = library['id']
if not library_id:
lib_create_data = {'name':data_library}
lib_create_data = {'name': data_library}
library = submit(api_key, api_url + 'libraries', lib_create_data, return_formatted=False)
library_id = library[0]['id']
folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted = False)
folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted=False)
for f in folders:
if f['name'] == "/":
library_folder_id = f['id']
workflow = display(api_key, api_url + 'workflows/%s' % workflow, return_formatted = False)
workflow = display(api_key, api_url + 'workflows/%s' % workflow, return_formatted=False)
if not workflow:
print "Workflow %s not found, terminating."
sys.exit(1)
@@ -52,8 +55,8 @@ def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
data['upload_option'] = 'upload_paths'
data['filesystem_paths'] = fullpath
data['create_type'] = 'file'
libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted = False)
#TODO Handle this better, but the datatype isn't always
libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted=False)
# TODO Handle this better, but the datatype isn't always
# set for the followup workflow execution without this
# pause.
time.sleep(5)
@@ -65,7 +68,7 @@ def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
wf_data['history'] = "%s - %s" % (fname, workflow['name'])
wf_data['ds_map'] = {}
for step_id, ds_in in workflow['inputs'].iteritems():
wf_data['ds_map'][step_id] = {'src':'ld', 'id':ds['id']}
wf_data['ds_map'][step_id] = {'src': 'ld', 'id': ds['id']}
res = submit( api_key, api_url + 'workflows', wf_data, return_formatted=False)
if res:
print res
@@ -85,4 +88,3 @@ if __name__ == '__main__':
print 'usage: %s key url in_folder out_folder data_library workflow' % os.path.basename( sys.argv[0] )
sys.exit( 1 )
main(api_key, api_url, in_folder, out_folder, data_library, workflow )
scripts/api/form_create_from_xml.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/history_create_history.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/history_delete_history.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import delete
try:
scripts/api/import_library_dataset_to_history.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/import_workflows_from_installed_tool_shed_repository.py
+4 -2
View File
@@ -13,6 +13,7 @@ sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import display
from common import submit
def clean_url( url ):
if url.find( '//' ) > 0:
# We have an url that includes a protocol, something like: http://localhost:9009
@@ -20,6 +21,7 @@ def clean_url( url ):
return items[ 1 ].rstrip( '/' )
return url.rstrip( '/' )
def main( options ):
api_key = options.api
base_galaxy_url = options.local_url.rstrip( '/' )
@@ -41,9 +43,9 @@ def main( options ):
url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/exported_workflows' % str( tool_shed_repository_id ) )
exported_workflows = display( api_key, url, return_formatted=False )
if exported_workflows:
# Import all of the workflows in the list of exported workflows.
# Import all of the workflows in the list of exported workflows.
data = {}
# NOTE: to import a single workflow, add an index to data (e.g.,
# NOTE: to import a single workflow, add an index to data (e.g.,
# data[ 'index' ] = 0
# and change the url to be ~/import_workflow (singular). For example,
# url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) )
scripts/api/install_tool_shed_repositories.py
+1
View File
@@ -23,6 +23,7 @@ import argparse
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
def main( options ):
"""Collect all user data and install the tools via the Galaxy API."""
data = {}
scripts/api/library_create_folder.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/library_create_library.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/library_upload_dir.py
+2 -1
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python
import sys
import argparse
import os
import sys
from bioblend import galaxy
scripts/api/library_upload_from_import_dir.py
+4 -1
View File
@@ -3,8 +3,11 @@
Example usage:
./library_upload_from_import_dir.py <key> http://127.0.0.1:8080/api/libraries/dda47097d9189f15/contents Fdda47097d9189f15 auto /Users/EnisAfgan/projects/pprojects/galaxy/lib_upload_dir ?
"""
import os, sys
import os
import sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/load_data_with_metadata.py
+9 -10
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env python
"""
This script scans a directory for files with companion '.json' files, then loads
This script scans a directory for files with companion '.json' files, then loads
the data from the file, and attaches the .json contents using the 'extended_metadata'
system in the library
@@ -10,16 +9,16 @@ python load_data_with_metadata.py <api_key> <api_url> /data/folder "API Imports"
NOTE: The upload method used requires the data library filesystem upload allow_library_path_paste
"""
import os
import shutil
import sys
import json
import time
import argparse
import json
import os
import sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit, display
def load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_field=None):
data = {}
data['folder_id'] = library_folder_id
@@ -38,7 +37,7 @@ def load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_fi
if uuid_field is not None and uuid_field in ext_meta:
data['uuid'] = ext_meta[uuid_field]
libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted = True)
libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted=True)
print libset
@@ -50,10 +49,10 @@ def main(api_key, api_url, in_folder, data_library, uuid_field=None):
if library['name'] == data_library:
library_id = library['id']
if not library_id:
lib_create_data = {'name':data_library}
lib_create_data = {'name': data_library}
library = submit(api_key, api_url + 'libraries', lib_create_data, return_formatted=False)
library_id = library['id']
folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted = False)
folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted=False)
for f in folders:
if f['name'] == "/":
library_folder_id = f['id']
scripts/api/repair_tool_shed_repository.py
+3 -1
View File
@@ -13,13 +13,15 @@ sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import display
from common import submit
def clean_url( url ):
if url.find( '//' ) > 0:
# We have an url that includes a protocol, something like: http://localhost:9009
items = url.split( '//' )
return items[ 1 ].rstrip( '/' )
return url.rstrip( '/' )
def main( options ):
"""Collect all user data and install the tools via the Galaxy API."""
api_key = options.api
scripts/api/request_type_create_from_xml.py
+3 -1
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
try:
scripts/api/requests_update_state.py
+3 -3
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys, traceback
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import update
try:
@@ -13,4 +14,3 @@ except IndexError:
sys.exit( 1 )
update( sys.argv[1], sys.argv[2], data, return_formatted=True )
scripts/api/reset_metadata_on_installed_repositories.py
+3 -2
View File
@@ -8,15 +8,16 @@ usage: reset_metadata_on_installed_repositories.py key
Here is a working example of how to use this script.
python ./reset_metadata_on_installed_repositories.py -a 22be3b -u http://localhost:8763/
"""
import argparse
import os
import sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
def main( options ):
api_key = options.api
base_galaxy_url = options.galaxy_url.rstrip( '/' )
url = '%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories' % base_galaxy_url
submit( options.api, url, {} )
scripts/api/sample_dataset_update_status.py
+5 -5
View File
@@ -1,11 +1,11 @@
#!/usr/bin/env python
import os
import sys
import os, sys, traceback
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import display
from common import submit
from common import update
try:
data = {}
data[ 'update_type' ] = 'sample_dataset_transfer_status'
@@ -17,6 +17,6 @@ except IndexError:
try:
data[ 'error_msg' ] = sys.argv[5]
except IndexError:
data[ 'error_msg' ] = ''
data[ 'error_msg' ] = ''
print data
update( sys.argv[1], sys.argv[2], data, return_formatted=True )
scripts/api/sample_update_state.py
+3 -4
View File
@@ -1,10 +1,9 @@
#!/usr/bin/env python
import os
import sys
import os, sys, traceback
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import display
from common import submit
from common import update
try:
scripts/api/search.py
+2 -2
View File
@@ -1,11 +1,11 @@
"""
Sample script for Galaxy Search API
"""
import json
import requests
import sys
class RemoteGalaxy(object):
def __init__(self, url, api_key):
@@ -23,7 +23,7 @@ class RemoteGalaxy(object):
c_url = self.url + path
params = {}
params['key'] = self.api_key
req = requests.post(c_url, data=json.dumps(payload), params=params, headers = {'Content-Type': 'application/json'} )
req = requests.post(c_url, data=json.dumps(payload), params=params, headers={'Content-Type': 'application/json'} )
return req.json()
if __name__ == "__main__":
scripts/api/sequencer_configuration_create.py
+11 -7
View File
@@ -1,32 +1,36 @@
#!/usr/bin/env python
import os, sys#, traceback
import os
import sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit, get
def create_sequencer_configuration( key, base_url, request_form_filename, sample_form_filename, request_type_filename, email_addresses, return_formatted=True ):
#create request_form
# create request_form
data = {}
data[ 'xml_text' ] = open( request_form_filename ).read()
request_form = submit( key, "%sforms" % base_url, data, return_formatted=False )[0]
#create sample_form
# create sample_form
data = {}
data[ 'xml_text' ] = open( sample_form_filename ).read()
sample_form = submit( key, "%sforms" % base_url, data, return_formatted=False )[0]
#get user ids
# get user ids
user_ids = [ user['id'] for user in get( key, "%susers" % base_url ) if user['email'] in email_addresses ]
#create role, assign to user
# create role, assign to user
data = {}
data[ 'name' ] = "request_type_role_%s_%s_%s name" % ( request_form['id'], sample_form['id'], '_'.join( email_addresses ) )
data[ 'description' ] = "request_type_role_%s_%s_%s description" % ( request_form['id'], sample_form['id'], '_'.join( email_addresses ) )
data[ 'user_ids' ] = user_ids
role_ids = [ role[ 'id' ] for role in submit( key, "%sroles" % base_url, data, return_formatted=False ) ]
#create request_type
# create request_type
data = {}
data[ 'request_form_id' ] = request_form[ 'id' ]
data[ 'sample_form_id' ] = sample_form[ 'id' ]
data[ 'role_ids' ] = role_ids
data[ 'xml_text' ] = open( request_type_filename ).read()
return submit( key, "%srequest_types" % base_url, data, return_formatted=return_formatted ) #create and print out results for request type
return submit( key, "%srequest_types" % base_url, data, return_formatted=return_formatted ) # create and print out results for request type
def main():
try:
scripts/api/update.py
+3 -1
View File
@@ -4,9 +4,11 @@ Generic PUT/update script
usage: create.py key url [key=value ...]
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import update
data = {}
scripts/api/upload_to_history.py
+9 -15
View File
@@ -2,21 +2,18 @@
"""
Upload a file to the desired history.
"""
_USAGE = ( "history_upload.py <api key> <galaxy base url> <history id> <filepath to upload>\n"
+ " (where galaxy base url is just the root url where your Galaxy is served; e.g. 'localhost:8080')" )
import os, sys, json, pprint
#sys.path.insert( 0, os.path.dirname( __file__ ) )
import json
import os
import sys
try:
import requests
except ImportError, imp_err:
log.error( "Could not import the requests module. See http://docs.python-requests.org/en/latest/ or "
+ "install with 'pip install requests'" )
print "Could not import the requests module. See http://docs.python-requests.org/en/latest/" + \
" or install with 'pip install requests'"
raise
# -----------------------------------------------------------------------------
def upload_file( base_url, api_key, history_id, filepath, **kwargs ):
full_url = base_url + '/api/tools'
@@ -28,12 +25,10 @@ def upload_file( base_url, api_key, history_id, filepath, **kwargs ):
inputs = {
'files_0|NAME' : kwargs.get( 'filename', os.path.basename( filepath ) ),
'files_0|type' : 'upload_dataset',
#TODO: the following doesn't work with tools.py
#'dbkey' : kwargs.get( 'dbkey', '?' ),
# TODO: the following doesn't work with tools.py
'dbkey' : '?',
'file_type' : kwargs.get( 'file_type', 'auto' ),
'ajax_upload' : u'true',
#'async_datasets': '1',
}
payload[ 'inputs' ] = json.dumps( inputs )
@@ -42,13 +37,12 @@ def upload_file( base_url, api_key, history_id, filepath, **kwargs ):
files = { 'files_0|file_data' : file_to_upload }
response = requests.post( full_url, data=payload, files=files )
return response.json()
# -----------------------------------------------------------------------------
if __name__ == '__main__':
if len( sys.argv ) < 5:
print _USAGE
print "history_upload.py <api key> <galaxy base url> <history id> <filepath to upload>\n" + \
" (where galaxy base url is just the root url where your Galaxy is served; e.g. 'localhost:8080')"
sys.exit( 1 )
api_key, base_url, history_id, filepath = sys.argv[1:5]
scripts/api/workflow_delete.py
+4 -2
View File
@@ -2,15 +2,17 @@
"""
# ---------------------------------------------- #
# PARKLAB, Author: RPARK
API example script for deleting workflows
API example script for deleting workflows
# ---------------------------------------------- #
Example calls:
python workflow_delete.py <api_key> <galaxy_url>/api/workflows/<workflow id> True
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import delete
try:
scripts/api/workflow_execute.py
+4 -3
View File
@@ -5,9 +5,11 @@ Example calls:
python workflow_execute.py <api_key> <galaxy_url>/api/workflows f2db41e1fa331b3e 'Test API History' '38=ldda=0qr350234d2d192f'
python workflow_execute.py <api_key> <galaxy_url>/api/workflows f2db41e1fa331b3e 'hist_id=a912e9e5d84530d4' '38=hda=03501d7626bd192f'
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
@@ -21,7 +23,7 @@ def main():
# mapping, just use it for everything?
for v in sys.argv[5:]:
step, src, ds_id = v.split('=')
data['ds_map'][step] = {'src':src, 'id':ds_id}
data['ds_map'][step] = {'src': src, 'id': ds_id}
except IndexError:
print 'usage: %s key url workflow_id history step=src=dataset_id' % os.path.basename(sys.argv[0])
sys.exit(1)
@@ -29,4 +31,3 @@ def main():
if __name__ == '__main__':
main()
scripts/api/workflow_execute_parameters.py
+16 -16
View File
@@ -6,44 +6,45 @@
Execute workflows from the command line.
Example calls:
python workflow_execute.py <api_key> <galaxy_url>/api/workflows <workflow_id> 'hist_id=<history_id>' '38=hda=<file_id>' 'param=tool=name=value'
python workflow_execute_parameters.py <api_key> http://localhost:8080/api/workflows 1cd8e2f6b131e891 'Test API' '69=ld=a799d38679e985db' '70=ld=33b43b4e7093c91f' 'param=peakcalling_spp=aligner=bowtie' 'param=bowtie_wrapper=suppressHeader=True' 'param=peakcalling_spp=window_size=1000'
python workflow_execute.py <api_key> <galaxy_url>/api/workflows <workflow_id> 'hist_id=<history_id>' '38=hda=<file_id>' 'param=tool=name=value'
python workflow_execute_parameters.py <api_key> http://localhost:8080/api/workflows 1cd8e2f6b131e891 'Test API' '69=ld=a799d38679e985db' '70=ld=33b43b4e7093c91f' 'param=peakcalling_spp=aligner=bowtie' 'param=bowtie_wrapper=suppressHeader=True' 'param=peakcalling_spp=window_size=1000'
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
def main():
try:
print("workflow_execute:py:");
print("workflow_execute:py:")
data = {}
data['workflow_id'] = sys.argv[3]
data['history'] = sys.argv[4]
data['ds_map'] = {}
#########################################################
### Trying to pass in parameter for my own dictionary ###
data['parameters'] = {};
# Trying to pass in parameter for my own dictionary
data['parameters'] = {}
# DBTODO If only one input is given, don't require a step
# mapping, just use it for everything?
for v in sys.argv[5:]:
print("Multiple arguments ");
print(v);
print("Multiple arguments ")
print(v)
try:
step, src, ds_id = v.split('=');
data['ds_map'][step] = {'src':src, 'id':ds_id};
step, src, ds_id = v.split('=')
data['ds_map'][step] = {'src': src, 'id': ds_id}
except ValueError:
print("VALUE ERROR:");
wtype, wtool, wparam, wvalue = v.split('=');
print("VALUE ERROR:")
wtype, wtool, wparam, wvalue = v.split('=')
try:
data['parameters'][wtool] = {'param':wparam, 'value':wvalue}
data['parameters'][wtool] = {'param': wparam, 'value': wvalue}
except ValueError:
print("TOOL ID ERROR:");
print("TOOL ID ERROR:")
except IndexError:
print 'usage: %s key url workflow_id history step=src=dataset_id' % os.path.basename(sys.argv[0])
@@ -52,4 +53,3 @@ def main():
if __name__ == '__main__':
main()
scripts/api/workflow_import.py
+6 -3
View File
@@ -4,11 +4,14 @@ Import workflows from the command line.
Example calls:
python workflow_import.py <api_key> <galaxy_url> '/path/to/workflow/file [--add_to_menu]'
"""
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
def main():
api_key = sys.argv[1]
api_base_url = sys.argv[2]
@@ -17,11 +20,11 @@ def main():
data = {}
data['installed_repository_file'] = sys.argv[3]
if len(sys.argv) > 4 and sys.argv[4] == "--add_to_menu":
data['add_to_menu'] = True
data['add_to_menu'] = True
except IndexError:
print 'usage: %s key galaxy_url workflow_file' % os.path.basename(sys.argv[0])
sys.exit(1)
#print display( api_key, api_base_url + "/api/workflows" )
# print display( api_key, api_base_url + "/api/workflows" )
submit( api_key, api_url, data, return_formatted=False )
if __name__ == '__main__':
scripts/api/workflow_import_from_file_rpark.py
+9 -15
View File
@@ -1,23 +1,21 @@
#!/usr/bin/env python
"""
python rpark_import_workflow_from_file.py 35a24ae2643785ff3d046c98ea362c7f http://localhost:8080/api/workflows/import 'spp_submodule.ga'
python rpark_import_workflow_from_file.py 35a24ae2643785ff3d046c98ea362c7f http://localhost:8080/api/workflows/import 'spp_submodule.ga'
"""
import json
import os
import sys
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
### Rpark edit ###
import json
def openWorkflow(in_file):
def openWorkflow(in_file):
with open(in_file) as f:
temp_data = json.load(f)
return temp_data;
return temp_data
try:
@@ -26,13 +24,9 @@ except IndexError:
print 'usage: %s key url [name] ' % os.path.basename( sys.argv[0] )
sys.exit( 1 )
try:
#data = {}
#data[ 'name' ] = sys.argv[3]
data = {};
workflow_dict = openWorkflow(sys.argv[3]);
data ['workflow'] = workflow_dict;
data = {}
workflow_dict = openWorkflow(sys.argv[3])
data['workflow'] = workflow_dict
except IndexError:
pass
scripts/check_galaxy.py
+3 -7
View File
@@ -3,7 +3,6 @@
check_galaxy can be run by hand, although it is meant to run from cron
via the check_galaxy.sh script in Galaxy's cron/ directory.
"""
import filecmp
import formatter
import getopt
@@ -14,6 +13,9 @@ import sys
import tempfile
import time
import twill
import twill.commands as tc
# options
if "DEBUG" in os.environ:
debug = os.environ["DEBUG"]
@@ -94,12 +96,6 @@ If the user does not exist, check_galaxy will create it for you.""" % login_file
sys.exit(message)
( username, password ) = f.readline().split()
# find/import twill
lib_dir = os.path.join( scripts_dir, os.pardir, "lib" )
sys.path.insert( 1, lib_dir )
import twill
import twill.commands as tc
# default timeout for twill browser is never
socket.setdefaulttimeout(300)
scripts/data_libraries/build_lucene_index.py
+16 -8
View File
@@ -10,19 +10,21 @@ Run from the ~/scripts/data_libraries directory:
%sh build_lucene_index.sh
"""
import sys
import os
import csv
import urllib, urllib2
import ConfigParser
import csv
import os
import sys
import urllib
import urllib2
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
import galaxy.model.mapping
from galaxy import config, model
def main( ini_file ):
sa_session, gconfig = get_sa_session( ini_file )
max_size = float( gconfig.get( "fulltext_max_size", 100 ) ) * 1048576
@@ -37,11 +39,13 @@ def main( ini_file ):
if os.path.exists( dataset_file ):
os.remove( dataset_file )
def build_index( search_url, dataset_file ):
url = "%s/index?%s" % ( search_url, urllib.urlencode( { "docfile" : dataset_file } ) )
request = urllib2.Request( url )
request.get_method = lambda: "PUT"
response = urllib2.urlopen( request )
urllib2.urlopen( request )
def create_dataset_file( dataset_iter ):
dataset_file = os.path.join( os.getcwd(), "full-text-search-files.csv" )
@@ -52,6 +56,7 @@ def create_dataset_file( dataset_iter ):
out_handle.close()
return dataset_file
def get_lddas( sa_session, max_size, ignore_exts ):
for ldda in sa_session.query( model.LibraryDatasetDatasetAssociation ).filter_by( deleted=False ):
if ( float( ldda.dataset.get_size() ) > max_size or ldda.extension in ignore_exts ):
@@ -60,6 +65,7 @@ def get_lddas( sa_session, max_size, ignore_exts ):
fname = ldda.dataset.get_file_name()
yield ldda.id, fname, _get_dataset_metadata(ldda).replace("\n", " ")
def _get_dataset_metadata(ldda):
"""Retrieve descriptions and information associated with a dataset.
"""
@@ -73,6 +79,7 @@ def _get_dataset_metadata(ldda):
return "%s %s %s %s %s" % (lds.name or "", lds_info, ldda.metadata.dbkey,
ldda.message, folder_info)
def _get_folder_info(folder):
"""Get names and descriptions for all parent folders except top level.
"""
@@ -81,11 +88,12 @@ def _get_folder_info(folder):
folder_info = _get_folder_info(folder.parent)
folder_info += " %s %s" % (
folder.name.replace("Unnamed folder", ""),
folder.description or "")
folder.description or "")
return folder_info
def get_sa_session( ini_file ):
conf_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } )
conf_parser = ConfigParser.ConfigParser( { 'here': os.getcwd() } )
conf_parser.read( ini_file )
kwds = dict()
for key, value in conf_parser.items( "app:main" ):
scripts/data_libraries/build_whoosh_index.py
+10 -7
View File
@@ -8,32 +8,33 @@ data library search section for more details.
Run from the ~/scripts/data_libraries directory:
%sh build_whoosh_index.sh
"""
import sys, os, csv, urllib, urllib2, ConfigParser
import ConfigParser
import os
import sys
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
# Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether search is enabled.
try:
from whoosh.filedb.filestore import FileStorage
from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT
from whoosh.index import Index
from whoosh.fields import Schema, STORED, TEXT
whoosh_search_enabled = True
schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT )
import galaxy.model.mapping
from galaxy import config, model
import pkg_resources
pkg_resources.require( "SQLAlchemy >= 0.4" )
except ImportError, e:
whoosh_search_enabled = False
schema = None
def build_index( sa_session, whoosh_index_dir ):
storage = FileStorage( whoosh_index_dir )
index = storage.create_index( schema )
writer = index.writer()
def to_unicode( a_basestr ):
if type( a_basestr ) is str:
return unicode( a_basestr, 'utf-8' )
@@ -50,6 +51,7 @@ def build_index( sa_session, whoosh_index_dir ):
writer.commit()
print "Number of active library datasets indexed: ", lddas_indexed
def get_lddas( sa_session ):
for ldda in sa_session.query( model.LibraryDatasetDatasetAssociation ).filter_by( deleted=False ):
id = ldda.id
@@ -66,6 +68,7 @@ def get_lddas( sa_session ):
message = ''
yield id, name, info, dbkey, message
def get_sa_session_and_needed_config_settings( ini_file ):
conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } )
conf_parser.read( ini_file )
scripts/galaxy-main
+3 -5
View File
@@ -15,15 +15,13 @@ a loggers section in Galaxy's ini file - this can be overridden with sensible
defaults logging to a single file with the following:
galaxy-main -d --server-name handler0 --daemon-log-file=handler0-daemon.log --pid-file handler0.pid --log-file handler0.log
"""
import logging
from logging.config import fileConfig
import functools
import logging
import os
import time
import sys
import time
from logging.config import fileConfig
try:
import ConfigParser as configparser
scripts/get_platforms.py
-1
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env python
import os
import sys
scripts/loc_files/create_all_fasta_loc.py
+17 -15
View File
@@ -1,6 +1,3 @@
import optparse, os, sys
from xml.etree.ElementTree import parse
"""
Generates a loc file containing names of all the fasta files that match the
name of the genome subdirectory they're in.
@@ -26,6 +23,10 @@ usage: %prog [options]
-a, --append=a: Append to existing all_fasta.loc file rather than create new
-p, --sample-text=p: Copy over text from all_fasta.loc.sample file (false if set to append)
"""
import optparse
import os
import sys
from xml.etree.ElementTree import parse
DEFAULT_TOOL_DATA_TABLE_CONF = 'tool_data_table_conf.xml'
DEFAULT_ALL_FASTA_LOC_BASE = 'all_fasta'
@@ -138,14 +139,13 @@ DBKEY_DESCRIPTION_MAP = { 'AaegL1': 'Mosquito (Aedes aegypti): AaegL1',
'tetNig2': 'Tetraodon (Tetraodon nigroviridis): tetNig2',
'tupBel1': 'Tree Shrew (Tupaia belangeri): tupBel1',
'venter1': 'Human (J. Craig Venter): venter1',
'xenTro2': 'Frog (Xenopus tropicalis): xenTro2'
}
'xenTro2': 'Frog (Xenopus tropicalis): xenTro2' }
VARIANT_MAP = { 'canon': 'Canonical',
'full': 'Full',
'female': 'Female',
'male': 'Male'
}
'male': 'Male' }
# alphabetize ignoring case
def caseless_compare( a, b ):
@@ -158,6 +158,7 @@ def caseless_compare( a, b ):
elif au < bu:
return -1
def __main__():
# command line variables
parser = optparse.OptionParser()
@@ -176,7 +177,7 @@ def __main__():
(options, args) = parser.parse_args()
exemptions = [ e.strip() for e in options.exemptions.split( ',' ) ]
fasta_exts = [ x.strip() for x in options.fasta_exts.split( ',' ) ]
fasta_exts = [ x.strip() for x in options.fasta_exts.split( ',' ) ]
variants = [ v.strip() for v in options.variants.split( ',' ) ]
variant_exclusions = {}
try:
@@ -199,7 +200,7 @@ def __main__():
# say what we're looking in
print '\nLooking in:\n\t%s' % '\n\t'.join( [ p % '<build_name>' for p in paths_to_look_in ] )
poss_names = [ '<build_name>%s' % v for v in variants ]
poss_names = [ '<build_name>%s' % _ for _ in variants ]
print 'for files that are named %s' % ', '.join( poss_names[:-1] ),
if len( poss_names ) > 1:
print 'or %s' % poss_names[-1],
@@ -226,7 +227,7 @@ def __main__():
if cols:
col_values = [ col.strip() for col in cols.split( ',' ) ]
if not col_values or not loc_path:
stop_err( 'No columns can be found for this data table (%s) in %s' % ( options.data_table, options.data_table_xml ) )
raise Exception( 'No columns can be found for this data table (%s) in %s' % ( options.data_table, options.data_table_xml ) )
# get all fasta paths under genome directory
fasta_locs = {}
@@ -234,7 +235,7 @@ def __main__():
genome_subdirs = [ dr for dr in os.listdir( options.genome_dir ) if dr not in exemptions ]
for genome_subdir in genome_subdirs:
possible_names = [ genome_subdir ]
possible_names.extend( [ '%s%s' % ( genome_subdir, v ) for v in variants ] )
possible_names.extend( [ '%s%s' % ( genome_subdir, _ ) for _ in variants ] )
# get paths to all fasta files
for path_to_look_in in paths_to_look_in:
for dirpath, dirnames, filenames in os.walk( path_to_look_in % genome_subdir ):
@@ -257,10 +258,10 @@ def __main__():
if variant_exclusions.keys():
for k in variant_exclusions.keys():
leave_in = '%s%s' % ( genome_subdir, k )
if fasta_locs.has_key( leave_in ):
if leave_in in fasta_locs:
to_remove = [ '%s%s' % ( genome_subdir, k ) for k in variant_exclusions[ k ] ]
for tr in to_remove:
if fasta_locs.has_key( tr ):
if tr in fasta_locs:
del fasta_locs[ tr ]
# output results
@@ -291,10 +292,11 @@ def __main__():
try:
out_line.append( fasta_locs[ fb ][ col ] )
except KeyError:
stop_err( 'Unexpected column (%s) encountered' % col )
raise Exception( 'Unexpected column (%s) encountered' % col )
if out_line:
all_fasta_loc.write( '%s\n' % '\t'.join( out_line ) )
# close up output loc file
all_fasta_loc.close()
if __name__=='__main__': __main__()
if __name__ == '__main__':
__main__()
scripts/microbes/BeautifulSoup.py
+285 -290
View File
File diff suppressed because it is too large Load Diff
scripts/microbes/create_bacteria_loc_file.py
+72 -69
View File
@@ -1,69 +1,72 @@
#!/usr/bin/env python
#Dan Blankenberg
import sys, os
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
pass
#print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir,sub_dirs,files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir,file),'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n","").split("=")
dict[fields[0]]="=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs':{},'base_dir':this_base_dir}
for key in dict.keys():
organisms[name][key]=dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs':{},'base_dir':this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']]=dict
for org in organisms:
org = organisms[org]
#if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
except: continue
if 'build' in org:
build = org['build']
print "ORG\t%s\t%s\t%s\t%s\t%s\t%s\tUCSC" % ( build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url'] )
else:
print "ORG\t%s\t%s\t%s\t%s\t%s\t%s\tNone" % ( build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url'] )
for chr in org['chrs']:
chr = org['chrs'][chr]
print "CHR\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], chr['name'], chr['length'], chr['gi'], chr['gb'], "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&val="+chr['refseq'] )
for feature in ['CDS','tRNA','rRNA']:
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], feature, build, chr['chromosome'], feature, "bed", os.path.join( org['base_dir'], "%s.%s.bed" % ( chr['chromosome'], feature ) ) )
#FASTA
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "seq", build, chr['chromosome'], "sequence", "fasta", os.path.join( org['base_dir'], "%s.fna" % chr['chromosome'] ) )
#GeneMark
if os.path.exists( os.path.join( org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'] ) ):
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "GeneMark", build, chr['chromosome'], "GeneMark", "bed", os.path.join( org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'] ) )
#GenMarkHMM
if os.path.exists( os.path.join( org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'] ) ):
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "GeneMarkHMM", build, chr['chromosome'], "GeneMarkHMM", "bed", os.path.join( org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'] ) )
#Glimmer3
if os.path.exists( os.path.join( org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'] ) ):
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "Glimmer3", build, chr['chromosome'], "Glimmer3", "bed", os.path.join( org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'] ) )
if __name__ == "__main__": __main__()
#!/usr/bin/env python
# Dan Blankenberg
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
pass
# print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
for org in organisms:
org = organisms[org]
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
except:
continue
if 'build' in org:
build = org['build']
print "ORG\t%s\t%s\t%s\t%s\t%s\t%s\tUCSC" % ( build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url'] )
else:
print "ORG\t%s\t%s\t%s\t%s\t%s\t%s\tNone" % ( build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url'] )
for chr in org['chrs']:
chr = org['chrs'][chr]
print "CHR\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], chr['name'], chr['length'], chr['gi'], chr['gb'], "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&val=" + chr['refseq'] )
for feature in ['CDS', 'tRNA', 'rRNA']:
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], feature, build, chr['chromosome'], feature, "bed", os.path.join( org['base_dir'], "%s.%s.bed" % ( chr['chromosome'], feature ) ) )
# FASTA
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "seq", build, chr['chromosome'], "sequence", "fasta", os.path.join( org['base_dir'], "%s.fna" % chr['chromosome'] ) )
# GeneMark
if os.path.exists( os.path.join( org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'] ) ):
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "GeneMark", build, chr['chromosome'], "GeneMark", "bed", os.path.join( org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'] ) )
# GenMarkHMM
if os.path.exists( os.path.join( org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'] ) ):
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "GeneMarkHMM", build, chr['chromosome'], "GeneMarkHMM", "bed", os.path.join( org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'] ) )
# Glimmer3
if os.path.exists( os.path.join( org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'] ) ):
print "DATA\t%s_%s_%s\t%s\t%s\t%s\t%s\t%s" % ( build, chr['chromosome'], "Glimmer3", build, chr['chromosome'], "Glimmer3", "bed", os.path.join( org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'] ) )
if __name__ == "__main__":
__main__()
scripts/microbes/create_bacteria_table.py
+78 -79
View File
@@ -1,79 +1,78 @@
#!/usr/bin/env python
#Dan Blankenberg
import sys, os
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
pass
#print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir,sub_dirs,files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir,file),'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n","").split("=")
dict[fields[0]]="=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs':{},'base_dir':this_base_dir}
for key in dict.keys():
organisms[name][key]=dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs':{},'base_dir':this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']]=dict
orgs = organisms.keys()
for org in orgs:
if 'name' not in organisms[org]:del organisms[org]
orgs = organisms.keys()
#need to sort by name
swap_test = False
for i in range(0, len(orgs) - 1):
for j in range(0, len(orgs) - i - 1):
if organisms[orgs[j]]['name'] > organisms[orgs[j + 1]]['name']:
orgs[j], orgs[j + 1] = orgs[j + 1], orgs[j]
swap_test = True
if swap_test is False:
break
print "||'''Organism'''||'''Kingdom'''||'''Group'''||'''Links to UCSC Archaea Browser'''||"
for org in orgs:
org = organisms[org]
at_ucsc = False
#if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
except: continue
if 'build' in org:
build = org['build']
at_ucsc = True
out_str = "||"+org['name']+"||"+org['kingdom']+"||"+org['group']+"||"
if at_ucsc:
out_str = out_str + "Yes"
out_str = out_str + "||"
print out_str
if __name__ == "__main__": __main__()
#!/usr/bin/env python
# Dan Blankenberg
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
pass
# print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
orgs = organisms.keys()
for org in orgs:
if 'name' not in organisms[org]:
del organisms[org]
orgs = organisms.keys()
# need to sort by name
swap_test = False
for i in range(0, len(orgs) - 1):
for j in range(0, len(orgs) - i - 1):
if organisms[orgs[j]]['name'] > organisms[orgs[j + 1]]['name']:
orgs[j], orgs[j + 1] = orgs[j + 1], orgs[j]
swap_test = True
if swap_test is False:
break
print "||'''Organism'''||'''Kingdom'''||'''Group'''||'''Links to UCSC Archaea Browser'''||"
for org in orgs:
org = organisms[org]
at_ucsc = False
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
org['genome project id']
except:
continue
if 'build' in org:
at_ucsc = True
out_str = "||" + org['name'] + "||" + org['kingdom'] + "||" + org['group'] + "||"
if at_ucsc:
out_str = out_str + "Yes"
out_str = out_str + "||"
print out_str
if __name__ == "__main__":
__main__()
scripts/microbes/create_nib_seq_loc_file.py
+87 -88
View File
@@ -1,88 +1,87 @@
#!/usr/bin/env python
#Dan Blankenberg
import sys, os
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
print "using default base_dir:", base_dir
loc_out = os.path.join( base_dir, "seq.loc" )
try:
loc_out = os.path.join( base_dir, sys.argv[2] )
except:
print "using default seq.loc:", loc_out
organisms = {}
loc_out = open( loc_out, 'wb' )
for result in os.walk( base_dir ):
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open( os.path.join( this_base_dir, file ), 'r' )
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace( "\n", "" ).split( "=" )
dict[fields[0]]="=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs':{}, 'base_dir':this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs':{},'base_dir':this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
for org in organisms:
org = organisms[org]
try:
build = org['genome project id']
except: continue
if 'build' in org:
build = org['build']
seq_path = os.path.join( org['base_dir'], "seq" )
#create seq dir, if exists go to next org
##TODO: add better checking, i.e. for updating
try:
os.mkdir( seq_path )
except:
print "Skipping", build
#continue
loc_out.write( "seq %s %s\n" % ( build, seq_path ) )
#print org info
for chr in org['chrs']:
chr = org['chrs'][chr]
fasta_file = os.path.join( org['base_dir'], "%s.fna" % chr['chromosome'] )
nib_out_file = os.path.join( seq_path, "%s.nib "% chr['chromosome'] )
#create nibs using faToNib binary
#TODO: when bx supports writing nib, use it here instead
command = "faToNib %s %s" % ( fasta_file, nib_out_file )
os.system( command )
loc_out.close()
if __name__ == "__main__": __main__()
#!/usr/bin/env python
# Dan Blankenberg
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
print "using default base_dir:", base_dir
loc_out = os.path.join( base_dir, "seq.loc" )
try:
loc_out = os.path.join( base_dir, sys.argv[2] )
except:
print "using default seq.loc:", loc_out
organisms = {}
loc_out = open( loc_out, 'wb' )
for result in os.walk( base_dir ):
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open( os.path.join( this_base_dir, file ), 'r' )
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace( "\n", "" ).split( "=" )
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
for org in organisms:
org = organisms[org]
try:
build = org['genome project id']
except:
continue
if 'build' in org:
build = org['build']
seq_path = os.path.join( org['base_dir'], "seq" )
# create seq dir, if exists go to next org
# TODO: add better checking, i.e. for updating
try:
os.mkdir( seq_path )
except:
print "Skipping", build
# continue
loc_out.write( "seq %s %s\n" % ( build, seq_path ) )
# print org info
for chr in org['chrs']:
chr = org['chrs'][chr]
fasta_file = os.path.join( org['base_dir'], "%s.fna" % chr['chromosome'] )
nib_out_file = os.path.join( seq_path, "%s.nib " % chr['chromosome'] )
# create nibs using faToNib binary
# TODO: when bx supports writing nib, use it here instead
command = "faToNib %s %s" % ( fasta_file, nib_out_file )
os.system( command )
loc_out.close()
if __name__ == "__main__":
__main__()
scripts/microbes/get_builds_lengths.py
+59 -56
View File
@@ -1,56 +1,59 @@
#!/usr/bin/env python
#Dan Blankenberg
import sys, os
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
pass
#print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir,sub_dirs,files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir,file),'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n","").split("=")
dict[fields[0]]="=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs':{},'base_dir':this_base_dir}
for key in dict.keys():
organisms[name][key]=dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs':{},'base_dir':this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']]=dict
for org in organisms:
org = organisms[org]
#if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
except: continue
if 'build' in org:
build = org['build']
chrs=[]
for chrom in org['chrs']:
chrom = org['chrs'][chrom]
chrs.append( "%s=%s" % ( chrom['chromosome'], chrom['length'] ) )
print "%s\t%s\t%s" % ( build, org['name'], ",".join( chrs ) )
if __name__ == "__main__": __main__()
#!/usr/bin/env python
# Dan Blankenberg
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
pass
# print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
for org in organisms:
org = organisms[org]
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
except:
continue
if 'build' in org:
build = org['build']
chrs = []
for chrom in org['chrs']:
chrom = org['chrs'][chrom]
chrs.append( "%s=%s" % ( chrom['chromosome'], chrom['length'] ) )
print "%s\t%s\t%s" % ( build, org['name'], ",".join( chrs ) )
if __name__ == "__main__":
__main__()
scripts/microbes/harvest_bacteria.py
+254 -246
View File
@@ -1,246 +1,254 @@
#!/usr/bin/env python
#Dan Blankenberg
#Harvest Bacteria
#Connects to NCBI's Microbial Genome Projects website and scrapes it for information.
#Downloads and converts annotations for each Genome
import sys, os, time
from urllib2 import urlopen
from urllib import urlretrieve
from ftplib import FTP
from BeautifulSoup import BeautifulSoup
from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark
assert sys.version_info[:2] >= ( 2, 4 )
#this defines the types of ftp files we are interested in, and how to process/convert them to a form for our use
desired_ftp_files = {'GeneMark':{'ext':'GeneMark-2.5f','parser':'process_GeneMark'},
'GeneMarkHMM':{'ext':'GeneMarkHMM-2.6m','parser':'process_GeneMarkHMM'},
'Glimmer3':{'ext':'Glimmer3','parser':'process_Glimmer3'},
'fna':{'ext':'fna','parser':'process_FASTA'},
'gbk':{'ext':'gbk','parser':'process_Genbank'} }
#number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url
def iter_genome_projects( url = "http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=" ):
for row in BeautifulSoup( urlopen( url ) ).findAll( name = 'tr', bgcolor = ["#EEFFDD", "#E8E8DD"] ):
row = str( row ).replace( "\n", "" ).replace( "\r", "" )
fields = row.split( "</td>" )
org_num = fields[0].split( "list_uids=" )[-1].split( "\"" )[0]
name = fields[1].split( "\">" )[-1].split( "<" )[0]
kingdom = "archaea"
if "<td class=\"bacteria\" align=\"center\">B" in fields[2]:
kingdom = "bacteria"
group = fields[3].split( ">" )[-1]
info_url = "%s%s" % ( info_url_base, org_num )
org_genbank = fields[7].split( "\">" )[-1].split( "<" )[0].split( "." )[0]
org_refseq = fields[8].split( "\">" )[-1].split( "<" )[0].split( "." )[0]
#seems some things donot have an ftp url, try and except it here:
try:
ftp_url = fields[22].split( "href=\"" )[1].split( "\"" )[0]
except:
print "FAILED TO AQUIRE FTP ADDRESS:", org_num, info_url
ftp_url = None
chroms = get_chroms_by_project_id( org_num )
yield org_num, name, chroms, kingdom, group, org_genbank, org_refseq, info_url, ftp_url
def get_chroms_by_project_id( org_num, base_url = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=" ):
html_count = 0
html = None
while html_count < 500 and html == None:
html_count += 1
url = "%s%s" % ( base_url, org_num )
try:
html = urlopen( url )
except:
print "GENOME PROJECT FAILED:", html_count, "org:", org_num, url
html = None
time.sleep( 1 ) #Throttle Connection
if html is None:
"GENOME PROJECT COMPLETELY FAILED TO LOAD", "org:", org_num,"http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="+org_num
return None
chroms = []
for chr_row in BeautifulSoup( html ).findAll( "tr", { "class" : "vvv" } ):
chr_row = str( chr_row ).replace( "\n","" ).replace( "\r", "" )
fields2 = chr_row.split( "</td>" )
refseq = fields2[1].split( "</a>" )[0].split( ">" )[-1]
#genbank = fields2[2].split( "</a>" )[0].split( ">" )[-1]
chroms.append( refseq )
return chroms
def get_ftp_contents( ftp_url ):
ftp_count = 0
ftp_contents = None
while ftp_count < 500 and ftp_contents == None:
ftp_count += 1
try:
ftp = FTP( ftp_url.split("/")[2] )
ftp.login()
ftp.cwd( ftp_url.split( ftp_url.split( "/" )[2] )[-1] )
ftp_contents = ftp.nlst()
ftp.close()
except:
ftp_contents = None
time.sleep( 1 ) #Throttle Connection
return ftp_contents
def scrape_ftp( ftp_contents, org_dir, org_num, refseq, ftp_url ):
for file_type, items in desired_ftp_files.items():
ext = items['ext']
ftp_filename = "%s.%s" % ( refseq, ext )
target_filename = os.path.join( org_dir, "%s.%s" % ( refseq, ext ) )
if ftp_filename in ftp_contents:
url_count = 0
url = "%s/%s" % ( ftp_url, ftp_filename )
results = None
while url_count < 500 and results is None:
url_count += 1
try:
results = urlretrieve( url, target_filename )
except:
results = None
time.sleep(1) #Throttle Connection
if results is None:
"URL COMPLETELY FAILED TO LOAD:", url
return
#do special processing for each file type:
if items['parser'] is not None:
parser_results = globals()[items['parser']]( target_filename, org_num, refseq )
else:
print "FTP filetype:", file_type, "not found for", org_num, refseq
#FTP Files have been Loaded
def process_FASTA( filename, org_num, refseq ):
fasta = []
fasta = [line.strip() for line in open( filename, 'rb' ).readlines()]
fasta_header = fasta.pop( 0 )[1:]
fasta_header_split = fasta_header.split( "|" )
chr_name = fasta_header_split.pop( -1 ).strip()
accesions = {fasta_header_split[0]:fasta_header_split[1], fasta_header_split[2]:fasta_header_split[3]}
fasta = "".join( fasta )
#Create Chrom Info File:
chrom_info_file = open( os.path.join( os.path.split( filename )[0], "%s.info" % refseq ), 'wb+' )
chrom_info_file.write( "chromosome=%s\nname=%s\nlength=%s\norganism=%s\n" % ( refseq, chr_name, len( fasta ), org_num ) )
try:
chrom_info_file.write( "gi=%s\n" % accesions['gi'] )
except:
chrom_info_file.write( "gi=None\n" )
try:
chrom_info_file.write( "gb=%s\n" % accesions['gb'] )
except:
chrom_info_file.write( "gb=None\n" )
try:
chrom_info_file.write( "refseq=%s\n" % refseq )
except:
chrom_info_file.write( "refseq=None\n" )
chrom_info_file.close()
def process_Genbank( filename, org_num, refseq ):
#extracts 'CDS', 'tRNA', 'rRNA' features from genbank file
features = get_bed_from_genbank( filename, refseq, ['CDS', 'tRNA', 'rRNA'] )
for feature in features.keys():
feature_file = open( os.path.join( os.path.split( filename )[0], "%s.%s.bed" % ( refseq, feature ) ), 'wb+' )
feature_file.write( '\n'.join( features[feature] ) )
feature_file.close()
print "Genbank extraction finished for chrom:", refseq, "file:", filename
def process_Glimmer3( filename, org_num, refseq ):
try:
glimmer3_bed = get_bed_from_glimmer3( filename, refseq )
except Exception, e:
print "Converting Glimmer3 to bed FAILED! For chrom:", refseq, "file:", filename, e
glimmer3_bed = []
glimmer3_bed_file = open( os.path.join( os.path.split( filename )[0], "%s.Glimmer3.bed" % refseq ), 'wb+' )
glimmer3_bed_file.write( '\n'.join( glimmer3_bed ) )
glimmer3_bed_file.close()
def process_GeneMarkHMM( filename, org_num, refseq ):
try:
geneMarkHMM_bed = get_bed_from_GeneMarkHMM( filename, refseq )
except Exception, e:
print "Converting GeneMarkHMM to bed FAILED! For chrom:", refseq, "file:", filename, e
geneMarkHMM_bed = []
geneMarkHMM_bed_bed_file = open( os.path.join( os.path.split( filename )[0], "%s.GeneMarkHMM.bed" % refseq ), 'wb+' )
geneMarkHMM_bed_bed_file.write( '\n'.join( geneMarkHMM_bed ) )
geneMarkHMM_bed_bed_file.close()
def process_GeneMark( filename, org_num, refseq ):
try:
geneMark_bed = get_bed_from_GeneMark( filename, refseq )
except Exception, e:
print "Converting GeneMark to bed FAILED! For chrom:", refseq, "file:", filename, e
geneMark_bed = []
geneMark_bed_bed_file = open( os.path.join( os.path.split( filename )[0], "%s.GeneMark.bed" % refseq ), 'wb+' )
geneMark_bed_bed_file.write( '\n'.join( geneMark_bed ) )
geneMark_bed_bed_file.close()
def __main__():
start_time = time.time()
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
print "using default base_dir:", base_dir
try:
os.mkdir( base_dir )
print "path '%s' has been created" % base_dir
except:
print "path '%s' seems to already exist" % base_dir
for org_num, name, chroms, kingdom, group, org_genbank, org_refseq, info_url, ftp_url in iter_genome_projects():
if chroms is None:
continue #No chrom information, we can't really do anything with this organism
#Create org directory, if exists, assume it is done and complete --> skip it
try:
org_dir = os.path.join( base_dir, org_num )
os.mkdir( org_dir )
except:
print "Organism %s already exists on disk, skipping" % org_num
continue
#get ftp contents
ftp_contents = get_ftp_contents( ftp_url )
if ftp_contents is None:
"FTP COMPLETELY FAILED TO LOAD", "org:", org_num, "ftp:", ftp_url
else:
for refseq in chroms:
ftp_result = scrape_ftp( ftp_contents, org_dir, org_num, refseq, ftp_url )
#FTP Files have been Loaded
print "Org:", org_num, "chrom:", refseq, "[", time.time() - start_time, "seconds elapsed. ]"
#Create org info file
info_file = open( os.path.join( org_dir, "%s.info" % org_num ), 'wb+' )
info_file.write("genome project id=%s\n" % org_num )
info_file.write("name=%s\n" % name )
info_file.write("kingdom=%s\n" % kingdom )
info_file.write("group=%s\n" % group )
info_file.write("chromosomes=%s\n" % ",".join( chroms ) )
info_file.write("info url=%s\n" % info_url )
info_file.write("ftp url=%s\n" % ftp_url )
info_file.close()
print "Finished Harvesting", "[", time.time() - start_time, "seconds elapsed. ]"
print "[", ( time.time() - start_time )/60, "minutes. ]"
print "[", ( time.time() - start_time )/60/60, "hours. ]"
if __name__ == "__main__": __main__()
#!/usr/bin/env python
# Dan Blankenberg
# Harvest Bacteria
# Connects to NCBI's Microbial Genome Projects website and scrapes it for information.
# Downloads and converts annotations for each Genome
import os
import sys
import time
from ftplib import FTP
from urllib2 import urlopen
from urllib import urlretrieve
from BeautifulSoup import BeautifulSoup
from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark
assert sys.version_info[:2] >= ( 2, 4 )
# this defines the types of ftp files we are interested in, and how to process/convert them to a form for our use
desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_GeneMark'},
'GeneMarkHMM': {'ext': 'GeneMarkHMM-2.6m', 'parser': 'process_GeneMarkHMM'},
'Glimmer3': {'ext': 'Glimmer3', 'parser': 'process_Glimmer3'},
'fna': {'ext': 'fna', 'parser': 'process_FASTA'},
'gbk': {'ext': 'gbk', 'parser': 'process_Genbank'} }
# number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url
def iter_genome_projects( url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=" ):
for row in BeautifulSoup( urlopen( url ) ).findAll( name='tr', bgcolor=["#EEFFDD", "#E8E8DD"] ):
row = str( row ).replace( "\n", "" ).replace( "\r", "" )
fields = row.split( "</td>" )
org_num = fields[0].split( "list_uids=" )[-1].split( "\"" )[0]
name = fields[1].split( "\">" )[-1].split( "<" )[0]
kingdom = "archaea"
if "<td class=\"bacteria\" align=\"center\">B" in fields[2]:
kingdom = "bacteria"
group = fields[3].split( ">" )[-1]
info_url = "%s%s" % ( info_url_base, org_num )
org_genbank = fields[7].split( "\">" )[-1].split( "<" )[0].split( "." )[0]
org_refseq = fields[8].split( "\">" )[-1].split( "<" )[0].split( "." )[0]
# seems some things donot have an ftp url, try and except it here:
try:
ftp_url = fields[22].split( "href=\"" )[1].split( "\"" )[0]
except:
print "FAILED TO AQUIRE FTP ADDRESS:", org_num, info_url
ftp_url = None
chroms = get_chroms_by_project_id( org_num )
yield org_num, name, chroms, kingdom, group, org_genbank, org_refseq, info_url, ftp_url
def get_chroms_by_project_id( org_num, base_url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=" ):
html_count = 0
html = None
while html_count < 500 and html is None:
html_count += 1
url = "%s%s" % ( base_url, org_num )
try:
html = urlopen( url )
except:
print "GENOME PROJECT FAILED:", html_count, "org:", org_num, url
html = None
time.sleep( 1 ) # Throttle Connection
if html is None:
"GENOME PROJECT COMPLETELY FAILED TO LOAD", "org:", org_num, "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=" + org_num
return None
chroms = []
for chr_row in BeautifulSoup( html ).findAll( "tr", { "class" : "vvv" } ):
chr_row = str( chr_row ).replace( "\n", "" ).replace( "\r", "" )
fields2 = chr_row.split( "</td>" )
refseq = fields2[1].split( "</a>" )[0].split( ">" )[-1]
# genbank = fields2[2].split( "</a>" )[0].split( ">" )[-1]
chroms.append( refseq )
return chroms
def get_ftp_contents( ftp_url ):
ftp_count = 0
ftp_contents = None
while ftp_count < 500 and ftp_contents is None:
ftp_count += 1
try:
ftp = FTP( ftp_url.split("/")[2] )
ftp.login()
ftp.cwd( ftp_url.split( ftp_url.split( "/" )[2] )[-1] )
ftp_contents = ftp.nlst()
ftp.close()
except:
ftp_contents = None
time.sleep( 1 ) # Throttle Connection
return ftp_contents
def scrape_ftp( ftp_contents, org_dir, org_num, refseq, ftp_url ):
for file_type, items in desired_ftp_files.items():
ext = items['ext']
ftp_filename = "%s.%s" % ( refseq, ext )
target_filename = os.path.join( org_dir, "%s.%s" % ( refseq, ext ) )
if ftp_filename in ftp_contents:
url_count = 0
url = "%s/%s" % ( ftp_url, ftp_filename )
results = None
while url_count < 500 and results is None:
url_count += 1
try:
results = urlretrieve( url, target_filename )
except:
results = None
time.sleep(1) # Throttle Connection
if results is None:
"URL COMPLETELY FAILED TO LOAD:", url
return
# do special processing for each file type:
if items['parser'] is not None:
globals()[items['parser']]( target_filename, org_num, refseq )
else:
print "FTP filetype:", file_type, "not found for", org_num, refseq
# FTP Files have been Loaded
def process_FASTA( filename, org_num, refseq ):
fasta = []
fasta = [line.strip() for line in open( filename, 'rb' ).readlines()]
fasta_header = fasta.pop( 0 )[1:]
fasta_header_split = fasta_header.split( "|" )
chr_name = fasta_header_split.pop( -1 ).strip()
accesions = {fasta_header_split[0]: fasta_header_split[1], fasta_header_split[2]: fasta_header_split[3]}
fasta = "".join( fasta )
# Create Chrom Info File:
chrom_info_file = open( os.path.join( os.path.split( filename )[0], "%s.info" % refseq ), 'wb+' )
chrom_info_file.write( "chromosome=%s\nname=%s\nlength=%s\norganism=%s\n" % ( refseq, chr_name, len( fasta ), org_num ) )
try:
chrom_info_file.write( "gi=%s\n" % accesions['gi'] )
except:
chrom_info_file.write( "gi=None\n" )
try:
chrom_info_file.write( "gb=%s\n" % accesions['gb'] )
except:
chrom_info_file.write( "gb=None\n" )
try:
chrom_info_file.write( "refseq=%s\n" % refseq )
except:
chrom_info_file.write( "refseq=None\n" )
chrom_info_file.close()
def process_Genbank( filename, org_num, refseq ):
# extracts 'CDS', 'tRNA', 'rRNA' features from genbank file
features = get_bed_from_genbank( filename, refseq, ['CDS', 'tRNA', 'rRNA'] )
for feature in features.keys():
feature_file = open( os.path.join( os.path.split( filename )[0], "%s.%s.bed" % ( refseq, feature ) ), 'wb+' )
feature_file.write( '\n'.join( features[feature] ) )
feature_file.close()
print "Genbank extraction finished for chrom:", refseq, "file:", filename
def process_Glimmer3( filename, org_num, refseq ):
try:
glimmer3_bed = get_bed_from_glimmer3( filename, refseq )
except Exception, e:
print "Converting Glimmer3 to bed FAILED! For chrom:", refseq, "file:", filename, e
glimmer3_bed = []
glimmer3_bed_file = open( os.path.join( os.path.split( filename )[0], "%s.Glimmer3.bed" % refseq ), 'wb+' )
glimmer3_bed_file.write( '\n'.join( glimmer3_bed ) )
glimmer3_bed_file.close()
def process_GeneMarkHMM( filename, org_num, refseq ):
try:
geneMarkHMM_bed = get_bed_from_GeneMarkHMM( filename, refseq )
except Exception, e:
print "Converting GeneMarkHMM to bed FAILED! For chrom:", refseq, "file:", filename, e
geneMarkHMM_bed = []
geneMarkHMM_bed_bed_file = open( os.path.join( os.path.split( filename )[0], "%s.GeneMarkHMM.bed" % refseq ), 'wb+' )
geneMarkHMM_bed_bed_file.write( '\n'.join( geneMarkHMM_bed ) )
geneMarkHMM_bed_bed_file.close()
def process_GeneMark( filename, org_num, refseq ):
try:
geneMark_bed = get_bed_from_GeneMark( filename, refseq )
except Exception, e:
print "Converting GeneMark to bed FAILED! For chrom:", refseq, "file:", filename, e
geneMark_bed = []
geneMark_bed_bed_file = open( os.path.join( os.path.split( filename )[0], "%s.GeneMark.bed" % refseq ), 'wb+' )
geneMark_bed_bed_file.write( '\n'.join( geneMark_bed ) )
geneMark_bed_bed_file.close()
def __main__():
start_time = time.time()
base_dir = os.path.join( os.getcwd(), "bacteria" )
try:
base_dir = sys.argv[1]
except:
print "using default base_dir:", base_dir
try:
os.mkdir( base_dir )
print "path '%s' has been created" % base_dir
except:
print "path '%s' seems to already exist" % base_dir
for org_num, name, chroms, kingdom, group, org_genbank, org_refseq, info_url, ftp_url in iter_genome_projects():
if chroms is None:
continue # No chrom information, we can't really do anything with this organism
# Create org directory, if exists, assume it is done and complete --> skip it
try:
org_dir = os.path.join( base_dir, org_num )
os.mkdir( org_dir )
except:
print "Organism %s already exists on disk, skipping" % org_num
continue
# get ftp contents
ftp_contents = get_ftp_contents( ftp_url )
if ftp_contents is None:
"FTP COMPLETELY FAILED TO LOAD", "org:", org_num, "ftp:", ftp_url
else:
for refseq in chroms:
scrape_ftp( ftp_contents, org_dir, org_num, refseq, ftp_url )
# FTP Files have been Loaded
print "Org:", org_num, "chrom:", refseq, "[", time.time() - start_time, "seconds elapsed. ]"
# Create org info file
info_file = open( os.path.join( org_dir, "%s.info" % org_num ), 'wb+' )
info_file.write("genome project id=%s\n" % org_num )
info_file.write("name=%s\n" % name )
info_file.write("kingdom=%s\n" % kingdom )
info_file.write("group=%s\n" % group )
info_file.write("chromosomes=%s\n" % ",".join( chroms ) )
info_file.write("info url=%s\n" % info_url )
info_file.write("ftp url=%s\n" % ftp_url )
info_file.close()
print "Finished Harvesting", "[", time.time() - start_time, "seconds elapsed. ]"
print "[", ( time.time() - start_time ) / 60, "minutes. ]"
print "[", ( time.time() - start_time ) / 60 / 60, "hours. ]"
if __name__ == "__main__":
__main__()
scripts/microbes/ncbi_to_ucsc.py
+55 -60
View File
@@ -1,15 +1,14 @@
#!/usr/bin/env python
"""
Walk downloaded Genome Projects and Convert, in place, IDs to match the UCSC Archaea browser, where applicable.
Uses UCSC Archaea DSN.
"""
import sys, os
import os
import sys
import urllib
from xml.etree import ElementTree
from BeautifulSoup import BeautifulSoup
from shutil import move
from xml.etree import ElementTree
def __main__():
base_dir = os.path.join( os.getcwd(), "bacteria" )
@@ -20,31 +19,30 @@ def __main__():
organisms = {}
for result in os.walk(base_dir):
this_base_dir,sub_dirs,files = result
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
info_file = open(os.path.join(this_base_dir,file),'r')
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n","").split("=")
dict[fields[0]]="=".join(fields[1:])
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
if dict['genome project id'] not in organisms.keys():
organisms[dict['genome project id']] = {'chrs':{},'base_dir':this_base_dir}
organisms[dict['genome project id']] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[dict['genome project id']][key]=dict[key]
organisms[dict['genome project id']][key] = dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs':{},'base_dir':this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']]=dict
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
##get UCSC data
# get UCSC data
URL = "http://archaea.ucsc.edu/cgi-bin/das/dsn"
try:
page = urllib.urlopen(URL)
except:
@@ -60,21 +58,19 @@ def __main__():
print "?\tunspecified (?)"
sys.exit(1)
builds = {}
#print "#Harvested from http://archaea.ucsc.edu/cgi-bin/das/dsn"
#print "?\tunspecified (?)"
# print "#Harvested from http://archaea.ucsc.edu/cgi-bin/das/dsn"
# print "?\tunspecified (?)"
for dsn in tree:
build = dsn.find("SOURCE").attrib['id']
try:
org_page = urllib.urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db="+build).read().replace("\n","").split("<table border=2 cellspacing=2 cellpadding=2>")[1].split("</table>")[0].split("</tr>")
org_page = urllib.urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build).read().replace("\n", "").split("<table border=2 cellspacing=2 cellpadding=2>")[1].split("</table>")[0].split("</tr>")
except:
print "NO CHROMS FOR",build
print "NO CHROMS FOR", build
continue
org_page.pop(0)
if org_page[-1]=="":
if org_page[-1] == "":
org_page.pop(-1)
for row in org_page:
@@ -82,65 +78,64 @@ def __main__():
refseq = row.split("</a>")[-2].split(">")[-1]
for org in organisms:
for org_chr in organisms[org]['chrs']:
if organisms[org]['chrs'][org_chr]['chromosome']==refseq:
if organisms[org]['chrs'][org_chr]['chromosome'] == refseq:
if org not in builds:
builds[org]={'chrs':{},'build':build}
builds[org]['chrs'][refseq]=chr
#print build,org,chr,refseq
builds[org] = {'chrs': {}, 'build': build}
builds[org]['chrs'][refseq] = chr
# print build,org,chr,refseq
print
ext_to_edit = ['bed', 'info', ]
for org in builds:
print org,"changed to",builds[org]['build']
print org, "changed to", builds[org]['build']
#org info file
info_file_old = os.path.join(base_dir+org,org+".info")
info_file_new = os.path.join(base_dir+org,builds[org]['build']+".info")
# org info file
info_file_old = os.path.join(base_dir + org, org + ".info")
info_file_new = os.path.join(base_dir + org, builds[org]['build'] + ".info")
old_dir = base_dir + org
new_dir = base_dir + builds[org]['build']
old_dir = base_dir+org
new_dir = base_dir+builds[org]['build']
#open and edit org info file
# open and edit org info file
info_file_contents = open(info_file_old).read()
info_file_contents = info_file_contents+"build="+builds[org]['build']+"\n"
info_file_contents = info_file_contents + "build=" + builds[org]['build'] + "\n"
for chrom in builds[org]['chrs']:
info_file_contents = info_file_contents.replace(chrom,builds[org]['chrs'][chrom])
for result in os.walk(base_dir+org):
this_base_dir,sub_dirs,files = result
info_file_contents = info_file_contents.replace(chrom, builds[org]['chrs'][chrom])
for result in os.walk(base_dir + org):
this_base_dir, sub_dirs, files = result
for file in files:
if file[0:len(chrom)]==chrom:
#rename file
old_name = os.path.join(this_base_dir,file)
new_name = os.path.join(this_base_dir,builds[org]['chrs'][chrom]+file[len(chrom):])
move(old_name,new_name)
if file[0:len(chrom)] == chrom:
# rename file
old_name = os.path.join(this_base_dir, file)
new_name = os.path.join(this_base_dir, builds[org]['chrs'][chrom] + file[len(chrom):])
move(old_name, new_name)
#edit contents of file, skiping those in list
if file.split(".")[-1] not in ext_to_edit: continue
# edit contents of file, skiping those in list
if file.split(".")[-1] not in ext_to_edit:
continue
file_contents = open(new_name).read()
file_contents = file_contents.replace(chrom,builds[org]['chrs'][chrom])
file_contents = file_contents.replace(chrom, builds[org]['chrs'][chrom])
#special case fixes...
# special case fixes...
if file[-5:] == ".info":
file_contents = file_contents.replace("organism="+org,"organism="+builds[org]['build'])
file_contents = file_contents.replace("refseq="+builds[org]['chrs'][chrom],"refseq="+chrom)
file_contents = file_contents.replace("organism=" + org, "organism=" + builds[org]['build'])
file_contents = file_contents.replace("refseq=" + builds[org]['chrs'][chrom], "refseq=" + chrom)
#write out new file
file_out = open(new_name,'w')
# write out new file
file_out = open(new_name, 'w')
file_out.write(file_contents)
file_out.close()
#write out org info file and remove old file
org_info_out = open(info_file_new,'w')
# write out org info file and remove old file
org_info_out = open(info_file_new, 'w')
org_info_out.write(info_file_contents)
org_info_out.close()
os.unlink(info_file_old)
#change org directory name
move(old_dir,new_dir)
if __name__ == "__main__": __main__()
# change org directory name
move(old_dir, new_dir)
if __name__ == "__main__":
__main__()
scripts/microbes/util.py
+235 -224
View File
@@ -1,224 +1,235 @@
#!/usr/bin/env python
#Dan Blankenberg
import sys
assert sys.version_info[:2] >= ( 2, 4 )
#genbank_to_bed
class Region:
def __init__( self ):
self.qualifiers = {}
self.start = None
self.end = None
self.strand = '+'
def set_coordinates_by_location( self, location ):
location = location.strip().lower().replace( '..', ',' )
if "complement(" in location: #if part of the sequence is on the negative strand, it all is?
self.strand = '-' #default of + strand
for remove_text in ["join(", "order(", "complement(", ")"]:
location = location.replace( remove_text, "" )
for number in location.split( ',' ):
number = number.strip('\n\r\t <>,()')
if number:
if "^" in number:
#a single point
#check that this is correct for points, ie: 413/NC_005027.gbk: misc_feature 6636286^6636287 ===> 6636285,6636286
end = int( number.split( '^' )[0] )
start = end - 1
else:
end = int( number )
start = end - 1 #match BED coordinates
if self.start is None or start < self.start:
self.start = start
if self.end is None or end > self.end:
self.end = end
class GenBankFeatureParser:
"""Parses Features from Single Locus GenBank file"""
def __init__( self, fh, features_list = [] ):
self.fh = fh
self.features = {}
fh.seek(0)
in_features = False
last_feature_name = None
base_indent = 0
last_indent = 0
last_attr_name = None
for line in fh:
if not in_features and line.startswith('FEATURES'):
in_features = True
continue
if in_features:
lstrip = line.lstrip()
if line and lstrip == line:
break #end of feature block
cur_indent = len( line ) - len( lstrip )
if last_feature_name is None:
base_indent = cur_indent
if cur_indent == base_indent:
#a new feature
last_attr_name = None
fields = lstrip.split( None, 1 )
last_feature_name = fields[0].strip()
if not features_list or ( features_list and last_feature_name in features_list ):
if last_feature_name not in self.features:
self.features[last_feature_name] = []
region = Region()
region.set_coordinates_by_location( fields[1] )
self.features[last_feature_name].append( region )
else:
#add info to last known feature
line = line.strip()
if line.startswith( '/' ):
fields = line[1:].split( '=', 1 )
if len( fields ) == 2:
last_attr_name, content = fields
else:
#No data
last_attr_name = line[1:]
content = ""
content = content.strip( '"' )
if last_attr_name not in self.features[last_feature_name][-1].qualifiers:
self.features[last_feature_name][-1].qualifiers[last_attr_name] = []
self.features[last_feature_name][-1].qualifiers[last_attr_name].append( content )
elif last_attr_name is None and last_feature_name:
# must still be working on location
self.features[last_feature_name][-1].set_coordinates_by_location( line )
else:
#continuation of multi-line qualifier content
if last_feature_name.lower() in ['translation']:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "%s%s" % ( self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip( '"' ) )
else:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "%s %s" % ( self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip( '"' ) )
def get_features_by_type( self, feature_type ):
if feature_type not in self.features:
return []
else:
return self.features[feature_type]
# Parse A GenBank file and return arrays of BED regions for the corresponding features
def get_bed_from_genbank(gb_file, chrom, feature_list):
genbank_parser = GenBankFeatureParser( open( gb_file ) )
features = {}
for feature_type in feature_list:
features[feature_type]=[]
for feature in genbank_parser.get_features_by_type( feature_type ):
name = ""
for name_tag in ['gene', 'locus_tag', 'db_xref']:
if name_tag in feature.qualifiers:
if name: name = name + ";"
name = name + feature.qualifiers[name_tag][0].replace(" ","_")
if not name:
name = "unknown"
features[feature_type].append( "%s\t%s\t%s\t%s\t%s\t%s" % ( chrom, feature.start, feature.end, name, 0, feature.strand ) )#append new bed field here
return features
#geneMark to bed
import sys
#converts GeneMarkHMM to bed
#returns an array of bed regions
def get_bed_from_GeneMark(geneMark_filename, chr):
orfs = open(geneMark_filename).readlines()
while True:
line = orfs.pop(0).strip()
if line.startswith("--------"):
orfs.pop(0)
break
orfs = "".join(orfs)
ctr = 0
regions = []
for block in orfs.split("\n\n"):
if block.startswith("List of Regions of interest"): break
best_block = {'start':0,'end':0,'strand':'+','avg_prob':-sys.maxint,'start_prob':-sys.maxint,'name':'DNE'}
ctr+=1
ctr2=0
for line in block.split("\n"):
ctr2+=1
fields = line.split()
start = int(fields.pop(0))-1
end = int(fields.pop(0))
strand = fields.pop(0)
if strand == 'complement':
strand = "-"
else:
strand = "+"
frame = fields.pop(0)
frame = frame + " " + fields.pop(0)
avg_prob = float(fields.pop(0))
try:
start_prob = float(fields.pop(0))
except:
start_prob = 0
name = "orf_"+str(ctr)+"_"+str(ctr2)
if avg_prob >= best_block['avg_prob']:
if start_prob > best_block['start_prob']:
best_block = {'start':start,'end':end,'strand':strand,'avg_prob':avg_prob,'start_prob':start_prob,'name':name}
regions.append(chr+"\t"+str(best_block['start'])+"\t"+str(best_block['end'])+"\t"+best_block['name']+"\t"+str(int(best_block['avg_prob']*1000))+"\t"+best_block['strand'])
return regions
#geneMarkHMM to bed
#converts GeneMarkHMM to bed
#returns an array of bed regions
def get_bed_from_GeneMarkHMM(geneMarkHMM_filename, chr):
orfs = open(geneMarkHMM_filename).readlines()
while True:
line = orfs.pop(0).strip()
if line == "Predicted genes":
orfs.pop(0)
orfs.pop(0)
break
regions = []
for line in orfs:
fields = line.split()
name = "gene_number_"+fields.pop(0)
strand = fields.pop(0)
start = fields.pop(0)
if start.startswith("<"): start = 1
start = int(start)-1
end = fields.pop(0)
if end.startswith(">"): end = end[1:]
end = int(end)
score = 0 # no scores provided
regions.append(chr+"\t"+str(start)+"\t"+str(end)+"\t"+name+"\t"+str(score)+"\t"+strand)
return regions
#glimmer3 to bed
#converts glimmer3 to bed, doing some linear scaling (probably not correct?) on scores
#returns an array of bed regions
def get_bed_from_glimmer3(glimmer3_filename, chr):
max_score = -sys.maxint
min_score = sys.maxint
orfs = []
for line in open(glimmer3_filename).readlines():
if line.startswith(">"): continue
fields = line.split()
name = fields.pop(0)
start = int(fields.pop(0))
end = int(fields.pop(0))
if int(fields.pop(0))<0:
strand = "-"
temp = start
start = end
end = temp
else:
strand = "+"
start = start - 1
score = (float(fields.pop(0)))
if score > max_score: max_score = score
if score < min_score: min_score = score
orfs.append((chr,start,end,name,score,strand))
delta = 0
if min_score < 0: delta = min_score * -1
regions = []
for (chr,start,end,name,score,strand) in orfs:
#need to cast to str because was having the case where 1000.0 was rounded to 999 by int, some sort of precision bug?
my_score = int(float(str( ( (score+delta) * (1000-0-(min_score+delta)) ) / ( (max_score + delta) + 0 ))))
regions.append(chr+"\t"+str(start)+"\t"+str(end)+"\t"+name+"\t"+str(my_score)+"\t"+strand)
return regions
#!/usr/bin/env python
# Dan Blankenberg
import sys
assert sys.version_info[:2] >= ( 2, 4 )
# genbank_to_bed
class Region:
def __init__( self ):
self.qualifiers = {}
self.start = None
self.end = None
self.strand = '+'
def set_coordinates_by_location( self, location ):
location = location.strip().lower().replace( '..', ',' )
if "complement(" in location: # if part of the sequence is on the negative strand, it all is?
self.strand = '-' # default of + strand
for remove_text in ["join(", "order(", "complement(", ")"]:
location = location.replace( remove_text, "" )
for number in location.split( ',' ):
number = number.strip('\n\r\t <>,()')
if number:
if "^" in number:
# a single point
# check that this is correct for points, ie: 413/NC_005027.gbk: misc_feature 6636286^6636287 ===> 6636285,6636286
end = int( number.split( '^' )[0] )
start = end - 1
else:
end = int( number )
start = end - 1 # match BED coordinates
if self.start is None or start < self.start:
self.start = start
if self.end is None or end > self.end:
self.end = end
class GenBankFeatureParser:
"""Parses Features from Single Locus GenBank file"""
def __init__( self, fh, features_list=[] ):
self.fh = fh
self.features = {}
fh.seek(0)
in_features = False
last_feature_name = None
base_indent = 0
last_attr_name = None
for line in fh:
if not in_features and line.startswith('FEATURES'):
in_features = True
continue
if in_features:
lstrip = line.lstrip()
if line and lstrip == line:
break # end of feature block
cur_indent = len( line ) - len( lstrip )
if last_feature_name is None:
base_indent = cur_indent
if cur_indent == base_indent:
# a new feature
last_attr_name = None
fields = lstrip.split( None, 1 )
last_feature_name = fields[0].strip()
if not features_list or ( features_list and last_feature_name in features_list ):
if last_feature_name not in self.features:
self.features[last_feature_name] = []
region = Region()
region.set_coordinates_by_location( fields[1] )
self.features[last_feature_name].append( region )
else:
# add info to last known feature
line = line.strip()
if line.startswith( '/' ):
fields = line[1:].split( '=', 1 )
if len( fields ) == 2:
last_attr_name, content = fields
else:
# No data
last_attr_name = line[1:]
content = ""
content = content.strip( '"' )
if last_attr_name not in self.features[last_feature_name][-1].qualifiers:
self.features[last_feature_name][-1].qualifiers[last_attr_name] = []
self.features[last_feature_name][-1].qualifiers[last_attr_name].append( content )
elif last_attr_name is None and last_feature_name:
# must still be working on location
self.features[last_feature_name][-1].set_coordinates_by_location( line )
else:
# continuation of multi-line qualifier content
if last_feature_name.lower() in ['translation']:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "%s%s" % ( self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip( '"' ) )
else:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "%s %s" % ( self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip( '"' ) )
def get_features_by_type( self, feature_type ):
if feature_type not in self.features:
return []
else:
return self.features[feature_type]
# Parse A GenBank file and return arrays of BED regions for the corresponding features
def get_bed_from_genbank(gb_file, chrom, feature_list):
genbank_parser = GenBankFeatureParser( open( gb_file ) )
features = {}
for feature_type in feature_list:
features[feature_type] = []
for feature in genbank_parser.get_features_by_type( feature_type ):
name = ""
for name_tag in ['gene', 'locus_tag', 'db_xref']:
if name_tag in feature.qualifiers:
if name:
name = name + ";"
name = name + feature.qualifiers[name_tag][0].replace(" ", "_")
if not name:
name = "unknown"
features[feature_type].append( "%s\t%s\t%s\t%s\t%s\t%s" % ( chrom, feature.start, feature.end, name, 0, feature.strand ) ) # append new bed field here
return features
# geneMark to bed
# converts GeneMarkHMM to bed
# returns an array of bed regions
def get_bed_from_GeneMark(geneMark_filename, chr):
orfs = open(geneMark_filename).readlines()
while True:
line = orfs.pop(0).strip()
if line.startswith("--------"):
orfs.pop(0)
break
orfs = "".join(orfs)
ctr = 0
regions = []
for block in orfs.split("\n\n"):
if block.startswith("List of Regions of interest"):
break
best_block = {'start': 0, 'end': 0, 'strand': '+', 'avg_prob': -sys.maxint, 'start_prob': -sys.maxint, 'name': 'DNE'}
ctr += 1
ctr2 = 0
for line in block.split("\n"):
ctr2 += 1
fields = line.split()
start = int(fields.pop(0)) - 1
end = int(fields.pop(0))
strand = fields.pop(0)
if strand == 'complement':
strand = "-"
else:
strand = "+"
frame = fields.pop(0)
frame = frame + " " + fields.pop(0)
avg_prob = float(fields.pop(0))
try:
start_prob = float(fields.pop(0))
except:
start_prob = 0
name = "orf_" + str(ctr) + "_" + str(ctr2)
if avg_prob >= best_block['avg_prob']:
if start_prob > best_block['start_prob']:
best_block = {'start': start, 'end': end, 'strand': strand, 'avg_prob': avg_prob, 'start_prob': start_prob, 'name': name}
regions.append(chr + "\t" + str(best_block['start']) + "\t" + str(best_block['end']) + "\t" + best_block['name'] + "\t" + str(int(best_block['avg_prob'] * 1000)) + "\t" + best_block['strand'])
return regions
# geneMarkHMM to bed
# converts GeneMarkHMM to bed
# returns an array of bed regions
def get_bed_from_GeneMarkHMM(geneMarkHMM_filename, chr):
orfs = open(geneMarkHMM_filename).readlines()
while True:
line = orfs.pop(0).strip()
if line == "Predicted genes":
orfs.pop(0)
orfs.pop(0)
break
regions = []
for line in orfs:
fields = line.split()
name = "gene_number_" + fields.pop(0)
strand = fields.pop(0)
start = fields.pop(0)
if start.startswith("<"):
start = 1
start = int(start) - 1
end = fields.pop(0)
if end.startswith(">"):
end = end[1:]
end = int(end)
score = 0 # no scores provided
regions.append(chr + "\t" + str(start) + "\t" + str(end) + "\t" + name + "\t" + str(score) + "\t" + strand)
return regions
# glimmer3 to bed
# converts glimmer3 to bed, doing some linear scaling (probably not correct?) on scores
# returns an array of bed regions
def get_bed_from_glimmer3(glimmer3_filename, chr):
max_score = -sys.maxint
min_score = sys.maxint
orfs = []
for line in open(glimmer3_filename).readlines():
if line.startswith(">"):
continue
fields = line.split()
name = fields.pop(0)
start = int(fields.pop(0))
end = int(fields.pop(0))
if int(fields.pop(0)) < 0:
strand = "-"
temp = start
start = end
end = temp
else:
strand = "+"
start = start - 1
score = (float(fields.pop(0)))
if score > max_score:
max_score = score
if score < min_score:
min_score = score
orfs.append((chr, start, end, name, score, strand))
delta = 0
if min_score < 0:
delta = min_score * -1
regions = []
for (chr, start, end, name, score, strand) in orfs:
# need to cast to str because was having the case where 1000.0 was rounded to 999 by int, some sort of precision bug?
my_score = int(float(str( ( (score + delta) * (1000 - 0 - (min_score + delta)) ) / ( (max_score + delta) + 0 ))))
regions.append(chr + "\t" + str(start) + "\t" + str(end) + "\t" + name + "\t" + str(my_score) + "\t" + strand)
return regions
scripts/others/incorrect_gops_jobs.py
+27 -31
View File
@@ -1,17 +1,21 @@
#!/usr/bin/env python
"""
Fetch jobs using gops_intersect, gops_merge, gops_subtract, gops_complement, gops_coverage
Fetch jobs using gops_intersect, gops_merge, gops_subtract, gops_complement, gops_coverage
wherein the second dataset doesn't have chr, start and end in standard columns 1, 2 and 3.
"""
import sys, os, ConfigParser, tempfile
import galaxy.app
import galaxy.model.mapping
import ConfigParser
import os
import sys
import tempfile
import sqlalchemy as sa
import galaxy.app
import galaxy.model.mapping
assert sys.version_info[:2] >= ( 2, 4 )
class TestApplication( object ):
"""Encapsulates the state of a Universe application"""
def __init__( self, database_connection=None, file_path=None ):
@@ -25,9 +29,10 @@ class TestApplication( object ):
# Setup the database engine and ORM
self.model = galaxy.model.mapping.init( self.file_path, self.database_connection, engine_options={}, create_tables=False )
def main():
ini_file = sys.argv[1]
conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} )
conf_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} )
conf_parser.read( ini_file )
configuration = {}
for key, value in conf_parser.items( "app:main" ):
@@ -39,21 +44,13 @@ def main():
try:
for job in app.model.Job.filter( sa.and_( app.model.Job.table.c.create_time.between( '2008-05-23', '2008-11-29' ),
app.model.Job.table.c.state == 'ok',
sa.or_(
sa.and_( sa.or_( app.model.Job.table.c.tool_id == 'gops_intersect_1',
sa.or_( sa.and_( sa.or_( app.model.Job.table.c.tool_id == 'gops_intersect_1',
app.model.Job.table.c.tool_id == 'gops_subtract_1',
app.model.Job.table.c.tool_id == 'gops_coverage_1',
),
sa.not_( app.model.Job.table.c.command_line.like( '%-2 1,2,3%' ) )
),
app.model.Job.table.c.tool_id == 'gops_coverage_1' ),
sa.not_( app.model.Job.table.c.command_line.like( '%-2 1,2,3%' ) ) ),
sa.and_( sa.or_( app.model.Job.table.c.tool_id == 'gops_complement_1',
app.model.Job.table.c.tool_id == 'gops_merge_1',
),
sa.not_( app.model.Job.table.c.command_line.like( '%-1 1,2,3%' ) )
)
)
)
).all():
app.model.Job.table.c.tool_id == 'gops_merge_1' ),
sa.not_( app.model.Job.table.c.command_line.like( '%-1 1,2,3%' ) ) ) ) ) ).all():
print "# processing job id %s" % str( job.id )
for jtoda in job.output_datasets:
print "# --> processing JobToOutputDatasetAssociation id %s" % str( jtoda.id )
@@ -67,17 +64,17 @@ def main():
if history.user_id:
cmd_line = str( job.command_line )
new_output = tempfile.NamedTemporaryFile('w')
if job.tool_id in ['gops_intersect_1','gops_subtract_1','gops_coverage_1']:
new_cmd_line = " ".join(map(str,cmd_line.split()[:4])) + " " + new_output.name + " " + " ".join(map(str,cmd_line.split()[5:]))
if job.tool_id in ['gops_intersect_1', 'gops_subtract_1', 'gops_coverage_1']:
new_cmd_line = " ".join(map(str, cmd_line.split()[:4])) + " " + new_output.name + " " + " ".join(map(str, cmd_line.split()[5:]))
job_output = cmd_line.split()[4]
else:
new_cmd_line = " ".join(map(str,cmd_line.split()[:3])) + " " + new_output.name + " " + " ".join(map(str,cmd_line.split()[4:]))
new_cmd_line = " ".join(map(str, cmd_line.split()[:3])) + " " + new_output.name + " " + " ".join(map(str, cmd_line.split()[4:]))
job_output = cmd_line.split()[3]
try:
os.system(new_cmd_line)
except:
pass
diff_status = os.system('diff %s %s >> /dev/null' %(new_output.name, job_output))
diff_status = os.system('diff %s %s >> /dev/null' % (new_output.name, job_output))
if diff_status == 0:
continue
print "# --------> Outputs differ"
@@ -86,26 +83,25 @@ def main():
jobs[ job.id ][ 'hda_id' ] = hda.id
jobs[ job.id ][ 'hda_name' ] = hda.name
jobs[ job.id ][ 'hda_info' ] = hda.info
jobs[ job.id ][ 'history_id' ] = history.id
jobs[ job.id ][ 'history_name' ] = history.name
jobs[ job.id ][ 'history_id' ] = history.id
jobs[ job.id ][ 'history_name' ] = history.name
jobs[ job.id ][ 'history_update_time' ] = history.update_time
jobs[ job.id ][ 'user_email' ] = user.email
except Exception, e:
print "# caught exception: %s" % str( e )
print "\n\n# Number of incorrect Jobs: %d\n\n" % ( len( jobs ) )
print "#job_id\thda_id\thda_name\thda_info\thistory_id\thistory_name\thistory_update_time\tuser_email"
for jid in jobs:
print '%s\t%s\t"%s"\t"%s"\t%s\t"%s"\t"%s"\t%s' % \
( str( jid ),
str( jobs[ jid ][ 'hda_id' ] ),
jobs[ jid ][ 'hda_name' ],
( str( jid ),
str( jobs[ jid ][ 'hda_id' ] ),
jobs[ jid ][ 'hda_name' ],
jobs[ jid ][ 'hda_info' ],
str( jobs[ jid ][ 'history_id' ] ),
jobs[ jid ][ 'history_name' ],
jobs[ jid ][ 'history_update_time' ],
jobs[ jid ][ 'user_email' ]
)
jobs[ jid ][ 'user_email' ] )
sys.exit(0)
if __name__ == "__main__":
scripts/others/incorrect_gops_join_jobs.py
+20 -18
View File
@@ -2,15 +2,19 @@
"""
Fetch gops_join wherein the use specified minimum coverage is not 1.
"""
import sys, os, ConfigParser, tempfile
import galaxy.app
import galaxy.model.mapping
import ConfigParser
import os
import sys
import tempfile
import sqlalchemy as sa
import galaxy.app
import galaxy.model.mapping
assert sys.version_info[:2] >= ( 2, 4 )
class TestApplication( object ):
"""Encapsulates the state of a Universe application"""
def __init__( self, database_connection=None, file_path=None ):
@@ -24,9 +28,10 @@ class TestApplication( object ):
# Setup the database engine and ORM
self.model = galaxy.model.mapping.init( self.file_path, self.database_connection, engine_options={}, create_tables=False )
def main():
ini_file = sys.argv[1]
conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} )
conf_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} )
conf_parser.read( ini_file )
configuration = {}
for key, value in conf_parser.items( "app:main" ):
@@ -39,9 +44,7 @@ def main():
for job in app.model.Job.filter( sa.and_( app.model.Job.table.c.create_time < '2008-12-16',
app.model.Job.table.c.state == 'ok',
app.model.Job.table.c.tool_id == 'gops_join_1',
sa.not_( app.model.Job.table.c.command_line.like( '%-m 1 %' ) )
)
).all():
sa.not_( app.model.Job.table.c.command_line.like( '%-m 1 %' ) ) ) ).all():
print "# processing job id %s" % str( job.id )
for jtoda in job.output_datasets:
print "# --> processing JobToOutputDatasetAssociation id %s" % str( jtoda.id )
@@ -55,13 +58,13 @@ def main():
if history.user_id:
cmd_line = str( job.command_line )
new_output = tempfile.NamedTemporaryFile('w')
new_cmd_line = " ".join(map(str,cmd_line.split()[:4])) + " " + new_output.name + " " + " ".join(map(str,cmd_line.split()[5:]))
new_cmd_line = " ".join(map(str, cmd_line.split()[:4])) + " " + new_output.name + " " + " ".join(map(str, cmd_line.split()[5:]))
job_output = cmd_line.split()[4]
try:
os.system(new_cmd_line)
except:
pass
diff_status = os.system('diff %s %s >> /dev/null' %(new_output.name, job_output))
diff_status = os.system('diff %s %s >> /dev/null' % (new_output.name, job_output))
if diff_status == 0:
continue
print "# --------> Outputs differ"
@@ -70,26 +73,25 @@ def main():
jobs[ job.id ][ 'hda_id' ] = hda.id
jobs[ job.id ][ 'hda_name' ] = hda.name
jobs[ job.id ][ 'hda_info' ] = hda.info
jobs[ job.id ][ 'history_id' ] = history.id
jobs[ job.id ][ 'history_name' ] = history.name
jobs[ job.id ][ 'history_id' ] = history.id
jobs[ job.id ][ 'history_name' ] = history.name
jobs[ job.id ][ 'history_update_time' ] = history.update_time
jobs[ job.id ][ 'user_email' ] = user.email
except Exception, e:
print "# caught exception: %s" % str( e )
print "\n\n# Number of incorrect Jobs: %d\n\n" % ( len( jobs ) )
print "#job_id\thda_id\thda_name\thda_info\thistory_id\thistory_name\thistory_update_time\tuser_email"
for jid in jobs:
print '%s\t%s\t"%s"\t"%s"\t%s\t"%s"\t"%s"\t%s' % \
( str( jid ),
str( jobs[ jid ][ 'hda_id' ] ),
jobs[ jid ][ 'hda_name' ],
( str( jid ),
str( jobs[ jid ][ 'hda_id' ] ),
jobs[ jid ][ 'hda_name' ],
jobs[ jid ][ 'hda_info' ],
str( jobs[ jid ][ 'history_id' ] ),
jobs[ jid ][ 'history_name' ],
jobs[ jid ][ 'history_update_time' ],
jobs[ jid ][ 'user_email' ]
)
jobs[ jid ][ 'user_email' ] )
sys.exit(0)
if __name__ == "__main__":
scripts/tool_shed/check_download_urls.py
+8 -9
View File
@@ -1,23 +1,22 @@
#!/usr/bin/env python
##Dan Blankenberg
##Script that checks toolshed tags to see if URLs are accessible.
##Does not currently handle 'download_binary'
# Dan Blankenberg
# Script that checks toolshed tags to see if URLs are accessible.
# Does not currently handle 'download_binary'
import os
import sys
from optparse import OptionParser
import xml.etree.ElementTree as ET
import urllib2
import xml.etree.ElementTree as ET
from optparse import OptionParser
FILENAMES = [ 'tool_dependencies.xml' ]
ACTION_TYPES = [ 'download_by_url', 'download_file' ]
def main():
parser = OptionParser()
parser.add_option( '-d', '--directory', dest='directory', action='store', type="string", default='.', help='Root directory' )
( options, args ) = parser.parse_args()
for (dirpath, dirnames, filenames) in os.walk( options.directory ):
for filename in filenames:
if filename in FILENAMES:
scripts/tool_shed/migrate_tools_to_repositories.py
+42 -36
View File
@@ -15,23 +15,29 @@ associated with them, and migrates old tool shed stuff to new tool shed stuff.
# Enable next-gen tool shed features
enable_next_gen_tool_shed = True
2. This script requires the Galaxy instance to use Postgres for database storage.
2. This script requires the Galaxy instance to use Postgres for database storage.
To run this script, use "sh migrate_tools_to_repositories.sh" from this directory
'''
import sys, os, subprocess, ConfigParser, shutil, tarfile, tempfile
import ConfigParser
import os
import shutil
import sys
import tarfile
import tempfile
from time import strftime
from mercurial import hg, ui
assert sys.version_info[:2] >= ( 2, 4 )
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
import psycopg2
import galaxy.webapps.tool_shed.app
from mercurial import hg, ui, httprepo, commands
from time import strftime
assert sys.version_info[:2] >= ( 2, 4 )
def directory_hash_id( id ):
s = str( id )
@@ -44,13 +50,14 @@ def directory_hash_id( id ):
# Drop the last three digits -- 1000 files per directory
padded = padded[:-3]
# Break into chunks of three
return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ]
return [ padded[i * 3:(i + 1) * 3] for i in range( len( padded ) // 3 ) ]
def get_versions( app, item ):
"""Get all versions of item whose state is a valid state"""
valid_states = [ app.model.Tool.states.NEW,
app.model.Tool.states.WAITING,
app.model.Tool.states.APPROVED,
valid_states = [ app.model.Tool.states.NEW,
app.model.Tool.states.WAITING,
app.model.Tool.states.APPROVED,
app.model.Tool.states.ARCHIVED ]
versions = [ item ]
this_item = item
@@ -65,6 +72,7 @@ def get_versions( app, item ):
item = item.older_version[ 0 ]
return versions
def get_approved_tools( app, sa_session ):
"""Get only the latest version of each tool from the database whose state is approved"""
tools = []
@@ -74,6 +82,7 @@ def get_approved_tools( app, sa_session ):
tools.append( tool )
return tools
def create_repository_from_tool( app, sa_session, tool ):
# Make the repository name a form of the tool's tool_id by
# lower-casing everything and replacing any blank spaces with underscores.
@@ -81,7 +90,7 @@ def create_repository_from_tool( app, sa_session, tool ):
print "Creating repository '%s' in database" % ( repo_name )
repository = app.model.Repository( name=repo_name,
description=tool.description,
user_id = tool.user_id )
user_id=tool.user_id )
# Flush to get the id
sa_session.add( repository )
sa_session.flush()
@@ -97,7 +106,7 @@ def create_repository_from_tool( app, sa_session, tool ):
os.makedirs( repository_path )
# Create the local hg repository
print "Creating repository '%s' on disk" % ( os.path.abspath( repository_path ) )
repo = hg.repository( ui.ui(), os.path.abspath( repository_path ), create=True )
hg.repository( ui.ui(), os.path.abspath( repository_path ), create=True )
# Add an entry in the hgweb.config file for the new repository - this enables calls to repository.repo_path
add_hgweb_config_entry( repository, repository_path )
# Migrate tool categories
@@ -113,14 +122,15 @@ def create_repository_from_tool( app, sa_session, tool ):
rra = app.model.RepositoryRatingAssociation( user=tra.user,
rating=tra.rating,
comment=tra.comment )
rra.repository=repository
rra.repository = repository
sa_session.add( rra )
sa_session.flush()
def add_hgweb_config_entry( repository, repository_path ):
# Add an entry in the hgweb.config file for a new repository. This enables calls to repository.repo_path.
# An entry looks something like: repos/test/mira_assembler = database/community_files/000/repo_123
hgweb_config = "%s/hgweb.config" % os.getcwd()
hgweb_config = "%s/hgweb.config" % os.getcwd()
entry = "repos/%s/%s = %s" % ( repository.user.username, repository.name, repository_path.lstrip( './' ) )
if os.path.exists( hgweb_config ):
output = open( hgweb_config, 'a' )
@@ -130,6 +140,7 @@ def add_hgweb_config_entry( repository, repository_path ):
output.write( "%s\n" % entry )
output.close()
def create_hgrc_file( repository ):
# At this point, an entry for the repository is required to be in the hgweb.config
# file so we can call repository.repo_path.
@@ -150,6 +161,7 @@ def create_hgrc_file( repository ):
output.flush()
output.close()
def add_tool_files_to_repository( app, sa_session, tool ):
current_working_dir = os.getcwd()
# Get the repository to which the tool will be migrated
@@ -169,7 +181,7 @@ def add_tool_files_to_repository( app, sa_session, tool ):
cmd = "hg clone %s" % repo_path
os.chdir( tmp_archive_dir )
os.system( cmd )
os.chdir( current_working_dir )
os.chdir( current_working_dir )
cloned_repo_dir = os.path.join( tmp_archive_dir, 'repo_%d' % repository.id )
# We want these change sets to be associated with the owner of the repository, so we'll
# set the HGUSER environment variable accordingly. We do this because in the mercurial
@@ -194,8 +206,8 @@ def add_tool_files_to_repository( app, sa_session, tool ):
# Don't visit .hg directories
dirs.remove( '.hg' )
if 'hgrc' in files:
# Don't include hgrc files in commit - should be impossible
# since we don't visit .hg dirs, but just in case...
# Don't include hgrc files in commit - should be impossible
# since we don't visit .hg dirs, but just in case...
files.remove( 'hgrc' )
for dir in dirs:
os.system( "hg add %s" % dir )
@@ -221,13 +233,16 @@ def add_tool_files_to_repository( app, sa_session, tool ):
# Remove tmp directory
shutil.rmtree( tmp_dir )
def get_repository_by_name( app, sa_session, repo_name ):
"""Get a repository from the database"""
return sa_session.query( app.model.Repository ).filter_by( name=repo_name ).one()
def contains( containing_str, contained_str ):
return containing_str.lower().find( contained_str.lower() ) >= 0
def tool_archive_extension( file_name ):
extension = None
if extension is None:
@@ -248,9 +263,11 @@ def tool_archive_extension( file_name ):
extension = 'tar'
return extension
def tool_archive_file_name( tool, file_name ):
return '%s_%s.%s' % ( tool.tool_id, tool.version, tool_archive_extension( file_name ) )
def main():
if len( sys.argv ) < 2:
print "Usage: python %s <Tool shed config file>" % sys.argv[0]
@@ -261,24 +278,13 @@ def main():
print "%s - Migrating current tool archives to new tool repositories" % now
# tool_shed_wsgi.ini file
ini_file = sys.argv[1]
conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} )
conf_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} )
conf_parser.read( ini_file )
try:
db_conn_str = conf_parser.get( "app:main", "database_connection" )
except ConfigParser.NoOptionError, e:
except ConfigParser.NoOptionError:
db_conn_str = conf_parser.get( "app:main", "database_file" )
print 'DB Connection: ', db_conn_str
# Determine db connection - only postgres is supported
if contains( db_conn_str, '///' ) and contains( db_conn_str, '?' ) and contains( db_conn_str, '&' ):
# postgres:///galaxy_test?user=postgres&password=postgres
db_str = db_conn_str.split( '///' )[1]
db_name = db_str.split( '?' )[0]
db_user = db_str.split( '?' )[1].split( '&' )[0].split( '=' )[1]
db_password = db_str.split( '?' )[1].split( '&' )[1].split( '=' )[1]
elif contains( db_conn_str, '//' ) and contains( db_conn_str, ':' ):
# dialect://user:password@host/db_name
db_name = db_conn_str.split('/')[-1]
db_user = db_conn_str.split('//')[1].split(':')[0]
# Instantiate app
configuration = {}
for key, value in conf_parser.items( "app:main" ):
@@ -286,7 +292,7 @@ def main():
app = galaxy.webapps.tool_shed.app.UniverseApplication( global_conf=dict( __file__=ini_file ), **configuration )
sa_session = app.model.context
# Remove the hgweb.config file if it exists
hgweb_config = "%s/hgweb.config" % os.getcwd()
hgweb_config = "%s/hgweb.config" % os.getcwd()
if os.path.exists( hgweb_config ):
print "Removing old file: ", hgweb_config
os.remove( hgweb_config )
@@ -300,7 +306,7 @@ def main():
if os.path.exists( dir ):
print "Removing old repository file directory: ", dir
shutil.rmtree( dir )
# Delete all records from db tables:
# Delete all records from db tables:
# repository_category_association, repository_rating_association, repository
print "Deleting db records for repository: ", repo.name
for rca in repo.categories:
@@ -315,7 +321,7 @@ def main():
print "Deleted %d rows from the repository table" % repo_records
print "Deleted %d rows from the repository_category_association table" % rca_records
print "Deleted %d rows from the repository_rating_association table" % rra_records
# Migrate database tool, tool category and tool rating records to new
# Migrate database tool, tool category and tool rating records to new
# database repository, repository category and repository rating records
# and create the hg repository on disk for each.
for tool in get_approved_tools( app, sa_session ):
scripts/tools/maf/check_loc_file.py
+7 -5
View File
@@ -7,6 +7,7 @@ import sys
assert sys.version_info[:2] >= ( 2, 4 )
def __main__():
index_location_file = sys.argv[ 1 ]
for i, line in enumerate( open( index_location_file ) ):
@@ -20,12 +21,12 @@ def __main__():
species_indexed_in_maf = []
species_found_in_maf = []
for maf_file in maf_files:
indexed_maf = bx.align.maf.MAFIndexedAccess( maf_file, keep_open = True, parse_e_rows = False )
indexed_maf = bx.align.maf.MAFIndexedAccess( maf_file, keep_open=True, parse_e_rows=False )
for key in indexed_maf.indexes.indexes.keys():
spec = maf_utilities.src_split( key )[0]
if spec not in species_indexed_in_maf:
species_indexed_in_maf.append( spec )
while True: #reading entire maf set will take some time
while True: # reading entire maf set will take some time
block = indexed_maf.read_at_current_offset( indexed_maf.f )
if block is None:
break
@@ -33,14 +34,14 @@ def __main__():
spec = maf_utilities.src_split( comp.src )[0]
if spec not in species_found_in_maf:
species_found_in_maf.append( spec )
#indexed species
# indexed species
for spec in indexed_for_species:
if spec not in species_indexed_in_maf:
print "Line %i, %s claims to be indexed for %s, but indexes do not exist." % ( i, uid, spec )
for spec in species_indexed_in_maf:
if spec not in indexed_for_species:
print "Line %i, %s is indexed for %s, but is not listed in loc file." % ( i, uid, spec )
#existing species
# existing species
for spec in species_exist:
if spec not in species_found_in_maf:
print "Line %i, %s claims to have blocks for %s, but was not found in MAF files." % ( i, uid, spec )
@@ -50,4 +51,5 @@ def __main__():
except Exception, e:
print "Line %i is invalid: %s" % ( i, e )
if __name__ == "__main__": __main__()
if __name__ == "__main__":
__main__()
scripts/tools/re_escape_output.py
+3 -4
View File
@@ -1,20 +1,19 @@
#!/usr/bin/env python
"""
Escapes a file into a form suitable for use with tool tests using re_match or re_match_multiline (when -m/--multiline option is used)
usage: re_escape_output.py [options] input_file [output_file]
-m: Use Multiline Matching
"""
import optparse
import re
import optparse, re
def __main__():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option( "-m", "--multiline", action="store_true", dest="multiline", default=False, help="Use Multiline Matching")
( options, args ) = parser.parse_args()
input = open( args[0] ,'rb' )
input = open( args[0], 'rb' )
if len( args ) > 1:
output = open( args[1], 'wb' )
else:
scripts/transfer.py
+44 -24
View File
@@ -4,7 +4,17 @@ Downloads files to temp locations. This script is invoked by the Transfer
Manager (galaxy.jobs.transfer_manager) and should not normally be invoked by
hand.
"""
import os, sys, optparse, ConfigParser, socket, SocketServer, threading, logging, random, urllib2, tempfile, time
import ConfigParser
import logging
import optparse
import os
import random
import SocketServer
import sys
import tempfile
import threading
import time
import urllib2
galaxy_root = os.path.abspath( os.path.join( os.path.dirname( __file__ ), os.pardir ) )
sys.path.insert( 0, os.path.abspath( os.path.join( galaxy_root, 'lib' ) ) )
@@ -14,9 +24,9 @@ try:
except ImportError:
pexpect = None
from sqlalchemy import *
from sqlalchemy.orm import *
from daemon import DaemonContext
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import scoped_session, sessionmaker
import galaxy.model
from galaxy.util import json, bunch
@@ -33,6 +43,7 @@ log.addHandler( handler )
debug = False
slow = False
class ArgHandler( object ):
"""
Collect command line flags.
@@ -40,10 +51,11 @@ class ArgHandler( object ):
def __init__( self ):
self.parser = optparse.OptionParser()
self.parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)',
default=os.path.abspath( os.path.join( galaxy_root, 'config/galaxy.ini' ) ) )
default=os.path.abspath( os.path.join( galaxy_root, 'config/galaxy.ini' ) ) )
self.parser.add_option( '-d', '--debug', action='store_true', dest='debug', help="Debug (don't detach)" )
self.parser.add_option( '-s', '--slow', action='store_true', dest='slow', help="Transfer slowly (for debugging)" )
self.opts = None
def parse( self ):
self.opts, args = self.parser.parse_args()
if len( args ) != 1:
@@ -62,19 +74,21 @@ class ArgHandler( object ):
global slow
slow = True
class GalaxyApp( object ):
"""
A shell Galaxy App to provide access to the Galaxy configuration and
model/database.
"""
def __init__( self, config_file ):
self.config = ConfigParser.ConfigParser( dict( database_file = 'database/universe.sqlite',
file_path = 'database/files',
transfer_worker_port_range = '12275-12675',
transfer_worker_log = None ) )
self.config = ConfigParser.ConfigParser( dict( database_file='database/universe.sqlite',
file_path='database/files',
transfer_worker_port_range='12275-12675',
transfer_worker_log=None ) )
self.config.read( config_file )
self.model = bunch.Bunch()
self.connect_database()
def connect_database( self ):
# Avoid loading the entire model since doing so is exceptionally slow
default_dburl = 'sqlite:///%s?isolation_level=IMMEDIATE' % self.config.get( 'app:main', 'database_file' )
@@ -87,9 +101,11 @@ class GalaxyApp( object ):
self.sa_session = scoped_session( sessionmaker( bind=engine, autoflush=False, autocommit=True ) )
self.model.TransferJob = galaxy.model.TransferJob
self.model.TransferJob.table = Table( "transfer_job", metadata, autoload=True )
def get_transfer_job( self, id ):
return self.sa_session.query( self.model.TransferJob ).get( int( id ) )
class ListenerServer( SocketServer.ThreadingTCPServer ):
"""
The listener will accept state requests and new transfers for as long as
@@ -110,6 +126,7 @@ class ListenerServer( SocketServer.ThreadingTCPServer ):
app.sa_session.add( transfer_job )
app.sa_session.flush()
class ListenerRequestHandler( SocketServer.BaseRequestHandler ):
"""
Handle state or transfer requests received on the socket.
@@ -127,6 +144,7 @@ class ListenerRequestHandler( SocketServer.BaseRequestHandler ):
log.error( error_msg )
log.debug( 'Original request was: %s' % request )
class StateResult( object ):
"""
A mutable container for the 'result' portion of JSON-RPC responses to state requests.
@@ -134,6 +152,7 @@ class StateResult( object ):
def __init__( self, result=None ):
self.result = result
def transfer( app, transfer_job_id ):
transfer_job = app.get_transfer_job( transfer_job_id )
if transfer_job is None:
@@ -149,14 +168,14 @@ def transfer( app, transfer_job_id ):
if protocol not in ( 'http', 'https', 'scp' ):
log.error( 'Unsupported protocol: %s' % protocol )
return False
state_result = StateResult( result = dict( state = transfer_job.states.RUNNING, info='Transfer process starting up.' ) )
state_result = StateResult( result=dict( state=transfer_job.states.RUNNING, info='Transfer process starting up.' ) )
listener_server = ListenerServer( range( port_range[0], port_range[1] + 1 ), ListenerRequestHandler, app, transfer_job, state_result )
# daemonize here (if desired)
if not debug:
daemon_context = DaemonContext( files_preserve=[ listener_server.fileno() ], working_directory=os.getcwd() )
daemon_context.open()
# If this fails, it'll never be detected. Hopefully it won't fail since it succeeded once.
app.connect_database() # daemon closed the database fd
app.connect_database() # daemon closed the database fd
transfer_job = app.get_transfer_job( transfer_job_id )
listener_thread = threading.Thread( target=listener_server.serve_forever )
listener_thread.setDaemon( True )
@@ -191,13 +210,14 @@ def transfer( app, transfer_job_id ):
app.sa_session.flush()
return True
def http_transfer( transfer_job ):
"""Plugin" for handling http(s) transfers."""
url = transfer_job.params['url']
try:
f = urllib2.urlopen( url )
except urllib2.URLError, e:
yield dict( state = transfer_job.states.ERROR, info = 'Unable to open URL: %s' % str( e ) )
yield dict( state=transfer_job.states.ERROR, info='Unable to open URL: %s' % str( e ) )
return
size = f.info().getheader( 'Content-Length' )
if size is not None:
@@ -210,7 +230,7 @@ def http_transfer( transfer_job ):
try:
fh, fn = tempfile.mkstemp()
except Exception, e:
yield dict( state = transfer_job.states.ERROR, info = 'Unable to create temporary file for transfer: %s' % str( e ) )
yield dict( state=transfer_job.states.ERROR, info='Unable to create temporary file for transfer: %s' % str( e ) )
return
log.debug( 'Writing %s to %s, size is %s' % ( url, fn, size or 'unknown' ) )
try:
@@ -223,19 +243,20 @@ def http_transfer( transfer_job ):
if size is not None and read < size:
percent = int( float( read ) / size * 100 )
if percent != last:
yield dict( state = transfer_job.states.PROGRESS, read = read, percent = '%s' % percent )
yield dict( state=transfer_job.states.PROGRESS, read=read, percent='%s' % percent )
last = percent
elif size is None:
yield dict( state = transfer_job.states.PROGRESS, read = read )
yield dict( state=transfer_job.states.PROGRESS, read=read )
if slow:
time.sleep( 1 )
os.close( fh )
yield dict( state = transfer_job.states.DONE, path = fn )
yield dict( state=transfer_job.states.DONE, path=fn )
except Exception, e:
yield dict( state = transfer_job.states.ERROR, info = 'Error during file transfer: %s' % str( e ) )
yield dict( state=transfer_job.states.ERROR, info='Error during file transfer: %s' % str( e ) )
return
return
def scp_transfer( transfer_job ):
"""Plugin" for handling scp transfers using pexpect"""
def print_ticks( d ):
@@ -245,24 +266,23 @@ def scp_transfer( transfer_job ):
password = transfer_job.params[ 'password' ]
file_path = transfer_job.params[ 'file_path' ]
if pexpect is None:
return dict( state = transfer_job.states.ERROR, info = PEXPECT_IMPORT_MESSAGE )
return dict( state=transfer_job.states.ERROR, info=PEXPECT_IMPORT_MESSAGE )
try:
fh, fn = tempfile.mkstemp()
except Exception, e:
return dict( state = transfer_job.states.ERROR, info = 'Unable to create temporary file for transfer: %s' % str( e ) )
return dict( state=transfer_job.states.ERROR, info='Unable to create temporary file for transfer: %s' % str( e ) )
try:
# TODO: add the ability to determine progress of the copy here like we do in the http_transfer above.
cmd = "scp %s@%s:'%s' '%s'" % ( user_name,
host,
file_path.replace( ' ', '\ ' ),
fn )
output = pexpect.run( cmd,
events={ '.ssword:*': password + '\r\n',
pexpect.TIMEOUT: print_ticks },
timeout=10 )
return dict( state = transfer_job.states.DONE, path = fn )
pexpect.run( cmd, events={ '.ssword:*': password + '\r\n',
pexpect.TIMEOUT: print_ticks },
timeout=10 )
return dict( state=transfer_job.states.DONE, path=fn )
except Exception, e:
return dict( state = transfer_job.states.ERROR, info = 'Error during file transfer: %s' % str( e ) )
return dict( state=transfer_job.states.ERROR, info='Error during file transfer: %s' % str( e ) )
if __name__ == '__main__':
arg_handler = ArgHandler()