opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into ini_to_yaml_config

Browse Source
This commit is contained in:
Nate Coraor
2018-01-18 12:30:05 -05:00
parent cdc6657bda a48d0e9a7f
commit 44c291bc66
531 changed files with 11658 additions and 6713 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/api/form_create_from_xml.py
-17
View File
@@ -1,17 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
from common import submit
try:
data = {}
data['xml_text'] = open(sys.argv[3]).read()
except IndexError:
print('usage: %s key url form_xml_description_file' % os.path.basename(sys.argv[0]))
sys.exit(1)
submit(sys.argv[1], sys.argv[2], data)
scripts/api/request_type_create_from_xml.py
-23
View File
@@ -1,23 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
from common import submit
try:
data = {}
data['request_form_id'] = sys.argv[3]
data['sample_form_id'] = sys.argv[4]
data['sequencer_id'] = sys.argv[5]
data['xml_text'] = open(sys.argv[6]).read()
except IndexError:
print('usage: %s key url request_form_id sample_form_id request_type_xml_description_file [access_role_ids,]' % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data['role_ids'] = [i for i in sys.argv[7].split(',') if i]
except IndexError:
data['role_ids'] = []
submit(sys.argv[1], sys.argv[2], data)
scripts/api/requests_update_state.py
-16
View File
@@ -1,16 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
from common import update
try:
data = {}
data['update_type'] = 'request_state'
except IndexError:
print('usage: %s key url' % os.path.basename(sys.argv[0]))
sys.exit(1)
update(sys.argv[1], sys.argv[2], data, return_formatted=True)
scripts/api/sample_configurations/sample_tracking/request_form.xml
-15
View File
@@ -1,15 +0,0 @@
<form type="request" name="Sample Request Form" description="Sample Request Form description">
<fields>
<field name="field1" type="text" label="Request form field1" description="Description of request form field1" value="" required="True"/>
<field name="field2" type="text" area="True" label="Request form field2" description="Description of request form field2" value=""/>
<field name="field3" type="select" label="Request form field3" description="Description of request form field3" value="">
<option value="option1" />
<option value="option2" />
</field>
<field name="field4" type="select" label="Request form field4" description="Description of request form field4" value="" checkboxes="True">
<option value="option1" />
<option value="option2" />
</field>
<field name="field5" type="address" label="Request form field5" description="Description of request form field5"/>
</fields>
</form>
scripts/api/sample_configurations/sample_tracking/request_type.xml
-9
View File
@@ -1,9 +0,0 @@
<sequencer name="Sample Sequencer configuration" description="Sample Sequencer configuration description">
<sample_states>
<state name="New" description="Sample entered into the system"/>
<state name="Received" description="Sample tube received"/>
<state name="Library Started" description="Sample library preparation"/>
<state name="Run Started" description="Sequence run in progress"/>
<state name="Done" description="Sequence run complete"/>
</sample_states>
</sequencer>
scripts/api/sample_configurations/sample_tracking/sample_form.xml
-18
View File
@@ -1,18 +0,0 @@
<form type="sample" name="Sample Sample Form" description="Sample Sample Form description">
<layout>
<grid name="Run details" />
</layout>
<fields>
<field name="field1" type="text" label="Sample form field1" description="Description of sample form field1" value="" required="True" layout="Run details"/>
<field name="field2" type="text" area="True" label="Sample form field2" description="Description of sample form field2" value="" layout="Run details"/>
<field name="field3" type="select" label="Sample form field3" description="Description of sample form field3" value="" layout="Run details">
<option value="option1" />
<option value="option2" />
</field>
<field name="field4" type="select" label="Sample form field4" description="Description of sample form field4" value="" checkboxes="True" layout="Run details">
<option value="option1" />
<option value="option2" />
</field>
<field name="field5" type="address" label="Sample form field5" description="Description of sample form field5" layout="Run details"/>
</fields>
</form>
scripts/api/sample_dataset_update_status.py
-22
View File
@@ -1,22 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
from common import update
try:
data = {}
data['update_type'] = 'sample_dataset_transfer_status'
data['sample_dataset_ids'] = sys.argv[3].split(',')
data['new_status'] = sys.argv[4]
except IndexError:
print('usage: %s key url sample_dataset_ids new_state [error msg]' % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data['error_msg'] = sys.argv[5]
except IndexError:
data['error_msg'] = ''
print(data)
update(sys.argv[1], sys.argv[2], data, return_formatted=True)
scripts/api/sample_update_state.py
-21
View File
@@ -1,21 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
from common import update
try:
data = {}
data['update_type'] = 'sample_state'
data['new_state'] = sys.argv[3]
except IndexError:
print('usage: %s key url new_state [comment]' % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data['comment'] = sys.argv[4]
except IndexError:
data['comment'] = ''
update(sys.argv[1], sys.argv[2], data, return_formatted=True)
scripts/api/sequencer_configuration_create.py
-51
View File
@@ -1,51 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
from common import get, submit
def create_sequencer_configuration(key, base_url, request_form_filename, sample_form_filename, request_type_filename, email_addresses, return_formatted=True):
# create request_form
data = {}
data['xml_text'] = open(request_form_filename).read()
request_form = submit(key, "%sforms" % base_url, data, return_formatted=False)[0]
# create sample_form
data = {}
data['xml_text'] = open(sample_form_filename).read()
sample_form = submit(key, "%sforms" % base_url, data, return_formatted=False)[0]
# get user ids
user_ids = [user['id'] for user in get(key, "%susers" % base_url) if user['email'] in email_addresses]
# create role, assign to user
data = {}
data['name'] = "request_type_role_%s_%s_%s name" % (request_form['id'], sample_form['id'], '_'.join(email_addresses))
data['description'] = "request_type_role_%s_%s_%s description" % (request_form['id'], sample_form['id'], '_'.join(email_addresses))
data['user_ids'] = user_ids
role_ids = [role['id'] for role in submit(key, "%sroles" % base_url, data, return_formatted=False)]
# create request_type
data = {}
data['request_form_id'] = request_form['id']
data['sample_form_id'] = sample_form['id']
data['role_ids'] = role_ids
data['xml_text'] = open(request_type_filename).read()
return submit(key, "%srequest_types" % base_url, data, return_formatted=return_formatted) # create and print out results for request type
def main():
try:
key = sys.argv[1]
base_url = sys.argv[2]
request_form_filename = sys.argv[3]
sample_form_filename = sys.argv[4]
request_type_filename = sys.argv[5]
email_addresses = sys.argv[6].split(',')
except IndexError:
print('usage: %s key base_url request_form_xml_description_file sample_form_xml_description_file request_type_xml_description_file email_address1[,email_address2]' % os.path.basename(sys.argv[0]))
sys.exit(1)
return create_sequencer_configuration(key, base_url, request_form_filename, sample_form_filename, request_type_filename, email_addresses, return_formatted=True)
if __name__ == "__main__":
main()
scripts/build_universe_config.py
+2 -1
View File
@@ -1,9 +1,10 @@
from ConfigParser import ConfigParser
from os import listdir
from os.path import join
from re import match
from sys import argv
from six.moves.configparser import ConfigParser
def merge():
"""
scripts/common_startup.sh
+28
View File
@@ -18,6 +18,7 @@ FETCH_WHEELS=1
CREATE_VENV=1
REPLACE_PIP=$SET_VENV
COPY_SAMPLE_FILES=1
SKIP_CLIENT_BUILD=0
for arg in "$@"; do
[ "$arg" = "--skip-eggs" ] && FETCH_WHEELS=0
@@ -28,6 +29,7 @@ for arg in "$@"; do
[ "$arg" = "--replace-pip" ] && REPLACE_PIP=1
[ "$arg" = "--stop-daemon" ] && FETCH_WHEELS=0
[ "$arg" = "--skip-samples" ] && COPY_SAMPLE_FILES=0
[ "$arg" = "--skip-client-build" ] && SKIP_CLIENT_BUILD=1
done
SAMPLES="
@@ -61,6 +63,32 @@ for rmfile in $RMFILES; do
[ -f "$rmfile" ] && rm -f "$rmfile"
done
# Check client build state.
if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
gitbranch=$(git rev-parse --abbrev-ref HEAD)
if [ "$gitbranch" = "dev" ]; then
# We're on dev. This branch (only, currently) doesn't have build
# artifacts. We should probabably swap to a list of releases?
# Compare hash.
if [ -f static/client_build_hash.txt ]; then
githash=$(git rev-parse HEAD)
statichash=$(cat static/client_build_hash.txt)
if [ "$githash" = "$statichash" ]; then
SKIP_CLIENT_BUILD=1
fi
fi
else
# Not on dev. We're not going to bug people about building.
SKIP_CLIENT_BUILD=1
fi
if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
echo "The Galaxy client build is out of date. Please run 'make client' or your choice of client build target (client-*)."
echo "If you're sure you'd like to skip this check, you can run galaxy with the --skip-client-build flag, though this is not recommended as the client and server code will potentially be out of sync."
echo "See ./client/README.md in the Galaxy repository for more information, including how to get help if you're having trouble."
exit 1
fi
fi
: ${GALAXY_CONFIG_FILE:=config/galaxy.yml}
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
scripts/common_startup_functions.sh
+1 -1
View File
@@ -6,7 +6,7 @@ parse_common_args() {
while :
do
case "$1" in
--skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip)
--skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip|--skip-client-build)
common_startup_args="$common_startup_args $1"
shift
;;
scripts/data_libraries/build_lucene_index.py
+4 -4
View File
@@ -9,13 +9,13 @@ details.
Run from the ~/scripts/data_libraries directory:
%sh build_lucene_index.sh
"""
import ConfigParser
import csv
import os
import sys
import urllib
import requests
from six.moves.configparser import ConfigParser
from six.moves.urllib.parse import urlencode
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
@@ -39,7 +39,7 @@ def main(ini_file):
def build_index(search_url, dataset_file):
url = "%s/index?%s" % (search_url, urllib.urlencode({"docfile": dataset_file}))
url = "%s/index?%s" % (search_url, urlencode({"docfile": dataset_file}))
requests.put(url)
@@ -89,7 +89,7 @@ def _get_folder_info(folder):
def get_sa_session(ini_file):
conf_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
conf_parser = ConfigParser({'here': os.getcwd()})
conf_parser.read(ini_file)
kwds = dict()
for key, value in conf_parser.items("app:main"):
scripts/db_shell.py
+4 -1
View File
@@ -1,12 +1,15 @@
# This script allows easy access to Galaxy's database layer via the
# Galaxy models. For example:
# % python -i scripts/db_shell.py
# % python -i scripts/db_shell.py -c config/galaxy.ini
# >>> new_user = User("admin@gmail.com")
# >>> new_user.set_password
# >>> sa_session.add(new_user)
# >>> sa_session.commit()
# >>> sa_session.query(User).all()
#
# If you use ipython use:
# % ipython -i scripts/db_shell.py -- -c config/galaxy.ini
#
# You can also use this script as a library, for instance see https://gist.github.com/1979583
# TODO: This script overlaps a lot with manage_db.py and create_db.py,
# these should maybe be refactored to remove duplication.
scripts/galaxy-main
+2 -5
View File
@@ -31,10 +31,7 @@ import threading
from argparse import ArgumentParser
from logging.config import fileConfig
try:
import ConfigParser as configparser
except ImportError:
import configparser
from six.moves.configparser import ConfigParser
try:
from daemonize import Daemonize
@@ -219,7 +216,7 @@ class GalaxyConfigBuilder(object):
if not self.config_file:
return
if self.config_is_ini:
raw_config = configparser.ConfigParser()
raw_config = ConfigParser()
raw_config.read([self.config_file])
if raw_config.has_section('loggers'):
config_file = os.path.abspath(self.config_file)
scripts/helper.py
+2 -1
View File
@@ -8,9 +8,10 @@ from __future__ import print_function
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
from six.moves.configparser import ConfigParser
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))
from galaxy.model import mapping
scripts/loc_files/create_all_fasta_loc.py
+9 -22
View File
@@ -149,18 +149,6 @@ VARIANT_MAP = {'canon': 'Canonical',
'male': 'Male'}
# alphabetize ignoring case
def caseless_compare(a, b):
au = a.upper()
bu = b.upper()
if au > bu:
return 1
elif au == bu:
return 0
elif au < bu:
return -1
def __main__():
# command line variables
parser = optparse.OptionParser()
@@ -257,14 +245,13 @@ def __main__():
else:
unmatching_fasta_paths.append(os.path.join(dirpath, fn))
# remove redundant fasta files
if variant_exclusions.keys():
for k in variant_exclusions.keys():
leave_in = '%s%s' % (genome_subdir, k)
if leave_in in fasta_locs:
to_remove = ['%s%s' % (genome_subdir, k) for k in variant_exclusions[k]]
for tr in to_remove:
if tr in fasta_locs:
del fasta_locs[tr]
for k, v in variant_exclusions.items():
leave_in = '%s%s' % (genome_subdir, k)
if leave_in in fasta_locs:
to_remove = ['%s%s' % (genome_subdir, _) for _ in v]
for tr in to_remove:
if tr in fasta_locs:
del fasta_locs[tr]
# output results
print('\nThere were %s fasta files found that were not included because they did not have the expected file names.' % len(unmatching_fasta_paths))
@@ -286,8 +273,8 @@ def __main__():
else:
all_fasta_loc.write('%s\n' % open('%s.sample' % loc_path, 'rb').read().strip())
# output list of fasta files in alphabetical order
fasta_bases = fasta_locs.keys()
fasta_bases.sort(caseless_compare)
fasta_bases = list(fasta_locs.keys())
fasta_bases.sort(key=str.upper)
for fb in fasta_bases:
out_line = []
for col in col_values:
scripts/microbes/create_bacteria_loc_file.py
+11 -11
View File
@@ -20,25 +20,25 @@ def __main__():
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
tmp_dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
for org in organisms:
org = organisms[org]
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
scripts/microbes/create_bacteria_table.py
+16 -18
View File
@@ -20,32 +20,31 @@ def __main__():
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
tmp_dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
orgs = organisms.keys()
for org in orgs:
if 'name' not in organisms[org]:
del organisms[org]
for org_name, org in list(organisms.items()):
if 'name' not in org:
del organisms[org_name]
orgs = organisms.keys()
orgs = list(organisms.keys())
# need to sort by name
swap_test = False
for i in range(0, len(orgs) - 1):
@@ -58,8 +57,7 @@ def __main__():
print("||'''Organism'''||'''Kingdom'''||'''Group'''||'''Links to UCSC Archaea Browser'''||")
for org in orgs:
org = organisms[org]
for org in organisms.values():
at_ucsc = False
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
scripts/microbes/create_nib_seq_loc_file.py
+11 -11
View File
@@ -29,25 +29,25 @@ def __main__():
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
tmp_dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
for org in organisms:
org = organisms[org]
scripts/microbes/get_builds_lengths.py
+11 -11
View File
@@ -20,25 +20,25 @@ def __main__():
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
tmp_dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
name = dict['genome project id']
if 'build' in dict.keys():
name = dict['build']
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[name][key] = dict[key]
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
for org in organisms:
org = organisms[org]
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
scripts/microbes/harvest_bacteria.py
+2 -2
View File
@@ -12,10 +12,10 @@ import time
from ftplib import FTP
import requests
from BeautifulSoup import BeautifulSoup
from six.moves.urllib.request import urlretrieve
from util import ( # noqa: I202
from BeautifulSoup import BeautifulSoup # noqa: I100, I202
from util import (
get_bed_from_genbank,
get_bed_from_GeneMark,
get_bed_from_GeneMarkHMM,
scripts/microbes/ncbi_to_ucsc.py
+14 -13
View File
@@ -7,10 +7,11 @@ from __future__ import print_function
import os
import sys
import urllib
from shutil import move
from xml.etree import ElementTree
from six.moves.urllib.request import urlopen
def __main__():
base_dir = os.path.join(os.getcwd(), "bacteria")
@@ -24,29 +25,29 @@ def __main__():
this_base_dir, sub_dirs, files = result
for file in files:
if file[-5:] == ".info":
dict = {}
tmp_dict = {}
info_file = open(os.path.join(this_base_dir, file), 'r')
info = info_file.readlines()
info_file.close()
for line in info:
fields = line.replace("\n", "").split("=")
dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in dict.keys():
if dict['genome project id'] not in organisms.keys():
organisms[dict['genome project id']] = {'chrs': {}, 'base_dir': this_base_dir}
for key in dict.keys():
organisms[dict['genome project id']][key] = dict[key]
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
if tmp_dict['genome project id'] not in organisms.keys():
organisms[tmp_dict['genome project id']] = {'chrs': {}, 'base_dir': this_base_dir}
for key in tmp_dict.keys():
organisms[tmp_dict['genome project id']][key] = tmp_dict[key]
else:
if dict['organism'] not in organisms.keys():
organisms[dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[dict['organism']]['chrs'][dict['chromosome']] = dict
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
# get UCSC data
URL = "http://archaea.ucsc.edu/cgi-bin/das/dsn"
try:
page = urllib.urlopen(URL)
page = urlopen(URL)
except Exception:
print("#Unable to open " + URL)
print("?\tunspecified (?)")
@@ -65,7 +66,7 @@ def __main__():
for dsn in tree:
build = dsn.find("SOURCE").attrib['id']
try:
org_page = urllib.urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build).read().replace("\n", "").split("<table border=2 cellspacing=2 cellpadding=2>")[1].split("</table>")[0].split("</tr>")
org_page = urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build).read().replace("\n", "").split("<table border=2 cellspacing=2 cellpadding=2>")[1].split("</table>")[0].split("</tr>")
except Exception:
print("NO CHROMS FOR", build)
continue
scripts/microbes/util.py
+5 -6
View File
@@ -137,7 +137,7 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
for block in orfs.split("\n\n"):
if block.startswith("List of Regions of interest"):
break
best_block = {'start': 0, 'end': 0, 'strand': '+', 'avg_prob': -sys.maxint, 'start_prob': -sys.maxint, 'name': 'DNE'}
best_block = {'start': 0, 'end': 0, 'strand': '+', 'avg_prob': -sys.maxsize, 'start_prob': -sys.maxsize, 'name': 'DNE'}
ctr += 1
ctr2 = 0
for line in block.split("\n"):
@@ -158,9 +158,8 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
except Exception:
start_prob = 0
name = "orf_" + str(ctr) + "_" + str(ctr2)
if avg_prob >= best_block['avg_prob']:
if start_prob > best_block['start_prob']:
best_block = {'start': start, 'end': end, 'strand': strand, 'avg_prob': avg_prob, 'start_prob': start_prob, 'name': name}
if avg_prob >= best_block['avg_prob'] and start_prob > best_block['start_prob']:
best_block = {'start': start, 'end': end, 'strand': strand, 'avg_prob': avg_prob, 'start_prob': start_prob, 'name': name}
regions.append(chr + "\t" + str(best_block['start']) + "\t" + str(best_block['end']) + "\t" + best_block['name'] + "\t" + str(int(best_block['avg_prob'] * 1000)) + "\t" + best_block['strand'])
return regions
@@ -198,8 +197,8 @@ def get_bed_from_GeneMarkHMM(geneMarkHMM_filename, chr):
# converts glimmer3 to bed, doing some linear scaling (probably not correct?) on scores
# returns an array of bed regions
def get_bed_from_glimmer3(glimmer3_filename, chr):
max_score = -sys.maxint
min_score = sys.maxint
max_score = -sys.maxsize
min_score = sys.maxsize
orfs = []
for line in open(glimmer3_filename).readlines():
if line.startswith(">"):
scripts/others/incorrect_gops_jobs.py
+2 -2
View File
@@ -5,12 +5,12 @@ wherein the second dataset doesn't have chr, start and end in standard columns 1
"""
from __future__ import print_function
import ConfigParser
import os
import sys
import tempfile
import sqlalchemy as sa
from six.moves.configparser import ConfigParser
import galaxy.app
import galaxy.model.mapping
@@ -34,7 +34,7 @@ class TestApplication(object):
def main():
ini_file = sys.argv[1]
conf_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
conf_parser = ConfigParser({'here': os.getcwd()})
conf_parser.read(ini_file)
configuration = {}
for key, value in conf_parser.items("app:main"):
scripts/others/incorrect_gops_join_jobs.py
+2 -2
View File
@@ -4,12 +4,12 @@ Fetch gops_join wherein the use specified minimum coverage is not 1.
"""
from __future__ import print_function
import ConfigParser
import os
import sys
import tempfile
import sqlalchemy as sa
from six.moves.configparser import ConfigParser
import galaxy.app
import galaxy.model.mapping
@@ -33,7 +33,7 @@ class TestApplication(object):
def main():
ini_file = sys.argv[1]
conf_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
conf_parser = ConfigParser({'here': os.getcwd()})
conf_parser.read(ini_file)
configuration = {}
for key, value in conf_parser.items("app:main"):
scripts/paster.py
+3 -2
View File
@@ -4,14 +4,15 @@ Bootstrap the Galaxy framework.
This should not be called directly! Use the run.sh script in Galaxy's
top level directly.
"""
from __future__ import absolute_import
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
from galaxy.util.pastescript import serve
from check_python import check_python # noqa: I100
from check_python import check_python # noqa: I100, I201
# ensure supported version
try:
scripts/set_dataset_sizes.py
+1 -1
View File
@@ -3,9 +3,9 @@ from __future__ import print_function
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
from six.moves.configparser import ConfigParser
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
scripts/set_user_disk_usage.py
+2 -1
View File
@@ -3,9 +3,10 @@ from __future__ import print_function
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
from six.moves.configparser import ConfigParser
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
import galaxy.config
scripts/summarize_timings.py
+1 -1
View File
@@ -15,7 +15,7 @@ TIMING_LINE_PATTERN = re.compile("\((\d+.\d+) ms\)")
def main(argv=None):
"""Entry point for script."""
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--file", default="paster.log")
arg_parser.add_argument("--file", default="galaxy.log")
arg_parser.add_argument("--print_lines", default=False, action="store_true")
arg_parser.add_argument("--pattern", default=None)
scripts/sync_reports_config.py
+2 -1
View File
@@ -1,6 +1,7 @@
from ConfigParser import ConfigParser
from sys import argv
from six.moves.configparser import ConfigParser
REPLACE_PROPERTIES = ["file_path", "database_connection", "new_file_path"]
MAIN_SECTION = "app:main"
scripts/tool_shed/api/get_filtered_repository_revisions.py
+2 -2
View File
@@ -74,9 +74,9 @@ def main(options):
if latest_revision_only:
latest_revision = repository_dict.get('latest_revision', hg_util.INITIAL_CHANGELOG_HASH)
if changeset_revision == latest_revision:
repository_dicts.append(dict(repository_dict.items() + baseline_repository_dict.items()))
repository_dicts.append(dict(list(repository_dict.items()) + list(baseline_repository_dict.items())))
else:
repository_dicts.append(dict(repository_dict.items() + baseline_repository_dict.items()))
repository_dicts.append(dict(list(repository_dict.items()) + list(baseline_repository_dict.items())))
print('\n\n', repository_dicts)
print('\nThe url:\n\n', api_url, '\n\nreturned ', len(repository_dicts), ' repository dictionaries...')
scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py
+5 -5
View File
@@ -1,15 +1,15 @@
#!/usr/bin/python
from __future__ import print_function
import ConfigParser
import optparse
import os
import sys
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
from six.moves.configparser import ConfigParser
from sqlalchemy.exc import OperationalError, ProgrammingError
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
import galaxy.webapps.tool_shed.model.mapping as tool_shed_model
from tool_shed.util import xml_util
@@ -50,7 +50,7 @@ def check_db(config_parser):
pass
if config_parser.has_option('app:main', 'hgweb_config_dir'):
hgweb_config_parser = ConfigParser.ConfigParser()
hgweb_config_parser = ConfigParser()
hgweb_dir = config_parser.get('app:main', 'hgweb_config_dir')
hgweb_config_file = os.path.join(hgweb_dir, 'hgweb.config')
if not os.path.exists(hgweb_config_file):
@@ -104,7 +104,7 @@ def get_local_tool_shed_url(config_parser):
def main(args):
config_parser = ConfigParser.ConfigParser()
config_parser = ConfigParser()
if os.path.exists(args.config):
config_parser.read(args.config)
scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py
+4 -4
View File
@@ -1,21 +1,21 @@
#!/usr/bin/env python
from __future__ import print_function
import ConfigParser
import logging
import optparse
import os
import re
import sys
from six.moves.configparser import ConfigParser
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__)))
import galaxy.webapps.tool_shed.config as tool_shed_config
from galaxy.web import security
from galaxy.webapps.tool_shed.model import mapping
from bootstrap_util import admin_user_info # noqa: I100
from bootstrap_util import admin_user_info # noqa: I100,I201
log = logging.getLogger(__name__)
@@ -124,7 +124,7 @@ if __name__ == "__main__":
parser.add_option('-c', dest='config', action='store', help='.ini file to retried toolshed configuration from')
(args, options) = parser.parse_args()
ini_file = args.config
config_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
config_parser = ConfigParser({'here': os.getcwd()})
print("Reading ini file: ", ini_file)
config_parser.read(ini_file)
config_dict = {}
scripts/tool_shed/check_download_urls.py
+6 -2
View File
@@ -5,10 +5,14 @@
from __future__ import print_function
import os
import urllib2
import xml.etree.ElementTree as ET
from optparse import OptionParser
from six.moves.urllib.request import (
Request,
urlopen
)
FILENAMES = ['tool_dependencies.xml']
ACTION_TYPES = ['download_by_url', 'download_file']
@@ -30,7 +34,7 @@ def main():
for element in root.findall(".//action[@type='%s']" % action_type):
url = element.text.strip()
try:
urllib2.urlopen(urllib2.Request(url))
urlopen(Request(url))
except Exception as e:
print("Bad URL '%s' in file '%s': %s" % (url, path, e))
except Exception as e:
scripts/tool_shed/migrate_tools_to_repositories.py
+3 -3
View File
@@ -20,7 +20,6 @@ To run this script, use "sh migrate_tools_to_repositories.sh" from this director
'''
from __future__ import print_function
import ConfigParser
import os
import shutil
import sys
@@ -29,6 +28,7 @@ import tempfile
from time import strftime
from mercurial import hg, ui
from six.moves import configparser
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
@@ -276,11 +276,11 @@ def main():
print("%s - Migrating current tool archives to new tool repositories" % now)
# tool_shed_wsgi.ini file
ini_file = sys.argv[1]
conf_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
conf_parser = configparser.ConfigParser({'here': os.getcwd()})
conf_parser.read(ini_file)
try:
db_conn_str = conf_parser.get("app:main", "database_connection")
except ConfigParser.NoOptionError:
except configparser.NoOptionError:
db_conn_str = conf_parser.get("app:main", "database_file")
print('DB Connection: ', db_conn_str)
# Instantiate app
scripts/transfer.py
+13 -10
View File
@@ -4,18 +4,15 @@ Downloads files to temp locations. This script is invoked by the Transfer
Manager (galaxy.jobs.transfer_manager) and should not normally be invoked by
hand.
"""
import ConfigParser
import json
import logging
import optparse
import os
import random
import SocketServer
import sys
import tempfile
import threading
import time
import urllib2
try:
import pexpect
@@ -23,6 +20,12 @@ except ImportError:
pexpect = None
from daemon import DaemonContext
from six.moves import (
configparser,
socketserver
)
from six.moves.urllib.error import URLError
from six.moves.urllib.request import urlopen
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import scoped_session, sessionmaker
@@ -82,7 +85,7 @@ class GalaxyApp(object):
model/database.
"""
def __init__(self, config_file):
self.config = ConfigParser.ConfigParser(dict(database_file='database/universe.sqlite',
self.config = configparser.ConfigParser(dict(database_file='database/universe.sqlite',
file_path='database/files',
transfer_worker_port_range='12275-12675',
transfer_worker_log=None))
@@ -95,7 +98,7 @@ class GalaxyApp(object):
default_dburl = 'sqlite:///%s?isolation_level=IMMEDIATE' % self.config.get('app:main', 'database_file')
try:
dburl = self.config.get('app:main', 'database_connection')
except ConfigParser.NoOptionError:
except configparser.NoOptionError:
dburl = default_dburl
engine = create_engine(dburl)
metadata = MetaData(engine)
@@ -107,7 +110,7 @@ class GalaxyApp(object):
return self.sa_session.query(self.model.TransferJob).get(int(id))
class ListenerServer(SocketServer.ThreadingTCPServer):
class ListenerServer(socketserver.ThreadingTCPServer):
"""
The listener will accept state requests and new transfers for as long as
the manager is running.
@@ -118,7 +121,7 @@ class ListenerServer(SocketServer.ThreadingTCPServer):
while True:
random_port = random.choice(port_range)
try:
SocketServer.ThreadingTCPServer.__init__(self, ('localhost', random_port), RequestHandlerClass)
super(ListenerServer, self).__init__(('localhost', random_port), RequestHandlerClass)
log.info('Listening on port %s' % random_port)
break
except Exception as e:
@@ -128,7 +131,7 @@ class ListenerServer(SocketServer.ThreadingTCPServer):
app.sa_session.flush()
class ListenerRequestHandler(SocketServer.BaseRequestHandler):
class ListenerRequestHandler(socketserver.BaseRequestHandler):
"""
Handle state or transfer requests received on the socket.
"""
@@ -217,8 +220,8 @@ def http_transfer(transfer_job):
url = transfer_job.params['url']
assert url.startswith('http://') or url.startswith('https://')
try:
f = urllib2.urlopen(url)
except urllib2.URLError as e:
f = urlopen(url)
except URLError as e:
yield dict(state=transfer_job.states.ERROR, info='Unable to open URL: %s' % str(e))
return
size = f.info().getheader('Content-Length')
scripts/update_shed_config_path.py
+2 -2
View File
@@ -1,10 +1,10 @@
from __future__ import print_function
import argparse
import ConfigParser
import os
import sys
from six.moves.configparser import SafeConfigParser
from sqlalchemy import create_engine, MetaData
from sqlalchemy.orm import scoped_session, sessionmaker
@@ -27,7 +27,7 @@ def main(opts, session, model):
def create_database(config_file):
parser = ConfigParser.SafeConfigParser()
parser = SafeConfigParser()
parser.read(config_file)
# Determine which database connection to use.
database_connection = parser.get('app:main', 'install_database_connection')