opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge changes from next-stable to stable.

Browse Source
This commit is contained in:
Nate Coraor
2013-06-03 15:44:31 -04:00
parent 8a15acb8a8 c04ccc33d4
commit 0bc8e9fb29
619 changed files with 35062 additions and 27482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.hgignore
+8 -1
View File
@@ -20,6 +20,7 @@ database/job_working_directory
database/pbs
database/tmp
database/*.sqlite
database/openid_consumer_cache
# Python bytecode
*.pyc
@@ -35,6 +36,11 @@ tool_shed_webapp.log
tool_shed_webapp.pid
hgweb.config*
# Reports Runtime Files
reports_webapp.lock
reports_webapp.log
reports_webapp.pid
# Config files
universe_wsgi.ini
reports_wsgi.ini
@@ -54,7 +60,7 @@ shed_tool_data_table_conf.xml
job_conf.xml
data_manager_conf.xml
shed_data_manager_conf.xml
visualizations_conf.xml
static/welcome.html.*
static/welcome.html
@@ -75,6 +81,7 @@ tool-data/genome/*
# Test output
run_functional_tests.html
test/tool_shed/tmp/*
# Project files
*.kpf
buildbot_setup.sh
+39 -16
View File
@@ -93,26 +93,49 @@ database/pbs
JARS="/galaxy/software/jars"
for link in $LINKS; do
echo "Linking $link"
rm -f tool-data/`basename $link`
ln -sf $link tool-data
done
if [ -d "$HYPHY" ]; then
echo "Linking $HYPHY"
rm -f tool-data/HYPHY
ln -sf $HYPHY tool-data/HYPHY
if [ ! $1 ]; then
type="standard"
elif [ $1 == "-ec2" ]; then
type="external-ec2"
else
type="unknown"
fi
if [ -d "$JARS" ]; then
echo "Linking $JARS"
rm -f tool-data/shared/jars
ln -sf $JARS tool-data/shared/jars
fi
case $type in
external*)
echo "Running standalone buildbot setup..."
for sample in tool-data/*.sample; do
basename=${sample%.sample}
if [ ! -f $basename ]; then
echo "Copying $sample to $basename"
cp "$sample" "$basename"
fi
done
;;
*)
echo "Running standard buildbot setup..."
for link in $LINKS; do
echo "Linking $link"
rm -f tool-data/`basename $link`
ln -sf $link tool-data
done
if [ -d "$HYPHY" ]; then
echo "Linking $HYPHY"
rm -f tool-data/HYPHY
ln -sf $HYPHY tool-data/HYPHY
fi
if [ -d "$JARS" ]; then
echo "Linking $JARS"
rm -f tool-data/shared/jars
ln -sf $JARS tool-data/shared/jars
fi
;;
esac
for sample in $SAMPLES; do
file=`echo $sample | sed -e 's/\.sample$//'`
file=${sample%.sample}
echo "Copying $sample to $file"
cp $sample $file
done
datatypes_conf.xml.sample
+15 -17
View File
@@ -3,16 +3,13 @@
<registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications">
<datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true"/>
<datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false"/>
<datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" />
<datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" />
<datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/>
<datatype extension="fli" type="galaxy.datatypes.tabular:FeatureLocationIndex" display_in_upload="false"/>
<datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true">
<converter file="bam_to_bai.xml" target_datatype="bai"/>
<converter file="bam_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<!--
Caution: (a) this converter requires bedtools to be installed and (b) it is very memory intensive and
is not recommended for most laptops/desktops.
<converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
-->
<converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
<display file="ucsc/bam.xml" />
<display file="ensembl/ensembl_bam.xml" />
<display file="igv/bam.xml" />
@@ -20,10 +17,9 @@
</datatype>
<datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true">
<converter file="bed_to_gff_converter.xml" target_datatype="gff"/>
<converter file="interval_to_coverage.xml" target_datatype="coverage"/>
<converter file="bed_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="bed_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
<converter file="bed_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="bed_to_fli_converter.xml" target_datatype="fli"/>
<!-- <display file="ucsc/interval_as_bed.xml" /> -->
<display file="igb/bed.xml" />
@@ -49,7 +45,7 @@
<datatype extension="chrint" type="galaxy.datatypes.interval:ChromatinInteractions" display_in_upload="True">
<converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
<converter file="interval_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
<!-- MSI added Datatypes -->
<datatype extension="csv" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="true" /> <!-- FIXME: csv is 'tabular'ized data, but not 'tab-delimited'; the class used here is intended for 'tab-delimited' -->
@@ -91,7 +87,7 @@
<datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true">
<converter file="gff_to_bed_converter.xml" target_datatype="bed"/>
<converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/>
<converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="gff_to_fli_converter.xml" target_datatype="fli"/>
<display file="ensembl/ensembl_gff.xml" inherit="True"/>
<!-- <display file="gbrowse/gbrowse_gff.xml" inherit="True" /> -->
@@ -101,7 +97,7 @@
<datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/>
<datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true">
<converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/>
<converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
<datatype extension="toolshed.gz" type="galaxy.datatypes.binary:Binary" mimetype="multipart/x-gzip" subclass="True" />
<datatype extension="h5" type="galaxy.datatypes.binary:Binary" mimetype="application/octet-stream" subclass="True" />
@@ -113,7 +109,7 @@
<converter file="interval_to_bed12_converter.xml" target_datatype="bed12"/>
<converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
<converter file="interval_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="interval_to_bigwig_converter.xml" target_datatype="bigwig"/>
<!-- <display file="ucsc/interval_as_bed.xml" inherit="True" /> -->
<display file="ensembl/ensembl_interval_as_bed.xml" inherit="True"/>
<display file="gbrowse/gbrowse_interval_as_bed.xml" inherit="True"/>
@@ -154,10 +150,13 @@
<datatype extension="encodepeak" type="galaxy.datatypes.interval:ENCODEPeak" display_in_upload="True">
<converter file="encodepeak_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
<converter file="encodepeak_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="encodepeak_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
<datatype extension="pdf" type="galaxy.datatypes.images:Pdf" mimetype="application/pdf"/>
<datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true" />
<datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true">
<converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
</datatype>
<datatype extension="png" type="galaxy.datatypes.images:Png" mimetype="image/png"/>
<datatype extension="qual" type="galaxy.datatypes.qualityscore:QualityScore" />
<datatype extension="qualsolexa" type="galaxy.datatypes.qualityscore:QualityScoreSolexa" display_in_upload="true"/>
@@ -167,7 +166,7 @@
<datatype extension="Roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/>
<datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true">
<converter file="sam_to_bam.xml" target_datatype="bam"/>
<converter file="sam_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
<datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true"/>
<datatype extension="Sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/>
@@ -185,7 +184,7 @@
<converter file="vcf_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="vcf_to_vcf_bgzip_converter.xml" target_datatype="vcf_bgzip"/>
<converter file="vcf_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
<converter file="vcf_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
<display file="ucsc/vcf.xml" />
<display file="igv/vcf.xml" />
<display file="rviewer/vcf.xml" inherit="True"/>
@@ -198,7 +197,6 @@
<!-- <display file="gbrowse/gbrowse_wig.xml" /> -->
<display file="igb/wig.xml" />
</datatype>
<datatype extension="summary_tree" type="galaxy.datatypes.binary:Binary" subclass="True" />
<datatype extension="interval_index" type="galaxy.datatypes.binary:Binary" subclass="True" />
<datatype extension="tabix" type="galaxy.datatypes.binary:Binary" subclass="True" />
<datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="True" />
doc/source/lib/galaxy.webapps.tool_shed.api.rst
+19
View File
@@ -0,0 +1,19 @@
api Package
===========
:mod:`repositories` Module
--------------------------
.. automodule:: galaxy.webapps.tool_shed.api.repositories
:members:
:undoc-members:
:show-inheritance:
:mod:`repository_revisions` Module
----------------------------------
.. automodule:: galaxy.webapps.tool_shed.api.repository_revisions
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/galaxy.webapps.tool_shed.controllers.rst
+59
View File
@@ -0,0 +1,59 @@
controllers Package
===================
:mod:`controllers` Package
--------------------------
.. automodule:: galaxy.webapps.tool_shed.controllers
:members:
:undoc-members:
:show-inheritance:
:mod:`admin` Module
-------------------
.. automodule:: galaxy.webapps.tool_shed.controllers.admin
:members:
:undoc-members:
:show-inheritance:
:mod:`hg` Module
----------------
.. automodule:: galaxy.webapps.tool_shed.controllers.hg
:members:
:undoc-members:
:show-inheritance:
:mod:`repository` Module
------------------------
.. automodule:: galaxy.webapps.tool_shed.controllers.repository
:members:
:undoc-members:
:show-inheritance:
:mod:`repository_review` Module
-------------------------------
.. automodule:: galaxy.webapps.tool_shed.controllers.repository_review
:members:
:undoc-members:
:show-inheritance:
:mod:`upload` Module
--------------------
.. automodule:: galaxy.webapps.tool_shed.controllers.upload
:members:
:undoc-members:
:show-inheritance:
:mod:`user` Module
------------------
.. automodule:: galaxy.webapps.tool_shed.controllers.user
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/galaxy.webapps.tool_shed.framework.middleware.rst
+27
View File
@@ -0,0 +1,27 @@
middleware Package
==================
:mod:`middleware` Package
-------------------------
.. automodule:: galaxy.webapps.tool_shed.framework.middleware
:members:
:undoc-members:
:show-inheritance:
:mod:`hg` Module
----------------
.. automodule:: galaxy.webapps.tool_shed.framework.middleware.hg
:members:
:undoc-members:
:show-inheritance:
:mod:`remoteuser` Module
------------------------
.. automodule:: galaxy.webapps.tool_shed.framework.middleware.remoteuser
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/galaxy.webapps.tool_shed.framework.rst
+18
View File
@@ -0,0 +1,18 @@
framework Package
=================
:mod:`framework` Package
------------------------
.. automodule:: galaxy.webapps.tool_shed.framework
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
galaxy.webapps.tool_shed.framework.middleware
doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst
+11
View File
@@ -0,0 +1,11 @@
migrate Package
===============
:mod:`check` Module
-------------------
.. automodule:: galaxy.webapps.tool_shed.model.migrate.check
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/galaxy.webapps.tool_shed.model.rst
+26
View File
@@ -0,0 +1,26 @@
model Package
=============
:mod:`model` Package
--------------------
.. automodule:: galaxy.webapps.tool_shed.model
:members:
:undoc-members:
:show-inheritance:
:mod:`mapping` Module
---------------------
.. automodule:: galaxy.webapps.tool_shed.model.mapping
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
galaxy.webapps.tool_shed.model.migrate
doc/source/lib/galaxy.webapps.tool_shed.rst
+47
View File
@@ -0,0 +1,47 @@
tool_shed Package
=================
:mod:`tool_shed` Package
------------------------
.. automodule:: galaxy.webapps.tool_shed
:members:
:undoc-members:
:show-inheritance:
:mod:`app` Module
-----------------
.. automodule:: galaxy.webapps.tool_shed.app
:members:
:undoc-members:
:show-inheritance:
:mod:`buildapp` Module
----------------------
.. automodule:: galaxy.webapps.tool_shed.buildapp
:members:
:undoc-members:
:show-inheritance:
:mod:`config` Module
--------------------
.. automodule:: galaxy.webapps.tool_shed.config
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
galaxy.webapps.tool_shed.api
galaxy.webapps.tool_shed.controllers
galaxy.webapps.tool_shed.framework
galaxy.webapps.tool_shed.model
galaxy.webapps.tool_shed.security
galaxy.webapps.tool_shed.util
doc/source/lib/galaxy.webapps.tool_shed.security.rst
+11
View File
@@ -0,0 +1,11 @@
security Package
================
:mod:`security` Package
-----------------------
.. automodule:: galaxy.webapps.tool_shed.security
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/galaxy.webapps.tool_shed.util.rst
+35
View File
@@ -0,0 +1,35 @@
util Package
============
:mod:`common_util` Module
-------------------------
.. automodule:: galaxy.webapps.tool_shed.util.common_util
:members:
:undoc-members:
:show-inheritance:
:mod:`container_util` Module
----------------------------
.. automodule:: galaxy.webapps.tool_shed.util.container_util
:members:
:undoc-members:
:show-inheritance:
:mod:`hgweb_config` Module
--------------------------
.. automodule:: galaxy.webapps.tool_shed.util.hgweb_config
:members:
:undoc-members:
:show-inheritance:
:mod:`shed_statistics` Module
-----------------------------
.. automodule:: galaxy.webapps.tool_shed.util.shed_statistics
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/tool_shed.galaxy_install.grids.rst
+11
View File
@@ -0,0 +1,11 @@
grids Package
=============
:mod:`admin_toolshed_grids` Module
----------------------------------
.. automodule:: tool_shed.galaxy_install.grids.admin_toolshed_grids
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/tool_shed.galaxy_install.migrate.rst
+19
View File
@@ -0,0 +1,19 @@
migrate Package
===============
:mod:`check` Module
-------------------
.. automodule:: tool_shed.galaxy_install.migrate.check
:members:
:undoc-members:
:show-inheritance:
:mod:`common` Module
--------------------
.. automodule:: tool_shed.galaxy_install.migrate.common
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/tool_shed.galaxy_install.rst
+44
View File
@@ -0,0 +1,44 @@
galaxy_install Package
======================
:mod:`galaxy_install` Package
-----------------------------
.. automodule:: tool_shed.galaxy_install
:members:
:undoc-members:
:show-inheritance:
:mod:`install_manager` Module
-----------------------------
.. automodule:: tool_shed.galaxy_install.install_manager
:members:
:undoc-members:
:show-inheritance:
:mod:`repository_util` Module
-----------------------------
.. automodule:: tool_shed.galaxy_install.repository_util
:members:
:undoc-members:
:show-inheritance:
:mod:`update_manager` Module
----------------------------
.. automodule:: tool_shed.galaxy_install.update_manager
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
tool_shed.galaxy_install.grids
tool_shed.galaxy_install.migrate
tool_shed.galaxy_install.tool_dependencies
doc/source/lib/tool_shed.galaxy_install.tool_dependencies.rst
+27
View File
@@ -0,0 +1,27 @@
tool_dependencies Package
=========================
:mod:`common_util` Module
-------------------------
.. automodule:: tool_shed.galaxy_install.tool_dependencies.common_util
:members:
:undoc-members:
:show-inheritance:
:mod:`fabric_util` Module
-------------------------
.. automodule:: tool_shed.galaxy_install.tool_dependencies.fabric_util
:members:
:undoc-members:
:show-inheritance:
:mod:`install_util` Module
--------------------------
.. automodule:: tool_shed.galaxy_install.tool_dependencies.install_util
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/tool_shed.grids.rst
+35
View File
@@ -0,0 +1,35 @@
grids Package
=============
:mod:`admin_grids` Module
-------------------------
.. automodule:: tool_shed.grids.admin_grids
:members:
:undoc-members:
:show-inheritance:
:mod:`repository_grids` Module
------------------------------
.. automodule:: tool_shed.grids.repository_grids
:members:
:undoc-members:
:show-inheritance:
:mod:`repository_review_grids` Module
-------------------------------------
.. automodule:: tool_shed.grids.repository_review_grids
:members:
:undoc-members:
:show-inheritance:
:mod:`util` Module
------------------
.. automodule:: tool_shed.grids.util
:members:
:undoc-members:
:show-inheritance:
doc/source/lib/tool_shed.rst
+20
View File
@@ -0,0 +1,20 @@
tool_shed Package
=================
:mod:`tool_shed_registry` Module
--------------------------------
.. automodule:: tool_shed.tool_shed_registry
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
tool_shed.galaxy_install
tool_shed.grids
tool_shed.util
doc/source/lib/tool_shed.util.rst
+107
View File
@@ -0,0 +1,107 @@
util Package
============
:mod:`common_install_util` Module
---------------------------------
.. automodule:: tool_shed.util.common_install_util
:members:
:undoc-members:
:show-inheritance:
:mod:`common_util` Module
-------------------------
.. automodule:: tool_shed.util.common_util
:members:
:undoc-members:
:show-inheritance:
:mod:`data_manager_util` Module
-------------------------------
.. automodule:: tool_shed.util.data_manager_util
:members:
:undoc-members:
:show-inheritance:
:mod:`datatype_util` Module
---------------------------
.. automodule:: tool_shed.util.datatype_util
:members:
:undoc-members:
:show-inheritance:
:mod:`encoding_util` Module
---------------------------
.. automodule:: tool_shed.util.encoding_util
:members:
:undoc-members:
:show-inheritance:
:mod:`metadata_util` Module
---------------------------
.. automodule:: tool_shed.util.metadata_util
:members:
:undoc-members:
:show-inheritance:
:mod:`readme_util` Module
-------------------------
.. automodule:: tool_shed.util.readme_util
:members:
:undoc-members:
:show-inheritance:
:mod:`repository_dependency_util` Module
----------------------------------------
.. automodule:: tool_shed.util.repository_dependency_util
:members:
:undoc-members:
:show-inheritance:
:mod:`review_util` Module
-------------------------
.. automodule:: tool_shed.util.review_util
:members:
:undoc-members:
:show-inheritance:
:mod:`shed_util_common` Module
------------------------------
.. automodule:: tool_shed.util.shed_util_common
:members:
:undoc-members:
:show-inheritance:
:mod:`tool_dependency_util` Module
----------------------------------
.. automodule:: tool_shed.util.tool_dependency_util
:members:
:undoc-members:
:show-inheritance:
:mod:`tool_util` Module
-----------------------
.. automodule:: tool_shed.util.tool_util
:members:
:undoc-members:
:show-inheritance:
:mod:`workflow_util` Module
---------------------------
.. automodule:: tool_shed.util.workflow_util
:members:
:undoc-members:
:show-inheritance:
eggs.ini
+4 -5
View File
@@ -14,7 +14,6 @@ no_auto = pbs_python DRMAA_python
[eggs:platform]
bx_python = 0.7.1
Cheetah = 2.2.2
ctypes = 1.0.2
DRMAA_python = 0.2
MarkupSafe = 0.12
mercurial = 2.2.3
@@ -29,6 +28,7 @@ python_lzo = 1.08_2.03_static
simplejson = 2.1.1
threadframe = 0.2
guppy = 0.1.8
SQLAlchemy = 0.7.9
; msgpack_python = 0.2.4
[eggs:noplatform]
@@ -46,17 +46,17 @@ Mako = 0.4.1
nose = 0.11.1
NoseHTML = 0.4.1
NoseTestDiff = 0.1
Parsley = 1.1
Paste = 1.7.5.1
PasteDeploy = 1.5.0
pexpect = 2.4
python_openid = 2.2.5
python_daemon = 1.5.5
Routes = 1.12.3
SQLAlchemy = 0.5.6
sqlalchemy_migrate = 0.5.4
sqlalchemy_migrate = 0.7.2
ssh = 1.7.14
SVGFig = 1.1.6
Tempita = 0.1
Tempita = 0.5.1
twill = 0.9
WebError = 0.8a
WebHelpers = 0.2
@@ -75,7 +75,6 @@ pysqlite = _3.6.17_static
MySQL_python = _5.1.41_static
bx_python = _7b95ff194725
GeneTrack = _dev_48da9e998f0caf01c5be731e926f4b0481f658f0
SQLAlchemy = _dev_r6498
pysam = _kanwei_b10f6e722e9a
; dependency source urls, necessary for scrambling. for an explanation, see
job_conf.xml.sample_advanced
+1 -2
View File
@@ -7,7 +7,7 @@
<plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/>
<plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/>
<plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/>
<plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr.LwrJobRunner" /> <!-- https://lwr.readthedocs.org -->
<plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner" /> <!-- https://lwr.readthedocs.org -->
<plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" />
<plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" />
</plugins>
@@ -40,7 +40,6 @@
</destination>
<destination id="dynamic" runner="dynamic">
<!-- A destination that represents a method in the dynamic runner. -->
<param id="type">python</param>
<param id="function">foo</param>
</destination>
<destination id="secure_lwr" runner="lwr">
lib/fpconst.py
-163
View File
@@ -1,163 +0,0 @@
"""Utilities for handling IEEE 754 floating point special values
This python module implements constants and functions for working with
IEEE754 double-precision special values. It provides constants for
Not-a-Number (NaN), Positive Infinity (PosInf), and Negative Infinity
(NegInf), as well as functions to test for these values.
The code is implemented in pure python by taking advantage of the
'struct' standard module. Care has been taken to generate proper
results on both big-endian and little-endian machines. Some efficiency
could be gained by translating the core routines into C.
See <http://babbage.cs.qc.edu/courses/cs341/IEEE-754references.html>
for reference material on the IEEE 754 floating point standard.
Further information on this package is available at
<http://www.analytics.washington.edu/statcomp/projects/rzope/fpconst/>.
Author: Gregory R. Warnes <gregory_r_warnes@groton.pfizer.com>
Date:: 2003-04-08
Copyright: (c) 2003, Pfizer, Inc.
"""
__version__ = "0.7.0"
ident = "$Id: fpconst.py,v 1.12 2004/05/22 04:38:17 warnes Exp $"
import struct, operator
# check endianess
_big_endian = struct.pack('i',1)[0] != '\x01'
# and define appropriate constants
if(_big_endian):
NaN = struct.unpack('d', '\x7F\xF8\x00\x00\x00\x00\x00\x00')[0]
PosInf = struct.unpack('d', '\x7F\xF0\x00\x00\x00\x00\x00\x00')[0]
NegInf = -PosInf
else:
NaN = struct.unpack('d', '\x00\x00\x00\x00\x00\x00\xf8\xff')[0]
PosInf = struct.unpack('d', '\x00\x00\x00\x00\x00\x00\xf0\x7f')[0]
NegInf = -PosInf
def _double_as_bytes(dval):
"Use struct.unpack to decode a double precision float into eight bytes"
tmp = list(struct.unpack('8B',struct.pack('d', dval)))
if not _big_endian:
tmp.reverse()
return tmp
##
## Functions to extract components of the IEEE 754 floating point format
##
def _sign(dval):
"Extract the sign bit from a double-precision floating point value"
bb = _double_as_bytes(dval)
return bb[0] >> 7 & 0x01
def _exponent(dval):
"""Extract the exponentent bits from a double-precision floating
point value.
Note that for normalized values, the exponent bits have an offset
of 1023. As a consequence, the actual exponentent is obtained
by subtracting 1023 from the value returned by this function
"""
bb = _double_as_bytes(dval)
return (bb[0] << 4 | bb[1] >> 4) & 0x7ff
def _mantissa(dval):
"""Extract the _mantissa bits from a double-precision floating
point value."""
bb = _double_as_bytes(dval)
mantissa = bb[1] & 0x0f << 48
mantissa += bb[2] << 40
mantissa += bb[3] << 32
mantissa += bb[4]
return mantissa
def _zero_mantissa(dval):
"""Determine whether the mantissa bits of the given double are all
zero."""
bb = _double_as_bytes(dval)
return ((bb[1] & 0x0f) | reduce(operator.or_, bb[2:])) == 0
##
## Functions to test for IEEE 754 special values
##
def isNaN(value):
"Determine if the argument is a IEEE 754 NaN (Not a Number) value."
return (_exponent(value)==0x7ff and not _zero_mantissa(value))
def isInf(value):
"""Determine if the argument is an infinite IEEE 754 value (positive
or negative inifinity)"""
return (_exponent(value)==0x7ff and _zero_mantissa(value))
def isFinite(value):
"""Determine if the argument is an finite IEEE 754 value (i.e., is
not NaN, positive or negative inifinity)"""
return (_exponent(value)!=0x7ff)
def isPosInf(value):
"Determine if the argument is a IEEE 754 positive infinity value"
return (_sign(value)==0 and _exponent(value)==0x7ff and \
_zero_mantissa(value))
def isNegInf(value):
"Determine if the argument is a IEEE 754 negative infinity value"
return (_sign(value)==1 and _exponent(value)==0x7ff and \
_zero_mantissa(value))
##
## Functions to test public functions.
##
def test_isNaN():
assert( not isNaN(PosInf) )
assert( not isNaN(NegInf) )
assert( isNaN(NaN ) )
assert( not isNaN( 1.0) )
assert( not isNaN( -1.0) )
def test_isInf():
assert( isInf(PosInf) )
assert( isInf(NegInf) )
assert( not isInf(NaN ) )
assert( not isInf( 1.0) )
assert( not isInf( -1.0) )
def test_isFinite():
assert( not isFinite(PosInf) )
assert( not isFinite(NegInf) )
assert( not isFinite(NaN ) )
assert( isFinite( 1.0) )
assert( isFinite( -1.0) )
def test_isPosInf():
assert( isPosInf(PosInf) )
assert( not isPosInf(NegInf) )
assert( not isPosInf(NaN ) )
assert( not isPosInf( 1.0) )
assert( not isPosInf( -1.0) )
def test_isNegInf():
assert( not isNegInf(PosInf) )
assert( isNegInf(NegInf) )
assert( not isNegInf(NaN ) )
assert( not isNegInf( 1.0) )
assert( not isNegInf( -1.0) )
# overall test
def test():
test_isNaN()
test_isInf()
test_isFinite()
test_isPosInf()
test_isNegInf()
if __name__ == "__main__":
test()
lib/galaxy/__init__.py
+9 -4
View File
@@ -95,10 +95,15 @@ except:
pkg_resources.Distribution._insert_on = pkg_resources.Distribution.insert_on
pkg_resources.Distribution.insert_on = _insert_on
# patch to add the NullHandler class to logging
if sys.version_info[:2] < ( 2, 7 ):
import logging
# compat: BadZipFile introduced in Python 2.7
import zipfile
if not hasattr( zipfile, 'BadZipFile' ):
zipfile.BadZipFile = zipfile.error
# compat: patch to add the NullHandler class to logging
import logging
if not hasattr( logging, 'NullHandler' ):
class NullHandler( logging.Handler ):
def emit( self, record ):
pass
logging.NullHandler = NullHandler
logging.NullHandler = NullHandler
lib/galaxy/app.py
+6 -1
View File
@@ -15,6 +15,7 @@ import galaxy.quota
from galaxy.tags.tag_handler import GalaxyTagHandler
from galaxy.visualization.genomes import Genomes
from galaxy.visualization.data_providers.registry import DataProviderRegistry
from galaxy.visualization.registry import VisualizationsRegistry
from galaxy.tools.imp_exp import load_history_imp_exp_tools
from galaxy.tools.genome_index import load_genome_index_tools
from galaxy.sample_tracking import external_service_types
@@ -61,7 +62,8 @@ class UniverseApplication( object ):
self.config.database_engine_options,
database_query_profiling_proxy = self.config.database_query_profiling_proxy,
object_store = self.object_store,
trace_logger=self.trace_logger )
trace_logger=self.trace_logger,
use_pbkdf2=self.config.get_bool( 'use_pbkdf2', True ) )
# Manage installed tool shed repositories.
self.installed_repository_manager = tool_shed.galaxy_install.InstalledRepositoryManager( self )
# Create an empty datatypes registry.
@@ -120,6 +122,9 @@ class UniverseApplication( object ):
load_history_imp_exp_tools( self.toolbox )
# Load genome indexer tool.
load_genome_index_tools( self.toolbox )
# visualizations registry: associates resources with visualizations, controls how to render
self.visualizations_registry = ( VisualizationsRegistry( self.config.root, self.config.visualizations_conf_path )
if self.config.visualizations_conf_path else None )
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions )
lib/galaxy/config.py
+7 -1
View File
@@ -86,7 +86,6 @@ class Configuration( object ):
self.galaxy_data_manager_data_path = kwargs.get( 'galaxy_data_manager_data_path', self.tool_data_path )
self.tool_secret = kwargs.get( "tool_secret", "" )
self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) )
self.retry_metadata_internally = string_as_bool( kwargs.get( "retry_metadata_internally", "True" ) )
self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) )
self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None )
@@ -155,6 +154,10 @@ class Configuration( object ):
self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",")
self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225" ).lower().split(",")
self.brand = kwargs.get( 'brand', None )
# Configuration for the message box directly below the masthead.
self.message_box_visible = kwargs.get( 'message_box_visible', False )
self.message_box_content = kwargs.get( 'message_box_content', None )
self.message_box_class = kwargs.get( 'message_box_class', 'info' )
self.support_url = kwargs.get( 'support_url', 'http://wiki.g2.bx.psu.edu/Support' )
self.wiki_url = kwargs.get( 'wiki_url', 'http://g2.trac.bx.psu.edu/' )
self.blog_url = kwargs.get( 'blog_url', None )
@@ -166,6 +169,7 @@ class Configuration( object ):
self.enable_whoosh_library_search = string_as_bool( kwargs.get( 'enable_whoosh_library_search', False ) )
self.whoosh_index_dir = resolve_path( kwargs.get( "whoosh_index_dir", "database/whoosh_indexes" ), self.root )
self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None )
self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' ) # attribute on user - email, username, id, etc...
self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
@@ -271,6 +275,8 @@ class Configuration( object ):
self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) )
self.fluent_host = kwargs.get( 'fluent_host', 'localhost' )
self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) )
# visualizations registry config path
self.visualizations_conf_path = kwargs.get( 'visualizations_conf_path', None )
@property
def sentry_dsn_public( self ):
lib/galaxy/datatypes/assembly.py
+7 -8
View File
@@ -5,14 +5,14 @@ for velvet assembler tool in galaxy
"""
import data
import logging
import os
import re
import sys
from galaxy.datatypes import sequence
import logging, os, sys, time, tempfile, shutil, string, glob, re
import galaxy.model
from galaxy.datatypes import metadata
from galaxy.datatypes.metadata import MetadataElement
from galaxy import util
from galaxy.datatypes.images import Html
from sniff import *
from galaxy.datatypes.metadata import MetadataElement
log = logging.getLogger(__name__)
@@ -174,7 +174,6 @@ class Velvet( Html ):
gen_msg = ''
try:
efp = dataset.extra_files_path
flist = os.listdir(efp)
log_path = os.path.join(efp,'Log')
f = open(log_path,'r')
log_content = f.read(1000)
@@ -223,5 +222,5 @@ class Velvet( Html ):
self.regenerate_primary_file(dataset)
if __name__ == '__main__':
import doctest, sys
import doctest
doctest.testmod(sys.modules[__name__])
lib/galaxy/datatypes/binary.py
+34 -17
View File
@@ -2,18 +2,26 @@
Binary classes
"""
import data, logging, binascii
import binascii
import data
import gzip
import logging
import os
import shutil
import struct
import subprocess
import tempfile
import zipfile
from urllib import urlencode, quote_plus
from galaxy import eggs
eggs.require( "bx-python" )
from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes import metadata
from galaxy.datatypes.sniff import *
from galaxy import eggs
import pkg_resources
pkg_resources.require( "bx-python" )
from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE
from urllib import urlencode, quote_plus
import zipfile, gzip
import os, subprocess, tempfile
import struct
log = logging.getLogger(__name__)
@@ -85,9 +93,18 @@ class Ab1( Binary ):
Binary.register_unsniffable_binary_ext("ab1")
class GenericAsn1Binary( Binary ):
"""Class for generic ASN.1 binary format"""
file_ext = "asn1-binary"
Binary.register_unsniffable_binary_ext("asn1-binary")
class Bam( Binary ):
"""Class describing a BAM binary file"""
file_ext = "bam"
track_type = "ReadTrack"
data_sources = { "data": "bai", "index": "bigwig" }
MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True )
def _get_samtools_version( self ):
@@ -238,9 +255,7 @@ class Bam( Binary ):
return dataset.peek
except:
return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) )
def get_track_type( self ):
return "ReadTrack", { "data": "bai", "index": [ "bigwig", "summary_tree" ] }
Binary.register_sniffable_binary_format("bam", "bam", Bam)
class H5( Binary ):
@@ -318,6 +333,9 @@ class BigWig(Binary):
The supplemental info in the paper has the binary details:
http://bioinformatics.oxfordjournals.org/cgi/content/abstract/btq351v1
"""
track_type = "LineTrack"
data_sources = { "data_standalone": "bigwig" }
def __init__( self, **kwd ):
Binary.__init__( self, **kwd )
self._magic = 0x888FFC26
@@ -342,19 +360,18 @@ class BigWig(Binary):
return dataset.peek
except:
return "Binary UCSC %s file (%s)" % ( self._name, data.nice_size( dataset.get_size() ) )
def get_track_type( self ):
return "LineTrack", {"data_standalone": "bigwig"}
Binary.register_sniffable_binary_format("bigwig", "bigwig", BigWig)
class BigBed(BigWig):
"""BigBed support from UCSC."""
data_sources = { "data_standalone": "bigbed" }
def __init__( self, **kwd ):
Binary.__init__( self, **kwd )
self._magic = 0x8789F2EB
self._name = "BigBed"
def get_track_type( self ):
return "LineTrack", {"data_standalone": "bigbed"}
Binary.register_sniffable_binary_format("bigbed", "bigbed", BigBed)
lib/galaxy/datatypes/chrominfo.py
-4
View File
@@ -1,7 +1,3 @@
import data
from galaxy import util
from galaxy.datatypes.sniff import *
from galaxy.web import url_for
from tabular import Tabular
from galaxy.datatypes import metadata
from galaxy.datatypes.metadata import MetadataElement
lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+8 -1
View File
@@ -1,7 +1,14 @@
<tool id="CONVERTER_bam_to_bigwig_0" name="Convert BAM to BigWig" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
bedtools genomecov -bg -split -ibam $input -g $chromInfo | wigToBigWig stdin $chromInfo $output
bedtools genomecov -bg -split -ibam $input -g $chromInfo
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo $output
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output
</command>
<inputs>
<param format="bam" name="input" type="data" label="Choose BAM file"/>
lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml
-14
View File
@@ -1,14 +0,0 @@
<tool id="CONVERTER_bam_to_summary_tree_0" name="Convert BAM to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">
sam_or_bam_to_summary_tree_converter.py --bam $input1 $input1.metadata.bam_index $output1
</command>
<inputs>
<param format="bam" name="input1" type="data" label="Choose BAM file"/>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
+25
View File
@@ -0,0 +1,25 @@
<tool id="CONVERTER_bed_gff_or_vcf_to_bigwig_0" name="Convert BED, GFF, or VCF to BigWig" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
## Remove comments and sort by chromosome.
grep -v '^#' $input | sort -k1,1 |
## Generate coverage bedgraph.
bedtools genomecov -bg -split -i stdin -g $chromInfo
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo $output
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output
</command>
<inputs>
<param format="bed,gff,vcf" name="input" type="data" label="Choose input file"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_to_summary_tree_converter.xml
-14
View File
@@ -1,14 +0,0 @@
<tool id="CONVERTER_bed_to_summary_tree_0" name="Convert BED to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_summary_tree_converter.py $input1 $output1</command>
<inputs>
<page>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</page>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/encodepeak_to_summary_tree_converter.xml
-20
View File
@@ -1,20 +0,0 @@
<tool id="CONVERTER_encodepeak_to_summary_tree_0" name="Convert ENCODEPeak to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_summary_tree_converter.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
$input1 $output1
</command>
<inputs>
<page>
<param format="ENCODEPeak" name="input1" type="data" label="Choose ENCODEPeak file"/>
</page>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_summary_tree_converter.xml
-14
View File
@@ -1,14 +0,0 @@
<tool id="CONVERTER_gff_to_summary_tree_0" name="Convert GFF to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_summary_tree_converter.py $input1 $output1 --gff</command>
<inputs>
<page>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</page>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
+33
View File
@@ -0,0 +1,33 @@
<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>
## Remove comments and sort by chromosome.
grep -v '^#' $input1 | sort -k${input1.metadata.chromCol},${input1.metadata.chromCol} |
## Create simple BED by cutting chrom, start, and end columns.
awk -v OFS=' ' '{print $${input1.metadata.chromCol},$${input1.metadata.startCol},$${input1.metadata.endCol} }' |
## Generate coverage bedgraph.
bedtools genomecov -bg -split -i stdin -g $chromInfo
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo $output
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output
</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.py
-63
View File
@@ -1,63 +0,0 @@
#!/usr/bin/env python
"""
Convert from interval file to summary tree file. Default input file format is BED (0-based, half-open intervals).
usage: %prog <options> in_file out_file
-c, --chr-col: chromosome column, default=1
-s, --start-col: start column, default=2
-e, --end-col: end column, default=3
-t, --strand-col: strand column, default=6
-G, --gff: input is GFF format, meaning start and end coordinates are 1-based, closed interval
"""
from __future__ import division
import sys, fileinput, optparse
from galaxy import eggs
import pkg_resources; pkg_resources.require( "bx-python" )
from galaxy.visualization.tracks.summary import *
from bx.intervals.io import *
from galaxy.datatypes.util.gff_util import *
def main():
# Read options, args.
parser = optparse.OptionParser()
parser.add_option( '-c', '--chr-col', type='int', dest='chrom_col', default=1 )
parser.add_option( '-s', '--start-col', type='int', dest='start_col', default=2 )
parser.add_option( '-e', '--end-col', type='int', dest='end_col', default=3 )
parser.add_option( '-t', '--strand-col', type='int', dest='strand_col', default=6 )
parser.add_option( '-G', '--gff', dest="gff_format", action="store_true" )
(options, args) = parser.parse_args()
input_fname, output_fname = args
# Convert column indices to 0-based.
options.chrom_col -= 1
options.start_col -= 1
options.end_col -= 1
options.strand_col -= 1
# Do conversion.
if options.gff_format:
reader_wrapper_class = GFFReaderWrapper
chr_col, start_col, end_col, strand_col = ( 0, 3, 4, 6 )
else:
reader_wrapper_class = NiceReaderWrapper
chr_col, start_col, end_col, strand_col = ( options.chrom_col, options.start_col, options.end_col, options.strand_col )
reader_wrapper = reader_wrapper_class( fileinput.FileInput( input_fname ),
chrom_col=chr_col,
start_col=start_col,
end_col=end_col,
strand_col=strand_col,
fix_strand=True )
st = SummaryTree()
for feature in list( reader_wrapper ):
if isinstance( feature, GenomicInterval ):
# Tree expects BED coordinates.
if type( feature ) is GFFFeature:
convert_gff_coords_to_bed( feature )
st.insert_range( feature.chrom, long( feature.start ), long( feature.end ) )
st.write( output_fname )
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.xml
-20
View File
@@ -1,20 +0,0 @@
<tool id="CONVERTER_interval_to_summary_tree_0" name="Convert Interval to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_summary_tree_converter.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
$input1 $output1
</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose Interval file"/>
</page>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python
"""
Convert from pileup file to interval index file.
usage: %prog <options> in_file out_file
"""
from __future__ import division
import sys, fileinput, optparse
from galaxy import eggs
import pkg_resources; pkg_resources.require( "bx-python" )
from galaxy.visualization.tracks.summary import *
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed
from bx.interval_index_file import Indexes
def main():
# Read options, args.
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
input_fname, output_fname = args
# Do conversion.
index = Indexes()
offset = 0
for line in open( input_fname, "r" ):
chrom, start = line.split()[ 0:2 ]
# Pileup format is 1-based.
start = int( start ) - 1
index.add( chrom, start, start + 1, offset )
offset += len( line )
index.write( open(output_fname, "w") )
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
+15
View File
@@ -0,0 +1,15 @@
<tool id="CONVERTER_pileup_to_interval_index_0" name="Convert Pileup to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">pileup_to_interval_index_converter.py $input $output
</command>
<inputs>
<page>
<param format="pileup" name="input" type="data" label="Choose Pileup file"/>
</page>
</inputs>
<outputs>
<data format="interval_index" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py
-42
View File
@@ -1,42 +0,0 @@
#!/usr/bin/env python
from __future__ import division
import sys, os, optparse
sys.stderr = open(os.devnull, 'w') # suppress stderr as cython produces warning on some systems:
# csamtools.so:6: RuntimeWarning: __builtin__.file size changed
from galaxy import eggs
import pkg_resources
if sys.version_info[:2] == (2, 4):
pkg_resources.require( "ctypes" )
pkg_resources.require( "pysam" )
from pysam import csamtools
from galaxy.visualization.tracks.summary import *
def main():
parser = optparse.OptionParser()
parser.add_option( '-S', '--sam', action="store_true", dest="is_sam" )
parser.add_option( '-B', '--bam', action="store_true", dest="is_bam" )
options, args = parser.parse_args()
if options.is_bam:
input_fname = args[0]
index_fname = args[1]
out_fname = args[2]
samfile = csamtools.Samfile( filename=input_fname, mode='rb', index_filename=index_fname )
elif options.is_sam:
input_fname = args[0]
out_fname = args[1]
samfile = csamtools.Samfile( filename=input_fname, mode='r' )
st = SummaryTree()
for read in samfile.fetch():
st.insert_range( samfile.getrname( read.rname ), read.pos, read.pos + read.rlen )
st.write(out_fname)
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
+20
View File
@@ -0,0 +1,20 @@
<tool id="CONVERTER_sam_to_bigwig_0" name="Convert SAM to BigWig" version="1.0.0" hidden="true">
<command>
samtools view -bh $input | bedtools genomecov -bg -split -ibam stdin -g $chromInfo
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo $output
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output
</command>
<inputs>
<param format="bam" name="input" type="data" label="Choose BAM file"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml
-14
View File
@@ -1,14 +0,0 @@
<tool id="CONVERTER_sam_to_summary_tree_0" name="Convert SAM to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">sam_or_bam_to_summary_tree_converter.py --sam $input1 $output1</command>
<inputs>
<page>
<param format="sam" name="input1" type="data" label="Choose sam file"/>
</page>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.py
-30
View File
@@ -1,30 +0,0 @@
#!/usr/bin/env python
"""
Convert from VCF file to summary tree file.
usage: %prog in_file out_file
"""
from __future__ import division
import optparse
import galaxy_utils.sequence.vcf
from galaxy.visualization.tracks.summary import SummaryTree
def main():
# Read options, args.
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
in_file, out_file = args
# Do conversion.
st = SummaryTree()
for line in list( galaxy_utils.sequence.vcf.Reader( open( in_file ) ) ):
# VCF format provides a chrom and 1-based position for each variant.
# SummaryTree expects 0-based coordinates.
st.insert_range( line.chrom, long( line.pos-1 ), long( line.pos ) )
st.write(out_file)
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml
-14
View File
@@ -1,14 +0,0 @@
<tool id="CONVERTER_vcf_to_summary_tree_0" name="Convert VCF to Summary Tree" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">vcf_to_summary_tree_converter.py $input1 $output1</command>
<inputs>
<page>
<param format="vcf" name="input1" type="data" label="Choose VCF file"/>
</page>
</inputs>
<outputs>
<data format="summary_tree" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/coverage.py
+5 -12
View File
@@ -2,21 +2,14 @@
Coverage datatypes
"""
import pkg_resources
pkg_resources.require( "bx-python" )
import logging, os, sys, time, tempfile, shutil
import data
from galaxy import util
from galaxy.datatypes.sniff import *
from galaxy.web import url_for
from cgi import escape
import urllib
from bx.intervals.io import *
import logging
import math
from galaxy import eggs
from galaxy.datatypes import metadata
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.tabular import Tabular
import math
log = logging.getLogger(__name__)
@@ -34,7 +27,7 @@ class LastzCoverage( Tabular ):
Assumes we have a numpy file.
"""
# Maybe if we import here people will still be able to use Galaxy when numpy kills it
pkg_resources.require("numpy>=1.2.1")
eggs.require("numpy>=1.2.1")
#from numpy.lib import format
import numpy
lib/galaxy/datatypes/data.py
+20 -15
View File
@@ -2,6 +2,7 @@ import logging
import metadata
import mimetypes
import os
import shutil
import sys
import tempfile
import zipfile
@@ -17,12 +18,6 @@ from galaxy import eggs
eggs.require( "Paste" )
import paste
if sys.version_info[:2] < ( 2, 6 ):
zipfile.BadZipFile = zipfile.error
if sys.version_info[:2] < ( 2, 5 ):
zipfile.LargeZipFile = zipfile.error
log = logging.getLogger(__name__)
tmpd = tempfile.mkdtemp()
@@ -103,6 +98,12 @@ class Data( object ):
#A per datatype setting (inherited): max file size (in bytes) for setting optional metadata
_max_optional_metadata_filesize = None
# Trackster track type.
track_type = None
# Data sources.
data_sources = {}
def __init__(self, **kwd):
"""Initialize the datatype"""
object.__init__(self, **kwd)
@@ -545,21 +546,21 @@ class Data( object ):
return False
def merge( split_files, output_file):
"""
TODO: Do we need to merge gzip files using gzjoin? cat seems to work,
but might be brittle. Need to revisit this.
Merge files with copy.copyfileobj() will not hit the
max argument limitation of cat. gz and bz2 files are also working.
"""
if not split_files:
raise ValueError('Asked to merge zero files as %s' % output_file)
elif len(split_files) == 1:
cmd = 'mv -f %s %s' % ( split_files[0], output_file )
shutil.copyfileobj(open(split_files[0], 'rb'), open(output_file, 'wb'))
else:
cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file )
result = os.system(cmd)
if result != 0:
raise Exception('Result %s from %s' % (result, cmd))
fdst = open(output_file, 'wb')
for fsrc in split_files:
shutil.copyfileobj(open(fsrc, 'rb'), fdst)
fdst.close()
merge = staticmethod(merge)
def get_visualizations( self, dataset ):
@@ -567,7 +568,7 @@ class Data( object ):
Returns a list of visualizations for datatype.
"""
if hasattr( self, 'get_track_type' ):
if self.track_type:
return [ 'trackster', 'circster' ]
return []
@@ -740,6 +741,10 @@ class Text( Data ):
f.close()
split = classmethod(split)
class GenericAsn1( Text ):
"""Class for generic ASN.1 text format"""
file_ext = 'asn1'
class LineCount( Text ):
"""
Dataset contains a single line with a single integer that denotes the
lib/galaxy/datatypes/display_applications/application.py
+2 -2
View File
@@ -12,7 +12,7 @@ from copy import deepcopy
log = logging.getLogger( __name__ )
#Any basic functions that we want to provide as a basic part of parameter dict should be added to this dict
BASE_PARAMS = { 'qp': quote_plus, 'url_for':url_for } #url_for has route memory...
BASE_PARAMS = { 'qp': quote_plus, 'url_for':url_for }
class DisplayApplicationLink( object ):
@classmethod
@@ -40,7 +40,7 @@ class DisplayApplicationLink( object ):
self.name = None
def get_display_url( self, data, trans ):
dataset_hash, user_hash = encode_dataset_user( trans, data, None )
return url_for( controller='/dataset',
return url_for( controller='dataset',
action="display_application",
dataset_id=dataset_hash,
user_id=user_hash,
lib/galaxy/datatypes/display_applications/link_generator.py
-161
View File
@@ -1,161 +0,0 @@
"""Classes to generate links for old-style display applications.
Separating Transaction based elements of display applications from datatypes.
"""
#FIXME: The code contained within this file is for old-style display applications, but
#this module namespace is intended to only handle the new-style display applications.
import urllib
# for the url_for hack
import pkg_resources
pkg_resources.require( "Routes" )
import routes
from galaxy import util
from galaxy.web import url_for
from galaxy.datatypes.interval import Interval, Gff, Wiggle, CustomTrack
#TODO: Ideally, these classes would be instantiated in the trans (or some other semi-persistant fixture)
# Currently, these are instantiated per HDA which is not the best solution
#TODO: these could be extended to handle file_function and parse/contain the builds.txt files
#HACK: these duplicate functionality from the individual datatype classes themselves
def get_display_app_link_generator( display_app_name ):
"""Returns an instance of the proper link generator class
based on the display_app_name or DisplayAppLinkGenerator
if the display_app_name is unrecognized.
"""
if display_app_name == 'ucsc':
return UCSCDisplayAppLinkGenerator()
elif display_app_name == 'gbrowse':
return GBrowseDisplayAppLinkGenerator()
return DisplayAppLinkGenerator()
class DisplayAppLinkGenerator( object ):
"""Base class for display application link generators.
This class returns an empty list of links for all datatypes.
"""
def __init__( self ):
self.display_app_name = ''
def no_links_available( self, dataset, app, base_url, url_for=url_for ):
"""Called when no display application links are available
for this display app name and datatype combination.
"""
return []
def _link_function_from_datatype( self, datatype ):
"""Dispatch to proper link generating function on datatype.
"""
return self.no_links_available
def generate_links( self, trans, dataset ):
# here's the hack - which is expensive (time)
web_url_for = routes.URLGenerator( trans.webapp.mapper, trans.environ )
link_function = self._link_function_from_datatype( dataset.datatype )
display_links = link_function( dataset, trans.app, trans.request.base, url_for=web_url_for )
return display_links
class UCSCDisplayAppLinkGenerator( DisplayAppLinkGenerator ):
"""Class for generating links to display data in the
UCSC genome browser.
This class returns links for the following datatypes and their subclasses:
Interval, Wiggle, Gff, CustomTrack
"""
def __init__( self ):
self.display_app_name = 'ucsc'
def _link_function_from_datatype( self, datatype ):
"""Dispatch to proper link generating function based on datatype.
"""
if( ( isinstance( datatype, Interval ) )
or ( isinstance( datatype, Wiggle ) )
or ( isinstance( datatype, Gff ) )
or ( isinstance( datatype, CustomTrack ) ) ):
return self.ucsc_links
else:
return super( UCSCDisplayAppLinkGenerator, self )._link_function_from_datatype( datatype )
def ucsc_links( self, dataset, app, base_url, url_for=url_for ):
"""Generate links to UCSC genome browser sites based on the dbkey
and content of dataset.
"""
# this is a refactor of Interval.ucsc_links, GFF.ucsc_links, Wiggle.ucsc_links, and CustomTrack.ucsc_links
#TODO: app vars can be moved into init (and base_url as well)
chrom, start, stop = dataset.datatype.get_estimated_display_viewport( dataset )
if chrom is None:
return []
ret_val = []
for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey):
if site_name in app.config.ucsc_display_sites:
internal_url = url_for( controller='dataset', dataset_id=dataset.id,
action='display_at', filename='%s_%s' % ( self.display_app_name, site_name ) )
base_url = app.config.get( "display_at_callback", base_url )
display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at"
% (base_url, url_for( controller='root' ), dataset.id, self.display_app_name) )
redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s"
% (site_url, dataset.dbkey, chrom, start, stop ) )
link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
ret_val.append( ( site_name, link ) )
return ret_val
class GBrowseDisplayAppLinkGenerator( DisplayAppLinkGenerator ):
"""Class for generating links to display data in the
GBrowse genome browser.
This class returns links for the following datatypes and their subclasses:
Gff, Wiggle
"""
def __init__( self ):
self.display_app_name = 'gbrowse'
def _link_function_from_datatype( self, datatype ):
"""Dispatch to proper link generating function based on datatype.
"""
if( ( isinstance( datatype, Gff ) )
or ( isinstance( datatype, Wiggle ) ) ):
return self.gbrowse_links
else:
return super( GBrowseDisplayAppLinkGenerator, self )._link_function_from_datatype( datatype )
def gbrowse_links( self, dataset, app, base_url, url_for=url_for ):
"""Generate links to GBrowse genome browser sites based on the dbkey
and content of dataset.
"""
# when normalized for var names, Gff.gbrowse_links and Wiggle.gbrowse_links are the same
# also: almost identical to ucsc_links except for the 'chr' stripping, sites_by_build, config key
# could be refactored even more
chrom, start, stop = dataset.datatype.get_estimated_display_viewport( dataset )
if chrom is None:
return []
ret_val = []
for site_name, site_url in util.get_gbrowse_sites_by_build( dataset.dbkey ):
if site_name in app.config.gbrowse_display_sites:
# strip chr from seqid
if chrom.startswith( 'chr' ) and len ( chrom ) > 3:
chrom = chrom[3:]
internal_url = url_for( controller='dataset', dataset_id=dataset.id,
action='display_at', filename='%s_%s' % ( self.display_app_name, site_name ) )
redirect_url = urllib.quote_plus( "%s/?q=%s:%s..%s&eurl=%%s" % ( site_url, chrom, start, stop ) )
base_url = app.config.get( "display_at_callback", base_url )
display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at"
% ( base_url, url_for( controller='root' ), dataset.id, self.display_app_name ) )
link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
ret_val.append( ( site_name, link ) )
return ret_val
lib/galaxy/datatypes/display_applications/parameters.py
+1 -1
View File
@@ -163,7 +163,7 @@ class DisplayParameterValueWrapper( object ):
if self.parameter.strip_https and base_url[ : 5].lower() == 'https':
base_url = "http%s" % base_url[ 5: ]
return "%s%s" % ( base_url,
url_for( controller='/dataset',
url_for( controller='dataset',
action="display_application",
dataset_id=self._dataset_hash,
user_id=self._user_hash,
lib/galaxy/datatypes/interval.py
+16 -25
View File
@@ -46,6 +46,8 @@ class Interval( Tabular ):
"""Tab delimited data containing interval information"""
file_ext = "interval"
line_class = "region"
track_type = "FeatureTrack"
data_sources = { "data": "tabix", "index": "bigwig" }
"""Add metadata elements"""
MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter )
@@ -242,7 +244,7 @@ class Interval( Tabular ):
# Accumulate links for valid sites
ret_val = []
for site_name, site_url in valid_sites:
internal_url = url_for( controller='/dataset', dataset_id=dataset.id,
internal_url = url_for( controller='dataset', dataset_id=dataset.id,
action='display_at', filename='ucsc_' + site_name )
display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at"
% (base_url, url_for( controller='root' ), dataset.id, type) )
@@ -328,17 +330,13 @@ class Interval( Tabular ):
def get_track_resolution( self, dataset, start, end):
return None
def get_track_type( self ):
return "FeatureTrack", {"data": "tabix", "index": "summary_tree"}
class BedGraph( Interval ):
"""Tab delimited chrom/start/end/datavalue dataset"""
file_ext = "bedgraph"
def get_track_type( self ):
return "LineTrack", { "data": "bigwig", "index": "bigwig" }
track_type = "LineTrack"
data_sources = { "data": "bigwig", "index": "bigwig" }
def as_ucsc_display_file( self, dataset, **kwd ):
"""
@@ -356,6 +354,8 @@ class BedGraph( Interval ):
class Bed( Interval ):
"""Tab delimited data in BED format"""
file_ext = "bed"
data_sources = { "data": "tabix", "index": "bigwig", "feature_search": "fli" }
track_type = Interval.track_type
"""Add metadata elements"""
MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter )
@@ -510,9 +510,6 @@ class Bed( Interval ):
else: return False
return True
except: return False
def get_track_type( self ):
return "FeatureTrack", {"data": "tabix", "index": "summary_tree", "feature_search": "fli"}
class BedStrict( Bed ):
"""Tab delimited data in strict BED format - no non-standard columns allowed"""
@@ -572,6 +569,8 @@ class Gff( Tabular, _RemoteCallMixin ):
"""Tab delimited data in Gff format"""
file_ext = "gff"
column_names = [ 'Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Group' ]
data_sources = { "data": "interval_index", "index": "bigwig", "feature_search": "fli" }
track_type = Interval.track_type
"""Add metadata elements"""
MetadataElement( name="columns", default=9, desc="Number of columns", readonly=True, visible=False )
@@ -783,10 +782,6 @@ class Gff( Tabular, _RemoteCallMixin ):
return True
except:
return False
def get_track_type( self ):
return "FeatureTrack", {"data": "interval_index", "index": "summary_tree", "feature_search": "fli"}
class Gff3( Gff ):
"""Tab delimited data in Gff3 format"""
@@ -794,6 +789,7 @@ class Gff3( Gff ):
valid_gff3_strand = ['+', '-', '.', '?']
valid_gff3_phase = ['.', '0', '1', '2']
column_names = [ 'Seqid', 'Source', 'Type', 'Start', 'End', 'Score', 'Strand', 'Phase', 'Attributes' ]
track_type = Interval.track_type
"""Add metadata elements"""
MetadataElement( name="column_types", default=['str','str','str','int','int','float','str','int','list'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False )
@@ -898,6 +894,7 @@ class Gtf( Gff ):
"""Tab delimited data in Gtf format"""
file_ext = "gtf"
column_names = [ 'Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Attributes' ]
track_type = Interval.track_type
"""Add metadata elements"""
MetadataElement( name="columns", default=9, desc="Number of columns", readonly=True, visible=False )
@@ -966,6 +963,8 @@ class Gtf( Gff ):
class Wiggle( Tabular, _RemoteCallMixin ):
"""Tab delimited data in wiggle format"""
file_ext = "wig"
track_type = "LineTrack"
data_sources = { "data": "bigwig", "index": "bigwig" }
MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False )
@@ -1146,9 +1145,6 @@ class Wiggle( Tabular, _RemoteCallMixin ):
resolution = min( resolution, 100000 )
resolution = max( resolution, 1 )
return resolution
def get_track_type( self ):
return "LineTrack", { "data": "bigwig", "index": "bigwig" }
class CustomTrack ( Tabular ):
"""UCSC CustomTrack"""
@@ -1292,6 +1288,7 @@ class ENCODEPeak( Interval ):
file_ext = "encodepeak"
column_names = [ 'Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'SignalValue', 'pValue', 'qValue', 'Peak' ]
data_sources = { "data": "tabix", "index": "bigwig" }
"""Add metadata elements"""
MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter )
@@ -1303,15 +1300,14 @@ class ENCODEPeak( Interval ):
def sniff( self, filename ):
return False
def get_track_type( self ):
return "FeatureTrack", {"data": "tabix", "index": "summary_tree"}
class ChromatinInteractions( Interval ):
'''
Chromatin interactions obtained from 3C/5C/Hi-C experiments.
'''
file_ext = "chrint"
track_type = "DiagonalHeatmapTrack"
data_sources = { "data": "tabix", "index": "bigwig" }
column_names = [ 'Chrom1', 'Start1', 'End1', 'Chrom2', 'Start2', 'End2', 'Value' ]
@@ -1328,11 +1324,6 @@ class ChromatinInteractions( Interval ):
def sniff( self, filename ):
return False
def get_track_type( self ):
return "DiagonalHeatmapTrack", {"data": "tabix", "index": "summary_tree"}
if __name__ == '__main__':
import doctest, sys
lib/galaxy/datatypes/metadata.py
+33 -32
View File
@@ -1,22 +1,33 @@
import sys, logging, copy, shutil, weakref, cPickle, tempfile, os
"""
Galaxy Metadata
"""
from galaxy import eggs
eggs.require("simplejson")
import copy
import cPickle
import logging
import os
import shutil
import simplejson
import sys
import tempfile
import weakref
from os.path import abspath
from galaxy.util import string_as_bool, stringify_dictionary_keys, listify
import galaxy.model
from galaxy.util import listify, stringify_dictionary_keys, string_as_bool
from galaxy.util.odict import odict
from galaxy.web import form_builder
import galaxy.model
from sqlalchemy.orm import object_session
import pkg_resources
pkg_resources.require("simplejson")
import simplejson
log = logging.getLogger( __name__ )
log = logging.getLogger(__name__)
STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored
DATABASE_CONNECTION_AVAILABLE = True #When False, certain metadata parameter types (see FileParameter) will behave differently
class Statement( object ):
"""
This class inserts its target into a list in the surrounding
@@ -74,8 +85,8 @@ class MetadataCollection( object ):
def __getattr__( self, name ):
if name in self.spec:
if name in self.parent._metadata:
return self.spec[name].wrap( self.parent._metadata[name] )
return self.spec[name].wrap( self.spec[name].default )
return self.spec[name].wrap( self.parent._metadata[name], object_session( self.parent ) )
return self.spec[name].wrap( self.spec[name].default, object_session( self.parent ) )
if name in self.parent._metadata:
return self.parent._metadata[name]
def __setattr__( self, name, value ):
@@ -202,7 +213,7 @@ class MetadataParameter( object ):
self.validate( value )
return value
def wrap( self, value ):
def wrap( self, value, session ):
"""
Turns a value into its usable form.
"""
@@ -245,11 +256,11 @@ class MetadataElementSpec( object ):
def get( self, name, default=None ):
return self.__dict__.get(name, default)
def wrap( self, value ):
def wrap( self, value, session ):
"""
Turns a stored value into its usable form.
"""
return self.param.wrap( value )
return self.param.wrap( value, session )
def unwrap( self, value ):
"""
@@ -312,7 +323,7 @@ class SelectParameter( MetadataParameter ):
return ", ".join( map( str, value ) )
return MetadataParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
def wrap( self, value ):
def wrap( self, value, session ):
value = self.marshal( value ) #do we really need this (wasteful)? - yes because we are not sure that all existing selects have been stored previously as lists. Also this will handle the case where defaults/no_values are specified and are single non-list values.
if self.multiple:
return value
@@ -424,26 +435,16 @@ class FileParameter( MetadataParameter ):
def get_html( self, value=None, context={}, other_values={}, **kwd ):
return "<div>No display available for Metadata Files</div>"
def wrap( self, value ):
def wrap( self, value, session ):
if value is None:
return None
if isinstance( value, galaxy.model.MetadataFile ) or isinstance( value, MetadataTempFile ):
return value
if DATABASE_CONNECTION_AVAILABLE:
try:
# FIXME: this query requires a monkey patch in assignmapper.py since
# MetadataParameters do not have a handle to the sqlalchemy session
return galaxy.model.MetadataFile.get( value )
except:
#value was not a valid id
return None
else:
mf = galaxy.model.MetadataFile()
mf.id = value #we assume this is a valid id, since we cannot check it
return mf
mf = session.query( galaxy.model.MetadataFile ).get( value )
return mf
def make_copy( self, value, target_context, source_context ):
value = self.wrap( value )
value = self.wrap( value, object_session( target_context.parent ) )
if value:
new_value = galaxy.model.MetadataFile( dataset = target_context.parent, name = self.spec.name )
object_session( target_context.parent ).add( new_value )
@@ -485,13 +486,13 @@ class FileParameter( MetadataParameter ):
return value
def new_file( self, dataset = None, **kwds ):
if DATABASE_CONNECTION_AVAILABLE:
if object_session( dataset ):
mf = galaxy.model.MetadataFile( name = self.spec.name, dataset = dataset, **kwds )
object_session( dataset ).add( mf )
object_session( dataset ).flush() #flush to assign id
return mf
else:
#we need to make a tmp file that is accessable to the head node,
#we need to make a tmp file that is accessable to the head node,
#we will be copying its contents into the MetadataFile objects filename after restoring from JSON
#we do not include 'dataset' in the kwds passed, as from_JSON_value() will handle this for us
return MetadataTempFile( **kwds )
lib/galaxy/datatypes/registry.py
+32 -6
View File
@@ -163,7 +163,7 @@ class Registry( object ):
# Use default mime type as per datatype spec
mimetype = self.datatypes_by_extension[ extension ].get_mime()
self.mimetypes_by_extension[ extension ] = mimetype
if hasattr( datatype_class, "get_track_type" ):
if datatype_class.track_type:
self.available_tracks.append( extension )
if display_in_upload:
self.upload_file_formats.append( extension )
@@ -379,6 +379,36 @@ class Registry( object ):
if not included:
self.sniff_order.append(datatype)
append_to_sniff_order()
def get_datatype_class_by_name( self, name ):
"""
Return the datatype class where the datatype's `type` attribute
(as defined in the datatype_conf.xml file) contains `name`.
"""
#TODO: too roundabout - would be better to generate this once as a map and store in this object
found_class = None
for ext, datatype_obj in self.datatypes_by_extension.items():
datatype_obj_class = datatype_obj.__class__
datatype_obj_class_str = str( datatype_obj_class )
#print datatype_obj_class_str
if name in datatype_obj_class_str:
return datatype_obj_class
return None
# these seem to be connected to the dynamic classes being generated in this file, lines 157-158
# they appear when a one of the three are used in inheritance with subclass="True"
#TODO: a possible solution is to def a fn in datatypes __init__ for creating the dynamic classes
#remap = {
# 'galaxy.datatypes.registry.Tabular' : galaxy.datatypes.tabular.Tabular,
# 'galaxy.datatypes.registry.Text' : galaxy.datatypes.data.Text,
# 'galaxy.datatypes.registry.Binary' : galaxy.datatypes.binary.Binary
#}
#datatype_str = str( datatype )
#if datatype_str in remap:
# datatype = remap[ datatype_str ]
#
#return datatype
def get_available_tracks(self):
return self.available_tracks
def get_mimetype_by_extension(self, ext, default = 'application/octet-stream' ):
@@ -397,7 +427,7 @@ class Registry( object ):
except KeyError:
builder = data.Text()
return builder
def change_datatype(self, data, ext, set_meta = True ):
def change_datatype(self, data, ext):
data.extension = ext
# call init_meta and copy metadata from itself. The datatype
# being converted *to* will handle any metadata copying and
@@ -405,10 +435,6 @@ class Registry( object ):
if data.has_data():
data.set_size()
data.init_meta( copy_from=data )
if set_meta:
#metadata is being set internally
data.set_meta( overwrite = False )
data.set_peek()
return data
def old_change_datatype(self, data, ext):
"""Creates and returns a new datatype based on an existing data and an extension"""
lib/galaxy/datatypes/tabular.py
+25 -9
View File
@@ -328,7 +328,6 @@ class Tabular( data.Text ):
"""
Returns a list of visualizations for datatype.
"""
# Can visualize tabular data as scatterplot if there are 2+ numerical
# columns.
num_numerical_cols = 0
@@ -358,6 +357,9 @@ class Taxonomy( Tabular ):
class Sam( Tabular ):
file_ext = 'sam'
track_type = "ReadTrack"
data_sources = { "data": "bam", "index": "bigwig" }
def __init__(self, **kwd):
"""Initialize taxonomy datatype"""
Tabular.__init__( self, **kwd )
@@ -467,17 +469,16 @@ class Sam( Tabular ):
raise Exception('Result %s from %s' % (result, cmd))
merge = staticmethod(merge)
def get_track_type( self ):
return "ReadTrack", {"data": "bam", "index": "summary_tree"}
class Pileup( Tabular ):
"""Tab delimited data in pileup (6- or 10-column) format"""
file_ext = "pileup"
line_class = "genomic coordinate"
data_sources = { "data": "tabix" }
"""Add metadata elements"""
MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter )
MetadataElement( name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter )
MetadataElement( name="endCol", default=2, desc="End column", param=metadata.ColumnParameter )
MetadataElement( name="baseCol", default=3, desc="Reference base column", param=metadata.ColumnParameter )
def init_meta( self, dataset, copy_from=None ):
@@ -525,8 +526,7 @@ class Pileup( Tabular ):
return True
except:
return False
class ElandMulti( Tabular ):
file_ext = 'elandmulti'
@@ -535,23 +535,39 @@ class ElandMulti( Tabular ):
class Vcf( Tabular ):
""" Variant Call Format for describing SNPs and other simple genome variations. """
track_type = "VariantTrack"
data_sources = { "data": "tabix", "index": "bigwig" }
file_ext = 'vcf'
column_names = [ 'Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data' ]
MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False )
MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False )
MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True )
MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True, visible=False )
MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] )
def sniff( self, filename ):
headers = get_headers( filename, '\n', count=1 )
return headers[0][0].startswith("##fileformat=VCF")
def display_peek( self, dataset ):
"""Returns formated html of peek"""
return Tabular.make_html_table( self, dataset, column_names=self.column_names )
def get_track_type( self ):
return "VcfTrack", {"data": "tabix", "index": "summary_tree"}
def set_meta( self, dataset, **kwd ):
Tabular.set_meta( self, dataset, **kwd )
source = open( dataset.file_name )
# Skip comments.
line = None
for line in source:
if not line.startswith( '##' ):
break
if line and line.startswith( '#' ):
# Found header line, get sample names.
dataset.metadata.sample_names = line.split()[ 9: ]
class Eland( Tabular ):
"""Support for the export.txt.gz file used by Illumina's ELANDv2e aligner"""
lib/galaxy/eggs/__init__.py
-1
View File
@@ -387,7 +387,6 @@ class GalaxyConfig( object ):
"guppy": lambda: self.config.get( "app:main", "use_memdump" ),
"python_openid": lambda: self.config.get( "app:main", "enable_openid" ),
"python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ),
"ctypes": lambda: ( "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(",") ) and sys.version_info[:2] == ( 2, 4 ),
"pysam": lambda: check_pysam()
}.get( egg_name, lambda: True )()
except:
lib/galaxy/jobs/__init__.py
+67 -35
View File
@@ -4,6 +4,7 @@ Support for running a tool in Galaxy via an internal job management system
import copy
import datetime
import galaxy
import logging
import os
import pwd
@@ -14,20 +15,21 @@ import subprocess
import sys
import threading
import traceback
import galaxy
from galaxy import util, model
from galaxy.util.bunch import Bunch
from galaxy import model, util
from galaxy.datatypes import metadata
from galaxy.util.json import from_json_string
from galaxy.util.expressions import ExpressionContext
from galaxy.jobs.actions.post import ActionBox
from galaxy.exceptions import ObjectInvalid
from galaxy.jobs.actions.post import ActionBox
from galaxy.jobs.mapper import JobRunnerMapper
from galaxy.jobs.runners import BaseJobRunner
from galaxy.util.bunch import Bunch
from galaxy.util.expressions import ExpressionContext
from galaxy.util.json import from_json_string
log = logging.getLogger( __name__ )
DATABASE_MAX_STRING_SIZE = util.DATABASE_MAX_STRING_SIZE
DATABASE_MAX_STRING_SIZE_PRETTY = util.DATABASE_MAX_STRING_SIZE_PRETTY
# This file, if created in the job's working directory, will be used for
# setting advanced metadata properties on the job and its associated outputs.
# This interface is currently experimental, is only used by the upload tool,
@@ -578,7 +580,7 @@ class JobWrapper( object ):
self.tool_provided_job_metadata = None
# Wrapper holding the info required to restore and clean up from files used for setting metadata externally
self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job )
self.job_runner_mapper = JobRunnerMapper( self, queue.dispatcher.url_to_destination )
self.job_runner_mapper = JobRunnerMapper( self, queue.dispatcher.url_to_destination, self.app.job_config )
self.params = None
if job.params:
self.params = from_json_string( job.params )
@@ -676,13 +678,16 @@ class JobWrapper( object ):
# These can be passed on the command line if wanted as $__user_*__
if job.history and job.history.user:
user = job.history.user
user_id = '%d' % job.history.user.id
user_email = str(job.history.user.email)
user_name = str(job.history.user.username)
else:
user = None
user_id = 'Anonymous'
user_email = 'Anonymous'
user_name = 'Anonymous'
incoming['__user__'] = user
incoming['__user_id__'] = incoming['userId'] = user_id
incoming['__user_email__'] = incoming['userEmail'] = user_email
incoming['__user_name__'] = user_name
@@ -774,13 +779,13 @@ class JobWrapper( object ):
job.info = message
# TODO: Put setting the stdout, stderr, and exit code in one place
# (not duplicated with the finish method).
if ( len( stdout ) > 32768 ):
stdout = stdout[:32768]
log.info( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id )
if ( len( stdout ) > DATABASE_MAX_STRING_SIZE ):
stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
job.stdout = stdout
if ( len( stderr ) > 32768 ):
stderr = stderr[:32768]
log.info( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id )
if ( len( stderr ) > DATABASE_MAX_STRING_SIZE ):
stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
job.stderr = stderr
# Let the exit code be Null if one is not provided:
if ( exit_code != None ):
@@ -914,14 +919,21 @@ class JobWrapper( object ):
return self.fail( "Job %s's output dataset(s) could not be read" % job.id )
job_context = ExpressionContext( dict( stdout = job.stdout, stderr = job.stderr ) )
#DBTODO unused
#job_tool = self.app.toolbox.tools_by_id.get( job.tool_id, None )
for dataset_assoc in job.output_datasets + job.output_library_datasets:
context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset )
#should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: #need to update all associated output hdas, i.e. history was shared with job running
dataset.blurb = 'done'
dataset.peek = 'no peek'
dataset.info = ( dataset.info or '' ) + context['stdout'] + context['stderr']
dataset.info = (dataset.info or '')
if context['stdout'].strip():
#Ensure white space between entries
dataset.info = dataset.info.rstrip() + "\n" + context['stdout'].strip()
if context['stderr'].strip():
#Ensure white space between entries
dataset.info = dataset.info.rstrip() + "\n" + context['stderr'].strip()
dataset.tool_version = self.version_string
dataset.set_size()
if 'uuid' in context:
@@ -941,9 +953,7 @@ class JobWrapper( object ):
#either use the metadata from originating output dataset, or call set_meta on the copies
#it would be quicker to just copy the metadata from the originating output dataset,
#but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
if not self.app.config.set_metadata_externally or \
( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) \
and self.app.config.retry_metadata_internally ):
if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ):
dataset.datatype.set_meta( dataset, overwrite = False ) #call datatype.set_meta directly for the initial set_meta call during dataset creation
elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state:
dataset._state = model.Dataset.states.FAILED_METADATA
@@ -998,12 +1008,12 @@ class JobWrapper( object ):
# will now be seen by the user.
self.sa_session.flush()
# Save stdout and stderr
if len( job.stdout ) > 32768:
log.info( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id )
job.stdout = job.stdout[:32768]
if len( job.stderr ) > 32768:
log.info( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id )
job.stderr = job.stderr[:32768]
if len( job.stdout ) > DATABASE_MAX_STRING_SIZE:
log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
job.stdout = util.shrink_string_by_size( job.stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
if len( job.stderr ) > DATABASE_MAX_STRING_SIZE:
log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
job.stderr = util.shrink_string_by_size( job.stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
# The exit code will be null if there is no exit code to be set.
# This is so that we don't assign an exit code, such as 0, that
# is either incorrect or has the wrong semantics.
@@ -1165,8 +1175,8 @@ class JobWrapper( object ):
# the job has an error, and the job is ok otherwise.
else:
# TODO: Add in the tool and job id:
log.debug( "Tool did not define exit code or stdio handling; "
+ "checking stderr for success" )
# log.debug( "Tool did not define exit code or stdio handling; "
# + "checking stderr for success" )
if stderr:
success = False
else:
@@ -1214,8 +1224,7 @@ class JobWrapper( object ):
try:
for fname in self.extra_filenames:
os.remove( fname )
if self.app.config.set_metadata_externally:
self.external_output_metadata.cleanup_external_metadata( self.sa_session )
self.external_output_metadata.cleanup_external_metadata( self.sa_session )
galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session )
galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job()
galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app )
@@ -1467,6 +1476,14 @@ class JobWrapper( object ):
self.__galaxy_system_pwent = pwd.getpwuid(os.getuid())
return self.__galaxy_system_pwent
def get_output_destination( self, output_path ):
"""
Destination for outputs marked as from_work_dir. This is the normal case,
just copy these files directly to the ulimate destination.
"""
return output_path
class TaskWrapper(JobWrapper):
"""
Extension of JobWrapper intended for running tasks.
@@ -1556,6 +1573,12 @@ class TaskWrapper(JobWrapper):
self.sa_session.flush()
# Build any required config files
config_filenames = self.tool.build_config_files( param_dict, self.working_directory )
for config_filename in config_filenames:
config_contents = open(config_filename, "r").read()
for k, v in fnames.iteritems():
config_contents = config_contents.replace(k, v)
open(config_filename, "w").write(config_contents)
# FIXME: Build the param file (might return None, DEPRECATED)
param_filename = self.tool.build_param_file( param_dict, self.working_directory )
# Build the job's command line
@@ -1652,12 +1675,12 @@ class TaskWrapper(JobWrapper):
task.state = task.states.ERROR
# Save stdout and stderr
if len( stdout ) > 32768:
log.error( "stdout for task %d is greater than 32K, only first part will be logged to database" % task.id )
task.stdout = stdout[:32768]
if len( stderr ) > 32768:
log.error( "stderr for job %d is greater than 32K, only first part will be logged to database" % task.id )
task.stderr = stderr[:32768]
if len( stdout ) > DATABASE_MAX_STRING_SIZE:
log.error( "stdout for task %d is greater than %s, only a portion will be logged to database" % ( task.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
task.stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
if len( stderr ) > DATABASE_MAX_STRING_SIZE:
log.error( "stderr for task %d is greater than %s, only a portion will be logged to database" % ( task.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
task.stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
task.exit_code = tool_exit_code
task.command_line = self.command_line
self.sa_session.flush()
@@ -1688,6 +1711,15 @@ class TaskWrapper(JobWrapper):
# There is no metadata setting for tasks. This is handled after the merge, at the job level.
return ""
def get_output_destination( self, output_path ):
"""
Destination for outputs marked as from_work_dir. These must be copied with
the same basenme as the path for the ultimate output destination. This is
required in the task case so they can be merged.
"""
return os.path.join( self.working_directory, os.path.basename( output_path ) )
class NoopQueue( object ):
"""
Implements the JobQueue / JobStopQueue interface but does nothing
lib/galaxy/jobs/actions/post.py
+2 -1
View File
@@ -1,4 +1,5 @@
import logging, datetime
import datetime
import logging
from galaxy.util import send_mail
from galaxy.util.json import to_json_string
lib/galaxy/jobs/deferred/data_transfer.py
+5 -9
View File
@@ -136,15 +136,11 @@ class DataTransfer( object ):
if name not in [ 'name', 'info', 'dbkey', 'base_name' ]:
if spec.get( 'default' ):
setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) )
if self.app.config.set_metadata_externally:
self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( self.app.datatypes_registry.set_external_metadata_tool,
FakeTrans( self.app,
history=sample.history,
user=sample.request.user ),
incoming = { 'input1':ldda } )
else:
ldda.set_meta()
ldda.datatype.after_setting_metadata( ldda )
self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( self.app.datatypes_registry.set_external_metadata_tool,
FakeTrans( self.app,
history=sample.history,
user=sample.request.user ),
incoming = { 'input1':ldda } )
ldda.state = ldda.states.OK
# TODO: not sure if this flush is necessary
self.sa_session.add( ldda )
lib/galaxy/jobs/deferred/genome_transfer.py
+8 -18
View File
@@ -165,28 +165,18 @@ class GenomeTransferPlugin( DataTransfer ):
for name in z.namelist():
if name.endswith('/'):
continue
if sys.version_info[:2] >= ( 2, 6 ):
zipped_file = z.open( name )
while 1:
try:
chunk = zipped_file.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing zipped data' )
return self.app.model.DeferredJob.states.INVALID
if not chunk:
break
os.write( fd, chunk )
zipped_file.close()
else:
zipped_file = z.open( name )
while 1:
try:
outfile = open( fd, 'wb' )
outfile.write( z.read( name ) )
outfile.close()
chunk = zipped_file.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing zipped data' )
return
return self.app.model.DeferredJob.states.INVALID
if not chunk:
break
os.write( fd, chunk )
zipped_file.close()
os.close( fd )
z.close()
elif data_type == 'fasta':
lib/galaxy/jobs/manager.py
+5 -6
View File
@@ -2,17 +2,16 @@
Top-level Galaxy job manager, moves jobs to handler(s)
"""
import os
import time
import random
import logging
import os
import random
import threading
from Queue import Queue, Empty
import time
from sqlalchemy.sql.expression import and_, or_
from Queue import Empty, Queue
from galaxy import model
from galaxy.jobs import handler, Sleeper, NoopQueue, JobWrapper
from galaxy.jobs import handler, JobWrapper, NoopQueue, Sleeper
from galaxy.util.json import from_json_string
log = logging.getLogger( __name__ )
lib/galaxy/jobs/mapper.py
+18 -28
View File
@@ -21,10 +21,10 @@ class JobRunnerMapper( object ):
(in the form of job_wrappers) to job runner url strings.
"""
def __init__( self, job_wrapper, url_to_destination ):
def __init__( self, job_wrapper, url_to_destination, job_config ):
self.job_wrapper = job_wrapper
self.url_to_destination = url_to_destination
self.rule_modules = self.__get_rule_modules( )
self.job_config = job_config
def __get_rule_modules( self ):
unsorted_module_names = self.__get_rule_module_names( )
@@ -103,8 +103,13 @@ class JobRunnerMapper( object ):
return dest
def __determine_expand_function_name( self, destination ):
# default look for function with same name as tool, unless one specified
expand_function_name = destination.params.get('function', self.job_wrapper.tool.id)
# default look for function with name matching an id of tool, unless one specified
expand_function_name = destination.params.get('function', None)
if not expand_function_name:
for tool_id in self.job_wrapper.tool.all_ids:
if self.__last_rule_module_with_function( tool_id ):
expand_function_name = tool_id
break
return expand_function_name
def __get_expand_function( self, expand_function_name ):
@@ -118,39 +123,24 @@ class JobRunnerMapper( object ):
def __last_rule_module_with_function( self, function_name ):
# self.rule_modules is sorted in reverse order, so find first
# wiht function
for rule_module in self.rule_modules:
for rule_module in self.__get_rule_modules( ):
if hasattr( rule_module, function_name ):
return rule_module
return None
def __handle_dynamic_job_destination( self, destination ):
expand_type = destination.params.get('type', None)
expand_type = destination.params.get('type', "python")
if expand_type == "python":
expand_function_name = self.__determine_expand_function_name( destination )
expand_function = self.__get_expand_function( expand_function_name )
rval = self.__invoke_expand_function( expand_function )
# TODO: test me extensively
if isinstance(rval, basestring):
# If the function returned a string, check if it's a URL, convert if necessary
if '://' in rval:
return self.__convert_url_to_destination(rval)
job_destination = self.__invoke_expand_function( expand_function )
if not isinstance(job_destination, galaxy.jobs.JobDestination):
job_destination_rep = str(job_destination) # Should be either id or url
if '://' in job_destination_rep:
job_destination = self.__convert_url_to_destination(job_destination_rep)
else:
return self.app.job_config.get_destination(rval)
elif isinstance(rval, galaxy.jobs.JobDestination):
# If the function generated a JobDestination, we'll use that
# destination directly. However, for advanced job limiting, a
# function may want to set the JobDestination's 'tags'
# attribute so that limiting can be done on a destination tag.
#id_or_tag = rval.get('id')
#if rval.get('tags', None):
# # functions that are generating destinations should only define one tag
# id_or_tag = rval.get('tags')[0]
#return id_or_tag, rval
return rval
else:
raise Exception( 'Dynamic function returned a value that could not be understood: %s' % rval )
elif expand_type is None:
raise Exception( 'Dynamic function type not specified (hint: add <param id="type">python</param> to your <destination>)' )
job_destination = self.job_config.get_destination(job_destination_rep)
return job_destination
else:
raise Exception( "Unhandled dynamic job runner type specified - %s" % expand_type )
lib/galaxy/jobs/runners/__init__.py
+30 -4
View File
@@ -7,11 +7,13 @@ import time
import string
import logging
import threading
import subprocess
from Queue import Queue, Empty
import galaxy.jobs
from galaxy import model
from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size
log = logging.getLogger( __name__ )
@@ -174,7 +176,7 @@ class BaseJobRunner( object ):
# Append metadata setting commands, we don't want to overwrite metadata
# that was copied over in init_meta(), as per established behavior
if include_metadata and self.app.config.set_metadata_externally:
if include_metadata:
commands += "; cd %s; " % os.path.abspath( os.getcwd() )
commands += job_wrapper.setup_external_metadata(
exec_dir = os.path.abspath( os.getcwd() ),
@@ -226,7 +228,7 @@ class BaseJobRunner( object ):
# Copy from working dir to HDA.
# TODO: move instead of copy to save time?
source_file = os.path.join( os.path.abspath( job_wrapper.working_directory ), hda_tool_output.from_work_dir )
destination = output_paths[ dataset.dataset_id ]
destination = job_wrapper.get_output_destination( output_paths[ dataset.dataset_id ] )
if in_directory( source_file, job_wrapper.working_directory ):
output_pairs.append( ( source_file, destination ) )
log.debug( "Copying %s to %s as directed by from_work_dir" % ( source_file, destination ) )
@@ -235,6 +237,30 @@ class BaseJobRunner( object ):
log.exception( "from_work_dir specified a location not in the working directory: %s, %s" % ( source_file, job_wrapper.working_directory ) )
return output_pairs
def _handle_metadata_externally(self, job_wrapper):
"""
Set metadata externally. Used by the local and lwr job runners where this
shouldn't be attached to command-line to execute.
"""
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames=job_wrapper.get_output_fnames(),
set_extension=True,
tmp_dir=job_wrapper.working_directory,
#we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
kwds={ 'overwrite' : False } )
log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
external_metadata_proc = subprocess.Popen( args=external_metadata_script,
shell=True,
env=os.environ,
preexec_fn=os.setpgrp )
job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
external_metadata_proc.wait()
log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id )
class AsynchronousJobState( object ):
"""
Encapsulate the state of an asynchronous job, this should be subclassed as
@@ -383,8 +409,8 @@ class AsynchronousJobRunner( BaseJobRunner ):
which_try = 0
while which_try < (self.app.config.retry_job_output_collection + 1):
try:
stdout = file( job_state.output_file, "r" ).read( 32768 )
stderr = file( job_state.error_file, "r" ).read( 32768 )
stdout = shrink_stream_by_size( file( job_state.output_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( file( job_state.error_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
which_try = (self.app.config.retry_job_output_collection + 1)
except Exception, e:
if which_try == self.app.config.retry_job_output_collection:
lib/galaxy/jobs/runners/drmaa.py
-2
View File
@@ -15,8 +15,6 @@ from galaxy import model
from galaxy.jobs import JobDestination
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
if sys.version_info[:2] == ( 2, 4 ):
eggs.require( "ctypes" )
eggs.require( "drmaa" )
# We foolishly named this file the same as the name exported by the drmaa
# library... 'import drmaa' imports itself.
lib/galaxy/jobs/runners/local.py
+5 -20
View File
@@ -12,6 +12,7 @@ from time import sleep
from galaxy import model
from galaxy.jobs.runners import BaseJobRunner
from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size
log = logging.getLogger( __name__ )
@@ -78,32 +79,16 @@ class LocalJobRunner( BaseJobRunner ):
exit_code = proc.wait()
stdout_file.seek( 0 )
stderr_file.seek( 0 )
stdout = stdout_file.read( 32768 )
stderr = stderr_file.read( 32768 )
stdout = shrink_stream_by_size( stdout_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( stderr_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stdout_file.close()
stderr_file.close()
log.debug('execution finished: %s' % command_line)
except Exception, exc:
except Exception:
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
set_extension = True,
tmp_dir = job_wrapper.working_directory,
kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
external_metadata_proc = subprocess.Popen( args = external_metadata_script,
shell = True,
env = os.environ,
preexec_fn = os.setpgrp )
job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
external_metadata_proc.wait()
log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id )
self._handle_metadata_externally( job_wrapper )
# Finish the job!
try:
job_wrapper.finish( stdout, stderr, exit_code )
lib/galaxy/jobs/runners/lwr.py
+13 -28
View File
@@ -1,5 +1,4 @@
import logging
import subprocess
from galaxy import model
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
@@ -7,6 +6,7 @@ from galaxy.jobs import JobDestination
import errno
from time import sleep
import os
from lwr_client import FileStager, Client, url_to_destination_params
@@ -34,15 +34,18 @@ class LwrJobRunner( AsynchronousJobRunner ):
def check_watched_item(self, job_state):
try:
client = self.get_client_from_state(job_state)
complete = client.check_complete()
status = client.get_status()
except Exception:
# An orphaned job was put into the queue at app startup, so remote server went down
# either way we are done I guess.
self.mark_as_finished(job_state)
return None
if complete:
if status == "complete":
self.mark_as_finished(job_state)
return None
if status == "running" and not job_state.running:
job_state.running = True
job_state.job_wrapper.change_state( model.Job.states.RUNNING )
return job_state
def queue_job(self, job_wrapper):
@@ -81,7 +84,7 @@ class LwrJobRunner( AsynchronousJobRunner ):
job_id = file_stager.job_id
client.launch( rebuilt_command_line )
job_wrapper.set_job_destination( job_destination, job_id )
job_wrapper.change_state( model.Job.states.RUNNING )
job_wrapper.change_state( model.Job.states.QUEUED )
except Exception, exc:
job_wrapper.fail( "failure running job", exception=True )
@@ -92,7 +95,7 @@ class LwrJobRunner( AsynchronousJobRunner ):
lwr_job_state.job_wrapper = job_wrapper
lwr_job_state.job_id = job_id
lwr_job_state.old_state = True
lwr_job_state.running = True
lwr_job_state.running = False
lwr_job_state.job_destination = job_destination
self.monitor_job(lwr_job_state)
@@ -140,23 +143,8 @@ class LwrJobRunner( AsynchronousJobRunner ):
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
set_extension = True,
kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
external_metadata_proc = subprocess.Popen( args = external_metadata_script,
shell = True,
env = os.environ,
preexec_fn = os.setpgrp )
job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
external_metadata_proc.wait()
log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id )
# Finish the job
self._handle_metadata_externally( job_wrapper )
# Finish the job
try:
job_wrapper.finish( stdout, stderr )
except:
@@ -222,12 +210,9 @@ class LwrJobRunner( AsynchronousJobRunner ):
job_state.job_destination = job_wrapper.job_destination
job_wrapper.command_line = job.get_command_line()
job_state.job_wrapper = job_wrapper
if job.get_state() == model.Job.states.RUNNING:
state = job.get_state()
if state in [model.Job.states.RUNNING, model.Job.states.QUEUED]:
log.debug( "(LWR/%s) is still in running state, adding to the LWR queue" % ( job.get_id()) )
job_state.old_state = True
job_state.running = True
job_state.running = state == model.Job.states.RUNNING
self.monitor_queue.put( job_state )
elif job.get_state() == model.Job.states.QUEUED:
# LWR doesn't queue currently, so this indicates galaxy was shutoff while
# job was being staged. Not sure how to recover from that.
job_state.job_wrapper.fail( "This job was killed when Galaxy was restarted. Please retry the job." )
lib/galaxy/jobs/runners/lwr_client/client.py
+16 -2
View File
@@ -210,11 +210,25 @@ class Client(object):
check_complete_response = self.__raw_execute("check_complete", {"job_id": self.job_id})
return check_complete_response
def check_complete(self):
def check_complete(self, response=None):
"""
Return boolean indicating whether the job is complete.
"""
return self.raw_check_complete()["complete"] == "true"
if response == None:
response = self.raw_check_complete()
return response["complete"] == "true"
def get_status(self):
check_complete_response = self.raw_check_complete()
# Older LWR instances won't set status so use 'complete', at some
# point drop backward compatibility.
complete = self.check_complete(check_complete_response)
old_status = "complete" if complete else "running"
status = check_complete_response.get("status", old_status)
# Bug in certains older LWR instances returned literal "status".
if status not in ["complete", "running", "queued"]:
status = old_status
return status
def clean(self):
"""
lib/galaxy/jobs/runners/lwr_client/stager.py
+4 -1
View File
@@ -155,6 +155,9 @@ class FileStager(object):
self.new_working_directory = job_config['working_directory']
self.new_outputs_directory = job_config['outputs_directory']
# Default configs_directory to match remote working_directory to mimic
# behavior of older LWR servers.
self.new_configs_drectory = job_config.get('configs_directory', self.new_working_directory)
self.remote_path_separator = job_config['path_separator']
# If remote LWR server assigned job id, use that otherwise
# just use local job_id assigned.
@@ -218,7 +221,7 @@ class FileStager(object):
def __initialize_config_file_renames(self):
for config_file in self.config_files:
self.file_renames[config_file] = r'%s%s%s' % (self.new_working_directory,
self.file_renames[config_file] = r'%s%s%s' % (self.new_configs_drectory,
self.remote_path_separator,
os.path.basename(config_file))
lib/galaxy/jobs/runners/pbs.py
+3 -2
View File
@@ -5,6 +5,7 @@ from Queue import Queue, Empty
from galaxy import model
from galaxy.datatypes.data import nice_size
from galaxy.util.bunch import Bunch
from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size
from galaxy.jobs import JobDestination
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
@@ -522,8 +523,8 @@ class PBSJobRunner( AsynchronousJobRunner ):
ofh = file(ofile, "r")
efh = file(efile, "r")
ecfh = file(ecfile, "r")
stdout = ofh.read( 32768 )
stderr = efh.read( 32768 )
stdout = shrink_stream_by_size( ofh, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( efh, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = ecfh.read(32)
except:
lib/galaxy/jobs/runners/tasks.py
+1 -1
View File
@@ -123,7 +123,7 @@ class TaskedJobRunner( BaseJobRunner ):
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
set_extension = True,
kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
lib/galaxy/jobs/splitters/multi.py
+8 -1
View File
@@ -1,5 +1,8 @@
import os, logging, shutil
import os
import logging
import shutil
import inspect
from galaxy import model, util
@@ -19,8 +22,10 @@ def do_split (job_wrapper):
split_inputs = [x.strip() for x in split_inputs.split(",")]
shared_inputs=parallel_settings.get("shared_inputs")
auto_shared_inputs = False
if shared_inputs is None:
shared_inputs = []
auto_shared_inputs = True
else:
shared_inputs = [x.strip() for x in shared_inputs.split(",")]
illegal_inputs = [x for x in shared_inputs if x in split_inputs]
@@ -45,6 +50,8 @@ def do_split (job_wrapper):
type_to_input_map.setdefault(input.dataset.datatype, []).append(input.name)
elif input.name in shared_inputs:
pass # pass original file name
elif auto_shared_inputs:
shared_inputs.append(input.name)
else:
log_error = "The input '%s' does not define a method for implementing parallelism" % str(input.name)
log.exception(log_error)
lib/galaxy/model/__init__.py
+327 -48
View File
@@ -5,20 +5,33 @@ Naming: try to use class names that have a distinct plural form so that
the relationship cardinalities are obvious (e.g. prefer Dataset to Data)
"""
import pkg_resources
pkg_resources.require("simplejson")
pkg_resources.require("pexpect")
import simplejson, os, errno, codecs, operator, socket, pexpect, logging, time
from galaxy import eggs
eggs.require("simplejson")
eggs.require("pexpect")
import codecs
import errno
import logging
import operator
import os
import pexpect
import simplejson
import socket
import time
import galaxy.datatypes
import galaxy.datatypes.registry
import galaxy.security.passwords
from galaxy.datatypes.metadata import MetadataCollection
from galaxy.model.item_attrs import APIItem, UsesAnnotations
from galaxy.security import get_permitted_actions
from galaxy import util
from galaxy.util import is_multi_byte, nice_size, Params, restore_text, send_mail
from galaxy.util.bunch import Bunch
from galaxy.util.hash_util import new_secure_hash
from galaxy.web.form_builder import (AddressField, CheckboxField, PasswordField, SelectField, TextArea, TextField,
WorkflowField, WorkflowMappingField, HistoryField)
from galaxy.model.item_attrs import UsesAnnotations, APIItem
from galaxy.web.framework.helpers import to_unicode
from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField,
PasswordField, SelectField, TextArea, TextField, WorkflowField,
WorkflowMappingField)
from sqlalchemy.orm import object_session
from sqlalchemy.sql.expression import func
@@ -47,9 +60,18 @@ def set_datatypes_registry( d_registry ):
global datatypes_registry
datatypes_registry = d_registry
class User( object, APIItem ):
use_pbkdf2 = True
"""
Data for a Galaxy user or admin and relations to their
histories, credentials, and roles.
"""
# attributes that will be accessed and returned when calling get_api_value( view='collection' )
api_collection_visible_keys = ( 'id', 'email' )
# attributes that will be accessed and returned when calling get_api_value( view='element' )
api_element_visible_keys = ( 'id', 'email', 'username', 'total_disk_usage', 'nice_total_disk_usage' )
def __init__( self, email=None, password=None ):
self.email = email
self.password = password
@@ -60,45 +82,81 @@ class User( object, APIItem ):
# Relationships
self.histories = []
self.credentials = []
#? self.roles = []
def set_password_cleartext( self, cleartext ):
"""Set 'self.password' to the digest of 'cleartext'."""
self.password = new_secure_hash( text_type=cleartext )
"""
Set user password to the digest of `cleartext`.
"""
if User.use_pbkdf2:
self.password = galaxy.security.passwords.hash_password( cleartext )
else:
self.password = new_secure_hash( text_type=cleartext )
def check_password( self, cleartext ):
"""Check if 'cleartext' matches 'self.password' when hashed."""
return self.password == new_secure_hash( text_type=cleartext )
"""
Check if `cleartext` matches user password when hashed.
"""
return galaxy.security.passwords.check_password( cleartext, self.password )
def all_roles( self ):
"""
Return a unique list of Roles associated with this user or any of their groups.
"""
roles = [ ura.role for ura in self.roles ]
for group in [ uga.group for uga in self.groups ]:
for role in [ gra.role for gra in group.roles ]:
if role not in roles:
roles.append( role )
return roles
def get_disk_usage( self, nice_size=False ):
"""
Return byte count of disk space used by user or a human-readable
string if `nice_size` is `True`.
"""
rval = 0
if self.disk_usage is not None:
rval = self.disk_usage
if nice_size:
rval = galaxy.datatypes.data.nice_size( rval )
return rval
def set_disk_usage( self, bytes ):
"""
Manually set the disk space used by a user to `bytes`.
"""
self.disk_usage = bytes
total_disk_usage = property( get_disk_usage, set_disk_usage )
@property
def nice_total_disk_usage( self ):
"""
Return byte count of disk space used in a human-readable string.
"""
return self.get_disk_usage( nice_size=True )
def calculate_disk_usage( self ):
"""
Return byte count total of disk space used by all non-purged, non-library
HDAs in non-purged histories.
"""
# maintain a list so that we don't double count
dataset_ids = []
total = 0
# this can be a huge number and can run out of memory, so we avoid the mappers
db_session = object_session( self )
for history in db_session.query( History ).enable_eagerloads( False ).filter_by( user_id=self.id, purged=False ).yield_per( 1000 ):
for hda in db_session.query( HistoryDatasetAssociation ).enable_eagerloads( False ).filter_by( history_id=history.id, purged=False ).yield_per( 1000 ):
#TODO: def hda.counts_toward_disk_usage():
# return ( not self.dataset.purged and not self.dataset.library_associations )
if not hda.dataset.id in dataset_ids and not hda.dataset.purged and not hda.dataset.library_associations:
dataset_ids.append( hda.dataset.id )
total += hda.dataset.get_total_size()
return total
class Job( object ):
"""
A job represents a request to run a tool given input datasets, tool
@@ -574,8 +632,10 @@ class UserGroupAssociation( object ):
self.group = group
class History( object, UsesAnnotations ):
api_collection_visible_keys = ( 'id', 'name' )
api_element_visible_keys = ( 'id', 'name' )
api_collection_visible_keys = ( 'id', 'name', 'published', 'deleted' )
api_element_visible_keys = ( 'id', 'name', 'published', 'deleted', 'genome_build', 'purged' )
def __init__( self, id=None, name=None, user=None ):
self.id = id
self.name = name or "Unnamed history"
@@ -588,6 +648,7 @@ class History( object, UsesAnnotations ):
self.user = user
self.datasets = []
self.galaxy_sessions = []
def _next_hid( self ):
# TODO: override this with something in the database that ensures
# better integrity
@@ -599,18 +660,21 @@ class History( object, UsesAnnotations ):
if dataset.hid > last_hid:
last_hid = dataset.hid
return last_hid + 1
def add_galaxy_session( self, galaxy_session, association=None ):
if association is None:
self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) )
else:
self.galaxy_sessions.append( association )
def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid=True, quota=True ):
if isinstance( dataset, Dataset ):
dataset = HistoryDatasetAssociation(dataset=dataset)
object_session( self ).add( dataset )
object_session( self ).flush()
elif not isinstance( dataset, HistoryDatasetAssociation ):
raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history ( you tried to add %s )." % str( dataset )
raise TypeError, ( "You can only add Dataset and HistoryDatasetAssociation instances to a history" +
" ( you tried to add %s )." % str( dataset ) )
if parent_id:
for data in self.datasets:
if data.id == parent_id:
@@ -629,6 +693,7 @@ class History( object, UsesAnnotations ):
self.genome_build = genome_build
self.datasets.append( dataset )
return dataset
def copy( self, name=None, target_user=None, activatable=False ):
# Create new history.
if not name:
@@ -646,7 +711,7 @@ class History( object, UsesAnnotations ):
# Copy annotation.
self.copy_item_annotation( db_session, self.user, self, target_user, new_history )
#Copy Tags
# Copy Tags
new_history.copy_tags_from(target_user=target_user, source_history=self)
# Copy HDAs.
@@ -666,12 +731,17 @@ class History( object, UsesAnnotations ):
db_session.add( new_history )
db_session.flush()
return new_history
@property
def activatable_datasets( self ):
# This needs to be a list
return [ hda for hda in self.datasets if not hda.dataset.deleted ]
def get_display_name( self ):
""" History name can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format. """
"""
History name can be either a string or a unicode object.
If string, convert to unicode object assuming 'utf-8' format.
"""
history_name = self.name
if isinstance(history_name, str):
history_name = unicode(history_name, 'utf-8')
@@ -681,6 +751,7 @@ class History( object, UsesAnnotations ):
if value_mapper is None:
value_mapper = {}
rval = {}
try:
visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' )
except AttributeError:
@@ -692,26 +763,62 @@ class History( object, UsesAnnotations ):
rval[key] = value_mapper.get( key )( rval[key] )
except AttributeError:
rval[key] = None
tags_str_list = []
for tag in self.tags:
tag_str = tag.user_tname
if tag.value is not None:
tag_str += ":" + tag.user_value
tags_str_list.append( tag_str )
rval['tags'] = tags_str_list
rval['model_class'] = self.__class__.__name__
return rval
def set_from_dict( self, new_data ):
#AKA: set_api_value
"""
Set object attributes to the values in dictionary new_data limiting
to only those keys in api_element_visible_keys.
Returns a dictionary of the keys, values that have been changed.
"""
# precondition: keys are proper, values are parsed and validated
changed = {}
# unknown keys are ignored here
for key in [ k for k in new_data.keys() if k in self.api_element_visible_keys ]:
new_val = new_data[ key ]
old_val = self.__getattribute__( key )
if new_val == old_val:
continue
self.__setattr__( key, new_val )
changed[ key ] = new_val
return changed
@property
def get_disk_size_bytes( self ):
return self.get_disk_size( nice_size=False )
def unhide_datasets( self ):
for dataset in self.datasets:
dataset.mark_unhidden()
def resume_paused_jobs( self ):
for dataset in self.datasets:
job = dataset.creating_job
if job is not None and job.state == Job.states.PAUSED:
job.set_state(Job.states.NEW)
def get_disk_size( self, nice_size=False ):
# unique datasets only
db_session = object_session( self )
rval = db_session.query( func.sum( db_session.query( HistoryDatasetAssociation.dataset_id, Dataset.total_size ).join( Dataset )
.filter( HistoryDatasetAssociation.table.c.history_id == self.id )
.filter( HistoryDatasetAssociation.purged != True )
.filter( Dataset.purged != True )
.distinct().subquery().c.total_size ) ).first()[0]
rval = db_session.query(
func.sum( db_session.query( HistoryDatasetAssociation.dataset_id, Dataset.total_size ).join( Dataset )
.filter( HistoryDatasetAssociation.table.c.history_id == self.id )
.filter( HistoryDatasetAssociation.purged != True )
.filter( Dataset.purged != True )
.distinct().subquery().c.total_size ) ).first()[0]
if rval is None:
rval = 0
if nice_size:
@@ -724,6 +831,7 @@ class History( object, UsesAnnotations ):
new_shta.user = target_user
self.tags.append(new_shta)
class HistoryUserShareAssociation( object ):
def __init__( self ):
self.history = None
@@ -795,7 +903,7 @@ class Quota( object, APIItem ):
if self.bytes == -1:
return "unlimited"
else:
return util.nice_size( self.bytes )
return nice_size( self.bytes )
class DefaultQuotaAssociation( Quota, APIItem ):
api_element_visible_keys = ( 'type', )
@@ -920,7 +1028,10 @@ class Dataset( object ):
return self.object_store.get_filename( self, dir_only=True, extra_dir=self._extra_files_path or "dataset_%d_files" % self.id )
def _calculate_size( self ):
if self.external_filename:
return os.path.getsize(self.external_filename)
try:
return os.path.getsize(self.external_filename)
except OSError:
return 0
else:
return self.object_store.size(self)
def get_size( self, nice_size=False ):
@@ -965,7 +1076,7 @@ class Dataset( object ):
if not self.has_data():
return False
try:
return util.is_multi_byte( codecs.open( self.file_name, 'r', 'utf-8' ).read( 100 ) )
return is_multi_byte( codecs.open( self.file_name, 'r', 'utf-8' ).read( 100 ) )
except UnicodeDecodeError:
return False
# FIXME: sqlalchemy will replace this
@@ -1321,10 +1432,9 @@ class DatasetInstance( object ):
with entries of type
(<datasource_type> : {<datasource_name>, <indexing_message>}).
"""
track_type, data_sources = self.datatype.get_track_type()
data_sources_dict = {}
msg = None
for source_type, source_list in data_sources.iteritems():
for source_type, source_list in self.datatype.data_sources.iteritems():
data_source = None
if source_type == "data_standalone":
# Nothing to do.
@@ -1376,7 +1486,11 @@ class DatasetInstance( object ):
return msg
class HistoryDatasetAssociation( DatasetInstance ):
class HistoryDatasetAssociation( DatasetInstance, UsesAnnotations ):
"""
Resource class that creates a relation between a dataset and a user history.
"""
def __init__( self,
hid = None,
history = None,
@@ -1384,6 +1498,9 @@ class HistoryDatasetAssociation( DatasetInstance ):
copied_from_library_dataset_dataset_association = None,
sa_session = None,
**kwd ):
"""
Create a a new HDA and associate it with the given history.
"""
# FIXME: sa_session is must be passed to DataSetInstance if the create_dataset
# parameter is True so that the new object can be flushed. Is there a better way?
DatasetInstance.__init__( self, sa_session=sa_session, **kwd )
@@ -1392,7 +1509,11 @@ class HistoryDatasetAssociation( DatasetInstance ):
self.history = history
self.copied_from_history_dataset_association = copied_from_history_dataset_association
self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
def copy( self, copy_children = False, parent_id = None ):
"""
Create a copy of this HDA.
"""
hda = HistoryDatasetAssociation( hid=self.hid,
name=self.name,
info=self.info,
@@ -1419,13 +1540,20 @@ class HistoryDatasetAssociation( DatasetInstance ):
hda.set_peek()
object_session( self ).flush()
return hda
def to_library_dataset_dataset_association( self, trans, target_folder, replace_dataset=None, parent_id=None, user=None, roles=[], ldda_message='' ):
def to_library_dataset_dataset_association( self, trans, target_folder,
replace_dataset=None, parent_id=None, user=None, roles=[], ldda_message='' ):
"""
Copy this HDA to a library optionally replacing an existing LDDA.
"""
if replace_dataset:
# The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version.
# The replace_dataset param ( when not None ) refers to a LibraryDataset that
# is being replaced with a new version.
library_dataset = replace_dataset
else:
# If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new
# LibraryDataset, and the current user's DefaultUserPermissions will be applied to the associated Dataset.
# If replace_dataset is None, the Library level permissions will be taken from the folder and
# applied to the new LibraryDataset, and the current user's DefaultUserPermissions will be applied
# to the associated Dataset.
library_dataset = LibraryDataset( folder=target_folder, name=self.name, info=self.info )
object_session( self ).add( library_dataset )
object_session( self ).flush()
@@ -1450,7 +1578,8 @@ class HistoryDatasetAssociation( DatasetInstance ):
object_session( self ).flush()
# If roles were selected on the upload form, restrict access to the Dataset to those roles
for role in roles:
dp = trans.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, ldda.dataset, role )
dp = trans.model.DatasetPermissions( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action,
ldda.dataset, role )
trans.sa_session.add( dp )
trans.sa_session.flush()
# Must set metadata after ldda flushed, as MetadataFiles require ldda.id
@@ -1475,30 +1604,47 @@ class HistoryDatasetAssociation( DatasetInstance ):
ldda.set_peek()
object_session( self ).flush()
return ldda
def clear_associated_files( self, metadata_safe = False, purge = False ):
"""
"""
# metadata_safe = True means to only clear when assoc.metadata_safe == False
for assoc in self.implicitly_converted_datasets:
if not assoc.deleted and ( not metadata_safe or not assoc.metadata_safe ):
assoc.clear( purge = purge )
for assoc in self.implicitly_converted_parent_datasets:
assoc.clear( purge = purge, delete_dataset = False )
def get_display_name( self ):
## Name can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format.
"""
Return the name of this HDA in either ascii or utf-8 encoding.
"""
# Name can be either a string or a unicode object.
# If string, convert to unicode object assuming 'utf-8' format.
hda_name = self.name
if isinstance(hda_name, str):
hda_name = unicode(hda_name, 'utf-8')
return hda_name
def get_access_roles( self, trans ):
"""
Return The access roles associated with this HDA's dataset.
"""
return self.dataset.get_access_roles( trans )
def quota_amount( self, user ):
"""
If the user has multiple instances of this dataset, it will not affect their disk usage statistic.
Return the disk space used for this HDA relevant to user quotas.
If the user has multiple instances of this dataset, it will not affect their
disk usage statistic.
"""
rval = 0
# Anon users are handled just by their single history size.
if not user:
return rval
# Gets an HDA and its children's disk usage, if the user does not already have an association of the same dataset
# Gets an HDA and its children's disk usage, if the user does not already
# have an association of the same dataset
if not self.dataset.library_associations and not self.purged and not self.dataset.purged:
for hda in self.dataset.history_associations:
if hda.id == self.id:
@@ -1510,14 +1656,18 @@ class HistoryDatasetAssociation( DatasetInstance ):
for child in self.children:
rval += child.get_disk_usage( user )
return rval
def get_api_value( self, view='collection' ):
"""
Return attributes of this HDA that are exposed using the API.
"""
# Since this class is a proxy to rather complex attributes we want to
# display in other objects, we can't use the simpler method used by
# other model classes.
hda = self
rval = dict( id = hda.id,
hda_ldda = 'hda',
uuid = ( lambda uuid: str( uuid ) if uuid else None )( hda.dataset.uuid ),
history_id = hda.history.id,
hid = hda.hid,
file_ext = hda.ext,
peek = ( lambda hda: hda.display_peek() if hda.peek and hda.peek != 'no peek' else None )( hda ),
@@ -1532,6 +1682,11 @@ class HistoryDatasetAssociation( DatasetInstance ):
genome_build = hda.dbkey,
misc_info = hda.info,
misc_blurb = hda.blurb )
if hda.history is not None:
rval['history_id'] = hda.history.id
rval[ 'peek' ] = to_unicode( hda.display_peek() )
for name, spec in hda.metadata.spec.items():
val = hda.metadata.get( name )
if isinstance( val, MetadataFile ):
@@ -1542,6 +1697,33 @@ class HistoryDatasetAssociation( DatasetInstance ):
rval['metadata_' + name] = val
return rval
def set_from_dict( self, new_data ):
#AKA: set_api_value
"""
Set object attributes to the values in dictionary new_data limiting
to only the following keys: name, deleted, visible, genome_build,
info, and blurb.
Returns a dictionary of the keys, values that have been changed.
"""
# precondition: keys are proper, values are parsed and validated
#NOTE!: does not handle metadata
editable_keys = ( 'name', 'deleted', 'visible', 'dbkey', 'info', 'blurb' )
changed = {}
# unknown keys are ignored here
for key in [ k for k in new_data.keys() if k in editable_keys ]:
new_val = new_data[ key ]
old_val = self.__getattribute__( key )
if new_val == old_val:
continue
self.__setattr__( key, new_val )
changed[ key ] = new_val
return changed
class HistoryDatasetAssociationDisplayAtAuthorization( object ):
def __init__( self, hda=None, user=None, site=None ):
self.history_dataset_association = hda
@@ -1557,7 +1739,7 @@ class HistoryDatasetAssociationSubset( object ):
class Library( object, APIItem ):
permitted_actions = get_permitted_actions( filter='LIBRARY' )
api_collection_visible_keys = ( 'id', 'name' )
api_element_visible_keys = ( 'name', 'description', 'synopsis' )
api_element_visible_keys = ( 'id', 'deleted', 'name', 'description', 'synopsis' )
def __init__( self, name=None, description=None, synopsis=None, root_folder=None ):
self.name = name or "Unnamed library"
self.description = description
@@ -1624,7 +1806,7 @@ class Library( object, APIItem ):
return name
class LibraryFolder( object, APIItem ):
api_element_visible_keys = ( 'id', 'name', 'description', 'item_count', 'genome_build' )
api_element_visible_keys = ( 'id', 'parent_id', 'name', 'description', 'item_count', 'genome_build' )
def __init__( self, name=None, description=None, item_count=0, order_id=None ):
self.name = name or "Unnamed folder"
self.description = description
@@ -1704,8 +1886,18 @@ class LibraryFolder( object, APIItem ):
else:
template = info_association.template
rval['data_template'] = template.name
rval['library_path'] = self.library_path
rval['parent_library_id'] = self.parent_library.id
return rval
@property
def library_path(self):
l_path = []
f = self
while f.parent:
l_path.insert(0, f.name)
f = f.parent
return l_path
@property
def parent_library( self ):
f = self
while f.parent:
@@ -1767,6 +1959,7 @@ class LibraryDataset( object ):
rval = dict( id = self.id,
ldda_id = ldda.id,
folder_id = self.folder_id,
model_class = self.__class__.__name__,
name = ldda.name,
file_name = ldda.file_name,
@@ -1880,6 +2073,45 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
if restrict:
return None, inherited
return self.library_dataset.folder.get_info_association( inherited=True )
def get_api_value( self, view='collection' ):
# Since this class is a proxy to rather complex attributes we want to
# display in other objects, we can't use the simpler method used by
# other model classes.
ldda = self
try:
file_size = int( ldda.get_size() )
except OSError:
file_size = 0
rval = dict( id = ldda.id,
hda_ldda = 'ldda',
model_class = self.__class__.__name__,
name = ldda.name,
deleted = ldda.deleted,
visible = ldda.visible,
state = ldda.state,
library_dataset_id = ldda.library_dataset_id,
file_size = file_size,
file_name = ldda.file_name,
data_type = ldda.ext,
genome_build = ldda.dbkey,
misc_info = ldda.info,
misc_blurb = ldda.blurb )
if ldda.dataset.uuid is None:
rval['uuid'] = None
else:
rval['uuid'] = str(ldda.dataset.uuid)
rval['parent_library_id'] = ldda.library_dataset.folder.parent_library.id
if ldda.extended_metadata is not None:
rval['extended_metadata'] = ldda.extended_metadata.data
for name, spec in ldda.metadata.spec.items():
val = ldda.metadata.get( name )
if isinstance( val, MetadataFile ):
val = val.file_name
# If no value for metadata, look in datatype for metadata.
elif val == None and hasattr( ldda.datatype, name ):
val = getattr( ldda.datatype, name )
rval['metadata_' + name] = val
return rval
def get_template_widgets( self, trans, get_contents=True ):
# See if we have any associated templatesThe get_contents
# param is passed by callers that are inheriting a template - these
@@ -2069,8 +2301,8 @@ class UCI( object ):
self.user = None
class StoredWorkflow( object, APIItem):
api_collection_visible_keys = ( 'id', 'name' )
api_element_visible_keys = ( 'id', 'name' )
api_collection_visible_keys = ( 'id', 'name', 'published' )
api_element_visible_keys = ( 'id', 'name', 'published' )
def __init__( self ):
self.id = None
self.user = None
@@ -2086,6 +2318,18 @@ class StoredWorkflow( object, APIItem):
new_swta.user = target_user
self.tags.append(new_swta)
def get_api_value( self, view='collection', value_mapper = None ):
rval = APIItem.get_api_value(self, view=view, value_mapper = value_mapper)
tags_str_list = []
for tag in self.tags:
tag_str = tag.user_tname
if tag.value is not None:
tag_str += ":" + tag.user_value
tags_str_list.append( tag_str )
rval['tags'] = tags_str_list
return rval
class Workflow( object ):
def __init__( self ):
self.user = None
@@ -2198,7 +2442,7 @@ class FormDefinition( object, APIItem ):
Return the list of widgets that comprise a form definition,
including field contents if any.
'''
params = util.Params( kwd )
params = Params( kwd )
widgets = []
for index, field in enumerate( self.fields ):
field_type = field[ 'type' ]
@@ -2214,7 +2458,7 @@ class FormDefinition( object, APIItem ):
if field_type == 'CheckboxField':
value = CheckboxField.is_checked( params.get( field_name, False ) )
else:
value = util.restore_text( params.get( field_name, '' ) )
value = restore_text( params.get( field_name, '' ) )
elif contents:
try:
# This field has a saved value.
@@ -2429,7 +2673,7 @@ All samples in state: %(sample_state)s
frm = 'galaxy-no-reply@' + host
subject = "Galaxy Sample Tracking notification: '%s' sequencing request" % self.name
try:
util.send_mail( frm, to, subject, body, trans.app.config )
send_mail( frm, to, subject, body, trans.app.config )
comments = "Email notification sent to %s." % ", ".join( to ).strip().strip( ',' )
except Exception,e:
comments = "Email notification failed. (%s)" % str(e)
@@ -2471,7 +2715,7 @@ class ExternalService( object ):
if data_transfer_protocol == self.data_transfer_protocol.SCP:
scp_configs = {}
automatic_transfer = data_transfer_obj.config.get( 'automatic_transfer', 'false' )
scp_configs[ 'automatic_transfer' ] = util.string_as_bool( automatic_transfer )
scp_configs[ 'automatic_transfer' ] = galaxy.util.string_as_bool( automatic_transfer )
scp_configs[ 'host' ] = self.form_values.content.get( data_transfer_obj.config.get( 'host', '' ), '' )
scp_configs[ 'user_name' ] = self.form_values.content.get( data_transfer_obj.config.get( 'user_name', '' ), '' )
scp_configs[ 'password' ] = self.form_values.content.get( data_transfer_obj.config.get( 'password', '' ), '' )
@@ -2481,7 +2725,7 @@ class ExternalService( object ):
if data_transfer_protocol == self.data_transfer_protocol.HTTP:
http_configs = {}
automatic_transfer = data_transfer_obj.config.get( 'automatic_transfer', 'false' )
http_configs[ 'automatic_transfer' ] = util.string_as_bool( automatic_transfer )
http_configs[ 'automatic_transfer' ] = galaxy.util.string_as_bool( automatic_transfer )
self.data_transfer[ self.data_transfer_protocol.HTTP ] = http_configs
def populate_actions( self, trans, item, param_dict=None ):
return self.get_external_service_type( trans ).actions.populate( self, item, param_dict=param_dict )
@@ -3210,6 +3454,31 @@ class ToolShedRepository( object ):
return 'repository_dependencies' in self.metadata
return False
@property
def requires_prior_installation_of( self ):
"""
Return a list of repository dependency tuples like (tool_shed, name, owner, changeset_revision, prior_installation_required) for this
repository's repository dependencies where prior_installation_required is True. By definition, repository dependencies are required to
be installed in order for this repository to function correctly. However, those repository dependencies that are defined for this
repository with prior_installation_required set to True place them in a special category in that the required repositories must be
installed before this repository is installed. Among other things, this enables these "special" repository dependencies to include
information that enables the successful intallation of this repository. This method is not used during the initial installation of
this repository, but only after it has been installed (metadata must be set for this repository in order for this method to be useful).
"""
required_rd_tups_that_must_be_installed = []
if self.has_repository_dependencies:
rd_tups = self.metadata[ 'repository_dependencies' ][ 'repository_dependencies' ]
for rd_tup in rd_tups:
if len( rd_tup ) == 4:
# For backward compatibility to the 12/20/12 Galaxy release, default prior_installation_required to False.
tool_shed, name, owner, changeset_revision = rd_tup
prior_installation_required = False
elif len( rd_tup ) == 5:
tool_shed, name, owner, changeset_revision, prior_installation_required = rd_tup
prior_installation_required = galaxy.util.asbool( str( prior_installation_required ) )
if prior_installation_required:
required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, prior_installation_required ) )
return required_rd_tups_that_must_be_installed
@property
def includes_data_managers( self ):
if self.metadata:
return bool( len( self.metadata.get( 'data_manager', {} ).get( 'data_managers', {} ) ) )
@@ -3301,7 +3570,7 @@ class ToolShedRepository( object ):
"""Return the repository's tool dependencies that are currently installed."""
installed_dependencies = []
for tool_dependency in self.tool_dependencies:
if tool_dependency.status == ToolDependency.installation_status.INSTALLED:
if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, ToolDependency.installation_status.ERROR ]:
installed_dependencies.append( tool_dependency )
return installed_dependencies
@property
@@ -3403,7 +3672,8 @@ class ToolDependency( object ):
self.tool_shed_repository.name,
self.tool_shed_repository.installed_changeset_revision )
class ToolVersion( object ):
class ToolVersion( object, APIItem ):
api_element_visible_keys = ( 'id', 'tool_shed_repository' )
def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ):
self.id = id
self.create_time = create_time
@@ -3460,6 +3730,15 @@ class ToolVersion( object ):
return version_ids
return [ tool_version.tool_id for tool_version in self.get_versions( app ) ]
def get_api_value( self, view='element' ):
rval = APIItem.get_api_value(self, view)
rval['tool_name'] = self.tool_id
for a in self.parent_tool_association:
rval['parent_tool_id'] = a.parent_id
for a in self.child_tool_association:
rval['child_tool_id'] = a.tool_id
return rval
class ToolVersionAssociation( object ):
def __init__( self, id=None, tool_id=None, parent_id=None ):
self.id = id
lib/galaxy/model/custom_types.py
+20 -22
View File
@@ -11,9 +11,9 @@ from galaxy.util.bunch import Bunch
from galaxy.util.aliaspickler import AliasPickleModule
# For monkeypatching BIGINT
import sqlalchemy.databases.sqlite
import sqlalchemy.databases.postgres
import sqlalchemy.databases.mysql
import sqlalchemy.dialects.sqlite
import sqlalchemy.dialects.postgresql
import sqlalchemy.dialects.mysql
import logging
log = logging.getLogger( __name__ )
@@ -39,7 +39,7 @@ class JSONType( TypeDecorator ):
Defines a JSONType for SQLAlchemy. Takes a primitive as input and
JSONifies it. This should replace PickleType throughout Galaxy.
"""
impl = Binary
impl = LargeBinary
def process_bind_param( self, value, dialect ):
if value is None:
@@ -86,8 +86,6 @@ class MetadataType( JSONType ):
ret = None
return ret
class UUIDType(TypeDecorator):
"""
Platform-independent UUID type.
@@ -128,24 +126,24 @@ class TrimmedString( TypeDecorator ):
return value
class BigInteger( Integer ):
"""
A type for bigger ``int`` integers.
#class BigInteger( Integer ):
#"""
#A type for bigger ``int`` integers.
Typically generates a ``BIGINT`` in DDL, and otherwise acts like
a normal :class:`Integer` on the Python side.
#Typically generates a ``BIGINT`` in DDL, and otherwise acts like
#a normal :class:`Integer` on the Python side.
"""
#"""
class BIGINT( BigInteger ):
"""The SQL BIGINT type."""
#class BIGINT( BigInteger ):
#"""The SQL BIGINT type."""
class SLBigInteger( BigInteger ):
def get_col_spec( self ):
return "BIGINT"
#class SLBigInteger( BigInteger ):
#def get_col_spec( self ):
#return "BIGINT"
sqlalchemy.databases.sqlite.SLBigInteger = SLBigInteger
sqlalchemy.databases.sqlite.colspecs[BigInteger] = SLBigInteger
sqlalchemy.databases.sqlite.ischema_names['BIGINT'] = SLBigInteger
sqlalchemy.databases.postgres.colspecs[BigInteger] = sqlalchemy.databases.postgres.PGBigInteger
sqlalchemy.databases.mysql.colspecs[BigInteger] = sqlalchemy.databases.mysql.MSBigInteger
#sqlalchemy.dialects.sqlite.SLBigInteger = SLBigInteger
#sqlalchemy.dialects.sqlite.colspecs[BigInteger] = SLBigInteger
#sqlalchemy.dialects.sqlite.ischema_names['BIGINT'] = SLBigInteger
#sqlalchemy.dialects.postgres.colspecs[BigInteger] = sqlalchemy.dialects.postgres.PGBigInteger
#sqlalchemy.dialects.mysql.colspecs[BigInteger] = sqlalchemy.dialects.mysql.MSBigInteger
lib/galaxy/model/item_attrs.py
+2
View File
@@ -188,4 +188,6 @@ class APIItem:
rval[key] = get_value( key, item )
except AttributeError:
rval[key] = None
rval['model_class'] = self.__class__.__name__
return rval
lib/galaxy/model/mapping.py
+772 -740
View File
File diff suppressed because it is too large Load Diff
lib/galaxy/model/migrate/check.py
+10 -6
View File
@@ -1,19 +1,23 @@
import sys, os.path, logging
import sys
import os.path
import logging
from galaxy import eggs
import pkg_resources
pkg_resources.require( "sqlalchemy-migrate" )
eggs.require( "SQLAlchemy" )
eggs.require( "decorator" ) #Required by sqlalchemy-migrate
eggs.require( "Tempita " ) #Required by sqlalchemy-migrate
eggs.require( "sqlalchemy-migrate" )
from migrate.versioning import repository, schema
from sqlalchemy import *
from sqlalchemy.exc import NoSuchTableError
from migrate.versioning import repository, schema
log = logging.getLogger( __name__ )
# path relative to galaxy
migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 )
migrate_repository = repository.Repository( migrate_repository_directory )
dialect_to_egg = {
dialect_to_egg = {
"sqlite" : "pysqlite>=2",
"postgres" : "psycopg2",
"mysql" : "MySQL_python"
@@ -34,7 +38,7 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N
try:
egg = dialect_to_egg[dialect]
try:
pkg_resources.require( egg )
eggs.require( egg )
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
lib/galaxy/model/migrate/versions/0001_initial_tables.py
+15 -13
View File
@@ -10,7 +10,7 @@ from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
# Tables as of changeset 1464:c7acaa1bb88f
User_table = Table( "galaxy_user", metadata,
@@ -20,7 +20,7 @@ User_table = Table( "galaxy_user", metadata,
Column( "email", TrimmedString( 255 ), nullable=False ),
Column( "password", TrimmedString( 40 ), nullable=False ),
Column( "external", Boolean, default=False ) )
History_table = Table( "history", metadata,
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
@@ -32,7 +32,7 @@ History_table = Table( "history", metadata,
Column( "purged", Boolean, index=True, default=False ),
Column( "genome_build", TrimmedString( 40 ) ) )
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata,
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
@@ -51,7 +51,7 @@ HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata
Column( "deleted", Boolean, index=True, default=False ),
Column( "visible", Boolean ) )
Dataset_table = Table( "dataset", metadata,
Dataset_table = Table( "dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
@@ -89,7 +89,7 @@ Job_table = Table( "job", metadata,
Column( "tool_version", TEXT, default="1.0.0" ),
Column( "state", String( 64 ) ),
Column( "info", TrimmedString( 255 ) ),
Column( "command_line", TEXT ),
Column( "command_line", TEXT ),
Column( "param_filename", String( 1024 ) ),
Column( "runner_name", String( 255 ) ),
Column( "stdout", TEXT ),
@@ -98,26 +98,26 @@ Job_table = Table( "job", metadata,
Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ),
Column( "job_runner_name", String( 255 ) ),
Column( "job_runner_external_id", String( 255 ) ) )
JobParameter_table = Table( "job_parameter", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
Column( "name", String(255) ),
Column( "value", TEXT ) )
JobToInputDatasetAssociation_table = Table( "job_to_input_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
JobToOutputDatasetAssociation_table = Table( "job_to_output_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
Event_table = Table( "event", metadata,
Event_table = Table( "event", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -200,12 +200,14 @@ StoredWorkflowUserShareAssociation_table = Table( "stored_workflow_user_share_co
StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata,
Column( "id", Integer, primary_key=True ),
Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "order_index", Integer ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.create_all()
## def downgrade():
## def downgrade(migrate_engine):
## metadata.bind = migrate_engine
## # Operations to reverse the above upgrade go here.
## pass
lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
+6 -4
View File
@@ -10,10 +10,10 @@ from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
# New table in changeset 1568:0b022adfdc34
MetadataFile_table = Table( "metadata_file", metadata,
MetadataFile_table = Table( "metadata_file", metadata,
Column( "id", Integer, primary_key=True ),
Column( "name", TEXT ),
Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ),
@@ -22,10 +22,12 @@ MetadataFile_table = Table( "metadata_file", metadata,
Column( "deleted", Boolean, index=True, default=False ),
Column( "purged", Boolean, index=True, default=False ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
MetadataFile_table.create()
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
MetadataFile_table.drop()
lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
+82 -78
View File
@@ -19,46 +19,7 @@ log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
if migrate_engine.name == 'postgres':
# http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html
from sqlalchemy.databases import postgres
class PGCascadeSchemaDropper(postgres.PGSchemaDropper):
def visit_table(self, table):
for column in table.columns:
if column.default is not None:
self.traverse_single(column.default)
self.append("\nDROP TABLE " +
self.preparer.format_table(table) +
" CASCADE")
self.execute()
postgres.dialect.schemadropper = PGCascadeSchemaDropper
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def localtimestamp():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def boolean_false():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
metadata = MetaData()
# New tables as of changeset 2341:5498ac35eedd
Group_table = Table( "galaxy_group", metadata,
@@ -68,7 +29,7 @@ Group_table = Table( "galaxy_group", metadata,
Column( "name", String( 255 ), index=True, unique=True ),
Column( "deleted", Boolean, index=True, default=False ) )
UserGroupAssociation_table = Table( "user_group_association", metadata,
UserGroupAssociation_table = Table( "user_group_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "group_id", Integer, ForeignKey( "galaxy_group.id" ), index=True ),
@@ -168,7 +129,7 @@ DefaultHistoryPermissions_table = Table( "default_history_permissions", metadata
Column( "action", TEXT ),
Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
LibraryDataset_table = Table( "library_dataset", metadata,
LibraryDataset_table = Table( "library_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected
Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ),
@@ -179,7 +140,7 @@ LibraryDataset_table = Table( "library_dataset", metadata,
Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "deleted", Boolean, index=True, default=False ) )
LibraryDatasetDatasetAssociation_table = Table( "library_dataset_dataset_association", metadata,
LibraryDatasetDatasetAssociation_table = Table( "library_dataset_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
@@ -236,7 +197,7 @@ LibraryItemInfoTemplateElement_table = Table( "library_item_info_template_elemen
Column( "options", JSONType() ),
Column( "library_item_info_template_id", Integer, ForeignKey( "library_item_info_template.id" ) ) )
Index( "ix_liite_library_item_info_template_id", LibraryItemInfoTemplateElement_table.c.library_item_info_template_id )
LibraryItemInfoTemplate_table = Table( "library_item_info_template", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
@@ -343,25 +304,52 @@ JobExternalOutputMetadata_table = Table( "job_external_output_metadata", metadat
Column( "job_runner_external_pid", String( 255 ) ) )
Index( "ix_jeom_library_dataset_dataset_association_id", JobExternalOutputMetadata_table.c.library_dataset_dataset_association_id )
def upgrade():
def upgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
# Load existing tables
metadata.reflect()
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def localtimestamp():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def boolean_false():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
# Add 2 new columns to the galaxy_user table
try:
User_table = Table( "galaxy_user", metadata, autoload=True )
except NoSuchTableError:
User_table = None
log.debug( "Failed loading table galaxy_user" )
if User_table:
if User_table is not None:
try:
col = Column( 'deleted', Boolean, index=True, default=False )
col.create( User_table )
col.create( User_table, index_name='ix_user_deleted')
assert col is User_table.c.deleted
except Exception, e:
log.debug( "Adding column 'deleted' to galaxy_user table failed: %s" % ( str( e ) ) )
try:
col = Column( 'purged', Boolean, index=True, default=False )
col.create( User_table )
col.create( User_table, index_name='ix_user_purged')
assert col is User_table.c.purged
except Exception, e:
log.debug( "Adding column 'purged' to galaxy_user table failed: %s" % ( str( e ) ) )
@@ -371,10 +359,10 @@ def upgrade():
except NoSuchTableError:
HistoryDatasetAssociation_table = None
log.debug( "Failed loading table history_dataset_association" )
if HistoryDatasetAssociation_table:
if HistoryDatasetAssociation_table is not None:
try:
col = Column( 'copied_from_library_dataset_dataset_association_id', Integer, index=True, nullable=True )
col.create( HistoryDatasetAssociation_table )
col = Column( 'copied_from_library_dataset_dataset_association_id', Integer, nullable=True )
col.create( HistoryDatasetAssociation_table)
assert col is HistoryDatasetAssociation_table.c.copied_from_library_dataset_dataset_association_id
except Exception, e:
log.debug( "Adding column 'copied_from_library_dataset_dataset_association_id' to history_dataset_association table failed: %s" % ( str( e ) ) )
@@ -384,23 +372,23 @@ def upgrade():
except NoSuchTableError:
MetadataFile_table = None
log.debug( "Failed loading table metadata_file" )
if MetadataFile_table:
if MetadataFile_table is not None:
try:
col = Column( 'lda_id', Integer, index=True, nullable=True )
col.create( MetadataFile_table )
col.create( MetadataFile_table, index_name='ix_metadata_file_lda_id')
assert col is MetadataFile_table.c.lda_id
except Exception, e:
log.debug( "Adding column 'lda_id' to metadata_file table failed: %s" % ( str( e ) ) )
# Add 1 new column to the stored_workflow table - changeset 2328
try:
StoredWorkflow_table = Table( "stored_workflow", metadata,
Column( "latest_workflow_id", Integer,
StoredWorkflow_table = Table( "stored_workflow", metadata,
Column( "latest_workflow_id", Integer,
ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ),
autoload=True, useexisting=True )
autoload=True, extend_existing=True )
except NoSuchTableError:
StoredWorkflow_table = None
log.debug( "Failed loading table stored_workflow" )
if StoredWorkflow_table:
if StoredWorkflow_table is not None:
try:
col = Column( 'importable', Boolean, default=False )
col.create( StoredWorkflow_table )
@@ -413,7 +401,7 @@ def upgrade():
except NoSuchTableError:
Job_table = None
log.debug( "Failed loading table job" )
if Job_table:
if Job_table is not None:
try:
i = Index( 'ix_job_state', Job_table.c.state )
i.create()
@@ -432,7 +420,7 @@ def upgrade():
except NoSuchTableError:
LibraryDatasetDatasetAssociation_table = None
log.debug( "Failed loading table library_dataset_dataset_association" )
if HistoryDatasetAssociation_table and LibraryDatasetDatasetAssociation_table:
if HistoryDatasetAssociation_table is not None and LibraryDatasetDatasetAssociation_table is not None:
try:
cons = ForeignKeyConstraint( [HistoryDatasetAssociation_table.c.copied_from_library_dataset_dataset_association_id],
[LibraryDatasetDatasetAssociation_table.c.id],
@@ -452,22 +440,24 @@ def upgrade():
except NoSuchTableError:
LibraryDatasetDatasetAssociation_table = None
log.debug( "Failed loading table library_dataset_dataset_association" )
if MetadataFile_table and LibraryDatasetDatasetAssociation_table:
try:
cons = ForeignKeyConstraint( [MetadataFile_table.c.lda_id],
[LibraryDatasetDatasetAssociation_table.c.id],
name='metadata_file_lda_id_fkey' )
# Create the constraint
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'metadata_file_lda_id_fkey' to table 'metadata_file' failed: %s" % ( str( e ) ) )
if migrate_engine.name != 'sqlite':
#Sqlite can't alter table add foreign key.
if MetadataFile_table is not None and LibraryDatasetDatasetAssociation_table is not None:
try:
cons = ForeignKeyConstraint( [MetadataFile_table.c.lda_id],
[LibraryDatasetDatasetAssociation_table.c.id],
name='metadata_file_lda_id_fkey' )
# Create the constraint
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'metadata_file_lda_id_fkey' to table 'metadata_file' failed: %s" % ( str( e ) ) )
# Make sure we have at least 1 user
cmd = "SELECT * FROM galaxy_user;"
users = db_session.execute( cmd ).fetchall()
if users:
cmd = "SELECT * FROM role;"
roles = db_session.execute( cmd ).fetchall()
if not roles:
if not roles:
# Create private roles for each user - pass 1
cmd = \
"INSERT INTO role " + \
@@ -513,7 +503,7 @@ def upgrade():
"ORDER BY galaxy_user.id;"
cmd = cmd % nextval('default_user_permissions')
db_session.execute( cmd )
# Create default history permissions for each active history associated with a user
# Create default history permissions for each active history associated with a user
cmd = \
"INSERT INTO default_history_permissions " + \
@@ -543,7 +533,21 @@ def upgrade():
cmd = cmd % ( nextval('dataset_permissions'), localtimestamp(), localtimestamp(), boolean_false() )
db_session.execute( cmd )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
if migrate_engine.name == 'postgres':
# http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html
from sqlalchemy.databases import postgres
class PGCascadeSchemaDropper(postgres.PGSchemaDropper):
def visit_table(self, table):
for column in table.columns:
if column.default is not None:
self.traverse_single(column.default)
self.append("\nDROP TABLE " +
self.preparer.format_table(table) +
" CASCADE")
self.execute()
postgres.dialect.schemadropper = PGCascadeSchemaDropper
# Load existing tables
metadata.reflect()
# NOTE: all new data added in the upgrade method is eliminated here via table drops
@@ -558,7 +562,7 @@ def downgrade():
except NoSuchTableError:
LibraryDatasetDatasetAssociation_table = None
log.debug( "Failed loading table library_dataset_dataset_association" )
if MetadataFile_table and LibraryDatasetDatasetAssociation_table:
if MetadataFile_table is not None and LibraryDatasetDatasetAssociation_table is not None:
try:
cons = ForeignKeyConstraint( [MetadataFile_table.c.lda_id],
[LibraryDatasetDatasetAssociation_table.c.id],
@@ -578,7 +582,7 @@ def downgrade():
except NoSuchTableError:
LibraryDatasetDatasetAssociation_table = None
log.debug( "Failed loading table library_dataset_dataset_association" )
if HistoryDatasetAssociation_table and LibraryDatasetDatasetAssociation_table:
if HistoryDatasetAssociation_table is not None and LibraryDatasetDatasetAssociation_table is not None:
try:
cons = ForeignKeyConstraint( [HistoryDatasetAssociation_table.c.copied_from_library_dataset_dataset_association_id],
[LibraryDatasetDatasetAssociation_table.c.id],
@@ -718,7 +722,7 @@ def downgrade():
except NoSuchTableError:
Job_table = None
log.debug( "Failed loading table job" )
if Job_table:
if Job_table is not None:
try:
i = Index( 'ix_job_state', Job_table.c.state )
i.drop()
@@ -730,7 +734,7 @@ def downgrade():
except NoSuchTableError:
StoredWorkflow_table = None
log.debug( "Failed loading table stored_workflow" )
if StoredWorkflow_table:
if StoredWorkflow_table is not None:
try:
col = StoredWorkflow_table.c.importable
col.drop()
@@ -742,7 +746,7 @@ def downgrade():
except NoSuchTableError:
MetadataFile_table = None
log.debug( "Failed loading table metadata_file" )
if MetadataFile_table:
if MetadataFile_table is not None:
try:
col = MetadataFile_table.c.lda_id
col.drop()
@@ -754,7 +758,7 @@ def downgrade():
except NoSuchTableError:
HistoryDatasetAssociation_table = None
log.debug( "Failed loading table history_dataset_association" )
if HistoryDatasetAssociation_table:
if HistoryDatasetAssociation_table is not None:
try:
col = HistoryDatasetAssociation_table.c.copied_from_library_dataset_dataset_association_id
col.drop()
@@ -766,7 +770,7 @@ def downgrade():
except NoSuchTableError:
User_table = None
log.debug( "Failed loading table galaxy_user" )
if User_table:
if User_table is not None:
try:
col = User_table.c.deleted
col.drop()
lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py
+15 -14
View File
@@ -11,20 +11,20 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
User_table = Table( "galaxy_user", metadata, autoload=True )
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
metadata = MetaData()
def boolean_false():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def upgrade():
def upgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
User_table = Table( "galaxy_user", metadata, autoload=True )
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
def boolean_false():
if migrate_engine.name == 'postgresql' or migrate_engine.name == 'mysql':
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name)
# Load existing tables
metadata.reflect()
# Add 2 indexes to the galaxy_user table
@@ -57,5 +57,6 @@ def upgrade():
i.create()
except Exception, e:
log.debug( "Adding index 'ix_hda_copied_from_library_dataset_dataset_association_id' to history_dataset_association table failed: %s" % ( str( e ) ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py
+96 -94
View File
@@ -8,21 +8,21 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
from migrate import migrate_engine
from sqlalchemy import and_
from sqlalchemy import *
now = datetime.datetime.utcnow
from sqlalchemy.orm import *
from galaxy.model.orm.ext.assignmapper import assign_mapper
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
from galaxy.util.bunch import Bunch
metadata = MetaData( migrate_engine )
metadata = MetaData()
context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) )
@@ -95,7 +95,7 @@ class Dataset( object ):
file_name = property( get_file_name, set_file_name )
@property
def extra_files_path( self ):
if self._extra_files_path:
if self._extra_files_path:
path = self._extra_files_path
else:
path = os.path.join( self.file_path, "dataset_%d_files" % self.id )
@@ -137,7 +137,7 @@ class DatasetInstance( object ):
"""A base class for all 'dataset instances', HDAs, LDAs, etc"""
states = Dataset.states
permitted_actions = Dataset.permitted_actions
def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None,
def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None,
dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None,
parent_id=None, validation_errors=None, visible=True, create_dataset = False ):
self.name = name or "Unnamed dataset"
@@ -198,9 +198,9 @@ class DatasetInstance( object ):
return dbkey[0]
def set_dbkey( self, value ):
if "dbkey" in self.datatype.metadata_spec:
if not isinstance(value, list):
if not isinstance(value, list):
self.metadata.dbkey = [value]
else:
else:
self.metadata.dbkey = value
dbkey = property( get_dbkey, set_dbkey )
def change_datatype( self, new_ext ):
@@ -299,11 +299,11 @@ class DatasetInstance( object ):
class HistoryDatasetAssociation( DatasetInstance ):
def __init__( self,
hid = None,
history = None,
copied_from_history_dataset_association = None,
copied_from_library_dataset_dataset_association = None,
def __init__( self,
hid = None,
history = None,
copied_from_history_dataset_association = None,
copied_from_library_dataset_dataset_association = None,
**kwd ):
DatasetInstance.__init__( self, **kwd )
self.hid = hid
@@ -312,17 +312,17 @@ class HistoryDatasetAssociation( DatasetInstance ):
self.copied_from_history_dataset_association = copied_from_history_dataset_association
self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
def copy( self, copy_children = False, parent_id = None, target_history = None ):
hda = HistoryDatasetAssociation( hid=self.hid,
name=self.name,
info=self.info,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset = self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
hda = HistoryDatasetAssociation( hid=self.hid,
name=self.name,
info=self.info,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset = self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
copied_from_history_dataset_association=self,
history = target_history )
context.add( hda )
@@ -343,21 +343,21 @@ class HistoryDatasetAssociation( DatasetInstance ):
# The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version.
library_dataset = replace_dataset
else:
# If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new
# If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new
# LibraryDataset, and the current user's DefaultUserPermissions will be applied to the associated Dataset.
library_dataset = LibraryDataset( folder=target_folder, name=self.name, info=self.info )
context.add( library_dataset )
context.flush()
ldda = LibraryDatasetDatasetAssociation( name=self.name,
ldda = LibraryDatasetDatasetAssociation( name=self.name,
info=self.info,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset=self.dataset,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset=self.dataset,
library_dataset=library_dataset,
visible=self.visible,
deleted=self.deleted,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
copied_from_history_dataset_association=self,
user=self.history.user )
@@ -402,16 +402,16 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
self.user = user
def to_history_dataset_association( self, target_history, parent_id=None ):
hid = target_history._next_hid()
hda = HistoryDatasetAssociation( name=self.name,
hda = HistoryDatasetAssociation( name=self.name,
info=self.info,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset=self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset=self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
copied_from_library_dataset_dataset_association=self,
history=target_history,
hid=hid )
@@ -425,16 +425,16 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
context.flush()
return hda
def copy( self, copy_children = False, parent_id = None, target_folder = None ):
ldda = LibraryDatasetDatasetAssociation( name=self.name,
info=self.info,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset=self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
ldda = LibraryDatasetDatasetAssociation( name=self.name,
info=self.info,
blurb=self.blurb,
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset=self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
copied_from_library_dataset_dataset_association=self,
folder=target_folder )
context.add( ldda )
@@ -514,7 +514,7 @@ class LibraryDataset( object ):
##tables
Dataset.table = Table( "dataset", metadata,
Dataset.table = Table( "dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
@@ -528,7 +528,7 @@ Dataset.table = Table( "dataset", metadata,
HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata,
HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
Column( "create_time", DateTime, default=now ),
@@ -548,7 +548,7 @@ HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata
Column( "visible", Boolean ) )
LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata,
LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_id", Integer, ForeignKey( "library_dataset.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
@@ -568,7 +568,7 @@ LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_associa
Column( "visible", Boolean ),
Column( "message", TrimmedString( 255 ) ) )
LibraryDataset.table = Table( "library_dataset", metadata,
LibraryDataset.table = Table( "library_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected
Column( "order_id", Integer ),
@@ -583,70 +583,70 @@ LibraryDataset.table = Table( "library_dataset", metadata,
##mappers
assign_mapper( context, Dataset, Dataset.table,
properties=dict(
history_associations=relation(
HistoryDatasetAssociation,
mapper( Dataset, Dataset.table,
properties=dict(
history_associations=relation(
HistoryDatasetAssociation,
primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ),
active_history_associations=relation(
HistoryDatasetAssociation,
active_history_associations=relation(
HistoryDatasetAssociation,
primaryjoin=( ( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) & ( HistoryDatasetAssociation.table.c.deleted == False ) ) ),
library_associations=relation(
LibraryDatasetDatasetAssociation,
library_associations=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) ),
active_library_associations=relation(
LibraryDatasetDatasetAssociation,
active_library_associations=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == False ) ) )
) )
assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.table,
properties=dict(
dataset=relation(
Dataset,
mapper( HistoryDatasetAssociation, HistoryDatasetAssociation.table,
properties=dict(
dataset=relation(
Dataset,
primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ), lazy=False ),
# .history defined in History mapper
copied_to_history_dataset_associations=relation(
HistoryDatasetAssociation,
copied_to_history_dataset_associations=relation(
HistoryDatasetAssociation,
primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ),
backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ),
copied_to_library_dataset_dataset_associations=relation(
LibraryDatasetDatasetAssociation,
copied_to_library_dataset_dataset_associations=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ),
backref=backref( "copied_from_history_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False ) ),
children=relation(
HistoryDatasetAssociation,
children=relation(
HistoryDatasetAssociation,
primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ),
backref=backref( "parent", primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ),
visible_children=relation(
HistoryDatasetAssociation,
visible_children=relation(
HistoryDatasetAssociation,
primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == True ) ) )
) )
assign_mapper( context, LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table,
properties=dict(
mapper( LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table,
properties=dict(
dataset=relation( Dataset ),
library_dataset = relation( LibraryDataset,
primaryjoin=( LibraryDatasetDatasetAssociation.table.c.library_dataset_id == LibraryDataset.table.c.id ) ),
copied_to_library_dataset_dataset_associations=relation(
LibraryDatasetDatasetAssociation,
copied_to_library_dataset_dataset_associations=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ),
backref=backref( "copied_from_library_dataset_dataset_association", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ),
copied_to_history_dataset_associations=relation(
HistoryDatasetAssociation,
copied_to_history_dataset_associations=relation(
HistoryDatasetAssociation,
primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ),
backref=backref( "copied_from_library_dataset_dataset_association", primaryjoin=( HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False ) ),
children=relation(
LibraryDatasetDatasetAssociation,
children=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ),
backref=backref( "parent", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ),
visible_children=relation(
LibraryDatasetDatasetAssociation,
visible_children=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == True ) ) )
) )
assign_mapper( context, LibraryDataset, LibraryDataset.table,
properties=dict(
mapper( LibraryDataset, LibraryDataset.table,
properties=dict(
library_dataset_dataset_association=relation( LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ),
expired_datasets = relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id,LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True )
) )
@@ -663,7 +663,8 @@ def __guess_dataset_by_filename( filename ):
pass #some parsing error, we can't guess Dataset
return None
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Fixing a discrepancy concerning deleted shared history items." )
affected_items = 0
start_time = time.time()
@@ -678,7 +679,7 @@ def upgrade():
context.flush()
log.debug( "%i items affected, and restored." % ( affected_items ) )
log.debug( "Time elapsed: %s" % ( time.time() - start_time ) )
#fix share before hda
log.debug( "Fixing a discrepancy concerning cleaning up deleted history items shared before HDAs." )
dataset_by_filename = {}
@@ -692,7 +693,7 @@ def upgrade():
if guessed_dataset and dataset.file_name != guessed_dataset.file_name:#not os.path.samefile( dataset.file_name, guessed_dataset.file_name ):
guessed_dataset = None
dataset_by_filename[ dataset.file_name ] = guessed_dataset
if guessed_dataset is not None and guessed_dataset.id != dataset.id: #could we have a self referential dataset?
for dataset_instance in dataset.history_associations + dataset.library_associations:
dataset_instance.dataset = guessed_dataset
@@ -705,6 +706,7 @@ def upgrade():
log.debug( "%i items affected, and restored." % ( changed_associations ) )
log.debug( "Time elapsed: %s" % ( time.time() - start_time ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Downgrade is not possible." )
lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py
+7 -5
View File
@@ -15,8 +15,7 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -24,10 +23,12 @@ def display_migration_details():
print "column, specifically 'qual' is chaged to be 'qual454'."
print "========================================"
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
def upgrade():
def upgrade(migrate_engine):
display_migration_details()
metadata.bind = migrate_engine
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
# Load existing tables
metadata.reflect()
# Add 2 indexes to the galaxy_user table
@@ -54,5 +55,6 @@ def upgrade():
except Exception, e:
log.debug( "Dropping index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0007_sharing_histories.py
+8 -6
View File
@@ -17,7 +17,7 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -32,7 +32,8 @@ HistoryUserShareAssociation_table = Table( "history_user_share_association", met
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
)
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -47,15 +48,16 @@ def upgrade():
except NoSuchTableError:
History_table = None
log.debug( "Failed loading table history" )
if History_table:
if History_table is not None:
try:
col = Column( 'importable', Boolean, index=True, default=False )
col.create( History_table )
col.create( History_table, index_name='ix_history_importable')
assert col is History_table.c.importable
except Exception, e:
log.debug( "Adding column 'importable' to history table failed: %s" % ( str( e ) ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
metadata.reflect()
# Drop 1 column from the history table
@@ -64,7 +66,7 @@ def downgrade():
except NoSuchTableError:
History_table = None
log.debug( "Failed loading table history" )
if History_table:
if History_table is not None:
try:
col = History_table.c.importable
col.drop()
lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
+24 -21
View File
@@ -30,8 +30,7 @@ log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -112,11 +111,13 @@ SampleEvent_table = Table('sample_event', metadata,
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "sample_id", Integer, ForeignKey( "sample.id" ), index=True ),
Column( "sample_id", Integer, ForeignKey( "sample.id" ), index=True ),
Column( "sample_state_id", Integer, ForeignKey( "sample_state.id" ), index=True ),
Column( "comment", TEXT ) )
def upgrade():
def upgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -125,13 +126,13 @@ def upgrade():
try:
FormDefinitionCurrent_table.create()
except Exception, e:
log.debug( "Creating form_definition_current table failed: %s" % str( e ) )
log.debug( "Creating form_definition_current table failed: %s" % str( e ) )
try:
FormDefinition_table.create()
except Exception, e:
log.debug( "Creating form_definition table failed: %s" % str( e ) )
# Add 1 foreign key constraint to the form_definition_current table
if FormDefinitionCurrent_table and FormDefinition_table:
if FormDefinitionCurrent_table is not None and FormDefinition_table is not None:
try:
cons = ForeignKeyConstraint( [FormDefinitionCurrent_table.c.latest_form_id],
[FormDefinition_table.c.id],
@@ -143,60 +144,62 @@ def upgrade():
try:
FormValues_table.create()
except Exception, e:
log.debug( "Creating form_values table failed: %s" % str( e ) )
log.debug( "Creating form_values table failed: %s" % str( e ) )
try:
RequestType_table.create()
except Exception, e:
log.debug( "Creating request_type table failed: %s" % str( e ) )
log.debug( "Creating request_type table failed: %s" % str( e ) )
try:
Request_table.create()
except Exception, e:
log.debug( "Creating request table failed: %s" % str( e ) )
log.debug( "Creating request table failed: %s" % str( e ) )
try:
Sample_table.create()
except Exception, e:
log.debug( "Creating sample table failed: %s" % str( e ) )
log.debug( "Creating sample table failed: %s" % str( e ) )
try:
SampleState_table.create()
except Exception, e:
log.debug( "Creating sample_state table failed: %s" % str( e ) )
log.debug( "Creating sample_state table failed: %s" % str( e ) )
try:
SampleEvent_table.create()
except Exception, e:
log.debug( "Creating sample_event table failed: %s" % str( e ) )
log.debug( "Creating sample_event table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
# Load existing tables
metadata.reflect()
try:
FormDefinition_table.drop()
except Exception, e:
log.debug( "Dropping form_definition table failed: %s" % str( e ) )
log.debug( "Dropping form_definition table failed: %s" % str( e ) )
try:
FormDefinitionCurrent_table.drop()
except Exception, e:
log.debug( "Dropping form_definition_current table failed: %s" % str( e ) )
log.debug( "Dropping form_definition_current table failed: %s" % str( e ) )
try:
FormValues_table.drop()
except Exception, e:
log.debug( "Dropping form_values table failed: %s" % str( e ) )
log.debug( "Dropping form_values table failed: %s" % str( e ) )
try:
Request_table.drop()
except Exception, e:
log.debug( "Dropping request table failed: %s" % str( e ) )
log.debug( "Dropping request table failed: %s" % str( e ) )
try:
RequestType_table.drop()
except Exception, e:
log.debug( "Dropping request_type table failed: %s" % str( e ) )
log.debug( "Dropping request_type table failed: %s" % str( e ) )
try:
Sample_table.drop()
except Exception, e:
log.debug( "Dropping sample table failed: %s" % str( e ) )
log.debug( "Dropping sample table failed: %s" % str( e ) )
try:
SampleState_table.drop()
except Exception, e:
log.debug( "Dropping sample_state table failed: %s" % str( e ) )
log.debug( "Dropping sample_state table failed: %s" % str( e ) )
try:
SampleEvent_table.drop()
except Exception, e:
log.debug( "Dropping sample_event table failed: %s" % str( e ) )
log.debug( "Dropping sample_event table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0009_request_table.py
+14 -18
View File
@@ -19,7 +19,7 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -28,36 +28,32 @@ def display_migration_details():
print "2) a new string type column named 'bar_code' to the 'sample' table"
print "========================================"
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
Request_table = Table( "request", metadata, autoload=True )
Sample_table = Table( "sample", metadata, autoload=True )
metadata.reflect()
# Add 1 column to the request table
try:
Request_table = Table( "request", metadata, autoload=True )
except NoSuchTableError:
Request_table = None
log.debug( "Failed loading table request" )
if Request_table:
if Request_table is not None:
try:
col = Column( "submitted", Boolean, index=True, default=False )
col.create( Request_table )
col = Column( 'submitted', Boolean, default=False )
col.create( Request_table)
#col.create( Request_table, index_name='ix_request_submitted')
assert col is Request_table.c.submitted
except Exception, e:
log.debug( "Adding column 'submitted' to request table failed: %s" % ( str( e ) ) )
# Add 1 column to the sample table
try:
Sample_table = Table( "sample", metadata, autoload=True )
except NoSuchTableError:
Sample_table = None
log.debug( "Failed loading table sample" )
if Sample_table:
if Sample_table is not None:
try:
col = Column( "bar_code", TrimmedString( 255 ), index=True )
col.create( Sample_table )
col.create( Sample_table, index_name='ix_sample_bar_code')
assert col is Sample_table.c.bar_code
except Exception, e:
log.debug( "Adding column 'bar_code' to sample table failed: %s" % ( str( e ) ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py
+8 -7
View File
@@ -5,7 +5,7 @@ If using mysql, this script will display the following error, which is corrected
migration script:
history_dataset_association_display_at_authorization table failed: (OperationalError)
(1059, "Identifier name 'ix_history_dataset_association_display_at_authorization_update_time'
(1059, "Identifier name 'ix_history_dataset_association_display_at_authorization_update_time'
is too long
"""
from sqlalchemy import *
@@ -29,8 +29,7 @@ log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -51,19 +50,21 @@ HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "site", TrimmedString( 255 ) ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
try:
HistoryDatasetAssociationDisplayAtAuthorization_table.create()
except Exception, e:
log.debug( "Creating history_dataset_association_display_at_authorization table failed: %s" % str( e ) )
log.debug( "Creating history_dataset_association_display_at_authorization table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
metadata.reflect()
try:
HistoryDatasetAssociationDisplayAtAuthorization_table.drop()
except Exception, e:
log.debug( "Dropping history_dataset_association_display_at_authorization table failed: %s" % str( e ) )
log.debug( "Dropping history_dataset_association_display_at_authorization table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py
+8 -7
View File
@@ -1,5 +1,5 @@
"""
This script fixes a problem introduced in 0010_hda_display_at_atuhz_table.py. MySQL has a
This script fixes a problem introduced in 0010_hda_display_at_atuhz_table.py. MySQL has a
name length limit and thus the index "ix_hdadaa_history_dataset_association_id" has to be
manually created.
"""
@@ -24,8 +24,7 @@ log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -42,7 +41,8 @@ HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "site", TrimmedString( 255 ) ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
if migrate_engine.name == 'mysql':
# Load existing tables
@@ -51,9 +51,10 @@ def upgrade():
try:
i.create()
except Exception, e:
log.debug( "Adding index 'ix_hdadaa_history_dataset_association_id' to table 'history_dataset_association_display_at_authorization' table failed: %s" % str( e ) )
log.debug( "Adding index 'ix_hdadaa_history_dataset_association_id' to table 'history_dataset_association_display_at_authorization' table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
if migrate_engine.name == 'mysql':
# Load existing tables
metadata.reflect()
@@ -61,4 +62,4 @@ def downgrade():
try:
i.drop()
except Exception, e:
log.debug( "Removing index 'ix_hdadaa_history_dataset_association_id' from table 'history_dataset_association_display_at_authorization' table failed: %s" % str( e ) )
log.debug( "Removing index 'ix_hdadaa_history_dataset_association_id' from table 'history_dataset_association_display_at_authorization' table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0012_user_address.py
+18 -18
View File
@@ -7,13 +7,13 @@ which is a string, allowing for more flexibility with request states.
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import sys, logging
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -23,8 +23,7 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -51,7 +50,9 @@ UserAddress_table = Table( "user_address", metadata,
Column( "deleted", Boolean, index=True, default=False ),
Column( "purged", Boolean, index=True, default=False ) )
def upgrade():
def upgrade(migrate_engine):
#raise Exception
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -59,37 +60,36 @@ def upgrade():
try:
UserAddress_table.create()
except Exception, e:
log.debug( "Creating user_address table failed: %s" % str( e ) )
log.debug( "Creating user_address table failed: %s" % str( e ) )
# Add 1 column to the request_type table
try:
RequestType_table = Table( "request_type", metadata, autoload=True )
except NoSuchTableError:
RequestType_table = None
log.debug( "Failed loading table request_type" )
if RequestType_table:
if RequestType_table is not None:
try:
col = Column( "deleted", Boolean, index=True, default=False )
col.create( RequestType_table )
col.create( RequestType_table, index_name='ix_request_type_deleted')
assert col is RequestType_table.c.deleted
except Exception, e:
log.debug( "Adding column 'deleted' to request_type table failed: %s" % ( str( e ) ) )
# Delete the submitted column
# This fails for sqlite, so skip the drop -- no conflicts in the future
try:
Request_table = Table( "request", metadata, autoload=True )
except NoSuchTableError:
Request_table = None
log.debug( "Failed loading table request" )
if Request_table:
try:
if Request_table is not None:
if migrate_engine.name != 'sqlite':
#DBTODO drop from table doesn't work in sqlite w/ sqlalchemy-migrate .6+
Request_table.c.submitted.drop()
except Exception, e:
log.debug( "Deleting column 'submitted' to request table failed: %s" % ( str( e ) ) )
try:
col = Column( "state", TrimmedString( 255 ), index=True )
col.create( Request_table )
assert col is Request_table.c.state
except Exception, e:
log.debug( "Adding column 'state' to request table failed: %s" % ( str( e ) ) )
col = Column( "state", TrimmedString( 255 ), index=True )
col.create( Request_table, index_name='ix_request_state')
assert col is Request_table.c.state
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py
+19 -18
View File
@@ -30,7 +30,7 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print "========================================"
@@ -50,20 +50,6 @@ def display_migration_details():
print "an index with a shortened name."
print "========================================"
if migrate_engine.name == 'postgres':
# http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html
from sqlalchemy.databases import postgres
class PGCascadeSchemaDropper(postgres.PGSchemaDropper):
def visit_table(self, table):
for column in table.columns:
if column.default is not None:
self.traverse_single(column.default)
self.append("\nDROP TABLE " +
self.preparer.format_table(table) +
" CASCADE")
self.execute()
postgres.dialect.schemadropper = PGCascadeSchemaDropper
LibraryInfoAssociation_table = Table( 'library_info_association', metadata,
Column( "id", Integer, primary_key=True),
Column( "library_id", Integer, ForeignKey( "library.id" ), index=True ),
@@ -81,11 +67,25 @@ LibraryDatasetDatasetInfoAssociation_table = Table( 'library_dataset_dataset_inf
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True, index=True ),
Column( "form_definition_id", Integer, ForeignKey( "form_definition.id" ), index=True ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
if migrate_engine.name == 'postgres':
# http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html
from sqlalchemy.databases import postgres
class PGCascadeSchemaDropper(postgres.PGSchemaDropper):
def visit_table(self, table):
for column in table.columns:
if column.default is not None:
self.traverse_single(column.default)
self.append("\nDROP TABLE " +
self.preparer.format_table(table) +
" CASCADE")
self.execute()
postgres.dialect.schemadropper = PGCascadeSchemaDropper
# Drop all of the original library_item_info tables
# NOTE: all existing library item into template data is eliminated here via table drops
try:
@@ -251,5 +251,6 @@ def upgrade():
except Exception, e:
log.debug( "Adding index 'ix_lddaia_ldda_id' to table 'library_dataset_dataset_info_association' table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Downgrade is not possible." )
lib/galaxy/model/migrate/versions/0014_pages.py
+7 -5
View File
@@ -14,7 +14,7 @@ now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
Page_table = Table( "page", metadata,
Column( "id", Integer, primary_key=True ),
@@ -36,7 +36,8 @@ PageRevision_table = Table( "page_revision", metadata,
Column( "content", TEXT )
)
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
try:
@@ -47,14 +48,15 @@ def upgrade():
PageRevision_table.create()
except:
log.debug( "Could not create page_revision table" )
# Add 1 column to the user table
User_table = Table( "galaxy_user", metadata, autoload=True )
col = Column( 'username', String(255), index=True, unique=True, default=False )
col.create( User_table )
col.create( User_table, index_name='ix_user_username', unique_name='username' )
assert col is User_table.c.username
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
Page_table.drop()
PageRevision_table.drop()
lib/galaxy/model/migrate/versions/0015_tagging.py
+10 -8
View File
@@ -22,7 +22,7 @@ from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print ""
@@ -35,14 +35,14 @@ def display_migration_details():
print "(OperationalError) (1059, 'Identifier name "
print "'ix_history_dataset_association_tag_association_history_dataset_association_id'"
print "is too long)"
# New tables to support tagging of histories, datasets, and history-dataset associations.
Tag_table = Table( "tag", metadata,
Column( "id", Integer, primary_key=True ),
Column( "type", Integer ),
Column( "parent_id", Integer, ForeignKey( "tag.id" ) ),
Column( "name", TrimmedString(255) ),
Column( "name", TrimmedString(255) ),
UniqueConstraint( "name" ) )
HistoryTagAssociation_table = Table( "history_tag_association", metadata,
@@ -51,7 +51,7 @@ HistoryTagAssociation_table = Table( "history_tag_association", metadata,
Column( "user_tname", TrimmedString(255), index=True),
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
DatasetTagAssociation_table = Table( "dataset_tag_association", metadata,
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
Column( "tag_id", Integer, ForeignKey( "tag.id" ), index=True ),
@@ -66,7 +66,8 @@ HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_associat
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
metadata.reflect()
try:
@@ -89,8 +90,9 @@ def upgrade():
except Exception, e:
print str(e)
log.debug( "Creating history_dataset_association_tag_association table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
try:
Tag_table.drop()
@@ -111,4 +113,4 @@ def downgrade():
HistoryDatasetAssociationTagAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping history_dataset_association_tag_association table failed: %s" % str( e ) )
log.debug( "Dropping history_dataset_association_tag_association table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py
+8 -6
View File
@@ -16,14 +16,14 @@ from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print ""
print "This script fixes a problem introduced in 0015_tagging.py. MySQL has a"
print "name length limit and thus the index 'ix_hda_ta_history_dataset_association_id'"
print "has to be manually created."
HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_association_tag_association", metadata,
Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "tag_id", Integer, ForeignKey( "tag.id" ), index=True ),
@@ -31,7 +31,8 @@ HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_associat
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
metadata.reflect()
i = Index( "ix_hda_ta_history_dataset_association_id", HistoryDatasetAssociationTagAssociation_table.c.history_dataset_association_id )
@@ -40,12 +41,13 @@ def upgrade():
except Exception, e:
print str(e)
log.debug( "Adding index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
i = Index( "ix_hda_ta_history_dataset_association_id", HistoryDatasetAssociationTagAssociation_table.c.history_dataset_association_id )
try:
i.drop()
except Exception, e:
print str(e)
log.debug( "Removing index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed: %s" % str( e ) )
log.debug( "Removing index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0017_library_item_indexes.py
+10 -9
View File
@@ -15,19 +15,19 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
LibraryFolder_table = Table( "library_folder", metadata, autoload=True )
LibraryDatasetDatasetAssociation_table = Table( "library_dataset_dataset_association", metadata, autoload=True )
LibraryDataset_table = Table( "library_dataset", metadata, autoload=True )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script adds 3 indexes to table columns: library_folder.name,"
print "library_dataset.name, library_dataset_dataset_association.name."
print "========================================"
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
LibraryFolder_table = Table( "library_folder", metadata, autoload=True )
LibraryDatasetDatasetAssociation_table = Table( "library_dataset_dataset_association", metadata, autoload=True )
LibraryDataset_table = Table( "library_dataset", metadata, autoload=True )
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -49,5 +49,6 @@ def upgrade():
i.create()
except Exception, e:
log.debug( "Adding index 'ix_library_dataset_name' to library_dataset table failed: %s" % ( str( e ) ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Downgrade is not possible." )
lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py
+5 -3
View File
@@ -18,7 +18,7 @@ from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print ""
@@ -57,7 +57,8 @@ PageTagAssociation_table = Table( "page_tag_association", metadata,
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
metadata.reflect()
@@ -103,7 +104,8 @@ def upgrade():
print str(e)
log.debug( "Creating page_tag_association table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
# No need to downgrade other tagging tables. They work fine with verision 16 code.
lib/galaxy/model/migrate/versions/0019_request_library_folder.py
+12 -11
View File
@@ -17,17 +17,17 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
print """This script creates a request.folder_id column which is a foreign
key to the library_folder table. This also adds a 'type' and 'layout' column
to the form_definition table."""
to the form_definition table."""
print "========================================"
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -37,10 +37,10 @@ def upgrade():
except NoSuchTableError:
Request_table = None
log.debug( "Failed loading table request" )
if Request_table:
if Request_table is not None:
try:
col = Column( "folder_id", Integer, index=True )
col.create( Request_table )
col.create( Request_table, index_name='ix_request_folder_id')
assert col is Request_table.c.folder_id
except Exception, e:
log.debug( "Adding column 'folder_id' to request table failed: %s" % ( str( e ) ) )
@@ -50,7 +50,7 @@ def upgrade():
LibraryFolder_table = None
log.debug( "Failed loading table library_folder" )
# Add 1 foreign key constraint to the library_folder table
if Request_table and LibraryFolder_table:
if migrate_engine.name != 'sqlite' and Request_table is not None and LibraryFolder_table is not None:
try:
cons = ForeignKeyConstraint( [Request_table.c.folder_id],
[LibraryFolder_table.c.id],
@@ -65,20 +65,21 @@ def upgrade():
except NoSuchTableError:
FormDefinition_table = None
log.debug( "Failed loading table form_definition" )
if FormDefinition_table:
if FormDefinition_table is not None:
try:
col = Column( "type", TrimmedString( 255 ), index=True )
col.create( FormDefinition_table )
col.create( FormDefinition_table, index_name='ix_form_definition_type')
assert col is FormDefinition_table.c.type
except Exception, e:
log.debug( "Adding column 'type' to form_definition table failed: %s" % ( str( e ) ) )
try:
col = Column( "layout", JSONType())
col = Column( "layout", JSONType())
col.create( FormDefinition_table )
assert col is FormDefinition_table.c.layout
except Exception, e:
log.debug( "Adding column 'layout' to form_definition table failed: %s" % ( str( e ) ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0020_library_upload_job.py
+19 -16
View File
@@ -17,8 +17,7 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print ""
@@ -36,7 +35,8 @@ JobToOutputLibraryDatasetAssociation_table = Table( "job_to_output_library_datas
Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -52,10 +52,10 @@ def upgrade():
except NoSuchTableError:
Job_table = None
log.debug( "Failed loading table job" )
if Job_table:
if Job_table is not None:
try:
col = Column( "library_folder_id", Integer, index=True )
col.create( Job_table )
col.create( Job_table, index_name='ix_job_library_folder_id')
assert col is Job_table.c.library_folder_id
except Exception, e:
log.debug( "Adding column 'library_folder_id' to job table failed: %s" % ( str( e ) ) )
@@ -65,15 +65,17 @@ def upgrade():
LibraryFolder_table = None
log.debug( "Failed loading table library_folder" )
# Add 1 foreign key constraint to the job table
if Job_table and LibraryFolder_table:
try:
cons = ForeignKeyConstraint( [Job_table.c.library_folder_id],
[LibraryFolder_table.c.id],
name='job_library_folder_id_fk' )
# Create the constraint
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'job_library_folder_id_fk' to table 'library_folder' failed: %s" % ( str( e ) ) )
if migrate_engine.name != 'sqlite':
#Sqlite can't alter-table-add-foreign-key
if Job_table is not None and LibraryFolder_table is not None:
try:
cons = ForeignKeyConstraint( [Job_table.c.library_folder_id],
[LibraryFolder_table.c.id],
name='job_library_folder_id_fk' )
# Create the constraint
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'job_library_folder_id_fk' to table 'library_folder' failed: %s" % ( str( e ) ) )
# Create the ix_dataset_state index
try:
Dataset_table = Table( "dataset", metadata, autoload=True )
@@ -87,7 +89,8 @@ def upgrade():
print str(e)
log.debug( "Adding index 'ix_dataset_state' to dataset table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
# Drop the library_folder_id column
try:
@@ -95,7 +98,7 @@ def downgrade():
except NoSuchTableError:
Job_table = None
log.debug( "Failed loading table job" )
if Job_table:
if Job_table is not None:
try:
col = Job_table.c.library_folder_id
col.drop()
lib/galaxy/model/migrate/versions/0021_user_prefs.py
+6 -4
View File
@@ -11,7 +11,7 @@ now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print ""
@@ -27,7 +27,8 @@ UserPreference_table = Table( "user_preference", metadata,
Column( "name", Unicode( 255 ), index=True),
Column( "value", Unicode( 1024 ) ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
metadata.reflect()
try:
@@ -36,10 +37,11 @@ def upgrade():
print str(e)
log.debug( "Creating user_preference table failed: %s" % str( e ) )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
try:
UserPreference_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping user_preference table failed: %s" % str( e ) )
log.debug( "Dropping user_preference table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0022_visualization_tables.py
+5 -3
View File
@@ -13,7 +13,7 @@ now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
Visualization_table = Table( "visualization", metadata,
Column( "id", Integer, primary_key=True ),
@@ -35,7 +35,8 @@ VisualizationRevision_table = Table( "visualization_revision", metadata,
Column( "config", TEXT )
)
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
try:
@@ -48,7 +49,8 @@ def upgrade():
log.debug( "Could not create page_revision table" )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
Visualization_table.drop()
VisualizationRevision_table.drop()
lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py
+12 -10
View File
@@ -10,24 +10,26 @@ from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
def upgrade():
print __doc__
metadata.reflect()
Page_table = Table( "page", metadata, autoload=True )
c = Column( "published", Boolean, index=True, default=False )
c.create( Page_table )
c.create( Page_table, index_name = 'ix_page_published' )
assert c is Page_table.c.published
c = Column( "deleted", Boolean, index=True, default=False )
c.create( Page_table )
c = Column( "deleted", Boolean, index=True, default=False )
c.create( Page_table, index_name='ix_page_deleted')
assert c is Page_table.c.deleted
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
Page_table = Table( "page", metadata, autoload=True )
lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py
+7 -5
View File
@@ -13,9 +13,10 @@ now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
@@ -24,10 +25,10 @@ def upgrade():
try:
# Sqlite doesn't support .alter, so we need to drop an recreate
i = Index( "ix_page_slug", Page_table.c.slug )
i.drop()
i = Index( "ix_page_slug", Page_table.c.slug, unique=False )
i.create()
@@ -37,7 +38,8 @@ def upgrade():
Page_table.c.slug.alter( unique=False )
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
#Page_table = Table( "page", metadata, autoload=True )
#Page_table.c.slug.alter( unique=True )
lib/galaxy/model/migrate/versions/0025_user_info.py
+18 -17
View File
@@ -3,7 +3,6 @@ This script adds a foreign key to the form_values table in the galaxy_user table
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exceptions import *
from migrate import *
from migrate.changeset import *
import datetime
@@ -20,14 +19,14 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script adds a foreign key to the form_values table in the galaxy_user table"
print "========================================"
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
@@ -36,10 +35,10 @@ def upgrade():
except NoSuchTableError:
User_table = None
log.debug( "Failed loading table galaxy_user" )
if User_table:
if User_table is not None:
try:
col = Column( "form_values_id", Integer, index=True )
col.create( User_table )
col.create( User_table, index_name='ix_user_form_values_id')
assert col is User_table.c.form_values_id
except Exception, e:
log.debug( "Adding column 'form_values_id' to galaxy_user table failed: %s" % ( str( e ) ) )
@@ -48,15 +47,17 @@ def upgrade():
except NoSuchTableError:
FormValues_table = None
log.debug( "Failed loading table form_values" )
# Add 1 foreign key constraint to the form_values table
if User_table and FormValues_table:
try:
cons = ForeignKeyConstraint( [User_table.c.form_values_id],
[FormValues_table.c.id],
name='user_form_values_id_fk' )
# Create the constraint
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'user_form_values_id_fk' to table 'galaxy_user' failed: %s" % ( str( e ) ) )
def downgrade():
if migrate_engine.name != 'sqlite':
# Add 1 foreign key constraint to the form_values table
if User_table is not None and FormValues_table is not None:
try:
cons = ForeignKeyConstraint( [User_table.c.form_values_id],
[FormValues_table.c.id],
name='user_form_values_id_fk' )
# Create the constraint
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'user_form_values_id_fk' to table 'galaxy_user' failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0026_cloud_tables.py
+18 -16
View File
@@ -10,15 +10,15 @@ from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
metadata = MetaData()
def display_migration_details():
print
print "========================================"
print "This script adds tables needed for Galaxy cloud functionality."
print "========================================"
CloudImage_table = Table( "cloud_image", metadata,
CloudImage_table = Table( "cloud_image", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -30,7 +30,7 @@ CloudImage_table = Table( "cloud_image", metadata,
Column( "deleted", Boolean, default=False ) )
""" UserConfiguredInstance (UCI) table """
UCI_table = Table( "cloud_uci", metadata,
UCI_table = Table( "cloud_uci", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -45,7 +45,7 @@ UCI_table = Table( "cloud_uci", metadata,
Column( "launch_time", DateTime ),
Column( "deleted", Boolean, default=False ) )
CloudInstance_table = Table( "cloud_instance", metadata,
CloudInstance_table = Table( "cloud_instance", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -64,7 +64,7 @@ CloudInstance_table = Table( "cloud_instance", metadata,
Column( "security_group", TEXT ),
Column( "availability_zone", TEXT ) )
CloudStore_table = Table( "cloud_store", metadata,
CloudStore_table = Table( "cloud_store", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -81,7 +81,7 @@ CloudStore_table = Table( "cloud_store", metadata,
Column( "error", TEXT ),
Column( "deleted", Boolean, default=False ) )
CloudSnapshot_table = Table( "cloud_snapshot", metadata,
CloudSnapshot_table = Table( "cloud_snapshot", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -94,7 +94,7 @@ CloudSnapshot_table = Table( "cloud_snapshot", metadata,
Column( "error", TEXT ),
Column( "deleted", Boolean, default=False ) )
CloudUserCredentials_table = Table( "cloud_user_credentials", metadata,
CloudUserCredentials_table = Table( "cloud_user_credentials", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -105,7 +105,7 @@ CloudUserCredentials_table = Table( "cloud_user_credentials", metadata,
Column( "secret_key", TEXT ),
Column( "deleted", Boolean, default=False ) )
CloudProvider_table = Table( "cloud_provider", metadata,
CloudProvider_table = Table( "cloud_provider", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
@@ -127,11 +127,12 @@ CloudProvider_table = Table( "cloud_provider", metadata,
Column( "path", TEXT ),
Column( "deleted", Boolean, default=False ) )
def upgrade():
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
CloudProvider_table.create()
CloudUserCredentials_table.create()
@@ -141,16 +142,17 @@ def upgrade():
CloudInstance_table.create()
CloudStore_table.create()
CloudSnapshot_table.create()
def downgrade():
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
CloudInstance_table.drop()
CloudSnapshot_table.drop()
CloudStore_table.drop()
UCI_table.drop()
CloudImage_table.drop()
CloudImage_table.drop()
CloudUserCredentials_table.drop()
CloudUserCredentials_table.drop()
CloudProvider_table.drop()

Some files were not shown because too many files have changed in this diff Show More