diff --git a/.ci/jenkins/api-py3 b/.ci/jenkins/api-py3 new file mode 120000 index 00000000000..9e5bfb42d23 --- /dev/null +++ b/.ci/jenkins/api-py3 @@ -0,0 +1 @@ +api \ No newline at end of file diff --git a/.ci/jenkins/api-py3/run_tests.sh b/.ci/jenkins/api-py3/run_tests.sh deleted file mode 100755 index 24790d0b720..00000000000 --- a/.ci/jenkins/api-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --skip_flakey_fails -api "$@" diff --git a/.ci/jenkins/framework-py3 b/.ci/jenkins/framework-py3 new file mode 120000 index 00000000000..e3939d229ce --- /dev/null +++ b/.ci/jenkins/framework-py3 @@ -0,0 +1 @@ +framework \ No newline at end of file diff --git a/.ci/jenkins/framework-py3/run_tests.sh b/.ci/jenkins/framework-py3/run_tests.sh deleted file mode 100755 index f8bfe97335d..00000000000 --- a/.ci/jenkins/framework-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --framework "$@" diff --git a/.ci/jenkins/integration-py3 b/.ci/jenkins/integration-py3 new file mode 120000 index 00000000000..c0bfe0d4f94 --- /dev/null +++ b/.ci/jenkins/integration-py3 @@ -0,0 +1 @@ +integration \ No newline at end of file diff --git a/.ci/jenkins/integration-py3/run_tests.sh b/.ci/jenkins/integration-py3/run_tests.sh deleted file mode 100755 index ffc6c87e98c..00000000000 --- a/.ci/jenkins/integration-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@" diff --git a/.ci/jenkins/main-tools-py3 b/.ci/jenkins/main-tools-py3 new file mode 120000 index 00000000000..49589fdd3a8 --- /dev/null +++ b/.ci/jenkins/main-tools-py3 @@ -0,0 +1 @@ +main-tools \ No newline at end of file diff --git a/.ci/jenkins/main-tools-py3/run_tests.sh b/.ci/jenkins/main-tools-py3/run_tests.sh deleted file mode 100755 index 442b15cdf2d..00000000000 --- a/.ci/jenkins/main-tools-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc -main "$@" diff --git a/.ci/jenkins/selenium-compose/run_galaxy.bash b/.ci/jenkins/selenium-compose/run_galaxy.bash index aaf10b77401..c16901324c1 100755 --- a/.ci/jenkins/selenium-compose/run_galaxy.bash +++ b/.ci/jenkins/selenium-compose/run_galaxy.bash @@ -20,8 +20,7 @@ virtualenv "$GALAXY_VIRTUAL_ENV" chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "$GALAXY_VIRTUAL_ENV" cd /galaxy -HOME=/galaxy -sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh || { echo "common_startup.sh failed"; exit 1; } +sudo -E -H -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh || { echo "common_startup.sh failed"; exit 1; } echo "Waiting for postgres to become available" while ! nc -z postgres 5432; @@ -34,7 +33,7 @@ echo "Creating postgres database for Galaxy" createdb -w -U postgres -h postgres galaxy echo "Starting and waiting for Galaxy daemon(s)" -sudo -E -u "#${GALAXY_TEST_UID}" GALAXY_RUN_ALL=1 bash "$GALAXY_ROOT/run.sh" --daemon --wait +sudo -E -H -u "#${GALAXY_TEST_UID}" GALAXY_RUN_ALL=1 bash "$GALAXY_ROOT/run.sh" --daemon --wait echo "Galaxy daemon ready, monitoring Galaxy logs" tail -f "$GALAXY_ROOT/main.log" diff --git a/.ci/jenkins/selenium-py3 b/.ci/jenkins/selenium-py3 new file mode 120000 index 00000000000..954f0db0bc0 --- /dev/null +++ b/.ci/jenkins/selenium-py3 @@ -0,0 +1 @@ +selenium \ No newline at end of file diff --git a/.ci/jenkins/selenium-py3/run_tests.sh b/.ci/jenkins/selenium-py3/run_tests.sh deleted file mode 100755 index 827151835f6..00000000000 --- a/.ci/jenkins/selenium-py3/run_tests.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Enable retries on tests to reduce chances of transient failures. -: ${GALAXY_TEST_SELENIUM_RETRIES:=1} - -# If in Jenkins environment, use it for artifacts. -if [ -n "$BUILD_NUMBER" ]; -then - : ${GALAXY_TEST_ERRORS_DIRECTORY:=${BUILD_NUMBER}-test-errors} - : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=${BUILD_NUMBER}-test-screenshots} -else - : ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors} - : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=database/test-screenshots} -fi - -mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY" -mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY" - -mkdir -p ~/.jenkins-yarn-cache -YARN_CACHE_FOLDER=~/.jenkins-yarn-cache - -# Start Selenium server in the test Docker container. -DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} --shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY}" -export DOCKER_RUN_EXTRA_ARGS - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --skip_flakey_fails --selenium "$@" diff --git a/.circleci/config.yml b/.circleci/config.yml index 3afbbc0bb98..b83111cc575 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,23 +81,6 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} paths: - ~/repo - py27_lint: - docker: - - image: circleci/python:2.7 - <<: *set_workdir - steps: - - *restore_repo_cache - - *install_tox - - run: tox -e py27-lint - py27_unit: - docker: - - image: circleci/python:2.7 - <<: *set_workdir - steps: - - *restore_repo_cache - # Ensure minimum virtualenv version due to https://github.com/pypa/virtualenv/issues/1670 - - run: sudo pip install tox 'virtualenv>=20.0.8' - - run: tox -e py27-unit py35_docstring: docker: - image: circleci/python:3.5 @@ -106,14 +89,6 @@ jobs: - *restore_repo_cache - *install_tox - run: tox -e py35-lint_docstring_include_list - py27_first_startup: - docker: - - image: circleci/python:2.7 - <<: *set_workdir - steps: - - *restore_repo_cache - - *install_tox - - run: tox -e py27-first_startup py35_lint: docker: - image: circleci/python:3.5 @@ -136,10 +111,9 @@ jobs: <<: *set_workdir steps: - *restore_repo_cache - - run: sh scripts/common_startup.sh - - run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite - - run: mv db_gx_rev_0127.sqlite database/universe.sqlite - - run: sh manage_db.sh -c ./config/galaxy.yml.sample upgrade + # Use this job to test the latest migrations + - run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite + - run: mv db_gx_rev_0141.sqlite database/universe.sqlite - *install_tox - run: tox -e py35-first_startup validate_test_tools: @@ -195,12 +169,6 @@ workflows: get_code_and_test: jobs: - get_code - - py27_lint: - <<: *requires_get_code - - py27_unit: - <<: *requires_get_code - - py27_first_startup: - <<: *requires_get_code - py35_docstring: <<: *requires_get_code - py35_lint: diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 7e72077fbb2..3bd0aaf4d69 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -27,10 +27,15 @@ jobs: steps: - name: Prune unused docker image, volumes and containers run: docker system prune -a -f + - name: Clean dotnet folder for space + if: matrix.subset == 'kubernetes' + run: rm -Rf /usr/share/dotnet - name: Setup Minikube if: matrix.subset == 'kubernetes' id: minikube - uses: CodingNagger/minikube-setup-action@v1.0.2 + uses: CodingNagger/minikube-setup-action@v1.0.3 + with: + minikube-version: "1.9.0-0_amd64" - name: Launch Minikube if: matrix.subset == 'kubernetes' run: eval ${{ steps.minikube.outputs.launcher }} diff --git a/.github/workflows/osx_startup.yaml b/.github/workflows/osx_startup.yaml new file mode 100644 index 00000000000..f59525e812b --- /dev/null +++ b/.github/workflows/osx_startup.yaml @@ -0,0 +1,28 @@ +name: OSX +on: [push, pull_request] +jobs: + + test: + name: Startup test + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/Library/Caches/pip + # scripts/common_startup.sh creates a conda env for Galaxy containing Python 3.6 + key: pip-cache-3.6-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Install tox + run: pip install tox + - name: Install and activate miniconda # use this job to test using Python from a conda environment + uses: goanpeca/setup-miniconda@v1 + with: + activate-environment: '' + - name: run tests + run: tox -e first_startup + shell: bash -l {0} # need this to have CONDA_EXE set + working-directory: 'galaxy root' diff --git a/.github/workflows/toolshed.yaml b/.github/workflows/toolshed.yaml new file mode 100644 index 00000000000..38628962cea --- /dev/null +++ b/.github/workflows/toolshed.yaml @@ -0,0 +1,37 @@ +name: Toolshed +on: [push, pull_request] +env: + GALAXY_TEST_DBURI: 'postgres://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8' + TOOL_SHED_TEST_DBURI: 'postgres://postgres:postgres@localhost:5432/toolshed?client_encoding=utf8' +jobs: + test: + name: Test + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + services: + postgres: + image: postgres:11 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Run tests + run: './run_tests.sh -toolshed' + working-directory: 'galaxy root' diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0f690edec63..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -os: osx -# No version of Python is available via virtualenv on OS X workers, see https://github.com/travis-ci/travis-ci/issues/2312 -language: generic - -env: - - TOX_ENV=py27-first_startup - - TOX_ENV=py37-first_startup - -install: - - set -e - - pip install tox - - | - if [ "$TOX_ENV" == "py27-first_startup" ]; then - # Use this job to test the latest migrations - wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite - mv db_gx_rev_0141.sqlite database/universe.sqlite - elif [ "$TOX_ENV" == "py37-first_startup" ]; then - # There is now a pre-installed python3 on osx Travis workers, but use this job to test using a conda environment - MINICONDA_URL="https://repo.anaconda.com/miniconda" - MINICONDA_FILE="Miniconda3-latest-MacOSX-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b - . ~/miniconda3/bin/activate - fi - -script: tox -e $TOX_ENV - -notifications: - email: false diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 800d4762594..d6507a7be42 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -86,6 +86,7 @@ The following individuals have contributed code to Galaxy: * Joachim Jacob * Xiaoqian Jiang * Jim Johnson +* Kaivan Kamali * Radhesh Kamath * Iyad Kandalaft * Jan Kanis diff --git a/client/galaxy/scripts/bundleEntries.js b/client/galaxy/scripts/bundleEntries.js index 2818689602f..8e6efed234d 100644 --- a/client/galaxy/scripts/bundleEntries.js +++ b/client/galaxy/scripts/bundleEntries.js @@ -95,6 +95,7 @@ export { mountJobMetrics } from "components/JobMetrics"; export { mountJobParameters } from "components/JobParameters"; export { mountWorkflowEditor } from "components/Workflow/Editor/mount"; export { mountPageDisplay } from "components/PageDisplay"; +export { mountDestinationParams } from "components/JobDestinationParams"; // Used in common.mako export { default as store } from "storemodern"; diff --git a/client/galaxy/scripts/components/JobDestinationParams/JobDestinationParams.test.js b/client/galaxy/scripts/components/JobDestinationParams/JobDestinationParams.test.js new file mode 100644 index 00000000000..1a0eb75321a --- /dev/null +++ b/client/galaxy/scripts/components/JobDestinationParams/JobDestinationParams.test.js @@ -0,0 +1,67 @@ +import Vuex from "vuex"; +import axios from "axios"; +import MockAdapter from "axios-mock-adapter"; +import { mount, createLocalVue } from "@vue/test-utils"; +import { createStore } from "../../store"; +import flushPromises from "flush-promises"; +import JobDestinationParams from "./JobDestinationParams"; +import jobDestinationResponse from "./testData/jobDestinationResponse"; + +const JOB_ID = "foo_job_id"; + +describe("JobDestinationParams/JobDestinationParams.vue", () => { + const localVue = createLocalVue(); + localVue.use(Vuex); + + const responseKeys = Object.keys(jobDestinationResponse); + + let testStore, axiosMock, wrapper; + + beforeEach(async () => { + axiosMock = new MockAdapter(axios); + testStore = createStore(); + const propsData = { + jobId: JOB_ID, + }; + axiosMock.onGet(`/api/jobs/${JOB_ID}/destination_params`).reply(200, jobDestinationResponse); + wrapper = mount(JobDestinationParams, { + store: testStore, + propsData, + localVue, + }); + await flushPromises(); + assert(responseKeys.length > 0, "test data is invalid!"); + }); + + afterEach(() => { + axiosMock.restore(); + }); + + it("destination parameters should exist", async () => { + expect(Object.keys(wrapper.vm.jobDestinationParams).length).to.equals(responseKeys.length); + expect(wrapper.vm.jobId).to.equals(JOB_ID); + expect(wrapper.vm.jobDestinationParams["docker_net"]).to.equals("bridge"); + expect(wrapper.vm.jobDestinationParams["docker_set_user"]).to.equals(null); + }); + + it("destination parameters should be rendered", async () => { + console.log(wrapper.html()); + const paramsTable = wrapper.find("#destination_parameters"); + expect(paramsTable.isVisible()).to.equals(true); + const params = paramsTable.findAll("tbody > tr"); + expect(params.length).to.equals(responseKeys.length); + + for (let counter = 0; counter < responseKeys.length - 1; counter++) { + const parameter = params.at(counter).findAll("td"); + const parameterTitle = parameter.at(0).text(); + const parameterValue = parameter.at(1).text(); + + assert(responseKeys.includes(parameterTitle), "rendered parameter should exist in test data!"); + // since we render null as an empty string, rendered empty string should always equal null in test data + assert( + jobDestinationResponse[parameterTitle] === (parameterValue === "" ? null : parameterValue), + "parameter value is not equal to test data!" + ); + } + }); +}); diff --git a/client/galaxy/scripts/components/JobDestinationParams/JobDestinationParams.vue b/client/galaxy/scripts/components/JobDestinationParams/JobDestinationParams.vue new file mode 100644 index 00000000000..7732edf8b90 --- /dev/null +++ b/client/galaxy/scripts/components/JobDestinationParams/JobDestinationParams.vue @@ -0,0 +1,36 @@ + + + diff --git a/client/galaxy/scripts/components/JobDestinationParams/index.js b/client/galaxy/scripts/components/JobDestinationParams/index.js new file mode 100644 index 00000000000..d7e3a304577 --- /dev/null +++ b/client/galaxy/scripts/components/JobDestinationParams/index.js @@ -0,0 +1,3 @@ +export { default as JobDestinationParams } from "./JobDestinationParams"; + +export { mountDestinationParams } from "./mount"; diff --git a/client/galaxy/scripts/components/JobDestinationParams/mount.js b/client/galaxy/scripts/components/JobDestinationParams/mount.js new file mode 100644 index 00000000000..271db1a40b9 --- /dev/null +++ b/client/galaxy/scripts/components/JobDestinationParams/mount.js @@ -0,0 +1,13 @@ +/** + * Endpoint for mounting job metrics from non-Vue environment. + */ +import $ from "jquery"; +import JobDestinationParams from "./JobDestinationParams.vue"; +import { mountVueComponent } from "utils/mountVueComponent"; + +export const mountDestinationParams = (propsData = {}) => { + $(".job-destination-parameters").each((index, el) => { + propsData.jobId = $(el).attr("job_id"); + mountVueComponent(JobDestinationParams)(propsData, el); + }); +}; diff --git a/client/galaxy/scripts/components/JobDestinationParams/testData/jobDestinationResponse.json b/client/galaxy/scripts/components/JobDestinationParams/testData/jobDestinationResponse.json new file mode 100644 index 00000000000..42070b6f77e --- /dev/null +++ b/client/galaxy/scripts/components/JobDestinationParams/testData/jobDestinationResponse.json @@ -0,0 +1,12 @@ +{ + "Runner": "local", + "Runner Job ID": "23027", + "Handler": "main.web.1", + "docker_auto_rm": "true", + "docker_enabled": "true", + "docker_net": "bridge", + "docker_set_user": null, + "docker_sudo": "false", + "docker_volumes": "$galaxy_root:ro,$tool_directory:ro,$job_directory:rw,$working_directory:rw,$default_file_path:ro", + "require_container": "true" +} diff --git a/client/galaxy/scripts/components/Panels/ToolBoxWorkflow.vue b/client/galaxy/scripts/components/Panels/ToolBoxWorkflow.vue index a44c15ee5ae..71ff9f6647f 100644 --- a/client/galaxy/scripts/components/Panels/ToolBoxWorkflow.vue +++ b/client/galaxy/scripts/components/Panels/ToolBoxWorkflow.vue @@ -77,12 +77,15 @@ export default { }, workflows: { type: Array, + required: true, }, dataManagers: { type: Array, + required: true, }, moduleSections: { type: Array, + required: true, }, }, computed: { diff --git a/client/galaxy/scripts/components/ToolRecommendation.vue b/client/galaxy/scripts/components/ToolRecommendation.vue index 674f62f0e5e..3e26c55e4fa 100644 --- a/client/galaxy/scripts/components/ToolRecommendation.vue +++ b/client/galaxy/scripts/components/ToolRecommendation.vue @@ -1,6 +1,6 @@ diff --git a/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue new file mode 100644 index 00000000000..fa51a329330 --- /dev/null +++ b/client/galaxy/scripts/components/Workflow/Editor/Recommendations.vue @@ -0,0 +1,145 @@ + + + diff --git a/client/galaxy/scripts/components/Workflow/Editor/services.js b/client/galaxy/scripts/components/Workflow/Editor/services.js index aeea83c9762..4c89baa8601 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/services.js +++ b/client/galaxy/scripts/components/Workflow/Editor/services.js @@ -67,3 +67,21 @@ export async function saveWorkflow(workflow, id) { } return {}; } + +export async function getDatatypeMapping() { + try { + const mappingRequest = await axios.get(`${getAppRoot()}api/datatypes/mapping`); + return mappingRequest.data; + } catch (e) { + rethrowSimple(e); + } +} + +export async function getToolPredictions(requestData) { + try { + const response = await axios.post(`${getAppRoot()}api/workflows/get_tool_predictions`, requestData); + return response.data; + } catch (e) { + rethrowSimple(e); + } +} diff --git a/client/galaxy/scripts/components/Workflow/Editor/utilities.js b/client/galaxy/scripts/components/Workflow/Editor/utilities.js index dcd01b05aaf..e2534b6072c 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/utilities.js +++ b/client/galaxy/scripts/components/Workflow/Editor/utilities.js @@ -144,9 +144,9 @@ export function getWorkflowParameters(nodes) { }); } if (node.post_job_actions) { - Object.entries(node.post_job_actions).forEach(([k, pja]) => { + Object.values(node.post_job_actions).forEach((pja) => { if (pja.action_arguments) { - Object.entries(pja.action_arguments).forEach(([k, action_argument]) => { + Object.values(pja.action_arguments).forEach((action_argument) => { if (typeof action_argument === "string") { const arg_matches = action_argument.match(parameter_re); if (arg_matches) { @@ -198,7 +198,8 @@ export function saveAs(workflow) { window.location = `${getAppRoot()}workflow/editor?id=${id}`; hide_modal(); }) - .fail(() => { + .fail((err) => { + console.debug(err); hide_modal(); alert("Saving this workflow failed. Please contact this site's administrator."); }); diff --git a/client/galaxy/scripts/entry/panels/admin-panel.js b/client/galaxy/scripts/entry/panels/admin-panel.js index c44603784d8..f972914eed3 100644 --- a/client/galaxy/scripts/entry/panels/admin-panel.js +++ b/client/galaxy/scripts/entry/panels/admin-panel.js @@ -87,6 +87,7 @@ const AdminPanel = Backbone.View.extend({ title: _l("Forms"), url: "admin/forms", target: "__use_router__", + id: "admin-link-forms", }, ], }, @@ -97,17 +98,20 @@ const AdminPanel = Backbone.View.extend({ title: _l("Install and Uninstall"), url: "admin/toolshed", target: "__use_router__", + id: "admin-link-toolshed", enabled: this.settings.is_tool_shed_installed, }, { title: _l("Manage Metadata"), url: "admin/reset_metadata", + id: "admin-link-metadata", enabled: this.settings.is_repo_installed, target: "__use_router__", }, { title: _l("Manage Whitelist"), url: "admin/sanitize_whitelist", + id: "admin-link-whitelist", }, { title: _l("Manage Dependencies"), @@ -123,14 +127,17 @@ const AdminPanel = Backbone.View.extend({ title: _l("View Lineage"), url: "admin/tool_versions", target: "__use_router__", + id: "admin-link-tool-versions", }, { title: _l("View Migration Stages"), url: "admin/review_tool_migration_stages", + id: "admin-link-migrations", }, { title: _l("View Error Logs"), url: "admin/error_stack", + id: "admin-link-error-stack", target: "__use_router__", }, ], diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index 084484ee74d..2724cdd4db8 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -24,7 +24,7 @@ export default FormBase.extend({ FormBase.prototype.initialize.call(this, options); // optional model update - this._update(this.model.get("initialmodel")); + this._update(); // listen to history panel if (this.model.get("listen_to_history") && Galaxy.currHistoryPanel) { @@ -39,9 +39,9 @@ export default FormBase.extend({ }, /** Allows tool form variation to update tool model */ - _update: function (callback) { + _update: function () { var self = this; - callback = callback || this.model.get("buildmodel"); + var callback = this.model.get("buildmodel"); if (callback) { this.deferred.reset(); this.deferred.execute((process) => { diff --git a/client/galaxy/scripts/mvc/workflow/workflow-forms.js b/client/galaxy/scripts/mvc/workflow/workflow-forms.js index c83d7f0dc64..21ddc881447 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-forms.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-forms.js @@ -1,4 +1,4 @@ -import $ from "jquery"; +import axios from "axios"; import { getAppRoot } from "onload/loadConfig"; import { getGalaxyInstance } from "app"; import _l from "utils/localization"; @@ -11,95 +11,85 @@ export class DefaultForm { constructor(options) { var self = this; var node = options.node; - this.form = new Form( - Utils.merge(options, { - onchange: function () { - Utils.request({ - type: "POST", - url: `${getAppRoot()}api/workflows/build_module`, - data: { - id: node.id, - type: node.type, - content_id: node.content_id, - inputs: self.form.data.create(), - }, - success: function (data) { - node.update_field_data(data); - }, + this.workflow = options.workflow; + _addLabelAnnotation(this, node); + this.form = new Form({ + ...options, + onchange() { + axios + .post(`${getAppRoot()}api/workflows/build_module`, { + id: node.id, + type: node.type, + content_id: node.content_id, + inputs: self.form.data.create(), + }) + .then((response) => { + const data = response.data; + node.update_field_data(data); }); - }, - }) - ); - _addLabelAnnotation(this.form); - this.form.render(); + }, + }); } } /** Tool form wrapper for the workflow editor. */ export class ToolForm { constructor(options) { - var self = this; - var node = options.node; - this.form = new ToolFormBase( - Utils.merge(options, { - text_enable: "Set in Advance", - text_disable: "Set at Runtime", - narrow: true, - initial_errors: true, - cls: "ui-portlet-section", - initialmodel: function (process, form) { - self._customize(form); - process.resolve(); - }, - buildmodel: function (process, form) { - form.model.get("postchange")(process, form); - }, - postchange: function (process, form) { - const Galaxy = getGalaxyInstance(); - var options = form.model.attributes; - var current_state = { - tool_id: options.id, - tool_version: options.version, - type: "tool", - inputs: $.extend(true, {}, form.data.create()), - }; - Galaxy.emit.debug("tool-form-workflow::postchange()", "Sending current state.", current_state); - Utils.request({ - type: "POST", - url: `${getAppRoot()}api/workflows/build_module`, - data: current_state, - success: function (data) { - form.model.set(data.config_form); - self._customize(form); - form.update(data.config_form); - form.errors(data.config_form); - // This hasn't modified the workflow, just returned - // module information for the tool to update the workflow - // state stored on the client with. User needs to save - // for this to take effect. - node.update_field_data(data); - Galaxy.emit.debug("tool-form-workflow::postchange()", "Received new model.", data); - process.resolve(); - }, - error: function (response) { - Galaxy.emit.debug("tool-form-workflow::postchange()", "Refresh request failed.", response); - process.reject(); - }, + const self = this; + const node = options.node; + this.workflow = options.workflow; + this.datatypes = options.datatypes; + this._customize(node); + this.form = new ToolFormBase({ + ...node.config_form, + text_enable: "Set in Advance", + text_disable: "Set at Runtime", + narrow: true, + initial_errors: true, + cls: "ui-portlet-section", + postchange(process, form) { + const Galaxy = getGalaxyInstance(); + const options = form.model.attributes; + const current_state = { + tool_id: options.id, + tool_version: options.version, + type: "tool", + inputs: Object.assign({}, form.data.create()), + }; + Galaxy.emit.debug("tool-form-workflow::postchange()", "Sending current state.", current_state); + axios + .post(`${getAppRoot()}api/workflows/build_module`, current_state) + .then((response) => { + const data = response.data; + self._customize(data); + self.form.model.set(data.config_form); + self.form.update(data.config_form); + self.form.errors(data.config_form); + // This hasn't modified the workflow, just returned + // module information for the tool to update the workflow + // state stored on the client with. User needs to save + // for this to take effect. + node.update_field_data(data); + Galaxy.emit.debug("tool-form-workflow::postchange()", "Received new model.", data); + process.resolve(); + }) + .catch((response) => { + Galaxy.emit.debug("tool-form-workflow::postchange()", "Refresh request failed.", response); + process.reject(); }); - }, - }) - ); + }, + }); } - _customize(form) { - var options = form.model.attributes; - Utils.deepeach(options.inputs, (input) => { + _customize(node) { + const inputs = node.config_form.inputs; + Utils.deepeach(inputs, (input) => { if (input.type) { - input.connectable = true; if (["data", "data_collection"].indexOf(input.type) != -1) { input.type = "hidden"; input.info = `Data input '${input.name}' (${Utils.textify(input.extensions)})`; input.value = { __class__: "RuntimeValue" }; } else if (!input.fixed) { + input.connectable = true; input.collapsible_value = { __class__: "RuntimeValue", }; @@ -108,23 +98,22 @@ export class ToolForm { } } }); - Utils.deepeach(options.inputs, (input) => { + Utils.deepeach(inputs, (input) => { if (input.type === "conditional") { input.connectable = false; input.test_param.collapsible_value = undefined; } }); - _addSections(form); - _addLabelAnnotation(form); + _addSections(this, node); + _addLabelAnnotation(this, node); } } /** Augments the module form definition by adding label and annotation fields */ -function _addLabelAnnotation(form) { - var options = form.model.attributes; - var workflow = options.workflow; - var node = options.node; - options.inputs.unshift({ +function _addLabelAnnotation(self, node) { + var workflow = self.workflow; + const inputs = node.config_form.inputs; + inputs.unshift({ type: "text", name: "__annotation", label: "Annotation", @@ -133,7 +122,7 @@ function _addLabelAnnotation(form) { area: true, help: "Add an annotation or notes to this step. Annotations are available when a workflow is viewed.", }); - options.inputs.unshift({ + inputs.unshift({ type: "text", name: "__label", label: "Label", @@ -149,20 +138,20 @@ function _addLabelAnnotation(form) { break; } } - var input_id = form.data.match("__label"); - var input_element = form.element_list[input_id]; + var input_id = self.form.data.match("__label"); + var input_element = self.form.element_list[input_id]; input_element.model.set( "error_text", duplicate && "Duplicate label. Please fix this before saving the workflow." ); - form.trigger("change"); + self.form.trigger("change"); }, }); } /** Visit input nodes and enrich by name/value pairs from server data */ -function _visit(head, head_list, output_id, options) { - var post_job_actions = options.node.post_job_actions; +function _visit(head, head_list, output_id, node) { + var post_job_actions = node.post_job_actions; head_list = head_list || []; head_list.push(head); for (var i in head.inputs) { @@ -192,7 +181,7 @@ function _visit(head, head_list, output_id, options) { } } if (input.inputs) { - _visit(input, head_list.slice(0), output_id, options); + _visit(input, head_list.slice(0), output_id, node); } } } @@ -215,13 +204,11 @@ function _makeRenameHelp(name_labels) { } /** Builds sub section with step actions/annotation */ -function _makeSection(output_id, label, options) { +function _makeSection(self, output_id, label, node) { var extensions = []; var name_label_map = []; - var datatypes = options.datatypes; - var node = options.node; - var workflow = options.workflow; - + var datatypes = self.datatypes; + var workflow = self.workflow; for (const key in datatypes) { extensions.push({ 0: datatypes[key], 1: datatypes[key] }); } @@ -350,18 +337,15 @@ function _makeSection(output_id, label, options) { }, ], }; - _visit(input_config, [], output_id, options); + _visit(input_config, [], output_id, node); return input_config; } /** Builds all sub sections */ -function _addSections(form) { - var options = form.model.attributes; - var inputs = options.inputs; - var node = options.node; +function _addSections(self, node) { + var inputs = node.config_form.inputs; var post_job_actions = node.post_job_actions; var output_id = node.output_terminals && Object.keys(node.output_terminals)[0]; - if (output_id) { inputs.push({ name: `pja__${output_id}__EmailAction`, @@ -385,7 +369,7 @@ function _addSections(form) { }); for (const output_id in node.output_terminals) { const label = node.output_terminals[output_id].label || output_id; - inputs.push(_makeSection(output_id, label, options)); + inputs.push(_makeSection(self, output_id, label, node)); } } } diff --git a/client/galaxy/scripts/mvc/workflow/workflow-manager.js b/client/galaxy/scripts/mvc/workflow/workflow-manager.js index 0fdf0b5b2d8..241b9646f81 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-manager.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-manager.js @@ -18,6 +18,7 @@ class Workflow extends EventEmitter { this.has_changes = false; this.workflowOutputLabels = {}; this.workflow_version = 0; + this.popover_counter = 0; // Canvas overview management this.canvas_manager = new WorkflowCanvas(this, $("#canvas-viewport"), $("#overview-container")); @@ -49,7 +50,7 @@ class Workflow extends EventEmitter { set_node(node, data) { node.init_field_data(data); node.update_field_data(data); - $.each(node.output_terminals, (ot_id, ot) => { + Object.values(node.output_terminals).forEach((ot) => { node.addWorkflowOutput(ot.name); node.markWorkflowOutput(ot.name); }); @@ -142,8 +143,10 @@ class Workflow extends EventEmitter { type: type, title: title_text, node: node, + nodeId: this.popover_counter, }); + this.popover_counter++; // Set initial scroll position $f.css("left", $(window).scrollLeft() + 20); $f.css("top", $(window).scrollTop() + 20); @@ -195,20 +198,20 @@ class Workflow extends EventEmitter { } remove_all() { var wf = this; - $.each(this.nodes, (k, v) => { - v.destroy(); - wf.remove_node(v); + Object.values(this.nodes).forEach((node) => { + node.destroy(); + wf.remove_node(node); }); } rectify_workflow_outputs() { // Find out if we're using workflow_outputs or not. var using_workflow_outputs = false; var has_existing_pjas = false; - $.each(this.nodes, (k, node) => { + Object.values(this.nodes).forEach((node) => { if (node.type === "tool" && node.workflow_outputs && node.workflow_outputs.length > 0) { using_workflow_outputs = true; } - $.each(node.post_job_actions, (pja_id, pja) => { + Object.values(node.post_job_actions).forEach((pja) => { if (pja.action_type === "HideDatasetAction") { has_existing_pjas = true; } @@ -216,26 +219,26 @@ class Workflow extends EventEmitter { }); if (using_workflow_outputs !== false || has_existing_pjas !== false) { // Using workflow outputs, or has existing pjas. Remove all PJAs and recreate based on outputs. - $.each(this.nodes, (k, node) => { + Object.values(this.nodes).forEach((node) => { var node_changed = false; if (node.post_job_actions === null) { node.post_job_actions = {}; node_changed = true; } var pjas_to_rem = []; - $.each(node.post_job_actions, (pja_id, pja) => { + Object.entries(node.post_job_actions).forEach(([pja_id, pja]) => { if (pja.action_type == "HideDatasetAction") { pjas_to_rem.push(pja_id); } }); if (pjas_to_rem.length > 0) { - $.each(pjas_to_rem, (i, pja_name) => { + pjas_to_rem.forEach((pja_name) => { node_changed = true; delete node.post_job_actions[pja_name]; }); } if (using_workflow_outputs) { - $.each(node.output_terminals, (ot_id, ot) => { + Object.values(node.output_terminals).forEach((ot) => { var create_pja = !node.isWorkflowOutput(ot.name); if (create_pja === true) { node_changed = true; @@ -258,14 +261,14 @@ class Workflow extends EventEmitter { } to_simple() { var nodes = {}; - $.each(this.nodes, (i, node) => { + Object.values(this.nodes).forEach((node) => { var input_connections = {}; - $.each(node.input_terminals, (k, t) => { + Object.values(node.input_terminals).forEach((t) => { input_connections[t.name] = null; // There should only be 0 or 1 connectors, so this is // really a sneaky if statement var cons = []; - $.each(t.connectors, (i, c) => { + t.connectors.forEach((c, i) => { if (c.handle1) { var con_dict = { id: c.handle1.node.id, @@ -282,8 +285,8 @@ class Workflow extends EventEmitter { }); var post_job_actions = {}; if (node.post_job_actions) { - $.each(node.post_job_actions, (i, act) => { - var pja = { + Object.values(node.post_job_actions).forEach((act) => { + const pja = { action_type: act.action_type, output_name: act.output_name, action_arguments: act.action_arguments, @@ -316,8 +319,7 @@ class Workflow extends EventEmitter { const report = this.report; return { steps: nodes, report: report }; } - from_simple(data, initialImport_) { - var initialImport = initialImport_ === undefined ? true : initialImport_; + from_simple(data, initialImport = true) { var wf = this; var offset = 0; if (initialImport) { @@ -330,13 +332,13 @@ class Workflow extends EventEmitter { var using_workflow_outputs = false; wf.workflow_version = data.version; wf.report = data.report || {}; - $.each(data.steps, (id, step) => { + Object.entries(data.steps).forEach(([id, step]) => { var node = wf.prebuildNode(step.type, step.name, step.content_id); // If workflow being copied into another, wipe UUID and let // Galaxy assign new ones. if (!initialImport) { step.uuid = null; - $.each(step.workflow_outputs, (name, workflow_output) => { + step.workflow_outputs.forEach((workflow_output) => { workflow_output.uuid = null; }); } @@ -358,7 +360,7 @@ class Workflow extends EventEmitter { if (node.workflow_outputs.length > 0) { using_workflow_outputs = true; } else { - $.each(node.post_job_actions || [], (pja_id, pja) => { + Object.values(node.post_job_actions).forEach((pja) => { if (pja.action_type === "HideDatasetAction") { using_workflow_outputs = true; } @@ -368,14 +370,14 @@ class Workflow extends EventEmitter { }); wf.id_counter = max_id + 1; // Second pass, connections - $.each(data.steps, (id, step) => { + Object.entries(data.steps).forEach(([id, step]) => { const node = wf.nodes[parseInt(id) + offset]; - $.each(step.input_connections, (k, v) => { + Object.entries(step.input_connections).forEach(([k, v]) => { if (v) { - if (!$.isArray(v)) { + if (!Array.isArray(v)) { v = [v]; } - $.each(v, (l, x) => { + v.forEach((x) => { const other_node = wf.nodes[parseInt(x.id) + offset]; const c = new Connector(this.canvas_manager); c.connect(other_node.output_terminals[x.output_name], node.input_terminals[k]); @@ -385,7 +387,7 @@ class Workflow extends EventEmitter { }); if (using_workflow_outputs) { // Ensure that every output terminal has a WorkflowOutput or HideDatasetAction. - $.each(node.output_terminals, (ot_id, ot) => { + Object.values(node.output_terminals).forEach((ot) => { if (node.post_job_actions[`HideDatasetAction${ot.name}`] === undefined) { node.addWorkflowOutput(ot.name); node.markWorkflowOutput(ot.name); @@ -451,7 +453,7 @@ class Workflow extends EventEmitter { var n_pred = {}; var successors = {}; // First pass to initialize arrays even for nodes with no connections - $.each(this.nodes, (id, node) => { + Object.keys(this.nodes).forEach((id) => { if (n_pred[id] === undefined) { n_pred[id] = 0; } @@ -460,9 +462,9 @@ class Workflow extends EventEmitter { } }); // Second pass to count predecessors and successors - $.each(this.nodes, (id, node) => { - $.each(node.input_terminals, (j, t) => { - $.each(t.connectors, (k, c) => { + Object.values(this.nodes).forEach((node) => { + Object.values(node.input_terminals).forEach((t) => { + t.connectors.forEach((c) => { // A connection exists from `other` to `node` var other = c.handle1.node; // node gains a predecessor @@ -505,14 +507,14 @@ class Workflow extends EventEmitter { var h_pad = 80; var v_pad = 30; var left = h_pad; - $.each(node_ids_by_level, (i, ids) => { + node_ids_by_level.forEach((ids) => { // We keep nodes in the same order in a level to give the user // some control over ordering ids.sort((a, b) => $(all_nodes[a].element).position().top - $(all_nodes[b].element).position().top); // Position each node var max_width = 0; var top = v_pad; - $.each(ids, (j, id) => { + ids.forEach((id) => { var node = all_nodes[id]; var element = $(node.element); $(element).css({ top: top, left: left }); @@ -522,7 +524,7 @@ class Workflow extends EventEmitter { left += max_width + h_pad; }); // Need to redraw all connectors - $.each(all_nodes, (_, node) => { + Object.values(all_nodes).forEach((node) => { node.redraw(); }); } @@ -532,7 +534,7 @@ class Workflow extends EventEmitter { var ymin = Infinity; var ymax = -Infinity; var p; - $.each(this.nodes, (id, node) => { + Object.values(this.nodes).forEach((node) => { var e = $(node.element); p = e.position(); xmin = Math.min(xmin, p.left); diff --git a/client/galaxy/scripts/mvc/workflow/workflow-node.js b/client/galaxy/scripts/mvc/workflow/workflow-node.js index fcdc22080e9..6fe45f1f9f2 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-node.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-node.js @@ -274,7 +274,7 @@ export class Node { x.destroy(); // Removes the noodle connectors } }); - nodeView.outputViews[unused_output].remove(); // removes the rendered output + nodeView.outputViews[unused_output].$el.remove(); // removes the rendered output delete nodeView.outputViews[unused_output]; // removes the reference to the output delete node.output_terminals[unused_output]; // removes the output terminal }); diff --git a/client/galaxy/scripts/store/index.js b/client/galaxy/scripts/store/index.js index ecdde654cba..0f89419a996 100644 --- a/client/galaxy/scripts/store/index.js +++ b/client/galaxy/scripts/store/index.js @@ -9,6 +9,7 @@ import createCache from "vuex-cache"; import { gridSearchStore } from "./gridSearchStore"; import { tagStore } from "./tagStore"; import { jobMetricsStore } from "./jobMetricsStore"; +import { jobDestinationParametersStore } from "./jobDestinationParametersStore"; import { invocationStore } from "./invocationStore"; import { historyStore } from "./historyStore"; import { userStore } from "./userStore"; @@ -31,6 +32,7 @@ export function createStore() { histories: historyStore, tags: tagStore, jobMetrics: jobMetricsStore, + destinationParameters: jobDestinationParametersStore, invocations: invocationStore, user: userStore, config: configStore, diff --git a/client/galaxy/scripts/store/jobDestinationParametersStore.js b/client/galaxy/scripts/store/jobDestinationParametersStore.js new file mode 100644 index 00000000000..4503f15cb17 --- /dev/null +++ b/client/galaxy/scripts/store/jobDestinationParametersStore.js @@ -0,0 +1,33 @@ +export const state = { + jobDestinationParametersByJobId: {}, +}; + +import Vue from "vue"; +import { getAppRoot } from "onload/loadConfig"; +import axios from "axios"; + +const getters = { + jobDestinationParams: (state) => (jobId) => { + return state.jobDestinationParametersByJobId[jobId] || []; + }, +}; + +const actions = { + fetchJobDestinationParams: async ({ commit }, jobId) => { + const { data } = await axios.get(`${getAppRoot()}api/jobs/${jobId}/destination_params`); + commit("saveJobDestinationParamsForJobId", { jobId, jobDestinationParams: data }); + }, +}; + +const mutations = { + saveJobDestinationParamsForJobId: (state, { jobId, jobDestinationParams }) => { + Vue.set(state.jobDestinationParametersByJobId, jobId, jobDestinationParams); + }, +}; + +export const jobDestinationParametersStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/galaxy/scripts/utils/simple-error.js b/client/galaxy/scripts/utils/simple-error.js index 1341c1301f4..9c6f3f4ccdd 100644 --- a/client/galaxy/scripts/utils/simple-error.js +++ b/client/galaxy/scripts/utils/simple-error.js @@ -9,5 +9,6 @@ export function errorMessageAsString(e) { } export function rethrowSimple(e) { + console.debug(e); throw errorMessageAsString(e); } diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index 05fdce4b813..8c0502e238f 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1656,31 +1656,29 @@ body.reports { .node { cursor: pointer; circle { - fill: lighten($brand-primary, 20%); - stroke: lighten($brand-primary, 20%); - stroke-width: 0.1rem; + fill: $brand-primary; + stroke: $brand-primary; } text { - font: 0.4rem sans-serif; + font-size: 0.4rem; } } - .node-enter { - fill-opacity: 1e-6; - } - - .node-update { - fill-opacity: 1; - } - .tree-size { width: 100%; - height: 50%; } .link { fill: none; - stroke: lighten($brand-primary, 20%); - stroke-width: 0.2rem; + stroke: $brand-primary; + stroke-width: 1; + } +} + +.workflow-recommendations { + display: block; + .header-background { + border-bottom: solid 1px $brand-primary; + margin-bottom: 0.5rem; } } diff --git a/client/galaxy/style/scss/workflow.scss b/client/galaxy/style/scss/workflow.scss index b89ca0bc69e..b4ecd68b854 100644 --- a/client/galaxy/style/scss/workflow.scss +++ b/client/galaxy/style/scss/workflow.scss @@ -7,6 +7,9 @@ display: flex; flex-direction: column; flex-grow: 1; + .workflow-recommendations { + height: 30rem; + } .workflow-node { @extend .card; @extend .position-absolute; diff --git a/client/gulpfile.js b/client/gulpfile.js index f31949a263c..af0d5c6369c 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -21,7 +21,7 @@ const paths = { // "../config/plugins/{visualizations,interactive_environments}/*/*/package.json" //], plugin_build_dirs: [ - "../config/plugins/visualizations/{annotate_image,hyphyvision,openlayers}/package.json", + "../config/plugins/visualizations/{annotate_image,hyphyvision,openlayers,editor}/package.json", ], lib_locs: { // This is a stepping stone towards having all this staged diff --git a/config/plugins/visualizations/editor/config/editor.xml b/config/plugins/visualizations/editor/config/editor.xml new file mode 100644 index 00000000000..48589917e99 --- /dev/null +++ b/config/plugins/visualizations/editor/config/editor.xml @@ -0,0 +1,16 @@ + + + + Manually edit text + + + HistoryDatasetAssociation + data.Data + dataset_id + + + + dataset_id + + editor.mako + diff --git a/config/plugins/visualizations/editor/package.json b/config/plugins/visualizations/editor/package.json new file mode 100644 index 00000000000..f10c514d629 --- /dev/null +++ b/config/plugins/visualizations/editor/package.json @@ -0,0 +1,21 @@ +{ + "devDependencies": { + "css-loader": "^3.4.2", + "file-loader": "^5.0.2", + "mini-css-extract-plugin": "^0.9.0", + "style-loader": "^1.1.3", + "webpack": "^4.41.6", + "webpack-cli": "^3.3.11", + "extract-text-webpack-plugin": "^3.0.2" + }, + "dependencies": { + "ace-builds": "^1.4.8" + }, + "name": "galaxy-vis-editor", + "version": "1.1.0", + "main": "index.js", + "license": "MIT", + "scripts": { + "build": "webpack --mode production" + } +} diff --git a/config/plugins/visualizations/editor/src/index.js b/config/plugins/visualizations/editor/src/index.js new file mode 100644 index 00000000000..b9c17d7a2bb --- /dev/null +++ b/config/plugins/visualizations/editor/src/index.js @@ -0,0 +1,4 @@ +import './styles/style.css'; +import "ace-builds/src-noconflict/ace"; +import "ace-builds/src-noconflict/mode-powershell"; +import "ace-builds/src-noconflict/theme-textmate"; diff --git a/config/plugins/visualizations/editor/src/styles/style.css b/config/plugins/visualizations/editor/src/styles/style.css new file mode 100644 index 00000000000..e187e15daac --- /dev/null +++ b/config/plugins/visualizations/editor/src/styles/style.css @@ -0,0 +1,41 @@ +#editor { + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +#export-btn { + position: relative; + z-index: 3; + left: 90%; +} + +.transparent_btn { + display: inline-block; + padding: 10px 14px; + color: #FFF; + border: 1px solid #FFF; + text-decoration: none; + font-size: 14px; + line-height: 120%; + background-color: rgba(255, 255, 255, 0); + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + -webkit-transition: 300ms ease; + -moz-transition: 300ms ease; + transition: 300ms ease; + cursor: pointer; +} + +.transparent_btn.blue { + color: #aeddf5; + border-color: #aeddf5; +} + +.transparent_btn.blue:hover { + color: #0062ff; + background-color: rgba(174, 221, 245, 0.3); +} diff --git a/config/plugins/visualizations/editor/templates/editor.mako b/config/plugins/visualizations/editor/templates/editor.mako new file mode 100755 index 00000000000..385edbba620 --- /dev/null +++ b/config/plugins/visualizations/editor/templates/editor.mako @@ -0,0 +1,72 @@ +<% + default_title = "JS Editor" + + # Use root for resource loading. + root = h.url_for( '/static/' ) + app_root = root + "plugins/visualizations/editor/static/" +%> +## ---------------------------------------------------------------------------- + + + + + + ${visualization_name} + ${h.javascript_link( app_root + "script.js" )} + ${h.stylesheet_link( app_root + "main.css" )} + + + +
+
+ + + + diff --git a/config/plugins/visualizations/editor/webpack.config.js b/config/plugins/visualizations/editor/webpack.config.js new file mode 100644 index 00000000000..0417541a1d0 --- /dev/null +++ b/config/plugins/visualizations/editor/webpack.config.js @@ -0,0 +1,39 @@ +const webpack = require("webpack"); +const path = require("path"); +const MiniCssExtractPlugin = require("mini-css-extract-plugin"); + +module.exports = { + mode: 'production', + entry: path.resolve(__dirname, "src/index.js"), + output: { + filename: "script.js", + path: path.resolve(__dirname, "static") + }, + plugins: [ + new MiniCssExtractPlugin(), + ], + module: { + rules: [ + { + test: /\.css$/i, + use: [MiniCssExtractPlugin.loader, 'css-loader'], + }, + { + test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/, + loader: "url-loader", + options: {limit: 10000, mimetype: "application/font-woff"} + }, + { + test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, + loader: "url-loader", + options: {limit: 10000, mimetype: "application/octet-stream"} + }, + {test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, loaders: "file-loader"}, + { + test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, + loaders: "url-loader", + options: {limit: 10000, mimetype: "image/svg+xml"} + } + ] + }, +}; diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 6d387d8ee36..736869828f1 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1376,6 +1376,27 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``interactivetools_proxy_host`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Proxy host - assumed to just be hosted on the same hostname and + port as Galaxy by default. +:Default: ``None`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~ +``interactivetools_map`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Map for interactivetool proxy. +:Default: ``interactivetools_map.sqlite`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~ ``visualizations_visible`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index b8b3388dfd5..75a7f4a72cf 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -116,9 +116,9 @@ class PSAAuthnz(IdentityProvider): # Secondary AuthZ with Google identities is currently supported if provider != "google": - if "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config: + if 'SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER' in self.config: del self.config["SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER"] - if "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config: + if 'SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT' in self.config: del self.config["SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT"] def _setup_idp(self, oidc_backend_config): diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index a9cb031c201..ccc1c4a08e4 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -118,6 +118,10 @@ class BaseAppConfiguration(object): self._create_attributes_from_raw_config() # Create attributes based on raw_config self._resolve_paths() # Overwrite attribute values with resolved paths + def resolve_path(self, path): + """Resolve a path relative to Galaxy's root.""" + return self._in_root_dir(path) + def _set_config_base(self, config_kwargs): def _set_global_conf(): @@ -245,17 +249,23 @@ class BaseAppConfiguration(object): for key in self.schema.paths_to_resolve: resolve(key) + def _in_root_dir(self, path): + return self._in_dir(self.root, path) + def _in_managed_config_dir(self, path): - return os.path.join(self.managed_config_dir, path) + return self._in_dir(self.managed_config_dir, path) def _in_config_dir(self, path): - return os.path.join(self.config_dir, path) + return self._in_dir(self.config_dir, path) def _in_sample_dir(self, path): - return os.path.join(self.sample_config_dir, path) + return self._in_dir(self.sample_config_dir, path) def _in_data_dir(self, path): - return os.path.join(self.data_dir, path) + return self._in_dir(self.data_dir, path) + + def _in_dir(self, _dir, path): + return os.path.join(_dir, path) if path else None def _parse_config_file_options(self, defaults, listify_defaults, config_kwargs): for var, values in defaults.items(): @@ -291,7 +301,47 @@ class BaseAppConfiguration(object): setattr(self, var, [os.path.join(self.root, x) for x in paths]) -class GalaxyAppConfiguration(BaseAppConfiguration): +class CommonConfigurationMixin(object): + """Shared configuration settings code for Galaxy and ToolShed.""" + + @property + def admin_users(self): + return self._admin_users + + @admin_users.setter + def admin_users(self, value): + self._admin_users = value + if value: + self.admin_users_list = [u.strip() for u in value.split(',') if u] + else: # provide empty list for convenience (check membership, etc.) + self.admin_users_list = [] + + def is_admin_user(self, user): + """Determine if the provided user is listed in `admin_users`.""" + return user is not None and user.email in self.admin_users_list + + @property + def sentry_dsn_public(self): + """ + Sentry URL with private key removed for use in client side scripts, + sentry server will need to be configured to accept events + """ + if self.sentry_dsn: + return re.sub(r"^([^:/?#]+:)?//(\w+):(\w+)", r"\1//\2", self.sentry_dsn) + + def get_bool(self, key, default): + # Warning: the value of self.config_dict['foo'] may be different from self.foo + if key in self.config_dict: + return string_as_bool(self.config_dict[key]) + else: + return default + + def get(self, key, default=None): + # Warning: the value of self.config_dict['foo'] may be different from self.foo + return self.config_dict.get(key, default) + + +class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): deprecated_options = ('database_file', 'track_jobs_in_database') default_config_file_name = 'galaxy.yml' deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'} @@ -317,12 +367,11 @@ class GalaxyAppConfiguration(BaseAppConfiguration): # Database related configuration self.check_migrate_databases = kwargs.get('check_migrate_databases', True) if not self.database_connection: # Provide default if not supplied by user - db_path = os.path.join(self.data_dir, 'universe.sqlite') + db_path = self._in_data_dir('universe.sqlite') self.database_connection = 'sqlite:///%s?isolation_level=IMMEDIATE' % db_path self.database_engine_options = get_database_engine_options(kwargs) self.database_create_tables = string_as_bool(kwargs.get('database_create_tables', 'True')) self.database_encoding = kwargs.get('database_encoding') # Create new databases with this encoding - self.database_log_query_counts = string_as_bool(kwargs.get("database_log_query_counts", 'False')) self.thread_local_log = None if self.enable_per_request_sql_debugging: self.thread_local_log = threading.local() @@ -334,10 +383,10 @@ class GalaxyAppConfiguration(BaseAppConfiguration): tempfile.tempdir = self.new_file_path self.shared_home_dir = kwargs.get("shared_home_dir") self.cookie_path = kwargs.get("cookie_path") - self.tool_path = os.path.join(self.root, self.tool_path) - self.tool_data_path = os.path.join(self.root, self.tool_data_path) + self.tool_path = self._in_root_dir(self.tool_path) + self.tool_data_path = self._in_root_dir(self.tool_data_path) if not running_from_source and kwargs.get("tool_data_path") is None: - self.tool_data_path = os.path.join(self.data_dir, "tool-data") + self.tool_data_path = self._in_data_dir(self.schema.defaults['tool_data_path']) self.builds_file_path = os.path.join(self.tool_data_path, self.builds_file_path) self.len_file_path = os.path.join(self.tool_data_path, self.len_file_path) # Galaxy OIDC settings. @@ -347,7 +396,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): self.integrated_tool_panel_config = self._in_managed_config_dir(self.integrated_tool_panel_config) integrated_tool_panel_tracking_directory = kwargs.get('integrated_tool_panel_tracking_directory') if integrated_tool_panel_tracking_directory: - self.integrated_tool_panel_tracking_directory = os.path.join(self.root, integrated_tool_panel_tracking_directory) + self.integrated_tool_panel_tracking_directory = self._in_root_dir(integrated_tool_panel_tracking_directory) else: self.integrated_tool_panel_tracking_directory = None self.toolbox_filter_base_modules = listify(self.toolbox_filter_base_modules) @@ -363,7 +412,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): self.password_expiration_period = timedelta(days=int(self.password_expiration_period)) if self.shed_tool_data_path: - self.shed_tool_data_path = os.path.join(self.root, self.shed_tool_data_path) + self.shed_tool_data_path = self._in_root_dir(self.shed_tool_data_path) else: self.shed_tool_data_path = self.tool_data_path @@ -398,21 +447,21 @@ class GalaxyAppConfiguration(BaseAppConfiguration): for ip in kwargs.get("fetch_url_whitelist", "").split(',') if len(ip.strip()) > 0 ] - self.template_path = os.path.join(self.root, kwargs.get("template_path", "templates")) + self.template_path = self._in_root_dir(kwargs.get("template_path", "templates")) self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5")) - self.cluster_files_directory = self.resolve_path(self.cluster_files_directory) + self.cluster_files_directory = self._in_root_dir(self.cluster_files_directory) # Fall back to legacy job_working_directory config variable if set. - self.jobs_directory = os.path.join(self.data_dir, kwargs.get("jobs_directory", self.job_working_directory)) + self.jobs_directory = self._in_data_dir(kwargs.get("jobs_directory", self.job_working_directory)) if self.preserve_python_environment not in ["legacy_only", "legacy_and_local", "always"]: log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only") self.preserve_python_environment = "legacy_only" self.nodejs_path = kwargs.get("nodejs_path") # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we # should probably drop the backward compatiblity to save the path check. - self.container_image_cache_path = os.path.join(self.data_dir, kwargs.get("container_image_cache_path", "container_images")) + self.container_image_cache_path = self._in_data_dir(kwargs.get("container_image_cache_path", "container_images")) if not os.path.exists(self.container_image_cache_path): - self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", os.path.join(self.data_dir, "container_cache"))) + self.container_image_cache_path = self._in_root_dir(kwargs.get("container_image_cache_path", self._in_data_dir("container_cache"))) self.output_size_limit = int(kwargs.get('output_size_limit', 0)) # activation_email was used until release_15.03 activation_email = kwargs.get('activation_email') @@ -421,7 +470,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): # Get the disposable email domains blacklist file and its contents self.blacklist_content = None if self.blacklist_file: - self.blacklist_file = os.path.join(self.root, self.blacklist_file) + self.blacklist_file = self._in_root_dir(self.blacklist_file) try: with open(self.blacklist_file) as f: self.blacklist_content = [line.rstrip() for line in f] @@ -449,7 +498,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): workflow_resource_params_mapper = None elif ":" not in workflow_resource_params_mapper: # Assume it is not a Python function, so a file - workflow_resource_params_mapper = self.resolve_path(workflow_resource_params_mapper) + workflow_resource_params_mapper = self._in_root_dir(workflow_resource_params_mapper) # else: a Python a function! self.workflow_resource_params_mapper = workflow_resource_params_mapper @@ -460,7 +509,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): _sanitize_whitelist_path = self._in_managed_config_dir(self.sanitize_whitelist_file) if not os.path.isfile(_sanitize_whitelist_path): # then check old default location - deprecated = os.path.join(self.root, 'config/sanitize_whitelist.txt') + deprecated = self._in_root_dir('config/sanitize_whitelist.txt') if os.path.isfile(deprecated): log.warning("The path '%s' for the 'sanitize_whitelist_file' config option is " "deprecated and will be no longer checked in a future release. Please consult " @@ -488,29 +537,26 @@ class GalaxyAppConfiguration(BaseAppConfiguration): # not needed on production systems but useful if running many functional tests. self.index_tool_help = string_as_bool(kwargs.get("index_tool_help", True)) self.tool_labels_boost = kwargs.get("tool_labels_boost", 1) - default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", os.path.join(self.root, "test-data")) + default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", self._in_root_dir("test-data")) self.tool_test_data_directories = kwargs.get("tool_test_data_directories", default_tool_test_data_directories) # Deployers may either specify a complete list of mapping files or get the default for free and just # specify a local mapping file to adapt and extend the default one. if "conda_mapping_files" not in kwargs: - conda_mapping_files = [ - self.local_conda_mapping_file, - os.path.join(self.root, "lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml"), - ] + _default_mapping = self._in_root_dir(os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml")) # dependency resolution options are consumed via config_dict - so don't populate # self, populate config_dict - self.config_dict["conda_mapping_files"] = conda_mapping_files + self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] if self.containers_resolvers_config_file: - self.containers_resolvers_config_file = os.path.join(self.config_dir, self.containers_resolvers_config_file) + self.containers_resolvers_config_file = self._in_config_dir(self.containers_resolvers_config_file) # tool_dependency_dir can be "none" (in old configs). If so, set it to None if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none': self.tool_dependency_dir = None if self.involucro_path is None: target_dir = self.tool_dependency_dir or self.schema.defaults['tool_dependency_dir'] - self.involucro_path = os.path.join(self.data_dir, target_dir, "involucro") - self.involucro_path = os.path.join(self.root, self.involucro_path) + self.involucro_path = self._in_data_dir(os.path.join(target_dir, "involucro")) + self.involucro_path = self._in_root_dir(self.involucro_path) if self.mulled_channels: self.mulled_channels = [c.strip() for c in self.mulled_channels.split(',')] @@ -524,18 +570,12 @@ class GalaxyAppConfiguration(BaseAppConfiguration): self.nginx_upload_store = os.path.abspath(self.nginx_upload_store) self.object_store = kwargs.get('object_store', 'disk') self.object_store_check_old_style = string_as_bool(kwargs.get('object_store_check_old_style', False)) - self.object_store_cache_path = self.resolve_path(kwargs.get("object_store_cache_path", os.path.join(self.data_dir, "object_store_cache"))) - object_store_store_by = kwargs.get('object_store_store_by', None) - if object_store_store_by is None: - if not self.file_path_set: - if self.file_path.endswith('objects'): - object_store_store_by = 'uuid' - else: - object_store_store_by = 'id' - else: - object_store_store_by = 'id' - assert object_store_store_by in ['id', 'uuid'], "Invalid value for object_store_store_by [%s]" % object_store_store_by - self.object_store_store_by = object_store_store_by + self.object_store_cache_path = self._in_root_dir(kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache"))) + if self.object_store_store_by is None: + self.object_store_store_by = 'id' + if not self.file_path_set and self.file_path.endswith('objects'): + self.object_store_store_by = 'uuid' + assert self.object_store_store_by in ['id', 'uuid'], "Invalid value for object_store_store_by [%s]" % self.object_store_store_by # Handle AWS-specific config options for backward compatibility if kwargs.get('aws_access_key') is not None: self.os_access_key = kwargs.get('aws_access_key') @@ -554,7 +594,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): self.object_store_cache_size = float(kwargs.get('object_store_cache_size', -1)) self.distributed_object_store_config_file = kwargs.get('distributed_object_store_config_file') if self.distributed_object_store_config_file is not None: - self.distributed_object_store_config_file = os.path.join(self.root, self.distributed_object_store_config_file) + self.distributed_object_store_config_file = self._in_root_dir(self.distributed_object_store_config_file) self.irods_root_collection_path = kwargs.get('irods_root_collection_path') self.irods_default_resource = kwargs.get('irods_default_resource') # Heartbeat log file name override @@ -600,7 +640,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration): elif 'database_connection' in kwargs: self.amqp_internal_connection = "sqlalchemy+" + self.database_connection else: - self.amqp_internal_connection = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % os.path.join(self.data_dir, "control.sqlite") + self.amqp_internal_connection = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % self._in_data_dir("control.sqlite") self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format) try: with open(self.user_preferences_extra_conf_path, 'r') as stream: @@ -617,7 +657,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration): # This is for testing new library browsing capabilities. self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False)) # Logging configuration with logging.config.configDict: - self.logging = kwargs.get('logging') # Statistics and profiling with statsd self.statsd_host = kwargs.get('statsd_host', '') @@ -630,8 +669,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration): self.manage_dynamic_proxy = self.dynamic_proxy_manage # Set to false if being launched externally # InteractiveTools propagator mapping file - self.interactivetool_map = self.resolve_path(kwargs.get("interactivetools_map", os.path.join(self.data_dir, "interactivetools_map.sqlite"))) + self.interactivetools_map = self._in_root_dir(kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))) self.interactivetool_prefix = kwargs.get("interactivetools_prefix", "interactivetool") + self.interactivetool_proxy_host = kwargs.get("interactivetool_proxy_host", None) self.containers_conf = parse_containers_config(self.containers_config_file) @@ -708,33 +748,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration): 'UWSGI_PORT': port }) - @property - def admin_users(self): - return self._admin_users - - @admin_users.setter - def admin_users(self, value): - self._admin_users = value - if value: - self.admin_users_list = [u.strip() for u in value.split(',') if u] - else: # provide empty list for convenience (check membership, etc.) - self.admin_users_list = [] - - @property - def sentry_dsn_public(self): - """ - Sentry URL with private key removed for use in client side scripts, - sentry server will need to be configured to accept events - """ - if self.sentry_dsn: - return re.sub(r"^([^:/?#]+:)?//(\w+):(\w+)", r"\1//\2", self.sentry_dsn) - else: - return None - def parse_config_file_options(self, kwargs): - """ - Backwards compatibility for config files moved to the config/ dir. - """ + """Backwards compatibility for config files moved to the config/ dir.""" defaults = dict( auth_config_file=[self._in_config_dir('auth_conf.xml')], build_sites_config_file=[self._in_config_dir('build_sites.yml'), self._in_sample_dir('build_sites.yml.sample')], @@ -792,15 +807,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration): if explicit: log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools whitelisted.", self.sanitize_whitelist_file) - def get(self, key, default=None): - return self.config_dict.get(key, default) - - def get_bool(self, key, default): - if key in self.config_dict: - return string_as_bool(self.config_dict[key]) - else: - return default - def ensure_tempdir(self): self._ensure_directory(self.new_file_path) @@ -838,20 +844,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration): if key in self.deprecated_options: log.warning("Config option '%s' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file." % key) - def is_admin_user(self, user): - """ - Determine if the provided user is listed in `admin_users`. - - NOTE: This is temporary, admin users will likely be specified in the - database in the future. - """ - return user is not None and user.email in self.admin_users_list - - def resolve_path(self, path): - """ Resolve a path relative to Galaxy's root. - """ - return os.path.join(self.root, path) - @staticmethod def _parse_allowed_origin_hostnames(kwargs): """ @@ -877,7 +869,7 @@ Configuration = GalaxyAppConfiguration def reload_config_options(current_config): - """ Reload modified reloadable config options """ + """Reload modified reloadable config options.""" modified_config = read_properties_from_file(current_config.config_file) for option in current_config.schema.reloadable_options: if option in modified_config: @@ -978,8 +970,7 @@ def configure_logging(config): class ConfiguresGalaxyMixin(object): - """ Shared code for configuring Galaxy-like app objects. - """ + """Shared code for configuring Galaxy-like app objects.""" def _configure_genome_builds(self, data_table_name="__dbkeys__", load_old_style=True): self.genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style) @@ -1116,9 +1107,7 @@ class ConfiguresGalaxyMixin(object): self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry() def _configure_models(self, check_migrate_databases=False, check_migrate_tools=False, config_file=None): - """ - Preconditions: object_store must be set on self. - """ + """Preconditions: object_store must be set on self.""" db_url = get_database_url(self.config) install_db_url = self.config.install_database_connection # TODO: Consider more aggressive check here that this is not the same diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 433b799404b..bbaef985a1b 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -17,6 +17,8 @@ + + @@ -228,19 +230,27 @@ + + + + - + + + + + @@ -248,6 +258,9 @@ + + + @@ -256,11 +269,18 @@ + + + + + + + @@ -284,6 +304,7 @@ + @@ -341,9 +362,7 @@ - - - + @@ -379,6 +398,7 @@ + @@ -798,7 +818,10 @@ + + + @@ -840,27 +863,48 @@ + + - - - - + + + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + @@ -897,6 +941,7 @@ + diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 5c6eefc5d46..ec01788720a 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -761,6 +761,13 @@ galaxy: # Enable InteractiveTools. #interactivetools_enable: false + # Proxy host - assumed to just be hosted on the same hostname and port + # as Galaxy by default. + #interactivetools_proxy_host: null + + # Map for interactivetool proxy. + #interactivetools_map: interactivetools_map.sqlite + # Show visualization tab and list in masthead. #visualizations_visible: true diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample index fd6f80066f5..286a0c9f1c2 100644 --- a/lib/galaxy/config/sample/tool_conf.xml.sample +++ b/lib/galaxy/config/sample/tool_conf.xml.sample @@ -5,6 +5,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 1f464bbdc9f..6af3924c341 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -249,6 +249,16 @@ class CompressedZipArchive(CompressedArchive): except Exception: return "Compressed zip file (%s)" % (nice_size(dataset.get_size())) + def sniff(self, filename): + with zipfile.ZipFile(filename) as zf: + zf_files = zf.infolist() + count = 0 + for f in zf_files: + if f.file_size > 0 and not f.filename.startswith('__MACOSX/') and not f.filename.endswith('.DS_Store'): + count += 1 + if count > 1: + return True + class GenericAsn1Binary(Binary): """Class for generic ASN.1 binary format""" @@ -1571,6 +1581,91 @@ class MzSQlite(SQlite): return False +class PQP(SQlite): + """ + Class describing a Peptide query parameters file + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.pqp') + >>> PQP().sniff(fname) + True + >>> fname = get_test_fname('test.osw') + >>> PQP().sniff(fname) + False + """ + file_ext = "pqp" + + def set_meta(self, dataset, overwrite=True, **kwd): + super(PQP, self).set_meta(dataset, overwrite=overwrite, **kwd) + + def sniff(self, filename): + """ + table definition according to https://github.com/grosenberger/OpenMS/blob/develop/src/openms/source/ANALYSIS/OPENSWATH/TransitionPQPFile.cpp#L264 + for now VERSION GENE PEPTIDE_GENE_MAPPING are excluded, since + there is test data wo these tables, see also here https://github.com/OpenMS/OpenMS/issues/4365 + """ + if not super(PQP, self).sniff(filename): + return False + table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', + 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', + 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING'] + osw_table_names = ['FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + return self.sniff_table_names(filename, table_names) and not self.sniff_table_names(filename, osw_table_names) + + +class OSW(SQlite): + """ + Class describing OpenSwath output + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.osw') + >>> OSW().sniff(fname) + True + >>> fname = get_test_fname('test.sqmass') + >>> OSW().sniff(fname) + False + """ + file_ext = "osw" + + def set_meta(self, dataset, overwrite=True, **kwd): + super(OSW, self).set_meta(dataset, overwrite=overwrite, **kwd) + + def sniff(self, filename): + # osw seems to be an extension of pqp (few tables are added) + # see also here https://github.com/OpenMS/OpenMS/issues/4365 + if not super(OSW, self).sniff(filename): + return False + table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', + 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', + 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING', + 'FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + return self.sniff_table_names(filename, table_names) + + +class SQmass(SQlite): + """ + Class describing a Sqmass database + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.sqmass') + >>> SQmass().sniff(fname) + True + >>> fname = get_test_fname('test.pqp') + >>> SQmass().sniff(fname) + False + """ + file_ext = "sqmass" + + def set_meta(self, dataset, overwrite=True, **kwd): + super(SQmass, self).set_meta(dataset, overwrite=overwrite, **kwd) + + def sniff(self, filename): + if super(SQmass, self).sniff(filename): + table_names = ["CHROMATOGRAM", "PRECURSOR", "RUN", "SPECTRUM", "DATA", "PRODUCT", "RUN_EXTRA"] + return self.sniff_table_names(filename, table_names) + return False + + class BlibSQlite(SQlite): """Class describing a Proteomics Spectral Library Sqlite database """ MetadataElement(name="blib_version", default='1.8', param=MetadataParameter, desc="Blib Version", diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index ee6c4bc8d7b..8abd76f90c6 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -7,8 +7,9 @@ import re from galaxy.datatypes import data from galaxy.datatypes.binary import Binary from galaxy.datatypes.data import Text +from galaxy.datatypes.sequence import Sequence from galaxy.datatypes.sniff import build_sniff_from_prefix -from galaxy.datatypes.tabular import Tabular +from galaxy.datatypes.tabular import Tabular, TabularData from galaxy.datatypes.xml import GenericXml from galaxy.util import nice_size @@ -53,6 +54,221 @@ class Wiff(Binary): return "\n".join(rval) +@build_sniff_from_prefix +class MzTab(Text): + """ + exchange format for proteomics and metabolomics results + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.mztab') + >>> MzTab().sniff(fname) + True + >>> fname = get_test_fname('test.mztab2') + >>> MzTab().sniff(fname) + False + """ + edam_data = "data_3681" + file_ext = "mztab" + # section names (except MTD) + _sections = ["PRH", "PRT", "PEH", "PEP", "PSH", "PSM", "SMH", "SML", "COM"] + # mandatory metadata fields and list of allowed entries (in lower case) + # (or None if everything is allowed) + _man_mtd = {"mzTab-mode": ["complete", "summary"], + "mzTab-type": ['quantification', 'identification'], + "description": None} + _version_re = r"(1)(\.[0-9])?(\.[0-9])?" + + def __init__(self, **kwd): + super(MzTab, self).__init__(**kwd) + + def set_peek(self, dataset, is_multi_byte=False): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek(dataset.file_name) + dataset.blurb = 'mzTab Format' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def sniff_prefix(self, file_prefix): + """ Determines whether the file is the correct type. """ + has_version = False + found_man_mtd = set() + contents = file_prefix.string_io() + for line in contents: + if re.match(r"^\s*$", line): + continue + line = line.strip("\r\n").split("\t") + if line[0] == "MTD": + if line[1] == "mzTab-version" and re.match(self._version_re, line[2]) is not None: + has_version = True + elif line[1] in self._man_mtd and (self._man_mtd[line[1]] is None or line[2].lower() in self._man_mtd[line[1]]): + found_man_mtd.add(line[1]) + elif not line[0] in self._sections: + return False + return has_version and found_man_mtd == set(self._man_mtd.keys()) + + +class MzTab2(MzTab): + """ + exchange format for proteomics and metabolomics results + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.mztab2') + >>> MzTab2().sniff(fname) + True + >>> fname = get_test_fname('test.mztab') + >>> MzTab2().sniff(fname) + False + """ + file_ext = "mztab2" + _sections = ["SMH", "SML", "SFH", "SMF", "SEH", "SME", "COM"] + _version_re = r"(2)(\.[0-9])?(\.[0-9])?-M$" + _man_mtd = {"mzTab-ID": None} + + def __init__(self, **kwd): + super(MzTab2, self).__init__(**kwd) + + def set_peek(self, dataset, is_multi_byte=False): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek(dataset.file_name) + dataset.blurb = 'mzTab2 Format' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + +@build_sniff_from_prefix +class Kroenik(Tabular): + """ + Kroenik (HardKloer sibling) files + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.kroenik') + >>> Kroenik().sniff(fname) + True + >>> fname = get_test_fname('test.peplist') + >>> Kroenik().sniff(fname) + False + """ + file_ext = "kroenik" + + def __init__(self, **kwd): + super(Kroenik, self).__init__(**kwd) + self.column_names = ["File", "First Scan", "Last Scan", "Num of Scans", "Charge", "Monoisotopic Mass", "Base Isotope Peak", "Best Intensity", "Summed Intensity", "First RTime", "Last RTime", "Best RTime", "Best Correlation", "Modifications"] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix): + fh = file_prefix.string_io() + line = [_.strip() for _ in fh.readline().split("\t")] + if line != self.column_names: + return False + line = fh.readline().split("\t") + try: + [int(_) for _ in line[1:5]] + [float(_) for _ in line[5:13]] + except ValueError: + return False + return True + + +@build_sniff_from_prefix +class PepList(Tabular): + """ + Peplist file as used in OpenMS + https://github.com/OpenMS/OpenMS/blob/0fc8765670a0ad625c883f328de60f738f7325a4/src/openms/source/FORMAT/FileHandler.cpp#L432 + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.peplist') + >>> PepList().sniff(fname) + True + >>> fname = get_test_fname('test.psms') + >>> PepList().sniff(fname) + False + """ + file_ext = "peplist" + + def __init__(self, **kwd): + super(PepList, self).__init__(**kwd) + self.column_names = ["m/z", "rt(min)", "snr", "charge", "intensity"] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix): + fh = file_prefix.string_io() + line = [_.strip() for _ in fh.readline().split("\t")] + if line == self.column_names: + return True + return False + + +@build_sniff_from_prefix +class PSMS(Tabular): + """ + Percolator tab-delimited output (PSM level, .psms) as used in OpenMS + https://github.com/OpenMS/OpenMS/blob/0fc8765670a0ad625c883f328de60f738f7325a4/src/openms/source/FORMAT/FileHandler.cpp#L453 + see also http://www.kojak-ms.org/docs/percresults.html + + Note that the data rows can have more columns than the header line + since ProteinIds are listed tab-separated. + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.psms') + >>> PSMS().sniff(fname) + True + >>> fname = get_test_fname('test.kroenik') + >>> PSMS().sniff(fname) + False + """ + file_ext = "psms" + + def __init__(self, **kwd): + super(PSMS, self).__init__(**kwd) + self.column_names = ["PSMId", "score", "q-value", "posterior_error_prob", "peptide", "proteinIds"] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix): + fh = file_prefix.string_io() + line = [_.strip() for _ in fh.readline().split("\t")] + if line == self.column_names: + return True + return False + + +@build_sniff_from_prefix +class PEFF(Sequence): + """ + PSI Extended FASTA Format + https://github.com/HUPO-PSI/PEFF + """ + file_ext = "peff" + + def sniff_prefix(self, file_prefix): + """ + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( 'test.peff' ) + >>> PEFF().sniff( fname ) + True + >>> fname = get_test_fname( 'sequence.fasta' ) + >>> PEFF().sniff( fname ) + False + """ + fh = file_prefix.string_io() + if re.match(r"# PEFF \d+.\d+", fh.readline()): + return True + else: + return False + + class PepXmlReport(Tabular): """pepxml converted to tabular report""" edam_data = "data_2536" @@ -92,6 +308,277 @@ class ProtXmlReport(Tabular): return self.make_html_table(dataset, column_names=self.column_names) +class Dta(TabularData): + """dta + The first line contains the singly protonated peptide mass (MH+) and the + peptide charge state separated by a space. Subsequent lines contain space + separated pairs of fragment ion m/z and intensity values. + """ + file_ext = "dta" + comment_lines = 0 + + def set_meta(self, dataset, **kwd): + column_types = [] + data_row = [] + data_lines = 0 + if dataset.has_data(): + with open(dataset.file_name, 'r') as dtafile: + for line in dtafile: + data_lines += 1 + + # Guess column types + for cell in data_row: + column_types.append(self.guess_type(cell)) + + # Set metadata + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = 0 + dataset.metadata.column_types = ['float', 'float'] + dataset.metadata.columns = 2 + dataset.metadata.column_names = ['m/z', 'intensity'] + dataset.metadata.delimiter = " " + + +@build_sniff_from_prefix +class Dta2d(TabularData): + """ + dta2d: files with three tab/space-separated columns. + The default format is: retention time (seconds) , m/z , intensity. + If the first line starts with '#', a different order is defined by the the + order of the keywords 'MIN' (retention time in minutes) or 'SEC' (retention + time in seconds), 'MZ', and 'INT'. + Example: '#MZ MIN INT' + The peaks of one retention time have to be in subsequent lines. + + Note: sniffer detects (tab or space separated) dta2d files with correct + header, wo header seems to generic + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.dta2d') + >>> Dta2d().sniff(fname) + True + >>> fname = get_test_fname('test.edta') + >>> Dta2d().sniff(fname) + False + """ + file_ext = "dta2d" + comment_lines = 0 + + def _parse_header(self, line): + if len(line) != 3 or len(line[0]) < 3 or not line[0].startswith("#"): + return None + line[0] = line[0].lstrip("#") + line = [_.strip() for _ in line] + if 'MZ' not in line or 'INT' not in line or ('MIN' not in line and 'SEC' not in line): + return None + return line + + def _parse_delimiter(self, line): + if len(line.split(" ")) == 3: + return " " + elif len(line.split("\t")) == 3: + return "\t" + return None + + def _parse_dataline(self, line): + try: + line = [float(_) for _ in line] + except ValueError: + return False + if not all(_ >= 0 for _ in line): + return False + return True + + def set_meta(self, dataset, **kwd): + data_lines = 0 + delim = None + if dataset.has_data(): + with open(dataset.file_name, 'r') as dtafile: + for line in dtafile: + if delim is None: + delim = self._parse_delimiter(line) + dataset.metadata.column_names = self._parse_header(line.split(delim)) + data_lines += 1 + + # Set metadata + if delim is not None: + dataset.metadata.delimiter = delim + + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = 0 + dataset.metadata.column_types = ['float', 'float', 'float'] + dataset.metadata.columns = 3 + if dataset.metadata.column_names is None or dataset.metadata.column_names == []: + dataset.metadata.comment_lines += 1 + dataset.metadata.data_lines -= 1 + dataset.metadata.column_names = ['SEC', 'MZ', 'INT'] + + def sniff_prefix(self, file_prefix): + sep = None + header = None + for idx, line in enumerate(file_prefix.line_iterator()): + line = line.strip() + if sep is None: + sep = self._parse_delimiter(line) + if sep is None: + return False + line = line.split(sep) + if len(line) != 3: + return False + if idx == 0: + header = self._parse_header(line) + if (header is None) and not self._parse_dataline(line): + return False + elif not self._parse_dataline(line): + return False + if sep is None or header is None: + return False + return True + + +@build_sniff_from_prefix +class Edta(TabularData): + """ + Input text file containing tab, space or comma separated columns. + The separator between columns is checked in the first line in this order. + + It supports three variants of this format. + + 1. Columns are: RT, MZ, Intensity A header is optional. + 2. Columns are: RT, MZ, Intensity, Charge, columns{0,} A header is mandatory. + 3. Columns are: (RT, MZ, Intensity, Charge){1,}, columns{0,} + Header is mandatory. First quadruplet is the consensus. All following + quadruplets describe the sub-features. This variant is discerned from + variant #2 by the name of the fifth column, which is required to be RT1 + (or rt1). All other column names for sub-features are faithfully ignored. + + Note the sniffer only detects files with header. + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('test.edta') + >>> Edta().sniff(fname) + True + >>> fname = get_test_fname('test.dta2d') + >>> Edta().sniff(fname) + False + """ + file_ext = "edta" + comment_lines = 0 + + def _parse_delimiter(self, line): + if len(line.split(" ")) >= 3: + return " " + elif len(line.split("\t")) >= 3: + return "\t" + elif len(line.split(",")) >= 3: + return "\t" + return None + + def _parse_type(self, line): + """ + parse the type from the header line + types 1-3 as in the class docs, 0: type 1 wo/wrong header + """ + if len(line) < 3: + return None + line = [_.lower().replace("/", "") for _ in line] + if len(line) == 3: + if line[0] == "rt" and line[1] == "mz" and (line[2] == "int" or line[2] == "intensity"): + return 1 + else: + return None + if line[0] != "rt" or line[1] != "mz" or (line[2] != "int" and line[2] != "intensity") or line[3] != "charge": + return None + if not line[4].startswith("rt"): + return 2 + else: + return 3 + + def _parse_dataline(self, line, tpe): + if tpe == 2 or tpe == 3: + idx = 4 + else: + idx = 3 + try: + line = [float(_) for _ in line[:idx]] + except ValueError: + return False + if not all(_ >= 0 for _ in line[:idx]): + return False + return True + + def _clean_header(self, line): + for idx, el in enumerate(line): + el = el.lower() + if el.startswith("rt"): + line[idx] = "RT" + elif el.startswith("int"): + line[idx] = "intensity" + elif el.startswith("mz"): + line[idx] = "m/z" + elif el.startswith("charge"): + line[idx] = "charge" + else: + break + if idx // 4 > 0: + line[idx] += str(idx // 4) + return line + + def set_meta(self, dataset, **kwd): + data_lines = 0 + delim = None + if dataset.has_data(): + with open(dataset.file_name, 'r') as dtafile: + for idx, line in enumerate(dtafile): + if idx == 0: + delim = self._parse_delimiter(line) + tpe = self._parse_type(line.split(delim)) + if tpe == 0: + dataset.metadata.column_names = ["RT", "m/z", "intensity"] + else: + dataset.metadata.column_names = self._clean_header(line.split(delim)) + data_lines += 1 + + # Set metadata + if delim is not None: + dataset.metadata.delimiter = delim + for c in dataset.metadata.column_names: + if any(c.startswith(_) for _ in ["RT", "m/z", "intensity", "charge"]): + dataset.metadata.column_types.append("float") + else: + dataset.metadata.column_types.append("str") + + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = 0 + dataset.metadata.columns = len(dataset.metadata.column_names) + if tpe > 0: + dataset.metadata.comment_lines += 1 + dataset.metadata.data_lines -= 1 + + def sniff_prefix(self, file_prefix): + sep = None + tpe = None + for idx, line in enumerate(file_prefix.line_iterator()): + line = line.strip("\r\n") + if sep is None: + sep = self._parse_delimiter(line) + if sep is None: + return False + line = line.split(sep) + + if idx == 0: + tpe = self._parse_type(line) + if tpe is None: + return False + elif tpe == 0 and not self._parse_dataline(line, tpe): + return False + elif not self._parse_dataline(line, tpe): + return False + if tpe is None: + return False + return True + + class ProteomicsXml(GenericXml): """ An enhanced XML datatype used to reuse code across several proteomic/mass-spec datatypes. """ @@ -102,12 +589,12 @@ class ProteomicsXml(GenericXml): """ Determines whether the file is the correct XML type. """ contents = file_prefix.string_io() while True: - line = contents.readline() + line = contents.readline().strip() if line is None or not line.startswith('sp:Q9Y2X3 \Length=1 +M diff --git a/lib/galaxy/datatypes/test/test.peplist b/lib/galaxy/datatypes/test/test.peplist new file mode 100644 index 00000000000..af747d21cab --- /dev/null +++ b/lib/galaxy/datatypes/test/test.peplist @@ -0,0 +1,36 @@ + m/z rt(min) snr charge intensity + 706.0656 47.997 70.443 3 3709.630 + 740.4292 60.127 28.895 4 2153.832 + 708.7055 63.321 28.970 3 2377.576 + 728.7255 56.390 28.530 3 1820.929 + 721.3855 57.410 25.965 3 1332.942 + 740.4400 64.272 16.639 2 1024.329 + 707.3855 63.423 20.248 3 1495.899 + 735.7055 50.275 29.838 3 804.873 + 726.7255 63.966 13.282 3 1364.349 + 723.4000 57.647 13.069 2 713.563 + 700.4400 61.588 11.435 2 579.007 + 736.3400 61.113 9.765 2 464.364 + 705.4000 54.454 12.855 2 457.507 + 709.0056 57.104 12.614 3 454.622 + 710.3000 49.459 18.364 2 534.519 + 719.4000 64.714 7.860 3 447.372 + 742.4400 64.170 6.799 2 357.583 + 752.4000 55.983 7.138 2 290.348 + 714.1492 47.114 22.442 4 576.568 + 753.4256 61.486 7.788 3 318.019 + 730.3800 53.536 8.914 2 284.038 + 708.8400 58.836 6.272 2 246.319 + 737.0400 50.207 12.257 3 309.824 + 716.9200 58.123 6.188 2 283.297 + 729.7456 50.308 9.760 3 428.650 + 712.3200 49.391 8.749 2 235.981 + 748.9400 51.294 9.114 2 250.202 + 712.8800 53.672 6.466 2 160.206 + 714.9000 53.604 6.417 2 202.450 + 716.8800 54.080 6.920 2 225.521 + 715.7055 49.017 10.866 3 198.560 + 728.3600 53.299 6.913 3 156.242 + 753.8800 49.527 6.406 2 113.352 + 724.0400 47.997 6.559 3 281.194 + 745.9200 47.521 6.732 2 87.974 diff --git a/lib/galaxy/datatypes/test/test.pqp b/lib/galaxy/datatypes/test/test.pqp new file mode 100644 index 00000000000..bd9d3930e25 Binary files /dev/null and b/lib/galaxy/datatypes/test/test.pqp differ diff --git a/lib/galaxy/datatypes/test/test.psms b/lib/galaxy/datatypes/test/test.psms new file mode 100644 index 00000000000..56ef1913514 --- /dev/null +++ b/lib/galaxy/datatypes/test/test.psms @@ -0,0 +1,122 @@ +PSMId score q-value posterior_error_prob peptide proteinIds +query:161610;rank:1;spectrum:1093.0258_2187.96_spectrum=40633_uteruspremenopause;rt:2187.96;mz:1093.0258;charge:2 11.0472 0 1.48741e-12 X.SLAGSSGPGASSGTSGDHGELVVR.X UniProt_E9PI39 UniProt_E9PIZ1 UniProt_E9PK01 UniProt_E9PL12 UniProt_E9PMW7 UniProt_E9PPR1 UniProt_E9PQ49 UniProt_E9PRY8 UniProt_H0YCK7 UniProt_P29692 UniProt_P29692-2 genCDS_ENST00000317198_8_143579792-143586845_-1 genCDS_ENST00000395119_8_143579792-143586845_-1 genCDS_ENST00000419152_8_143579792-143586845_-1 genCDS_ENST00000423316_8_143579792-143590081_-1 genCDS_ENST00000442189_8_143579792-143590081_-1 genCDS_ENST00000524397_8_143580630-143586845_-1 genCDS_ENST00000529007_8_143580630-143586845_-1 genCDS_ENST00000529272_8_143579792-143586845_-1 genCDS_ENST00000530191_8_143580506-143586845_-1 genCDS_ENST00000530445_8_143580620-143586845_-1 genCDS_ENST00000530616_8_143580630-143589222_-1 genCDS_ENST00000531218_8_143580522-143586845_-1 genCDS_ENST00000532741_8_143579792-143590231_-1 genCDS_ENST00000533204_8_143580506-143586845_-1 genCDS_ENST00000533494_8_143580613-143586845_-1 genCDS_ENST00000533749_8_143581079-143599321_-1 genCDS_ENST00000534380_8_143580034-143586845_-1 genCDS_ENST00000618139_8_143580019-143590081_-1 +query:132093;rank:1;spectrum:832.40479_2605.29_spectrum=72602_uteruspremenopause;rt:2605.29;mz:832.40479;charge:2 9.36298 0 7.2763e-11 X.AAAFEEQENETVVVK.X UniProt_Q5TCU6 UniProt_Q9Y490 genCDS_ENST00000314888_9_35697791-35725694_-1 +query:119552;rank:1;spectrum:769.39142_3320.48_spectrum=167622_uteruspremenopause;rt:3320.48;mz:769.39142;charge:2 8.72204 0 3.19783e-10 X.FALQDLSVEETSAK.X UniProt_B2RCS5 UniProt_G3V2E8 UniProt_G3V2N5 UniProt_G3V2W4 UniProt_G3V2X9 UniProt_G3V5M4 UniProt_H9KV75 UniProt_O43707 UniProt_P12814 UniProt_P12814-2 UniProt_P12814-3 UniProt_P12814-4 UniProt_P35609 UniProt_Q08043 genCDS_ENST00000193403_14_68874859-68979056_-1 genCDS_ENST00000252699_19_38647746-38729432_1 genCDS_ENST00000366578_1_236686674-236762619_1 genCDS_ENST00000376839_14_68874859-68925582_-1 genCDS_ENST00000394419_14_68874859-68979056_-1 genCDS_ENST00000438964_14_68874859-68979056_-1 genCDS_ENST00000502692_11_66546511-66563193_1 genCDS_ENST00000513398_11_66546938-66563193_1 genCDS_ENST00000538545_14_68874859-68979056_-1 genCDS_ENST00000542672_1_236686674-236762619_1 genCDS_ENST00000553370_14_68904702-68925582_-1 genCDS_ENST00000553779_14_68910020-68925582_-1 genCDS_ENST00000555616_14_68892228-68925582_-1 genCDS_ENST00000556433_14_68893665-68978258_-1 genCDS_ENST00000556571_14_68910022-68978051_-1 +query:105717;rank:1;spectrum:715.89587_3127.2_spectrum=82140_uteruspremenopause;rt:3127.2;mz:715.89587;charge:2 8.58807 0 4.35755e-10 X.LDSLSAQLSQLQK.X UniProt_P02545 UniProt_P02545-2 UniProt_P02545-3 UniProt_P02545-4 UniProt_P02545-5 UniProt_P02545-6 UniProt_Q5TCI8 genCDS_ENST00000347559_1_156114919-156139106_1 genCDS_ENST00000361308_1_156114919-156137764_1 genCDS_ENST00000368297_1_156126803-156137764_1 genCDS_ENST00000368299_1_156114919-156139106_1 genCDS_ENST00000368300_1_156114919-156139106_1 genCDS_ENST00000368301_1_156114919-156137764_1 genCDS_ENST00000448611_1_156126204-156139839_1 genCDS_ENST00000473598_1_156129850-156139106_1 +query:121144;rank:1;spectrum:776.86639_1943.52_spectrum=62937_uteruspremenopause;rt:1943.52;mz:776.86639;charge:2 8.31993 0 8.09508e-10 X.TSTTSSMVASAEQPR.X UniProt_E7EMN6 UniProt_E7EUI7 UniProt_P41236 UniProt_Q6NXS1 genCDS_ENST00000413183_3_195527890-195543025_-1 genCDS_ENST00000438848_3_195516925-195543025_-1 genCDS_ENST00000618156_3_195516896-195543025_-1 +query:91084;rank:1;spectrum:661.3443_2685.51_spectrum=127861_uteruspremenopause;rt:2685.51;mz:661.3443;charge:2 7.84396 0 2.43037e-09 X.STSGGTAALGCLVK.X UniProt_P01857 UniProt_P01860 genCDS_ENST00000390542_14_105741473-105743071_-1 genCDS_ENST00000390548_14_105737762-105743071_-1 genCDS_ENST00000390549_14_105741473-105743071_-1 genCDS_ENST00000390551_14_105769237-105771405_-1 genCDS_ENST00000612473_14_105741473-106005532_-1 genCDS_ENST00000613152_14_105741473-106211391_-1 genCDS_ENST00000615822_14_105741473-106062604_-1 genCDS_ENST00000616127_14_105769237-106038345_-1 genCDS_ENST00000618145_14_105741473-106012356_-1 genCDS_ENST00000618756_14_105668245-106538265_-1 genCDS_ENST00000619212_14_105741473-106211391_-1 genCDS_ENST00000621473_14_105769237-106005532_-1 +query:128131;rank:1;spectrum:813.40948_2244.71_spectrum=40773_uteruspremenopause;rt:2244.71;mz:813.40948;charge:2 7.38397 0 7.03248e-09 X.DALQNPNDLQLQEK.X UniProt_C9IYV6 UniProt_C9JPE5 UniProt_Q9H0Q0 genCDS_ENST00000381323_2_16552936-16588119_-1 genCDS_ENST00000406434_2_16552936-16588119_-1 genCDS_ENST00000445605_2_16564005-16588119_-1 genCDS_ENST00000451689_2_16564024-16588119_-1 +query:141775;rank:1;spectrum:893.9129_2874.06_spectrum=158864_uteruspremenopause;rt:2874.06;mz:893.9129;charge:2 7.1148 0 1.30952e-08 X.NKDQGTYEDYVEGLR.X Augustus2_AUGUSTUS00000061902_12_56160654-56153919_1 UniProt_B7Z6Z4 UniProt_F8VPF3 UniProt_F8VZU9 UniProt_F8W180 UniProt_F8W1R7 UniProt_G3V1V0 UniProt_G3V1Y7 UniProt_G8JLA2 UniProt_J3KND3 UniProt_P60660 UniProt_P60660-2 genCDS_ENST00000293422_12_56158402-56161415_1 genCDS_ENST00000348108_12_56158402-56160654_1 genCDS_ENST00000536128_12_56158402-56160331_1 genCDS_ENST00000547408_12_56158402-56162322_1 genCDS_ENST00000547649_12_56158402-56161415_1 genCDS_ENST00000548293_12_56158402-56160654_1 genCDS_ENST00000548400_12_56158402-56160654_1 genCDS_ENST00000548580_12_56158402-56161415_1 genCDS_ENST00000549392_12_56158402-56160320_1 genCDS_ENST00000549566_12_56158402-56161415_1 genCDS_ENST00000550697_12_56158402-56160654_1 genCDS_ENST00000551589_12_56158402-56160331_1 +query:71406;rank:1;spectrum:404.21942_2506.81_spectrum=119549_uteruspremenopause;rt:2506.81;mz:404.21942;charge:3 7.03952 0 1.55822e-08 X.TANDAVELHLK.X UniProt_Q9P2B2 genCDS_ENST00000393203_1_116910204-116986967_1 +query:129027;rank:1;spectrum:817.40759_3010.12_spectrum=35161_uteruspremenopause;rt:3010.12;mz:817.40759;charge:2 7.02828 0 1.5992e-08 X.EGCTVSPETLSLNVK.X UniProt_E5KLJ5 UniProt_E5KLJ6 UniProt_E5KLJ9 UniProt_E5KLK1 UniProt_O60313 UniProt_O60313-2 genCDS_ENST00000361150_3_193593378-193692127_1 genCDS_ENST00000361510_3_193593378-193692127_1 genCDS_ENST00000361715_3_193593378-193692127_1 genCDS_ENST00000361828_3_193593378-193692127_1 genCDS_ENST00000361908_3_193593378-193692127_1 genCDS_ENST00000392438_3_193593378-193692127_1 +query:79758;rank:1;spectrum:627.82471_1646.41_spectrum=69987_uteruspremenopause;rt:1646.41;mz:627.82471;charge:2 6.79995 0 2.70985e-08 X.LEPGGGAEAQAVR.X UniProt_Q14315 UniProt_Q14315-2 genCDS_ENST00000325888_7_128830638-128858523_1 genCDS_ENST00000346177_7_128830638-128858523_1 +query:107233;rank:1;spectrum:721.87775_2878.99_spectrum=96857_uteruspremenopause;rt:2878.99;mz:721.87775;charge:2 6.53355 0 5.01392e-08 X.EAGAGGLSLAVEGPSK.X UniProt_E7EN95 UniProt_O75369 UniProt_O75369-2 UniProt_O75369-3 UniProt_O75369-6 UniProt_O75369-7 UniProt_O75369-8 UniProt_O75369-9 UniProt_Q14315 UniProt_Q14315-2 genCDS_ENST00000295956_3_58008565-58170762_1 genCDS_ENST00000325888_7_128830638-128858523_1 genCDS_ENST00000346177_7_128830638-128858523_1 genCDS_ENST00000358537_3_58008565-58170762_1 genCDS_ENST00000429972_3_58008565-58170762_1 genCDS_ENST00000490882_3_58008565-58170762_1 genCDS_ENST00000493452_3_58078486-58170762_1 +query:146872;rank:1;spectrum:619.28918_2152.13_spectrum=141583_uteruspremenopause;rt:2152.13;mz:619.28918;charge:3 6.13383 0 1.26226e-07 X.SEETKENEGFTVTAEGK.X UniProt_M0QYC8 UniProt_P01024 genCDS_ENST00000245907_19_6677882-6720589_-1 genCDS_ENST00000596548_19_6680159-6685038_-1 +query:172404;rank:1;spectrum:871.76086_2726.33_spectrum=42005_uteruspremenopause;rt:2726.33;mz:871.76086;charge:3 5.99128 0 1.75446e-07 X.SEAEEALTSFN(Deamidated)GHKPPGSSEPLTVK.X UniProt_B4DVB8 UniProt_Q15717 genCDS_ENST00000351593_19_7963483-7991815_-1 genCDS_ENST00000407627_19_7963483-7991815_-1 genCDS_ENST00000596459_19_7963483-7991815_-1 +query:105836;rank:1;spectrum:716.36646_2072.32_spectrum=141400_uteruspremenopause;rt:2072.32;mz:716.36646;charge:2 5.83486 0 2.51791e-07 X.VASSPVM(Oxidation)VSNPATR.X UniProt_A6NEM2 UniProt_P51610 UniProt_P51610-2 UniProt_P51610-4 genCDS_ENST00000310441_X_153949347-153970840_-1 genCDS_ENST00000369984_X_153949347-153970840_-1 +query:89905;rank:1;spectrum:657.87396_3407.52_spectrum=152816_uteruspremenopause;rt:3407.52;mz:657.87396;charge:2 5.75831 0 3.00493e-07 X.EHALLAYTLGVK.X Augustus2_AUGUSTUS00000028250_3_184026960-184027756_-1 UniProt_P68104 UniProt_Q05639 UniProt_Q5VTE0 calCuffs_CUFF.68550.1_19_35873852-35875252_1_1_ORF2 ensBodymap_RNASEQT00000006703_5_14651962-14653630_-1 ensBodymap_RNASEQT00000053260_5_14650973-14653630_-1 ensBodymap_RNASEQT00000095825_9_135894020-135896638_1 ensBodymap_RNASEQT00000213862_9_135894020-135896638_1 genCDS_ENST00000217182_20_63488298-63497763_-1 genCDS_ENST00000298049_20_63488298-63497763_-1 genCDS_ENST00000309268_6_73517810-73520026_-1 genCDS_ENST00000316292_6_73517810-73520026_-1 genCDS_ENST00000331523_6_73517810-73520026_-1 genCDS_ENST00000615060_6_73517916-73520026_-1 genpseudogene_ENST00000415278_1_96446930-96448318_1_1_ORF2 genpseudogene_ENST00000419025_3_184026369-184027756_-1_2_ORF4 genpseudogene_ENST00000436459_9_133019486-133020874_1_1_ORF1 genpseudogene_ENST00000514975_4_105484698-105486080_1_1_ORF1 genpseudogene_ENST00000596811_19_35382172-35384052_1_1_ORF5 yalePseudo_PGOHUM00000234671_19_35873074-35874951_1_2_ORF13 yalePseudo_PGOHUM00000237893_3_183744160-183745544_-1_2_ORF4 yalePseudo_PGOHUM00000244804_1_96912486-96913871_1_1_ORF2 yalePseudo_PGOHUM00000246017_4_106405855-106407234_1_1_ORF1 +query:120224;rank:1;spectrum:772.3432_1845.25_spectrum=39781_uteruspremenopause;rt:1845.25;mz:772.3432;charge:2 5.59137 0 4.41859e-07 X.FYEQM(Oxidation)N(Deamidated)GPVAGASR.X UniProt_E9PI39 UniProt_E9PIZ1 UniProt_E9PK01 UniProt_E9PK06 UniProt_E9PK72 UniProt_E9PKK3 UniProt_E9PL12 UniProt_E9PL71 UniProt_E9PMW7 UniProt_E9PN91 UniProt_E9PPR1 UniProt_E9PQ49 UniProt_E9PQZ1 UniProt_E9PRY8 UniProt_H0YCK7 UniProt_P29692 UniProt_P29692-2 UniProt_P29692-3 UniProt_P29692-4 genCDS_ENST00000317198_8_143579792-143586845_-1 genCDS_ENST00000395119_8_143579792-143586845_-1 genCDS_ENST00000419152_8_143579792-143586845_-1 genCDS_ENST00000423316_8_143579792-143590081_-1 genCDS_ENST00000442189_8_143579792-143590081_-1 genCDS_ENST00000524397_8_143580630-143586845_-1 genCDS_ENST00000524624_8_143579792-143586845_-1 genCDS_ENST00000525223_8_143586729-143586845_-1 genCDS_ENST00000526340_8_143586219-143586845_-1 genCDS_ENST00000526838_8_143579792-143586845_-1 genCDS_ENST00000528610_8_143579792-143586845_-1 genCDS_ENST00000529007_8_143580630-143586845_-1 genCDS_ENST00000529272_8_143579792-143586845_-1 genCDS_ENST00000529516_8_143580642-143586845_-1 genCDS_ENST00000530191_8_143580506-143586845_-1 genCDS_ENST00000530445_8_143580620-143586845_-1 genCDS_ENST00000530616_8_143580630-143589222_-1 genCDS_ENST00000531218_8_143580522-143586845_-1 genCDS_ENST00000531621_8_143579792-143586845_-1 genCDS_ENST00000532543_8_143586729-143586845_-1 genCDS_ENST00000532741_8_143579792-143590231_-1 genCDS_ENST00000533204_8_143580506-143586845_-1 genCDS_ENST00000533494_8_143580613-143586845_-1 genCDS_ENST00000533749_8_143581079-143599321_-1 genCDS_ENST00000533833_8_143580630-143586845_-1 genCDS_ENST00000534377_8_143580186-143586845_-1 genCDS_ENST00000534380_8_143580034-143586845_-1 genCDS_ENST00000618139_8_143580019-143590081_-1 genpseudogene_ENST00000433698_9_92836826-92837668_-1_1_ORF1 yalePseudo_PGOHUM00000236264_9_95599111-95599989_-1_1_ORF1 +query:83950;rank:1;spectrum:640.32916_1950.76_spectrum=62954_uteruspremenopause;rt:1950.76;mz:640.32916;charge:2 5.43278 0 6.3732e-07 X.YLAEVAAGDDKK.X UniProt_B0AZS6 UniProt_B7Z2E6 UniProt_E7EX29 UniProt_H0YB80 UniProt_P63104 UniProt_P63104-2 genCDS_ENST00000353245_8_100920693-100948889_-1 genCDS_ENST00000395948_8_100920693-100948658_-1 genCDS_ENST00000395951_8_100920693-100948889_-1 genCDS_ENST00000395953_8_100920693-100948889_-1 genCDS_ENST00000395956_8_100920693-100948889_-1 genCDS_ENST00000395957_8_100920693-100948889_-1 genCDS_ENST00000395958_8_100920693-100948889_-1 genCDS_ENST00000419477_8_100920693-100948889_-1 genCDS_ENST00000457309_8_100920693-100948889_-1 genCDS_ENST00000521309_8_100920693-100924973_-1 genCDS_ENST00000521607_8_100920716-100948889_-1 genCDS_ENST00000522542_8_100920693-100948145_-1 genCDS_ENST00000522819_8_100920693-100924973_-1 genCDS_ENST00000523848_8_100920693-100924988_-1 genpseudogene_ENST00000415292_10_23136924-23137661_1_1_ORF1 +query:70327;rank:1;spectrum:602.85034_3098.02_spectrum=159460_uteruspremenopause;rt:3098.02;mz:602.85034;charge:2 5.3382 0 7.92901e-07 X.TLMALGSLAVTK.X UniProt_H0YCU9 UniProt_Q01995 genCDS_ENST00000278968_11_117203014-117204359_1 genCDS_ENST00000392951_11_117203014-117204359_1 genCDS_ENST00000525531_11_117203014-117204359_1 genCDS_ENST00000529622_11_117203144-117204359_1 genCDS_ENST00000530649_11_117203014-117204359_1 genCDS_ENST00000532870_11_117203014-117204359_1 +query:161278;rank:1;spectrum:725.35535_3439.5_spectrum=82985_uteruspremenopause;rt:3439.5;mz:725.35535;charge:3 5.26056 0 9.48615e-07 X.LEWLESHQDADLEDFKAK.X UniProt_P11021 genCDS_ENST00000324460_9_125236592-125241126_-1 +query:61382;rank:1;spectrum:578.83197_2603.98_spectrum=142644_uteruspremenopause;rt:2603.98;mz:578.83197;charge:2 5.23568 0 1.00471e-06 X.VKGDVDVSLPK.X UniProt_Q09666 genCDS_ENST00000378024_11_62516744-62536098_-1 +query:121082;rank:1;spectrum:776.46204_3500.05_spectrum=153073_uteruspremenopause;rt:3500.05;mz:776.46204;charge:2 4.84174 0 2.49494e-06 X.LVQVALNGLENLLR.X UniProt_F5GYL8 UniProt_F5H4G7 UniProt_H0Y3K0 UniProt_O15131 UniProt_O60684 UniProt_P52294 genCDS_ENST00000344337_3_122426985-122496565_-1 genCDS_ENST00000356348_6_116681335-116732323_1 genCDS_ENST00000368564_6_116681335-116732323_1 genCDS_ENST00000373625_1_32108131-32170894_1 genCDS_ENST00000392517_6_116729563-116732323_1 +query:93986;rank:1;spectrum:671.32318_2044.34_spectrum=17669_uteruspremenopause;rt:2044.34;mz:671.32318;charge:2 4.67543 0 3.66237e-06 X.YELEETETVTK.X UniProt_C9J813 UniProt_E7EX44 UniProt_E9PGZ1 UniProt_F5H1Z9 UniProt_Q05682 UniProt_Q05682-2 UniProt_Q05682-3 UniProt_Q05682-4 UniProt_Q05682-5 UniProt_Q05682-6 genCDS_ENST00000361675_7_134867734-134968345_1 genCDS_ENST00000361901_7_134867734-134968345_1 genCDS_ENST00000393118_7_134891621-134968345_1 genCDS_ENST00000417172_7_134867734-134968345_1 genCDS_ENST00000422748_7_134867734-134968345_1 genCDS_ENST00000424922_7_134891621-134968345_1 genCDS_ENST00000436461_7_134867734-134960056_1 genCDS_ENST00000443197_7_134891621-134962918_1 genCDS_ENST00000495522_7_134891621-134968345_1 +query:108382;rank:1;spectrum:363.20432_2784.99_spectrum=158641_uteruspremenopause;rt:2784.99;mz:363.20432;charge:4 4.56709 0 4.70247e-06 X.VVAGVANALAHKYH.X UniProt_P02042 UniProt_P68871 genCDS_ENST00000335295_11_5225598-5227021_-1 genCDS_ENST00000380299_11_5232964-5234433_-1 +query:102060;rank:1;spectrum:701.8067_912.458_spectrum=14837_uteruspremenopause;rt:912.458;mz:701.8067;charge:2 4.56217 0 4.75611e-06 X.SDESDQQESLHK.X UniProt_G3V5E5 UniProt_H0YJJ8 UniProt_P49770 genCDS_ENST00000266126_14_75002991-75009188_1 genCDS_ENST00000553401_14_75003018-75005873_1 genCDS_ENST00000556028_14_75002991-75005920_1 +query:41841;rank:1;spectrum:522.80469_1137.9_spectrum=15355_uteruspremenopause;rt:1137.9;mz:522.80469;charge:2 4.55643 0 4.8195e-06 X.TSTGAPAALKK.X UniProt_B8ZZL6 UniProt_E7ENU9 UniProt_P40121 UniProt_P40121-2 genCDS_ENST00000263867_2_85394893-85402145_-1 genCDS_ENST00000409275_2_85399146-85402145_-1 genCDS_ENST00000409670_2_85394893-85402145_-1 genCDS_ENST00000409724_2_85394893-85402145_-1 genCDS_ENST00000409921_2_85394893-85402145_-1 genCDS_ENST00000439385_2_85398690-85402145_-1 genCDS_ENST00000447219_2_85398690-85402145_-1 genCDS_ENST00000449030_2_85398690-85402145_-1 +query:67749;rank:1;spectrum:596.77612_2303.32_spectrum=10843_uteruspremenopause;rt:2303.32;mz:596.77612;charge:2 4.29846 0 8.73504e-06 X.(Acetyl)EAESSPFVER.X UniProt_P14625 genCDS_ENST00000299767_12_103930516-103947662_1 +query:113387;rank:1;spectrum:745.88556_2835.91_spectrum=65124_uteruspremenopause;rt:2835.91;mz:745.88556;charge:2 4.24758 0 9.82097e-06 X.NQYVTLHDM(Oxidation)LLK.X UniProt_B3KQ25 UniProt_K7ESG5 UniProt_P61289 UniProt_P61289-2 genCDS_ENST00000293362_17_42833632-42841578_1 genCDS_ENST00000441946_17_42834182-42841578_1 genCDS_ENST00000541124_17_42833632-42841578_1 genCDS_ENST00000545225_17_42834817-42841578_1 genCDS_ENST00000590720_17_42833632-42841578_1 genCDS_ENST00000592169_17_42833632-42841578_1 genCDS_ENST00000622892_17_42833632-42841578_1 +query:125401;rank:1;spectrum:532.63049_2532.44_spectrum=80569_uteruspremenopause;rt:2532.44;mz:532.63049;charge:3 4.23343 0 1.01462e-05 X.GLHQSTLDLKNELK.X UniProt_Q14258 genCDS_ENST00000316881_17_56891700-56913988_-1 genCDS_ENST00000537230_17_56891700-56913988_-1 +query:164495;rank:1;spectrum:1143.5902_3304.6_spectrum=121470_uteruspremenopause;rt:3304.6;mz:1143.5902;charge:2 4.20795 0 1.07591e-05 X.YTPVQQGPVGVNVTYGGDPLPK.X UniProt_P21333 UniProt_P21333-2 UniProt_Q5HY54 genCDS_ENST00000344736_X_154348849-154371245_-1 genCDS_ENST00000360319_X_154348849-154371245_-1 genCDS_ENST00000369850_X_154348849-154371245_-1 genCDS_ENST00000369856_X_154348849-154371164_-1 genCDS_ENST00000422373_X_154348849-154371245_-1 genCDS_ENST00000610817_X_154348849-154371164_-1 +query:139545;rank:1;spectrum:586.2818_2698.67_spectrum=72867_uteruspremenopause;rt:2698.67;mz:586.2818;charge:3 3.86858 0 2.34555e-05 X.SPFEVYVDKSQGDASK.X UniProt_F8WE98 UniProt_P21333 UniProt_P21333-2 UniProt_Q5HY54 genCDS_ENST00000344736_X_154348849-154371245_-1 genCDS_ENST00000360319_X_154348849-154371245_-1 genCDS_ENST00000369850_X_154348849-154371245_-1 genCDS_ENST00000369856_X_154348849-154371164_-1 genCDS_ENST00000420627_X_154364689-154371203_-1 genCDS_ENST00000422373_X_154348849-154371245_-1 genCDS_ENST00000610817_X_154348849-154371164_-1 +query:20233;rank:1;spectrum:453.24112_1930.62_spectrum=171610_uteruspremenopause;rt:1930.62;mz:453.24112;charge:2 3.73764 0 3.16384e-05 X.ATDVMLAGK.X UniProt_P23526 UniProt_P23526-2 genCDS_ENST00000217426_20_34281034-34303270_-1 genCDS_ENST00000538132_20_34281034-34295529_-1 +query:81528;rank:1;spectrum:422.55518_3442.78_spectrum=98225_uteruspremenopause;rt:3442.78;mz:422.55518;charge:3 3.73141 0 3.20907e-05 X.TEFSLLHYAGK.X UniProt_P35749 UniProt_P35749-2 UniProt_P35749-3 UniProt_P35749-4 genCDS_ENST00000300036_16_15703991-15838252_-1 genCDS_ENST00000396324_16_15703991-15838252_-1 genCDS_ENST00000452625_16_15708811-15838252_-1 genCDS_ENST00000576790_16_15708811-15838252_-1 genCDS_ENST00000611087_16_15703879-15838252_-1 genCDS_ENST00000616439_16_15703879-15838252_-1 +query:116580;rank:1;spectrum:505.94232_2944.82_spectrum=120611_uteruspremenopause;rt:2944.82;mz:505.94232;charge:3 3.69185 0 3.51185e-05 X.TLRDLEVVEGSAAR.X UniProt_Q15746 UniProt_Q15746-10 UniProt_Q15746-11 UniProt_Q15746-2 UniProt_Q15746-3 UniProt_Q15746-4 UniProt_Q15746-5 UniProt_Q15746-6 UniProt_Q15746-7 UniProt_Q15746-8 UniProt_Q15746-9 genCDS_ENST00000346322_3_123614105-123793841_-1 genCDS_ENST00000354792_3_123614105-123793841_-1 genCDS_ENST00000359169_3_123614105-123793841_-1 genCDS_ENST00000360304_3_123614105-123793841_-1 genCDS_ENST00000360772_3_123614105-123793841_-1 genCDS_ENST00000418370_3_123614105-123620294_-1 genCDS_ENST00000475616_3_123614105-123793841_-1 genCDS_ENST00000578202_3_123614105-123620294_-1 genCDS_ENST00000583087_3_123614105-123620294_-1 +query:68430;rank:1;spectrum:598.2746_1630.22_spectrum=54409_uteruspremenopause;rt:1630.22;mz:598.2746;charge:2 3.62415 0 4.09639e-05 X.WCALSHHER.X UniProt_J3KN47 UniProt_P02787 genCDS_ENST00000402696_3_133746441-133778620_1 +query:74880;rank:1;spectrum:614.34271_3247.74_spectrum=43355_uteruspremenopause;rt:3247.74;mz:614.34271;charge:2 3.44948 0 6.07926e-05 X.EVM(Oxidation)LLLTGAHK.X UniProt_B7Z3X4 UniProt_D6R9P4 UniProt_D6RFF8 UniProt_E7EVU7 UniProt_P46926 UniProt_Q8TDQ7 UniProt_Q8TDQ7-2 UniProt_Q8TDQ7-3 UniProt_Q8TDQ7-4 UniProt_Q8TDQ7-5 UniProt_V9GYK3 genCDS_ENST00000295448_4_44703081-44722207_-1 genCDS_ENST00000311337_5_142002029-142012035_-1 genCDS_ENST00000500692_5_142002029-142012035_-1 genCDS_ENST00000503794_5_142002029-142012035_-1 genCDS_ENST00000505689_5_142002090-142012035_-1 genCDS_ENST00000507534_4_44703081-44718324_-1 genCDS_ENST00000507917_4_44703081-44722207_-1 genCDS_ENST00000508177_5_142002029-142012035_-1 genCDS_ENST00000509756_4_44707741-44722207_-1 genCDS_ENST00000513454_5_141991865-142012035_-1 genCDS_ENST00000609092_4_44682339-44711023_-1 +query:28620;rank:1;spectrum:480.78506_3493.94_spectrum=83136_uteruspremenopause;rt:3493.94;mz:480.78506;charge:2 3.4276 0 6.3855e-05 X.FQNALLVR.X CON_P02768-1 UniProt_B7WNR0 UniProt_C9JKR2 UniProt_D6RHD5 UniProt_H0YA55 UniProt_P02768 UniProt_P02768-2 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000401494_4_73404328-73420298_1 genCDS_ENST00000415165_4_73404328-73420298_1 genCDS_ENST00000503124_4_73406730-73420298_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000511370_4_73409340-73420298_1 genCDS_ENST00000621085_4_73404328-73420298_1 genCDS_ENST00000621628_4_73404328-73420298_1 +query:103766;rank:1;spectrum:473.23645_1715.59_spectrum=163581_uteruspremenopause;rt:1715.59;mz:473.23645;charge:3 3.41086 0 6.62984e-05 X.DLSTNYYASQKK.X UniProt_H0YIV0 UniProt_P14625 genCDS_ENST00000299767_12_103930516-103947662_1 genCDS_ENST00000550595_12_103943796-103953394_1 +query:150850;rank:1;spectrum:961.95966_2525.59_spectrum=88300_uteruspremenopause;rt:2525.59;mz:961.95966;charge:2 3.3544 0 7.52191e-05 X.DLQM(Oxidation)TQSPSSLSVSVGDR.X UniProt_P01605 +query:42641;rank:1;spectrum:525.26251_2586.03_spectrum=158158_uteruspremenopause;rt:2586.03;mz:525.26251;charge:2 3.24482 0 9.5912e-05 X.HTNFVEFR.X UniProt_M0QYZ2 UniProt_M0QZ21 UniProt_M0R0N4 UniProt_P53680 genCDS_ENST00000263270_19_46838447-46850766_-1 genCDS_ENST00000597020_19_46838447-46846085_-1 genCDS_ENST00000599990_19_46838447-46850664_-1 genCDS_ENST00000601498_19_46838447-46850184_-1 +query:81555;rank:1;spectrum:633.38092_3300.56_spectrum=13202_uteruspremenopause;rt:3300.56;mz:633.38092;charge:2 3.13577 0 0.000121753 X.VVLAPQDVVVAR.X UniProt_Q13308 UniProt_Q13308-2 UniProt_Q13308-3 UniProt_Q13308-4 UniProt_Q13308-5 UniProt_Q13308-6 UniProt_Q86X91 genCDS_ENST00000230418_6_43076489-43146661_1 genCDS_ENST00000230419_6_43076489-43160881_1 genCDS_ENST00000345201_6_43076489-43160881_1 genCDS_ENST00000349241_6_43076489-43160881_1 genCDS_ENST00000352931_6_43076489-43160881_1 genCDS_ENST00000471863_6_43076489-43132839_1 genCDS_ENST00000481273_6_43076884-43160881_1 +query:155440;rank:1;spectrum:673.72321_3106.63_spectrum=143874_uteruspremenopause;rt:3106.63;mz:673.72321;charge:3 3.10231 2.28599e-05 0.00013089 X.LKPEDLTQLQPQQLVLR.X UniProt_C9JPK5 UniProt_E7EQW5 UniProt_E7ERX5 UniProt_E7EUI6 UniProt_E9PLR6 UniProt_P05556 UniProt_P05556-2 UniProt_P05556-3 UniProt_P05556-4 UniProt_P05556-5 UniProt_Q5T3E6 genCDS_ENST00000302278_10_32901570-32935558_-1 genCDS_ENST00000396033_10_32901570-32935558_-1 genCDS_ENST00000423113_10_32907069-32935558_-1 genCDS_ENST00000437302_10_32928232-32935558_-1 genCDS_ENST00000474568_10_32928174-32930026_-1 genCDS_ENST00000475184_10_32929824-32935558_-1 genCDS_ENST00000480226_10_32928150-32935558_-1 genCDS_ENST00000488494_10_32928193-32935558_-1 genCDS_ENST00000534049_10_32928212-32935558_-1 +query:23764;rank:1;spectrum:465.2739_1931.52_spectrum=118184_uteruspremenopause;rt:1931.52;mz:465.2739;charge:2 2.96175 4.11168e-05 0.000176679 X.QVNLTVQK.X UniProt_H3BPZ1 UniProt_H3BS72 UniProt_Q9P035 genCDS_ENST00000261875_15_65530632-65576379_1 genCDS_ENST00000565299_15_65530632-65576379_1 genCDS_ENST00000568793_15_65530632-65576379_1 +query:113164;rank:1;spectrum:744.89307_1335.79_spectrum=53683_uteruspremenopause;rt:1335.79;mz:744.89307;charge:2 2.94029 4.11168e-05 0.000184873 X.RKPDTLEVQQM(Oxidation)K.X Augustus2_AUGUSTUS00000009543_5_25910749-25909503_1 UniProt_E7EQR4 UniProt_P15311 UniProt_P26038 UniProt_P35241 UniProt_P35241-4 UniProt_P35241-5 ensBodymap_RNASEQT00000007907_5_25909357-26027537_1 ensBodymap_RNASEQT00000020282_5_25909357-26027537_1 ensBodymap_RNASEQT00000153980_5_25909357-25913455_1 genCDS_ENST00000337147_6_158766914-158818093_-1 genCDS_ENST00000343115_11_110231869-110279692_-1 genCDS_ENST00000360270_X_65667842-65739893_1 genCDS_ENST00000367075_6_158766914-158818093_-1 genCDS_ENST00000392177_6_158766914-158789474_-1 genCDS_ENST00000405097_11_110199612-110279692_-1 genCDS_ENST00000528498_11_110199612-110279692_-1 genCDS_ENST00000530749_11_110199612-110279692_-1 genCDS_ENST00000544551_11_110231869-110272594_-1 genpseudogene_ENST00000367074_X_27517884-27519759_1_3_ORF10 genpseudogene_ENST00000511640_5_25909503-25911234_1_3_ORF3 yalePseudo_PGOHUM00000235534_5_25909612-25911337_1_3_ORF3 +query:88804;rank:1;spectrum:655.29285_1964.03_spectrum=141151_uteruspremenopause;rt:1964.03;mz:655.29285;charge:2 2.93857 4.11168e-05 0.000185547 X.EVEVEVESM(Oxidation)DK.X UniProt_Q7KZF4 genCDS_ENST00000354725_7_127652374-128092058_1 +query:86060;rank:1;spectrum:646.33569_2494.06_spectrum=33876_uteruspremenopause;rt:2494.06;mz:646.33569;charge:2 2.87421 4.11168e-05 0.000212455 X.AQSLEPYGTGLR.X UniProt_K7ERU2 UniProt_Q63ZY3 UniProt_Q63ZY3-2 UniProt_Q63ZY3-3 genCDS_ENST00000586659_19_11166558-11194511_-1 genCDS_ENST00000589359_19_11166558-11194511_-1 genCDS_ENST00000589894_19_11169853-11194511_-1 genCDS_ENST00000592675_19_11192881-11193248_-1 +query:90688;rank:1;spectrum:660.26282_1490.8_spectrum=109667_uteruspremenopause;rt:1490.8;mz:660.26282;charge:2 2.87129 4.11168e-05 0.000213763 X.YTM(Oxidation)GDAPDYDR.X UniProt_B9ZVX7 UniProt_E7EWW9 UniProt_E9PHN6 UniProt_E9PHN7 UniProt_E9PLF1 UniProt_F6XZQ7 UniProt_H3BQT3 UniProt_P09488 UniProt_P09488-2 UniProt_P28161 UniProt_P28161-2 UniProt_Q03013 UniProt_Q03013-2 UniProt_Q03013-3 genCDS_ENST00000241337_1_109668116-109674836_1 genCDS_ENST00000309851_1_109687874-109693295_1 genCDS_ENST00000326729_1_109656390-109665031_1 genCDS_ENST00000336075_1_109656390-109674836_1 genCDS_ENST00000349334_1_109687874-109693295_1 genCDS_ENST00000369819_1_109687874-109693295_1 genCDS_ENST00000369823_1_109687874-109693295_1 genCDS_ENST00000369827_1_109668116-109674836_1 genCDS_ENST00000369829_1_109668116-109673217_1 genCDS_ENST00000369831_1_109668116-109708576_1 genCDS_ENST00000369836_1_109656390-109661254_1 genCDS_ENST00000442650_1_109668116-109681817_1 genCDS_ENST00000460717_1_109668116-109681817_1 genCDS_ENST00000467579_1_109668116-109671486_1 genCDS_ENST00000483399_1_109687874-109689305_1 +query:127987;rank:1;spectrum:813.32916_1266.09_spectrum=69023_uteruspremenopause;rt:1266.09;mz:813.32916;charge:2 2.77394 5.68587e-05 0.000262028 X.LECDDKGDGSCDVR.X UniProt_P21333 UniProt_P21333-2 UniProt_Q14315 UniProt_Q14315-2 UniProt_Q5HY54 genCDS_ENST00000325888_7_128830638-128858523_1 genCDS_ENST00000344736_X_154348849-154371245_-1 genCDS_ENST00000346177_7_128830638-128858523_1 genCDS_ENST00000360319_X_154348849-154371245_-1 genCDS_ENST00000369850_X_154348849-154371245_-1 genCDS_ENST00000369856_X_154348849-154371164_-1 genCDS_ENST00000422373_X_154348849-154371245_-1 genCDS_ENST00000610817_X_154348849-154371164_-1 +query:154012;rank:1;spectrum:663.6684_2273.32_spectrum=79890_uteruspremenopause;rt:2273.32;mz:663.6684;charge:3 2.74607 5.68587e-05 0.000277701 X.VNPFRPGDSEPPPAPGAQR.X UniProt_C9IZ41 UniProt_H0Y2Y8 UniProt_Q15942 genCDS_ENST00000322764_7_143381572-143390682_1 genCDS_ENST00000354434_7_143381575-143390682_1 genCDS_ENST00000457235_7_143381572-143382676_1 +query:136470;rank:1;spectrum:572.59875_1543.09_spectrum=101367_uteruspremenopause;rt:1543.09;mz:572.59875;charge:3 2.67373 5.68587e-05 0.000322851 X.QEPERN(Deamidated)ECFLQHK.X CON_P02768-1 UniProt_H7C013 UniProt_P02768 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000441319_4_73397114-73409457_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000621085_4_73404328-73420298_1 genCDS_ENST00000621628_4_73404328-73420298_1 +query:106081;rank:1;spectrum:717.35742_2615.44_spectrum=165819_uteruspremenopause;rt:2615.44;mz:717.35742;charge:2 2.64505 5.68587e-05 0.00034272 X.AYGPGLEPTGNMVK.X UniProt_F8WE98 UniProt_P21333 UniProt_P21333-2 UniProt_Q5HY54 genCDS_ENST00000344736_X_154348849-154371245_-1 genCDS_ENST00000360319_X_154348849-154371245_-1 genCDS_ENST00000369850_X_154348849-154371245_-1 genCDS_ENST00000369856_X_154348849-154371164_-1 genCDS_ENST00000420627_X_154364689-154371203_-1 genCDS_ENST00000422373_X_154348849-154371245_-1 genCDS_ENST00000610817_X_154348849-154371164_-1 +query:135079;rank:1;spectrum:425.48462_2830.36_spectrum=96739_uteruspremenopause;rt:2830.36;mz:425.48462;charge:4 2.64252 5.68587e-05 0.000344529 X.TGVELGKPTHFTVNAK.X UniProt_P21333 UniProt_P21333-2 UniProt_Q5HY54 genCDS_ENST00000344736_X_154348849-154371245_-1 genCDS_ENST00000360319_X_154348849-154371245_-1 genCDS_ENST00000369850_X_154348849-154371245_-1 genCDS_ENST00000369856_X_154348849-154371164_-1 genCDS_ENST00000422373_X_154348849-154371245_-1 genCDS_ENST00000610817_X_154348849-154371164_-1 +query:133212;rank:1;spectrum:558.92828_2461.58_spectrum=18781_uteruspremenopause;rt:2461.58;mz:558.92828;charge:3 2.52804 6.99035e-05 0.000437424 X.LSPQFPNEEDSFHK.X UniProt_P49770 genCDS_ENST00000266126_14_75002991-75009188_1 +query:139472;rank:1;spectrum:585.95319_3414.38_spectrum=137257_uteruspremenopause;rt:3414.38;mz:585.95319;charge:3 2.48232 6.99035e-05 0.000481293 X.VFDKDGN(Deamidated)GYLSAAELR.X UniProt_E7EMB3 UniProt_E7ETZ0 UniProt_G3V361 UniProt_H0Y7A7 UniProt_P62158 UniProt_Q96HY3 genCDS_ENST00000272298_2_47160776-47176443_-1 genCDS_ENST00000291295_19_46601435-46609153_1 genCDS_ENST00000356978_14_90397231-90404717_1 genCDS_ENST00000391918_19_46608271-46609153_1 genCDS_ENST00000409563_2_47160776-47167650_-1 genCDS_ENST00000447653_14_90399082-90404717_1 genCDS_ENST00000456319_2_47160776-47176511_-1 genCDS_ENST00000544280_14_90401333-90404717_1 genCDS_ENST00000553542_14_90401333-90404717_1 genCDS_ENST00000557020_14_90401333-90404496_1 genCDS_ENST00000594523_19_46608271-46609153_1 genCDS_ENST00000596362_19_46602208-46609153_1 genCDS_ENST00000598871_19_46608271-46609153_1 genCDS_ENST00000599839_19_46608271-46609153_1 +query:75760;rank:1;spectrum:411.56995_2371.79_spectrum=149868_uteruspremenopause;rt:2371.79;mz:411.56995;charge:3 2.46586 6.99035e-05 0.000498166 X.SNFKPSLLAQK.X UniProt_C9JJ47 UniProt_E9PFW3 UniProt_Q96CW1 UniProt_Q96CW1-2 genCDS_ENST00000292807_3_184176994-184183616_1 genCDS_ENST00000382456_3_184176994-184183616_1 genCDS_ENST00000411763_3_184176994-184183616_1 genCDS_ENST00000432591_3_184176994-184182054_1 genCDS_ENST00000439647_3_184176994-184183616_1 genCDS_ENST00000621863_3_184176994-184183616_1 +query:13577;rank:1;spectrum:430.2381_2030.02_spectrum=133876_uteruspremenopause;rt:2030.02;mz:430.2381;charge:2 2.43497 6.99035e-05 0.000531473 X.LQLPNM(Oxidation)K.X UniProt_G3V281 UniProt_G3V5R2 UniProt_H0YJ34 UniProt_Q96AC1 UniProt_Q96AC1-2 UniProt_Q96AC1-3 genCDS_ENST00000341590_14_52858377-52950568_-1 genCDS_ENST00000343279_14_52858377-52950568_-1 genCDS_ENST00000395631_14_52858377-52950568_-1 genCDS_ENST00000399304_14_52859561-52950568_-1 genCDS_ENST00000553373_14_52858377-52950568_-1 genCDS_ENST00000554152_14_52858377-52919339_-1 genCDS_ENST00000554712_14_52893359-52950568_-1 genCDS_ENST00000555692_14_52893288-52928044_-1 +query:23073;rank:1;spectrum:463.27121_2707.23_spectrum=57220_uteruspremenopause;rt:2707.23;mz:463.27121;charge:2 2.37455 6.99035e-05 0.000603366 X.VPSGLPDLK.X UniProt_P21810 genCDS_ENST00000331595_X_153504632-153508445_1 +query:35867;rank:1;spectrum:503.7821_2660.72_spectrum=19332_uteruspremenopause;rt:2660.72;mz:503.7821;charge:2 2.36667 6.99035e-05 0.000613454 X.LVLEYVDR.X UniProt_E7ETH0 UniProt_G3XAM2 UniProt_P05156 genCDS_ENST00000394634_4_109740893-109801971_-1 genCDS_ENST00000394635_4_109740893-109801971_-1 genCDS_ENST00000512148_4_109740893-109801971_-1 +query:776;rank:1;spectrum:365.2341_2550.23_spectrum=11418_uteruspremenopause;rt:2550.23;mz:365.2341;charge:2 2.30486 8.72312e-05 0.000698664 X.ATVGLLR.X UniProt_B4DGU4 UniProt_P35222 calCuffs_CUFF.144442.10_7_134101990-134117256_-1_3_ORF25 calCuffs_CUFF.144442.11_7_134101990-134117266_-1_2_ORF13 calCuffs_CUFF.144442.12_7_134101990-134117277_-1_1_ORF18 calCuffs_CUFF.144442.13_7_134101990-134117277_-1_2_ORF23 calCuffs_CUFF.144442.9_7_134101990-134117207_-1_1_ORF15 genCDS_ENST00000349496_3_41224069-41239342_1 genCDS_ENST00000396183_3_41224069-41239342_1 genCDS_ENST00000396185_3_41224069-41239342_1 genCDS_ENST00000405570_3_41224069-41239342_1 genCDS_ENST00000453024_3_41224534-41239342_1 +query:18439;rank:1;spectrum:447.25858_2796.88_spectrum=73148_uteruspremenopause;rt:2796.88;mz:447.25858;charge:2 2.29139 0.000102579 0.000718741 X.TLLFSGQK.X UniProt_Q07954 genCDS_ENST00000243077_12_57128965-57212555_1 +query:16494;rank:1;spectrum:440.72366_1790.49_spectrum=86445_uteruspremenopause;rt:1790.49;mz:440.72366;charge:2 2.26159 0.000102579 0.00076526 X.AEFAEVSK.X CON_P02768-1 UniProt_B7WNR0 UniProt_C9JKR2 UniProt_D6RHD5 UniProt_H0YA55 UniProt_P02768 UniProt_P02768-2 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000401494_4_73404328-73420298_1 genCDS_ENST00000415165_4_73404328-73420298_1 genCDS_ENST00000503124_4_73406730-73420298_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000511370_4_73409340-73420298_1 +query:146021;rank:1;spectrum:614.65961_2560.2_spectrum=34039_uteruspremenopause;rt:2560.2;mz:614.65961;charge:3 2.19403 0.000116791 0.00088215 X.HVVPAQVHVN(Deamidated)GGALASER.X UniProt_E7ERH1 UniProt_E9PF55 UniProt_E9PGF5 UniProt_Q9HBL0 genCDS_ENST00000171887_2_217804459-217897965_-1 genCDS_ENST00000419504_2_217804459-217897965_-1 genCDS_ENST00000430930_2_217804459-217897965_-1 genCDS_ENST00000446903_2_217848176-218002872_-1 genCDS_ENST00000611415_2_217804459-217897965_-1 genCDS_ENST00000615025_2_217804459-217880904_-1 +query:40708;rank:1;spectrum:519.25708_2498.24_spectrum=127342_uteruspremenopause;rt:2498.24;mz:519.25708;charge:2 2.15281 0.000132074 0.00096205 X.(Acetyl)VNFAM(Oxidation)NVGK.X UniProt_P14618 UniProt_P14618-2 UniProt_P14618-3 UniProt_Q504U3 genCDS_ENST00000319622_15_72199650-72219097_-1 genCDS_ENST00000335181_15_72199650-72219097_-1 genCDS_ENST00000389093_15_72199650-72219097_-1 genCDS_ENST00000449901_15_72199650-72221201_-1 genCDS_ENST00000565154_15_72199650-72219097_-1 genCDS_ENST00000565184_15_72199650-72219097_-1 genCDS_ENST00000568459_15_72199650-72219097_-1 genCDS_ENST00000568883_15_72199650-72219097_-1 +query:101279;rank:1;spectrum:465.91821_2796_spectrum=73145_uteruspremenopause;rt:2796;mz:465.91821;charge:3 2.15095 0.000132074 0.000965824 X.VEHGSVALPALM(Oxidation)R.X UniProt_Q5TCU6 UniProt_Q9Y490 genCDS_ENST00000314888_9_35697791-35725694_-1 +query:65246;rank:1;spectrum:589.34381_3432.2_spectrum=182979_uteruspremenopause;rt:3432.2;mz:589.34381;charge:2 2.1457 0.000132074 0.00097655 X.EKGDYLLLVK.X UniProt_Q14315 UniProt_Q14315-2 genCDS_ENST00000325888_7_128830638-128858523_1 genCDS_ENST00000346177_7_128830638-128858523_1 +query:111686;rank:1;spectrum:492.8963_1268.73_spectrum=15674_uteruspremenopause;rt:1268.73;mz:492.8963;charge:3 2.14508 0.000132074 0.00097782 X.LEEAEKAADESER.X Augustus2_AUGUSTUS00000096365_2_230573165-230575961_-1 CON_Q3SX28 UniProt_B7Z596 UniProt_D6R904 UniProt_F5H7S3 UniProt_H0YK48 UniProt_H0YKP3 UniProt_H0YKX5 UniProt_H0YL52 UniProt_H0YL80 UniProt_H0YNC7 UniProt_H7BYY1 UniProt_J3KN67 UniProt_K7ENT6 UniProt_K7EP68 UniProt_K7ERG3 UniProt_P06753 UniProt_P06753-2 UniProt_P06753-3 UniProt_P06753-4 UniProt_P06753-5 UniProt_P07951 UniProt_P07951-2 UniProt_P07951-3 UniProt_P09493 UniProt_P09493-10 UniProt_P09493-2 UniProt_P09493-3 UniProt_P09493-4 UniProt_P09493-5 UniProt_P09493-6 UniProt_P09493-7 UniProt_P09493-8 UniProt_P09493-9 UniProt_P67936 UniProt_P67936-2 UniProt_Q5TCU3 UniProt_Q5TCU8 UniProt_Q5VU58 UniProt_Q5VU61 UniProt_Q6ZN40 genCDS_ENST00000267996_15_63042830-63071172_1 genCDS_ENST00000271850_1_154157639-154192018_-1 genCDS_ENST00000288398_15_63042830-63064146_1 genCDS_ENST00000300933_19_16076566-16101346_1 genCDS_ENST00000317516_15_63048576-63069943_1 genCDS_ENST00000323144_1_154158969-154183119_-1 genCDS_ENST00000328159_1_154157718-154183119_-1 genCDS_ENST00000329305_9_35682081-35689817_-1 genCDS_ENST00000330188_1_154157639-154183119_-1 genCDS_ENST00000334895_15_63048576-63069943_1 genCDS_ENST00000341372_1_154157639-154183119_-1 genCDS_ENST00000344824_19_16067625-16101346_1 genCDS_ENST00000357980_15_63042830-63071172_1 genCDS_ENST00000358278_15_63042830-63071172_1 genCDS_ENST00000360958_9_35683159-35689817_-1 genCDS_ENST00000368530_1_154167937-154192018_-1 genCDS_ENST00000368531_1_154158969-154183119_-1 genCDS_ENST00000368533_1_154157639-154183119_-1 genCDS_ENST00000378292_9_35682081-35689817_-1 genCDS_ENST00000378300_9_35682708-35689817_-1 genCDS_ENST00000403994_15_63042830-63065899_1 genCDS_ENST00000404484_15_63048576-63071172_1 genCDS_ENST00000509601_1_154172070-154183119_-1 genCDS_ENST00000558347_15_63042830-63061226_1 genCDS_ENST00000559281_15_63048576-63065899_1 genCDS_ENST00000559397_15_63042830-63071172_1 genCDS_ENST00000559556_15_63042830-63071172_1 genCDS_ENST00000559831_15_63044141-63061730_1 genCDS_ENST00000560959_15_63048576-63062809_1 genCDS_ENST00000560970_15_63042888-63064146_1 genCDS_ENST00000561266_15_63043776-63064146_1 genCDS_ENST00000561395_15_63057006-63061730_1 genCDS_ENST00000586499_19_16067897-16088093_1 genCDS_ENST00000586833_19_16075707-16093544_1 genCDS_ENST00000588410_19_16080953-16088066_1 genCDS_ENST00000611659_1_154157639-154183119_-1 genpseudogene_ENST00000330554_2_230573167-230573809_-1_1_ORF1 genpseudogene_ENST00000368528_3_27632976-27633720_1_2_ORF2 genpseudogene_ENST00000600996_19_41506152-41506898_1_3_ORF3 yalePseudo_PGOHUM00000240545_2_231437883-231438626_-1_1_ORF1 +query:48481;rank:1;spectrum:361.85358_1714.94_spectrum=140581_uteruspremenopause;rt:1714.94;mz:361.85358;charge:3 2.12562 0.000132074 0.00101867 X.VYLYHSSSK.X UniProt_B4DDT0 UniProt_H0YA32 UniProt_K7EMU3 UniProt_P26006 UniProt_P26006-1 genCDS_ENST00000007722_17_50056440-50089265_1 genCDS_ENST00000320031_17_50056440-50088335_1 genCDS_ENST00000510809_17_50072160-50074284_1 genCDS_ENST00000512553_17_50070873-50074514_1 +query:29295;rank:1;spectrum:483.24051_2183.66_spectrum=149333_uteruspremenopause;rt:2183.66;mz:483.24051;charge:2 2.06983 0.000144798 0.00114547 X.VDFNVPM(Oxidation)K.X UniProt_E7ERH5 UniProt_P00558 UniProt_P07205 genCDS_ENST00000304801_6_49785934-49787187_-1 genCDS_ENST00000373316_X_78104341-78125830_1 +query:36143;rank:1;spectrum:504.74222_1377.49_spectrum=31223_uteruspremenopause;rt:1377.49;mz:504.74222;charge:2 2.02503 0.000173319 0.00125862 X.SLM(Oxidation)SADNVR.X UniProt_O43143 genCDS_ENST00000336812_4_24527924-24584393_-1 +query:123327;rank:1;spectrum:525.58398_2676.85_spectrum=49516_uteruspremenopause;rt:2676.85;mz:525.58398;charge:3 1.99921 0.000173319 0.00132883 X.QVDVTSFAGHPCTR.X UniProt_H0Y5U1 UniProt_O00468 UniProt_O00468-2 UniProt_O00468-3 UniProt_O00468-4 UniProt_O00468-5 UniProt_O00468-6 UniProt_O00468-7 genCDS_ENST00000379370_1_1020173-1054981_1 genCDS_ENST00000419249_1_1050556-1053950_1 genCDS_ENST00000620552_1_1022414-1054981_1 +query:59938;rank:1;spectrum:575.31378_2777.5_spectrum=112771_uteruspremenopause;rt:2777.5;mz:575.31378;charge:2 1.9623 0.000173319 0.00143597 X.LVNEVTEFAK.X CON_P02768-1 UniProt_H7C013 UniProt_P02768 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000441319_4_73397114-73409457_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000621085_4_73404328-73420298_1 genCDS_ENST00000621628_4_73404328-73420298_1 +query:23238;rank:1;spectrum:464.21664_1155.55_spectrum=169721_uteruspremenopause;rt:1155.55;mz:464.21664;charge:2 1.87479 0.000198798 0.00172556 X.CQYVTEK.X UniProt_H3BPS8 UniProt_H3BQN4 UniProt_J3KPS3 UniProt_P04075 UniProt_P04075-2 UniProt_P09972 genCDS_ENST00000226253_17_28573526-28575532_-1 genCDS_ENST00000338110_16_30067255-30070212_1 genCDS_ENST00000395240_16_30067255-30070212_1 genCDS_ENST00000395248_16_30066898-30070212_1 genCDS_ENST00000395321_17_28573526-28575532_-1 genCDS_ENST00000412304_16_30067255-30070212_1 genCDS_ENST00000562679_16_30066898-30069547_1 genCDS_ENST00000563060_16_30067255-30070212_1 genCDS_ENST00000564546_16_30067255-30070212_1 genCDS_ENST00000564595_16_30066898-30070212_1 genCDS_ENST00000566897_16_30067255-30070212_1 genCDS_ENST00000569545_16_30067255-30070212_1 genCDS_ENST00000569798_16_30067255-30070116_1 +query:57302;rank:1;spectrum:379.21075_3171.84_spectrum=50657_uteruspremenopause;rt:3171.84;mz:379.21075;charge:3 1.8485 0.000212479 0.00182324 X.FKM(Oxidation)PELNLK.X UniProt_Q09666 genCDS_ENST00000378024_11_62516744-62536098_-1 +query:17102;rank:1;spectrum:442.76447_2988.18_spectrum=166780_uteruspremenopause;rt:2988.18;mz:442.76447;charge:2 1.7864 0.000268723 0.00207556 X.(Acetyl)VATVSLPR.X CON_P00761 +query:86128;rank:1;spectrum:431.55176_762.55_spectrum=60182_uteruspremenopause;rt:762.55;mz:431.55176;charge:3 1.68922 0.000320239 0.00253864 X.HM(Oxidation)QANPEPPKK.X UniProt_Q15404 UniProt_Q15404-2 genCDS_ENST00000345264_10_16593394-16817081_-1 genCDS_ENST00000377921_10_16593394-16817081_-1 genCDS_ENST00000602389_10_16593394-16782034_-1 +query:5783;rank:1;spectrum:395.23944_2103.35_spectrum=103066_uteruspremenopause;rt:2103.35;mz:395.23944;charge:2 1.63365 0.000371014 0.00284701 X.LVTDLTK.X CON_P02768-1 CON_P02769 UniProt_B7WNR0 UniProt_C9JKR2 UniProt_D6RHD5 UniProt_H0YA55 UniProt_P02768 UniProt_P02768-2 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000401494_4_73404328-73420298_1 genCDS_ENST00000415165_4_73404328-73420298_1 genCDS_ENST00000503124_4_73406730-73420298_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000511370_4_73409340-73420298_1 +query:95400;rank:1;spectrum:450.9129_2783.38_spectrum=4821_uteruspremenopause;rt:2783.38;mz:450.9129;charge:3 1.62591 0.000371014 0.00289274 X.TAAENEFVTLKK.X CON_P02538 CON_P04259 CON_P12035 CON_P48668 CON_P50446 UniProt_J3QST3 UniProt_P02538 UniProt_P04259 UniProt_P12035 UniProt_P48668 genCDS_ENST00000252250_12_52469062-52473737_-1 genCDS_ENST00000252252_12_52447190-52452078_-1 genCDS_ENST00000330722_12_52487720-52493188_-1 genCDS_ENST00000417996_12_52790042-52796042_-1 +query:16267;rank:1;spectrum:439.74139_3590.06_spectrum=75323_uteruspremenopause;rt:3590.06;mz:439.74139;charge:2 1.62109 0.000371014 0.00292163 X.FVADLWK.X UniProt_P35749 UniProt_P35749-2 UniProt_P35749-3 UniProt_P35749-4 genCDS_ENST00000300036_16_15703991-15838252_-1 genCDS_ENST00000396324_16_15703991-15838252_-1 genCDS_ENST00000452625_16_15708811-15838252_-1 genCDS_ENST00000576790_16_15708811-15838252_-1 genCDS_ENST00000611087_16_15703879-15838252_-1 genCDS_ENST00000616439_16_15703879-15838252_-1 +query:96057;rank:1;spectrum:452.58298_2886.61_spectrum=166517_uteruspremenopause;rt:2886.61;mz:452.58298;charge:3 1.60885 0.000371014 0.00299626 X.VTGEVHLGGVM(Oxidation)LK.X UniProt_G3V281 UniProt_G3V379 UniProt_Q96AC1 UniProt_Q96AC1-2 UniProt_Q96AC1-3 genCDS_ENST00000341590_14_52858377-52950568_-1 genCDS_ENST00000343279_14_52858377-52950568_-1 genCDS_ENST00000395631_14_52858377-52950568_-1 genCDS_ENST00000399304_14_52859561-52950568_-1 genCDS_ENST00000553373_14_52858377-52950568_-1 genCDS_ENST00000554712_14_52893359-52950568_-1 genCDS_ENST00000557562_14_52919335-52950568_-1 +query:47514;rank:1;spectrum:360.18124_1413.26_spectrum=124451_uteruspremenopause;rt:1413.26;mz:360.18124;charge:3 1.48919 0.000510326 0.00383368 X.FLENEDRR.X UniProt_P01009 UniProt_P01009-2 UniProt_P01009-3 genCDS_ENST00000355814_14_94378449-94383237_-1 genCDS_ENST00000393087_14_94378449-94383237_-1 genCDS_ENST00000393088_14_94378449-94383237_-1 genCDS_ENST00000402629_14_94379449-94383237_-1 genCDS_ENST00000404814_14_94378449-94383237_-1 genCDS_ENST00000437397_14_94378449-94383237_-1 genCDS_ENST00000440909_14_94378449-94383237_-1 genCDS_ENST00000448921_14_94378449-94383237_-1 genCDS_ENST00000449399_14_94378449-94383237_-1 genCDS_ENST00000489769_14_94380867-94383237_-1 +query:31310;rank:1;spectrum:489.31235_2716.22_spectrum=127949_uteruspremenopause;rt:2716.22;mz:489.31235;charge:2 1.36197 0.000578424 0.0050008 X.KLELHLPK.X UniProt_P29622 genCDS_ENST00000298841_14_94563483-94569595_1 genCDS_ENST00000555095_14_94563483-94569595_1 genCDS_ENST00000557004_14_94563483-94569595_1 +query:129404;rank:1;spectrum:546.57996_1004.75_spectrum=161953_uteruspremenopause;rt:1004.75;mz:546.57996;charge:3 1.28237 0.000726094 0.005938 X.LGREEPAM(Oxidation)SM(Oxidation)DANGK.X UniProt_B4DZI8 UniProt_P35606 genCDS_ENST00000333188_3_139357863-139389550_-1 genCDS_ENST00000507777_3_139357863-139383351_-1 +query:40260;rank:1;spectrum:517.28729_2307.21_spectrum=95484_uteruspremenopause;rt:2307.21;mz:517.28729;charge:2 1.25301 0.000760157 0.00633558 X.YVPGVGNVTK.X UniProt_Q8IWV7 UniProt_Q8IWV7-2 genCDS_ENST00000290650_15_42945329-43106022_-1 genCDS_ENST00000546274_15_43015684-43106022_-1 genCDS_ENST00000569066_15_43026631-43037829_-1 +query:19664;rank:1;spectrum:451.27832_3344.97_spectrum=106687_uteruspremenopause;rt:3344.97;mz:451.27832;charge:2 1.24376 0.000772162 0.00646756 X.GPFLVALGK.X UniProt_F5H7Y0 UniProt_H0Y8Y3 UniProt_Q96HC4 UniProt_Q96HC4-4 UniProt_Q96HC4-6 UniProt_Q96HC4-7 genCDS_ENST00000317968_4_94455289-94664067_1 genCDS_ENST00000437932_4_94618059-94664067_1 genCDS_ENST00000503974_4_94455289-94666026_1 genCDS_ENST00000506632_4_94585627-94656820_1 genCDS_ENST00000514743_4_94455289-94664067_1 genCDS_ENST00000542407_4_94575691-94664067_1 genCDS_ENST00000615540_4_94455289-94664067_1 +query:25040;rank:1;spectrum:469.26465_1783.6_spectrum=54803_uteruspremenopause;rt:1783.6;mz:469.26465;charge:2 1.2271 0.000782926 0.00671406 X.LLHTYYK.X UniProt_B5ME19 UniProt_H3BRV0 UniProt_Q99613 genCDS_ENST00000331666_16_28711687-28735513_1 genCDS_ENST00000380876_16_28379798-28403618_-1 genCDS_ENST00000395587_16_28711687-28735513_1 genCDS_ENST00000398944_16_28379798-28403618_-1 genCDS_ENST00000564243_16_28711687-28735513_1 genCDS_ENST00000566501_16_28711687-28735513_1 genCDS_ENST00000566866_16_28711687-28735513_1 +query:136411;rank:1;spectrum:429.48587_1838.93_spectrum=163885_uteruspremenopause;rt:1838.93;mz:429.48587;charge:4 1.16869 0.000861273 0.00767686 X.LDSEDKDKEGKPLLK.X UniProt_P13639 genCDS_ENST00000309311_19_3976554-3985380_-1 +query:11971;rank:1;spectrum:423.73737_1581.19_spectrum=24615_uteruspremenopause;rt:1581.19;mz:423.73737;charge:2 1.15731 0.00087327 0.00788465 X.ATEVTVAR.X UniProt_F5GZL7 UniProt_F8W8Q1 UniProt_H0Y390 UniProt_H3BPE1 UniProt_H3BQK9 UniProt_Q9UPN3 UniProt_Q9UPN3-2 UniProt_Q9UPN3-3 UniProt_Q9UPN3-4 UniProt_Q9UPN3-5 genCDS_ENST00000289893_1_39331269-39485794_1 genCDS_ENST00000361689_1_39084219-39485794_1 genCDS_ENST00000372915_1_39084219-39485794_1 genCDS_ENST00000372925_1_39300314-39485794_1 genCDS_ENST00000564288_1_39205023-39485794_1 genCDS_ENST00000567887_1_39084219-39485794_1 +query:1271;rank:1;spectrum:368.20523_2167.57_spectrum=33073_uteruspremenopause;rt:2167.57;mz:368.20523;charge:2 1.14788 0.000884565 0.00806222 X.GDVAFVK.X CON_Q0IIK2 CON_Q29443 CON_Q2HJF0 UniProt_H7C5E8 UniProt_J3KN47 UniProt_P02787 genCDS_ENST00000402696_3_133746441-133778620_1 genCDS_ENST00000461695_3_133766278-133775445_1 +query:95023;rank:1;spectrum:450.21964_926.016_spectrum=108365_uteruspremenopause;rt:926.016;mz:450.21964;charge:3 1.14691 0.000884565 0.00808068 X.SDDNRESLEKR.X UniProt_P30085 UniProt_P30085-2 genCDS_ENST00000371873_1_47333946-47376745_1 genCDS_ENST00000450808_1_47333946-47376745_1 +query:78779;rank:1;spectrum:625.27612_1963.17_spectrum=17448_uteruspremenopause;rt:1963.17;mz:625.27612;charge:2 1.11901 0.000928303 0.0086392 X.DNVDDPTGNFR.X UniProt_Q12907 genCDS_ENST00000303127_5_177332086-177351647_-1 +query:32633;rank:1;spectrum:493.74869_1021.76_spectrum=139019_uteruspremenopause;rt:1021.76;mz:493.74869;charge:2 1.11186 0.000928303 0.00879018 X.ASPEAASTPR.X UniProt_H3BT29 UniProt_H3BT57 UniProt_H3BUJ5 UniProt_H3BVD2 UniProt_P29590 UniProt_P29590-10 UniProt_P29590-11 UniProt_P29590-12 UniProt_P29590-13 UniProt_P29590-14 UniProt_P29590-2 UniProt_P29590-3 UniProt_P29590-4 UniProt_P29590-5 UniProt_P29590-8 UniProt_P29590-9 genCDS_ENST00000268058_15_73994813-74045008_1 genCDS_ENST00000268059_15_73994813-74035951_1 genCDS_ENST00000354026_15_73994813-74035951_1 genCDS_ENST00000359928_15_73994813-74034495_1 genCDS_ENST00000395132_15_73994813-74036060_1 genCDS_ENST00000395135_15_73994813-74043180_1 genCDS_ENST00000435786_15_73994813-74034656_1 genCDS_ENST00000436891_15_73994813-74034495_1 genCDS_ENST00000563500_15_73994813-74033608_1 genCDS_ENST00000564428_15_73994813-74043180_1 genCDS_ENST00000565239_15_74022912-74036060_1 genCDS_ENST00000565898_15_73994813-74045008_1 genCDS_ENST00000566068_15_74023295-74035154_1 genCDS_ENST00000567543_15_73994813-74034495_1 genCDS_ENST00000567606_15_74022981-74034495_1 genCDS_ENST00000569477_15_73994813-74035347_1 genCDS_ENST00000569965_15_73994813-74034495_1 +query:38217;rank:1;spectrum:511.28668_2167.71_spectrum=55788_uteruspremenopause;rt:2167.71;mz:511.28668;charge:2 1.03421 0.00103826 0.0106727 X.DLPEHAVLK.X UniProt_Q00839 UniProt_Q00839-2 genCDS_ENST00000283179_1_244854450-244864307_-1 genCDS_ENST00000444376_1_244854450-244864307_-1 +query:162959;rank:1;spectrum:1114.509_3262.17_spectrum=159859_uteruspremenopause;rt:3262.17;mz:1114.509;charge:2 1.02402 0.00105016 0.0109563 X.AM(Oxidation)LSGPGQ(Deamidated)FAENETNEVNFR.X UniProt_E5RHG8 UniProt_Q15369 UniProt_Q15369-2 calCuffs_CUFF.49941.1_15_41849132-41849866_1_1_ORF2 genCDS_ENST00000284811_8_73946630-73959768_-1 genCDS_ENST00000518127_8_73946630-73959768_-1 genCDS_ENST00000519082_8_73946700-73959768_-1 genCDS_ENST00000519487_8_73946630-73959768_-1 genCDS_ENST00000520210_8_73946630-73956010_-1 genCDS_ENST00000520242_8_73946630-73959768_-1 genCDS_ENST00000522337_8_73946630-73959768_-1 genCDS_ENST00000523815_8_73946630-73959768_-1 genCDS_ENST00000622804_8_73946630-73959768_-1 +query:493;rank:1;spectrum:364.2103_2722.7_spectrum=34432_uteruspremenopause;rt:2722.7;mz:364.2103;charge:2 1.01234 0.00107207 0.011293 X.YLYLR.X CON_Q05443 UniProt_A2RUS2 UniProt_A2RUS2-2 UniProt_E9PF32 UniProt_H0YAY3 UniProt_P51884 UniProt_Q8IZT6 XXX_1317687 XXX_1318739 XXX_1319802 XXX_1320698 XXX_1321015 XXX_1322425 XXX_1508340 XXX_1759976 XXX_1802721 XXX_1908615 XXX_244137 XXX_24581 XXX_2923480 XXX_2928492 XXX_2928493 XXX_2928495 XXX_2928496 XXX_2949234 XXX_2981109 XXX_2981110 XXX_3632075 XXX_3860754 XXX_3986952 XXX_3986953 XXX_3986954 XXX_3986955 XXX_3987789 XXX_4011861 XXX_4011867 XXX_4011870 XXX_815771 XXX_816106 calCuffs_CUFF.103119.2_3_161317860-161356881_1_2_ORF21 calCuffs_CUFF.103119.5_3_161333860-161356881_1_3_ORF17 calCuffs_CUFF.103119.6_3_161334114-161356881_1_1_ORF14 calCuffs_CUFF.103119.8_3_161346370-161356881_1_3_ORF15 calCuffs_CUFF.115499.1_5_1473232-1481829_1_2_ORF7 calCuffs_CUFF.128368.1_6_31802244-31811541_-1_1_ORF46 calCuffs_CUFF.128368.2_6_31802244-31811541_-1_1_ORF46 calCuffs_CUFF.1336.1_1_22047526-22055216_1_1_ORF1 calCuffs_CUFF.165574.1_X_121355709-121383470_1_3_ORF111 calCuffs_CUFF.165574.2_X_121355709-121383470_1_3_ORF155 calCuffs_CUFF.165574.3_X_121355857-121383470_1_1_ORF180 calCuffs_CUFF.165574.4_X_121355857-121383470_1_3_ORF146 calCuffs_CUFF.165574.6_X_121356178-121383470_1_3_ORF139 calCuffs_CUFF.165574.7_X_121363426-121383470_1_2_ORF112 calCuffs_CUFF.46425.1_14_68586652-68634192_-1_3_ORF32 calCuffs_CUFF.46425.2_14_68586652-68634288_-1_1_ORF52 calCuffs_CUFF.64507.5_18_33515874-33530500_-1_1_ORF63 calCuffs_CUFF.65492.2_18_61137255-61172334_-1_2_ORF176 calCuffs_CUFF.65972.3_18_70992026-71027992_1_1_ORF37 calCuffs_CUFF.65972.4_18_70992026-71027992_1_1_ORF19 calCuffs_CUFF.65972.7_18_70992235-71027992_1_3_ORF16 calCuffs_CUFF.87619.6_20_49983340-50039104_1_1_ORF17 calCuffs_CUFF.9273.1_1_173606791-173638011_1_1_ORF112 calCuffs_CUFF.9273.2_1_173606791-173638011_1_1_ORF106 genCDS_ENST00000262585_8_141136647-141194233_1 genCDS_ENST00000266718_12_91104165-91108979_-1 genCDS_ENST00000367409_1_197084324-197146437_-1 genCDS_ENST00000424248_8_141136647-141194233_1 genCDS_ENST00000518668_8_141128779-141194233_1 genCDS_ENST00000519811_8_141128708-141194233_1 genlncRNA_ENST00000449713_21_44485577-44490288_1_3_ORF3 genlncRNA_ENST00000562834_6_54943167-54945099_1_3_ORF15 genlncRNA_ENST00000614289_7_9082557-9189785_-1_1_ORF3 +query:172844;rank:1;spectrum:659.81531_2195.32_spectrum=87449_uteruspremenopause;rt:2195.32;mz:659.81531;charge:4 0.865178 0.00138098 0.0168505 X.QEPERNECFLQHKDDNPNLPR.X CON_P02768-1 UniProt_H7C013 UniProt_P02768 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000441319_4_73397114-73409457_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000621085_4_73404328-73420298_1 genCDS_ENST00000621628_4_73404328-73420298_1 +query:3314;rank:1;spectrum:381.23285_2666.74_spectrum=19351_uteruspremenopause;rt:2666.74;mz:381.23285;charge:2 0.532423 0.00282806 0.0455454 X.LM(Oxidation)VALAK.X Augustus2_AUGUSTUS00000042791_4_153307709-153311274_-1 UniProt_A6NMY6 UniProt_H0YKS4 UniProt_H0YL33 UniProt_H0YM50 UniProt_H0YMD0 UniProt_H0YMU9 UniProt_H0YN28 UniProt_H0YN42 UniProt_H0YN52 UniProt_H0YNA0 UniProt_H0YNP5 UniProt_P07355 UniProt_P07355-2 genCDS_ENST00000332680_15_60347630-60397913_-1 genCDS_ENST00000396024_15_60347630-60386075_-1 genCDS_ENST00000421017_15_60347630-60386075_-1 genCDS_ENST00000451270_15_60347630-60386075_-1 genCDS_ENST00000557906_15_60355919-60386075_-1 genCDS_ENST00000558132_15_60351782-60386075_-1 genCDS_ENST00000558985_15_60351724-60360946_-1 genCDS_ENST00000558998_15_60354154-60360946_-1 genCDS_ENST00000559113_15_60355919-60360946_-1 genCDS_ENST00000559176_15_60351192-60374505_-1 genCDS_ENST00000559818_15_60351733-60386075_-1 genCDS_ENST00000560014_15_60351813-60386075_-1 genCDS_ENST00000560165_15_60355919-60386075_-1 genCDS_ENST00000560367_15_60352383-60386075_-1 genCDS_ENST00000560389_15_60355919-60386075_-1 genCDS_ENST00000560466_15_60351733-60360946_-1 genpseudogene_ENST00000435128_9_33624274-33625293_1_1_ORF1 +query:14008;rank:1;spectrum:431.7608_3415_spectrum=82918_uteruspremenopause;rt:3415;mz:431.7608;charge:2 0.482234 0.0031261 0.0531809 X.FLLSNLR.X UniProt_E9PGM4 UniProt_Q04446 XXX_3648561 genCDS_ENST00000429644_3_81490407-81761517_-1 genCDS_ENST00000489715_3_81490407-81743576_-1 +query:26472;rank:1;spectrum:474.24692_2020.57_spectrum=133852_uteruspremenopause;rt:2020.57;mz:474.24692;charge:2 0.378973 0.00408 0.0731188 X.CLLVEEGK.X Augustus2_AUGUSTUS00000097311_8_73984804-73984493_1 UniProt_F6RFD5 UniProt_P60981 UniProt_P60981-2 genCDS_ENST00000246069_20_17570209-17607146_1 genCDS_ENST00000449141_20_17570209-17605129_1 genCDS_ENST00000474024_20_17600786-17607146_1 genpseudogene_ENST00000399472_3_39214199-39214672_1_1_ORF1 genpseudogene_ENST00000517767_8_73984493-73984883_1_1_ORF1 yalePseudo_PGOHUM00000249715_8_74896728-74897132_1_1_ORF1 +query:25242;rank:1;spectrum:470.24841_1903.97_spectrum=25332_uteruspremenopause;rt:1903.97;mz:470.24841;charge:2 0.374892 0.00415655 0.0740455 X.VSSYGGTLR.X UniProt_O15230 genCDS_ENST00000252999_20_62309336-62367245_-1 +query:14390;rank:1;spectrum:433.73349_2095.35_spectrum=156931_uteruspremenopause;rt:2095.35;mz:433.73349;charge:2 0.337635 0.00458149 0.0830843 X.(Carbamidomethyl)YLSALN(Deamidated)K.X genlncRNA_ENST00000433639_3_6490479-6736129_1_3_ORF9 +query:59374;rank:1;spectrum:574.29547_1290.25_spectrum=46381_uteruspremenopause;rt:1290.25;mz:574.29547;charge:2 0.216141 0.00585082 0.121453 X.ESKPAQGQFR.X UniProt_Q05707 UniProt_Q05707-2 UniProt_Q05707-3 UniProt_Q4G0W3 genCDS_ENST00000297848_8_120147843-120371231_1 genCDS_ENST00000309791_8_120147843-120370367_1 genCDS_ENST00000498051_8_120147843-120213942_1 genCDS_ENST00000537875_8_120147843-120213942_1 +query:139088;rank:1;spectrum:584.30249_2663.68_spectrum=181151_uteruspremenopause;rt:2663.68;mz:584.30249;charge:3 0.201002 0.00605916 0.127366 X.(Acetyl)LRTEGDGVYTLNDKK.X Augustus2_AUGUSTUS00000011721_16_72077081-72063214_1 UniProt_H0Y300 UniProt_J3KRH2 UniProt_J3KTC3 UniProt_J3QLC9 UniProt_J3QR68 UniProt_P00738 UniProt_P00739 UniProt_P00739-2 genCDS_ENST00000355906_16_72054653-72060890_1 genCDS_ENST00000357763_16_72054653-72060890_1 genCDS_ENST00000540303_16_72063256-72077081_1 genCDS_ENST00000561690_16_72063256-72076993_1 genCDS_ENST00000565574_16_72054653-72060890_1 genCDS_ENST00000567185_16_72056161-72060890_1 genCDS_ENST00000567612_16_72056161-72060890_1 genCDS_ENST00000576168_16_72056161-72059199_1 +query:41161;rank:1;spectrum:520.79297_2617.01_spectrum=165822_uteruspremenopause;rt:2617.01;mz:520.79297;charge:2 0.17738 0.00651168 0.137133 X.SVPM(Oxidation)VPPGLK.X CON_Q05443 UniProt_P51884 genCDS_ENST00000266718_12_91104165-91108979_-1 +query:46620;rank:1;spectrum:537.77405_1721.6_spectrum=2503_uteruspremenopause;rt:1721.6;mz:537.77405;charge:2 0.107507 0.00780292 0.169915 X.LDELRDEGK.X CON_P02768-1 UniProt_B7WNR0 UniProt_D6RHD5 UniProt_H0YA55 UniProt_P02768 genCDS_ENST00000295897_4_73404328-73420298_1 genCDS_ENST00000401494_4_73404328-73420298_1 genCDS_ENST00000503124_4_73406730-73420298_1 genCDS_ENST00000509063_4_73404328-73421121_1 genCDS_ENST00000511370_4_73409340-73420298_1 +query:70107;rank:1;spectrum:602.77112_906.603_spectrum=45528_uteruspremenopause;rt:906.603;mz:602.77112;charge:2 0.100629 0.00788342 0.173457 X.QQQQMEQER.X UniProt_H0YDN1 UniProt_Q15149 UniProt_Q15149-2 UniProt_Q15149-3 UniProt_Q15149-4 UniProt_Q15149-5 UniProt_Q15149-6 UniProt_Q15149-7 UniProt_Q15149-8 UniProt_Q15149-9 genCDS_ENST00000322810_8_143916177-143950706_-1 genCDS_ENST00000345136_8_143916177-143939461_-1 genCDS_ENST00000354589_8_143916177-143943890_-1 genCDS_ENST00000354958_8_143916177-143953771_-1 genCDS_ENST00000356346_8_143916177-143973472_-1 genCDS_ENST00000357649_8_143916177-143942515_-1 genCDS_ENST00000398774_8_143916177-143944663_-1 genCDS_ENST00000436759_8_143916177-143975369_-1 genCDS_ENST00000527096_8_143916177-143975369_-1 genCDS_ENST00000527303_8_143921932-143927011_-1 +query:58423;rank:1;spectrum:571.82373_3329.4_spectrum=28523_uteruspremenopause;rt:3329.4;mz:571.82373;charge:2 -0.0664699 0.0118764 0.276689 X.VKGDVDVSVPK.X UniProt_Q09666 genCDS_ENST00000378024_11_62516744-62536098_-1 +query:4273;rank:1;spectrum:388.21448_1441.57_spectrum=147307_uteruspremenopause;rt:1441.57;mz:388.21448;charge:2 -0.0871283 0.0123296 0.291726 X.LASLDEK.X UniProt_E9PNR6 UniProt_H0YE29 UniProt_Q07960 genCDS_ENST00000311956_11_46679037-46696107_-1 genCDS_ENST00000525488_11_46681069-46696107_-1 genCDS_ENST00000528837_11_46679217-46696099_-1 +query:981;rank:1;spectrum:366.20657_1827.57_spectrum=78717_uteruspremenopause;rt:1827.57;mz:366.20657;charge:2 -0.266624 0.0180136 0.438278 X.DNTLLR.X UniProt_A6NN80 UniProt_H0YG46 UniProt_J3KNE4 UniProt_P08133 UniProt_P08133-2 UniProt_Q8WZ42 UniProt_Q8WZ42-11 UniProt_Q8WZ42-12 UniProt_Q8WZ42-13 UniProt_Q8WZ42-2 UniProt_Q8WZ42-4 UniProt_Q8WZ42-5 UniProt_Q8WZ42-7 UniProt_Q8WZ42-8 UniProt_Q99550 UniProt_Q99550-2 UniProt_U3KQ28 XXX_3776517 XXX_3780596 XXX_3818260 XXX_3846030 XXX_3896565 XXX_4087153 XXX_4095718 XXX_4139454 XXX_4158810 XXX_4177746 genCDS_ENST00000302373_12_123161368-123221847_-1 genCDS_ENST00000342992_2_178527012-178804642_-1 genCDS_ENST00000354546_5_151101448-151147901_-1 genCDS_ENST00000523714_5_151101448-151140165_-1 genCDS_ENST00000539024_12_123161368-123218478_-1 genCDS_ENST00000541076_12_123156807-123230364_-1 genCDS_ENST00000589042_2_178527012-178804642_-1 genCDS_ENST00000591111_2_178527012-178804642_-1 genCDS_ENST00000606320_12_123156807-123230364_-1 genCDS_ENST00000615779_2_178527012-178804642_-1 +query:33052;rank:1;spectrum:494.77515_1390.27_spectrum=155237_uteruspremenopause;rt:1390.27;mz:494.77515;charge:2 -0.390915 0.0229485 0.545281 X.KQVENKN(Deamidated)K.X UniProt_Q15431 UniProt_Q5VXJ5 genCDS_ENST00000369518_1_114855465-114995019_1 genCDS_ENST00000369522_1_114855465-114995019_1 genCDS_ENST00000455987_1_114855465-114977610_1 genCDS_ENST00000613524_1_114855465-114995019_1 genCDS_ENST00000618516_1_114855465-114995019_1 +query:15821;rank:1;spectrum:438.23367_2620.9_spectrum=88556_uteruspremenopause;rt:2620.9;mz:438.23367;charge:2 -0.554281 0.0308507 0.671862 X.QGGPEFLK.X calCuffs_CUFF.56789.1_16_67551854-67597900_-1_1_ORF3 calCuffs_CUFF.56789.2_16_67551854-67562677_-1_1_ORF3 calCuffs_CUFF.56789.3_16_67551854-67562643_-1_3_ORF2 calCuffs_CUFF.56789.4_16_67551854-67564404_-1_2_ORF5 calCuffs_CUFF.56789.5_16_67551854-67556024_-1_1_ORF2 ensBodymap_RNASEQT00000108700_16_67551703-67597612_-1 genlncRNA_ENST00000613438_16_67517862-67528632_-1_3_ORF2 genlncRNA_ENST00000621378_16_67517950-67528675_-1_2_ORF4 mitSBM_kidney_16_67551858-67562264_-1_3_ORF1 mitSBM_lymphNode_16_67551861-67562262_-1_1_ORF1 +query:149496;rank:1;spectrum:951.48413_3058.62_spectrum=89727_uteruspremenopause;rt:3058.62;mz:951.48413;charge:2 -0.634256 0.0352042 0.722362 X.(Acetyl)KPGAAGQHPAPFDPQSVR.X genlncRNA_ENST00000449990_9_90463659-90582744_-1_3_ORF1 +query:106872;rank:1;spectrum:720.35449_2699.4_spectrum=150809_uteruspremenopause;rt:2699.4;mz:720.35449;charge:2 -0.816145 0.0457039 0.814517 X.STDNVFLACWVK.X calCuffs_CUFF.6079.5_1_107514034-107541205_-1_3_ORF3 +query:44599;rank:1;spectrum:531.28_2604.3_spectrum=19171_uteruspremenopause;rt:2604.3;mz:531.28;charge:2 -0.98233 0.0555745 0.881893 X.KQ(Deamidated)WERTGR.X calCuffs_CUFF.84105.1_2_227648862-227677259_1_3_ORF138 calCuffs_CUFF.84105.2_2_227653754-227677259_1_1_ORF74 +query:2044;rank:1;spectrum:374.22897_2891.89_spectrum=135978_uteruspremenopause;rt:2891.89;mz:374.22897;charge:2 -1.04589 0.0593188 0.912796 X.FLLQAR.X UniProt_P07360 UniProt_Q5SQ08 XXX_2911659 XXX_2911660 XXX_3992543 XXX_4073578 calCuffs_CUFF.65457.2_18_60989274-60994029_1_2_ORF3 genCDS_ENST00000224181_9_136945321-136946781_1 genCDS_ENST00000371634_9_136945321-136946361_1 +query:154157;rank:1;spectrum:664.31238_3596.01_spectrum=176082_uteruspremenopause;rt:3596.01;mz:664.31238;charge:3 -1.09757 0.0622534 0.94086 X.(Acetyl)SKCFASN(Deamidated)SQ(Deamidated)LLYSQGEK.X genlncRNA_ENST00000428520_10_10934524-10952095_-1_1_ORF2 +query:109576;rank:1;spectrum:730.34552_2652.62_spectrum=142764_uteruspremenopause;rt:2652.62;mz:730.34552;charge:2 -1.13897 0.0647164 0.964364 X.(Carbamidomethyl)NN(Deamidated)PVMSLQDQ(Deamidated)VR.X UniProt_A6ND99 UniProt_K7EP71 UniProt_Q7LGA3 UniProt_Q7LGA3-2 UniProt_Q7LGA3-3 genCDS_ENST00000370548_1_86993082-87168176_1 genCDS_ENST00000370550_1_86915037-87104696_1 genCDS_ENST00000370551_1_86915037-87097939_1 genCDS_ENST00000591456_1_87072984-87092588_1 +query:10757;rank:1;spectrum:419.25119_2212.53_spectrum=126558_uteruspremenopause;rt:2212.53;mz:419.25119;charge:2 -1.35549 0.0765402 1 X.KSRFTAK.X calCuffs_CUFF.117687.1_5_43436710-43444350_-1_3_ORF32 +query:81984;rank:1;spectrum:634.33673_1983.63_spectrum=10106_uteruspremenopause;rt:1983.63;mz:634.33673;charge:2 -1.63897 0.0885601 1 X.(Acetyl)KGLDVAEPGPSR.X UniProt_Q01433 genCDS_ENST00000256578_1_109621014-109631152_1 genCDS_ENST00000528667_1_109621014-109631152_1 +query:58730;rank:1;spectrum:572.7724_1350.11_spectrum=155143_uteruspremenopause;rt:1350.11;mz:572.7724;charge:2 -1.85044 0.0949322 1 X.LGGQ(Deamidated)M(Oxidation)Q(Deamidated)VHQ(Deamidated)K.X genpseudogene_ENST00000442645_7_14985378-14986074_-1_3_ORF2 +query:175487;rank:1;spectrum:943.40503_1694.59_spectrum=101873_uteruspremenopause;rt:1694.59;mz:943.40503;charge:3 -2.1027 0.100208 1 X.(Acetyl)YEDSQQ(Deamidated)EEAQ(Deamidated)YGAMFQ(Deamidated)EQLM(Oxidation)TLK.X UniProt_F5H4J1 UniProt_Q14980 UniProt_Q14980-2 UniProt_Q14980-3 UniProt_Q14980-4 genCDS_ENST00000358965_11_72003527-72035943_-1 genCDS_ENST00000393695_11_72003527-72035943_-1 genCDS_ENST00000542977_11_72014948-72035943_-1 genCDS_ENST00000616538_11_72003527-72035943_-1 genCDS_ENST00000620566_11_72003527-72035943_-1 +query:90871;rank:1;spectrum:660.8255_2297.68_spectrum=111599_uteruspremenopause;rt:2297.68;mz:660.8255;charge:2 -2.46345 0.104155 1 X.(Carbamidomethyl)AHN(Deamidated)FDERVFK.X UniProt_Q96M60 UniProt_Q96M60-2 genCDS_ENST00000299338_15_49328568-49615171_-1 genCDS_ENST00000561064_15_49422710-49615171_-1 diff --git a/lib/galaxy/datatypes/test/test.sqmass b/lib/galaxy/datatypes/test/test.sqmass new file mode 100644 index 00000000000..e03d7192cba Binary files /dev/null and b/lib/galaxy/datatypes/test/test.sqmass differ diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index cbb82492cfd..820363665ae 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -730,3 +730,84 @@ class IQTree(Text): False """ return file_prefix.startswith("IQ-TREE") + + +@build_sniff_from_prefix +class Paf(Text): + """ + PAF: a Pairwise mApping Format + + https://github.com/lh3/miniasm/blob/master/PAF.md + """ + file_ext = "paf" + + def sniff_prefix(self, file_prefix): + """ + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('A-3105.paf') + >>> Paf().sniff(fname) + True + """ + found_valid_lines = False + for line in iter_headers(file_prefix, "\t"): + if len(line) < 12: + return False + for i in (1, 2, 3, 6, 7, 8, 9, 10, 11): + int(line[i]) + if line[4] not in ('+', '-'): + return False + if not (0 <= int(line[11]) <= 255): + return False + # Check that the optional columns after the 12th contain SAM-like typed key-value pairs + for i in range(12, len(line)): + if len(line[i].split(':')) != 3: + return False + found_valid_lines = True + return found_valid_lines + + +@build_sniff_from_prefix +class Gfa1(Text): + """ + Graphical Fragment Assembly (GFA) 1.0 + + http://gfa-spec.github.io/GFA-spec/GFA1.html + """ + file_ext = "gfa1" + + def sniff_prefix(self, file_prefix): + """ + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('big.gfa1') + >>> Gfa1().sniff(fname) + True + """ + found_valid_lines = False + for line in iter_headers(file_prefix, "\t"): + if line[0].startswith('#'): + continue + if line[0] == 'H': + return len(line) == 2 and line[1] == 'VN:Z:1.0' + elif line[0] == 'S': + if len(line) < 3: + return False + elif line[0] == 'L': + if len(line) < 6: + return False + for i in (2, 4): + if line[i] not in ('+', '-'): + return False + elif line[0] == 'C': + if len(line) < 7: + return False + for i in (2, 4): + if line[i] not in ('+', '-'): + return False + int(line[5]) + elif line[0] == 'P': + if len(line) < 4: + return False + else: + return False + found_valid_lines = True + return found_valid_lines diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 328e553cb73..dc8bbab484a 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -175,6 +175,9 @@ class ConditionalDependencies(object): def check_kamaki(self): return 'pithos' in self.object_stores + def check_python_irodsclient(self): + return 'irods' in self.object_stores + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling'} return (self.config['watch_tools'] in install_set or diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index b1f96df9525..a595b75a6f8 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -8,7 +8,7 @@ drmaa statsd docker azure-storage==0.32.0 -# PyRods not in PyPI +python-irodsclient==0.8.2 python-ldap==3.2.0 python-pam galaxycloudrunner @@ -31,4 +31,4 @@ influxdb # Deep learning packages for tool recommendation keras==2.2.4 -tensorflow==1.12.2 +tensorflow==1.15.2 diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 5cf77f61560..0b669e32aaa 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -25,8 +25,9 @@ Sphinx = "*" sphinx_markdown_tables = "*" sphinx_rtd_theme = "*" testfixtures = "*" -twill = {version = "==0.9.1", markers = "python_version < '3'"} +twill = "*" watchdog = "*" +python-irodsclient = "*" [packages] numpy = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 8dffa3ebf0e..5d27d5abf3d 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -4,20 +4,20 @@ alabaster==0.7.12 atomicwrites==1.3.0 attrs==19.3.0 babel==2.8.0 -certifi==2019.11.28 +certifi==2020.4.5.1 chardet==3.0.4 commonmark==0.9.1 configparser==4.0.2 ; python_version < '3.2' contextlib2==0.6.0.post1 ; python_version < '3.5' -coverage==5.0.3 -deprecated==1.2.7 +coverage==5.0.4 +deprecated==1.2.9 docutils==0.15.2 funcsigs==1.0.2 ; python_version < '3.3' future==0.18.2 gunicorn==19.10.0 idna==2.9 imagesize==1.2.0 -importlib-metadata==1.5.0 ; python_version < '3.8' +importlib-metadata==1.6.0 ; python_version < '3.8' jinja2==2.11.1 lxml==4.5.0 markdown==3.1.1 @@ -27,23 +27,25 @@ mock==3.0.5 more-itertools==5.0.0 nose==1.3.7 nosehtml==0.4.5 -packaging==20.1 +packaging==20.3 pathlib2==2.3.5 ; python_version < '3.6' pathtools==0.1.2 pluggy==0.13.1 port-for==0.4 +prettytable==0.7.2 psutil==5.7.0 py==1.8.1 pygithub==1.45 pygments==2.5.2 pyjwt==1.7.1 -pyparsing==2.4.6 +pyparsing==2.4.7 pytest-cov==2.8.1 pytest-html==1.22.1 pytest-metadata==1.8.0 pytest-postgresql==1.4.1 pytest-pythonpath==0.7.3 pytest==4.6.9 +python-irodsclient==0.8.2 pytz==2019.3 recommonmark==0.6.0 requests==2.23.0 @@ -56,10 +58,11 @@ sphinx-rtd-theme==0.4.3 sphinx==1.8.5 sphinxcontrib-websupport==1.1.2 testfixtures==6.14.0 -twill==0.9.1 ; python_version < '3' +twill==2.0 typing==3.7.4.1 ; python_version < '3.5' urllib3==1.25.8 watchdog==0.10.2 -wcwidth==0.1.8 -wrapt==1.12.0 +wcwidth==0.1.9 +wrapt==1.12.1 +xmlrunner==1.7.7 zipp==1.2.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index a2deae34b45..948a30e797e 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -23,8 +23,8 @@ bcrypt==3.1.7 bdbag==1.5.6 beaker==1.11.0 bioblend==0.13.0 -bleach==3.1.1 -boltons==20.0.0 +bleach==3.1.4 +boltons==20.1.0 boto3==1.9.114 boto==2.49.0 botocore==1.12.253 @@ -32,7 +32,7 @@ bx-python==0.8.8 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 cachetools==3.1.1 -certifi==2019.11.28 +certifi==2020.4.5.1 cffi==1.14.0 chardet==3.0.4 cheetah3==3.2.4 @@ -43,18 +43,18 @@ cmd2==0.8.9 coloredlogs==14.0 configparser==4.0.2 ; python_version < '3.2' contextlib2==0.6.0.post1 ; python_version < '3.5' -cryptography==2.8 +cryptography==2.9 cwltool==1.0.20191225192155 debtcollector==1.22.0 decorator==4.4.2 -deprecated==1.2.7 +deprecated==1.2.9 deprecation==2.0.7 dictobj==0.4 docopt==0.6.2 docutils==0.15.2 dogpile.cache==0.9.0 ecdsa==0.15 -enum34==1.1.9 ; python_version < '3.4' +enum34==1.1.10 ; python_version < '3.4' fabric3==1.14.post1 funcsigs==1.0.2 ; python_version < '3.3' functools32==3.2.3.post2 ; python_version < '3.2' @@ -63,13 +63,13 @@ futures==3.3.0 ; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.5 google-api-python-client==1.7.8 google-auth-httplib2==0.0.3 -google-auth==1.11.2 -gxformat2==0.10.1 +google-auth==1.13.1 +gxformat2==0.11.1 h5py==2.10.0 -httplib2==0.17.0 -humanfriendly==7.1.1 +httplib2==0.17.2 +humanfriendly==8.1 idna==2.9 -importlib-metadata==1.5.0 ; python_version < '3.8' +importlib-metadata==1.6.0 ; python_version < '3.8' ipaddress==1.0.23 ; python_version < '3.3' isa-rwval==0.10.7 iso8601==0.1.12 @@ -78,14 +78,14 @@ jmespath==0.9.5 jsonpatch==1.25 jsonpointer==2.0 jsonschema==3.2.0 -keystoneauth1==3.18.0 -kombu==4.6.7 +keystoneauth1==4.0.0 +kombu==4.6.8 lockfile==0.12.2 lxml==4.5.0 -mako==1.1.1 +mako==1.1.2 markdown==3.1.1 markupsafe==1.1.1 -mercurial==5.3 +mercurial==5.3.2 mistune==0.8.4 monotonic==1.5 msgpack==1.0.0 @@ -102,7 +102,7 @@ numpy==1.16.6 oauth2client==4.1.3 oauthlib==3.1.0 openstacksdk==0.17.0 -os-client-config==2.0.0 +os-client-config==2.1.0 os-service-types==1.7.0 osc-lib==2.0.0 oslo.config==7.0.0 @@ -111,21 +111,21 @@ oslo.i18n==3.25.1 oslo.log==3.45.2 oslo.serialization==2.29.2 oslo.utils==3.42.1 -packaging==20.1 +packaging==20.3 paramiko==2.7.1 parsley==1.3 paste==3.4.0 pastedeploy==2.1.0 pastescript==3.2.0 pathlib2==2.3.5 ; python_version < '3.6' -pbr==5.4.4 +pbr==5.4.5 prettytable==0.7.2 prov==1.5.1 psutil==5.7.0 pulsar-galaxy-lib==0.14.0.dev1 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pycparser==2.19 +pycparser==2.20 pycryptodome==3.9.7 pyeventsystem==0.1.0 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' @@ -133,9 +133,9 @@ pyjwt==1.7.1 pykwalify==1.7.0 pynacl==1.3.0 pyopenssl==19.1.0 -pyparsing==2.4.6 -pyperclip==1.7.0 -pyrsistent==0.15.7 +pyparsing==2.4.7 +pyperclip==1.8.0 +pyrsistent==0.16.0 pysam==0.15.2 pysftp==0.2.9 python-cinderclient==4.0.0 @@ -149,7 +149,7 @@ python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2019.3 pyuwsgi==2.0.18.post0 -pyyaml==5.3 +pyyaml==5.3.1 rdflib-jsonld==0.4.0 rdflib==4.2.2 repoze.lru==0.7 @@ -157,7 +157,7 @@ requests-oauthlib==1.3.0 requests-toolbelt==0.9.1 requests==2.23.0 requestsexceptions==1.4.0 -rfc3986==1.3.2 +rfc3986==1.4.0 routes==2.4.1 rsa==4.0 ruamel.ordereddict==0.4.14 ; platform_python_implementation == 'CPython' and python_version <= '2.7' @@ -172,15 +172,15 @@ simplejson==3.17.0 six==1.11.0 social-auth-core[openidconnect]==3.3.0 sqlalchemy-migrate==0.13.0 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.13 +sqlalchemy-utils==0.36.3 +sqlalchemy==1.3.16 sqlparse==0.3.1 stevedore==1.32.0 subprocess32==3.5.4 ; python_version < '3.0' svgwrite==1.3.1 tempita==0.5.2 tenacity==4.12.0 -typing-extensions==3.7.4.1 +typing-extensions==3.7.4.2 typing==3.7.4.1 ; python_version < '3.5' tzlocal==2.0.0 unicodecsv==0.14.1 ; python_version < '3.0' @@ -188,9 +188,9 @@ uritemplate==3.0.1 urllib3==1.25.8 vine==1.3.0 warlock==1.3.3 -wcwidth==0.1.8 +wcwidth==0.1.9 webencodings==0.5.1 webob==1.8.6 whoosh==2.7.4 -wrapt==1.12.0 +wrapt==1.12.1 zipp==1.2.0 diff --git a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt index 0a1aafdbde5..aa48e32250e 100644 --- a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt @@ -1,7 +1,7 @@ -i https://pypi.python.org/simple configparser==4.0.2 ; python_version < '3.2' entrypoints==0.3 -enum34==1.1.9 ; python_version < '3.4' +enum34==1.1.10 ; python_version < '3.4' flake8-import-order==0.18.1 flake8==3.7.9 functools32==3.2.3.post2 ; python_version < '3.2' diff --git a/lib/galaxy/managers/interactivetool.py b/lib/galaxy/managers/interactivetool.py index a6504c5c587..3c7c2d1aa5a 100644 --- a/lib/galaxy/managers/interactivetool.py +++ b/lib/galaxy/managers/interactivetool.py @@ -139,7 +139,7 @@ class InteractiveToolManager(object): self.security = app.security self.sa_session = app.model.context self.job_manager = app.job_manager - self.propagator = InteractiveToolSqlite(app.config.interactivetool_map, app.security.encode_id) + self.propagator = InteractiveToolSqlite(app.config.interactivetools_map, app.security.encode_id) def create_entry_points(self, job, tool, entry_points=None, flush=True): entry_points = entry_points or tool.ports @@ -260,8 +260,13 @@ class InteractiveToolManager(object): def target_if_active(self, trans, entry_point): if entry_point.active and not entry_point.deleted: request_host = trans.request.host - rval = '%s//%s-%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], trans.security.encode_id(entry_point.id), - entry_point.token, entry_point.__class__.__name__.lower(), self.app.config.interactivetool_prefix, request_host) + protocol = trans.request.host_url.split('//', 1)[0] + entry_point_encoded_id = trans.security.encode_id(entry_point.id) + entry_point_class = entry_point.__class__.__name__.lower() + entry_point_prefix = self.app.config.interactivetool_prefix + interactivetool_proxy_host = self.app.config.interactivetool_proxy_host or request_host + rval = '%s//%s-%s.%s.%s.%s/' % (protocol, entry_point_encoded_id, + entry_point.token, entry_point_class, entry_point_prefix, interactivetool_proxy_host) if entry_point.entry_url: rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/')) return rval diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 54b119f9033..f1c24ed7567 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -498,6 +498,20 @@ def summarize_job_metrics(trans, job): return list(map(metric_to_dict, metrics)) +def summarize_destination_params(trans, job): + """Produce a dict-ified version of job destination parameters ready for tabular rendering. + + Precondition: the caller has verified the job is accessible to the user + represented by the trans parameter. + """ + + destination_params = {'Runner': job.job_runner_name, + 'Runner Job ID': job.job_runner_external_id, + 'Handler': job.handler} + destination_params.update(job.destination_params) + return destination_params + + def summarize_job_parameters(trans, job): """Produce a dict-ified version of job parameters ready for tabular rendering. diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 0dc8075ed46..0d760ba884c 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -617,8 +617,8 @@ class WorkflowContentsManager(UsesAnnotations): else: data['upgrade_messages'][step.order_index] = {module.tool.name: "\n".join(module.version_changes)} # Get user annotation. - annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or '' config_form = module.get_config_form(step=step) + annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or '' # Pack attributes into plain dictionary step_dict = { 'id': step.order_index, diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index a7fffd7381a..0888d192579 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -12,6 +12,10 @@ from sqlalchemy import ( Table ) from sqlalchemy.exc import NoSuchTableError +from sqlalchemy_utils import ( + create_database, + database_exists, +) from galaxy.model.tool_shed_install import mapping @@ -27,6 +31,11 @@ def create_or_verify_database(url, engine_options={}, app=None): """ """ # Create engine and metadata + if not database_exists(url): + message = "Creating database for URI [%s]" % url + log.info(message) + create_database(url) + engine = create_engine(url, **engine_options) def migrate(): diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index b1dccf8b249..b986a0e81eb 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -5,6 +5,7 @@ all providers ensure that data can be accessed on the filesystem for running tools """ +import abc import logging import os import random @@ -39,8 +40,9 @@ log = logging.getLogger(__name__) class ObjectStore(object): + __metaclass__ = abc.ABCMeta - """ObjectStore abstract interface. + """ObjectStore interface. FIELD DESCRIPTIONS (these apply to all the methods in this class): @@ -82,6 +84,120 @@ class ObjectStore(object): 000/obj.id) """ + @abc.abstractmethod + def exists(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): + """Return True if the object identified by `obj` exists, False otherwise.""" + raise NotImplementedError() + + @abc.abstractmethod + def create(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Mark the object (`obj`) as existing in the store, but with no content. + + This method will create a proper directory structure for + the file if the directory does not already exist. + """ + raise NotImplementedError() + + @abc.abstractmethod + def empty(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Test if the object identified by `obj` has content. + + If the object does not exist raises `ObjectNotFound`. + """ + raise NotImplementedError() + + @abc.abstractmethod + def size(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Return size of the object identified by `obj`. + + If the object does not exist, return 0. + """ + raise NotImplementedError() + + @abc.abstractmethod + def delete(self, obj, entire_dir=False, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Delete the object identified by `obj`. + + :type entire_dir: boolean + :param entire_dir: If True, delete the entire directory pointed to by + extra_dir. For safety reasons, this option applies + only for and in conjunction with the extra_dir or + obj_dir options. + """ + raise NotImplementedError() + + @abc.abstractmethod + def get_data(self, obj, start=0, count=-1, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Fetch `count` bytes of data offset by `start` bytes using `obj.id`. + + If the object does not exist raises `ObjectNotFound`. + + :type start: int + :param start: Set the position to start reading the dataset file + + :type count: int + :param count: Read at most `count` bytes from the dataset + """ + raise NotImplementedError() + + @abc.abstractmethod + def get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Get the expected filename with absolute path for object with id `obj.id`. + + This can be used to access the contents of the object. + """ + raise NotImplementedError() + + @abc.abstractmethod + def update_from_file(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False, file_name=None, create=False): + """ + Inform the store that the file associated with `obj.id` has been updated. + + If `file_name` is provided, update from that file instead of the + default. + If the object does not exist raises `ObjectNotFound`. + + :type file_name: string + :param file_name: Use file pointed to by `file_name` as the source for + updating the dataset identified by `obj` + + :type create: boolean + :param create: If True and the default dataset does not exist, create + it first. + """ + raise NotImplementedError() + + @abc.abstractmethod + def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + """ + Return the URL for direct acces if supported, otherwise return None. + + Note: need to be careful to not bypass dataset security with this. + """ + raise NotImplementedError() + + @abc.abstractmethod + def get_store_usage_percent(self): + """Return the percentage indicating how full the store is.""" + raise NotImplementedError() + + @abc.abstractmethod + def get_store_by(self, obj): + """Return how object is stored (by 'uuid', 'id', or None if not yet saved). + + Certain Galaxy remote data features aren't available if objects are stored by 'id'. + """ + raise NotImplementedError() + + +class BaseObjectStore(ObjectStore): + def __init__(self, config, config_dict=None, **kwargs): """ :type config: object @@ -111,10 +227,6 @@ class ObjectStore(object): """Close any connections for this ObjectStore.""" self.running = False - def exists(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): - """Return True if the object identified by `obj` exists, False otherwise.""" - raise NotImplementedError() - def file_ready(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Check if a file corresponding to a dataset is ready to be used. @@ -123,102 +235,6 @@ class ObjectStore(object): """ return True - def create(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Mark the object (`obj`) as existing in the store, but with no content. - - This method will create a proper directory structure for - the file if the directory does not already exist. - """ - raise NotImplementedError() - - def empty(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Test if the object identified by `obj` has content. - - If the object does not exist raises `ObjectNotFound`. - """ - raise NotImplementedError() - - def size(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Return size of the object identified by `obj`. - - If the object does not exist, return 0. - """ - raise NotImplementedError() - - def delete(self, obj, entire_dir=False, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Delete the object identified by `obj`. - - :type entire_dir: boolean - :param entire_dir: If True, delete the entire directory pointed to by - extra_dir. For safety reasons, this option applies - only for and in conjunction with the extra_dir or - obj_dir options. - """ - raise NotImplementedError() - - def get_data(self, obj, start=0, count=-1, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Fetch `count` bytes of data offset by `start` bytes using `obj.id`. - - If the object does not exist raises `ObjectNotFound`. - - :type start: int - :param start: Set the position to start reading the dataset file - - :type count: int - :param count: Read at most `count` bytes from the dataset - """ - raise NotImplementedError() - - def get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Get the expected filename with absolute path for object with id `obj.id`. - - This can be used to access the contents of the object. - """ - raise NotImplementedError() - - def update_from_file(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False, file_name=None, create=False): - """ - Inform the store that the file associated with `obj.id` has been updated. - - If `file_name` is provided, update from that file instead of the - default. - If the object does not exist raises `ObjectNotFound`. - - :type file_name: string - :param file_name: Use file pointed to by `file_name` as the source for - updating the dataset identified by `obj` - - :type create: boolean - :param create: If True and the default dataset does not exist, create - it first. - """ - raise NotImplementedError() - - def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): - """ - Return the URL for direct acces if supported, otherwise return None. - - Note: need to be careful to not bypass dataset security with this. - """ - raise NotImplementedError() - - def get_store_usage_percent(self): - """Return the percentage indicating how full the store is.""" - raise NotImplementedError() - - def get_store_by(self, obj): - """Return how object is stored (by 'uuid', 'id', or None if not yet saved). - - Certain Galaxy remote data features aren't available if objects are stored by 'id'. - """ - raise NotImplementedError() - @classmethod def parse_xml(clazz, config_xml): """Parse an XML description of a configuration for this object store. @@ -251,8 +267,44 @@ class ObjectStore(object): # job working directories. return obj.id + def _invoke(self, delegate, obj=None, **kwargs): + return self.__getattribute__("_" + delegate)(obj=obj, **kwargs) -class ConcreteObjectStore(ObjectStore): + def exists(self, obj, **kwargs): + return self._invoke('exists', obj, **kwargs) + + def create(self, obj, **kwargs): + return self._invoke('create', obj, **kwargs) + + def empty(self, obj, **kwargs): + return self._invoke('empty', obj, **kwargs) + + def size(self, obj, **kwargs): + return self._invoke('size', obj, **kwargs) + + def delete(self, obj, **kwargs): + return self._invoke('delete', obj, **kwargs) + + def get_data(self, obj, **kwargs): + return self._invoke('get_data', obj, **kwargs) + + def get_filename(self, obj, **kwargs): + return self._invoke('get_filename', obj, **kwargs) + + def update_from_file(self, obj, **kwargs): + return self._invoke('update_from_file', obj, **kwargs) + + def get_object_url(self, obj, **kwargs): + return self._invoke('get_object_url', obj, **kwargs) + + def get_store_usage_percent(self): + return self._invoke('get_store_usage_percent') + + def get_store_by(self, obj, **kwargs): + return self._invoke('get_store_by', obj, **kwargs) + + +class ConcreteObjectStore(BaseObjectStore): """Subclass of ObjectStore for stores that don't delegate (non-nested). Currently only adds store_by functionality. Which doesn't make @@ -280,7 +332,7 @@ class ConcreteObjectStore(ObjectStore): rval["store_by"] = self.store_by return rval - def get_store_by(self, obj): + def _get_store_by(self, obj): return self.store_by @@ -343,7 +395,7 @@ class DiskObjectStore(ConcreteObjectStore): as_dict["files_dir"] = self.file_path return as_dict - def _get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + def __get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ Return the absolute path for the file corresponding to the `obj.id`. @@ -425,7 +477,7 @@ class DiskObjectStore(ConcreteObjectStore): path = os.path.join(path, alt_name if alt_name else "dataset_%s.dat" % obj_id) return os.path.abspath(path) - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Override `ObjectStore`'s stub and check on disk.""" if self.check_old_style: path = self._construct_path(obj, old_style=True, **kwargs) @@ -435,9 +487,9 @@ class DiskObjectStore(ConcreteObjectStore): return True return os.path.exists(self._construct_path(obj, **kwargs)) - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Override `ObjectStore`'s stub by creating any files and folders on disk.""" - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): path = self._construct_path(obj, **kwargs) dir_only = kwargs.get('dir_only', False) # Create directory if it does not exist @@ -448,18 +500,18 @@ class DiskObjectStore(ConcreteObjectStore): open(path, 'w').close() # Should be rb? umask_fix_perms(path, self.config.umask, 0o666) - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): """Override `ObjectStore`'s stub by checking file size on disk.""" return self.size(obj, **kwargs) == 0 - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): """Override `ObjectStore`'s stub by return file size on disk. Returns 0 if the object doesn't exist yet or other error. """ - if self.exists(obj, **kwargs): + if self._exists(obj, **kwargs): try: - filepath = self.get_filename(obj, **kwargs) + filepath = self._get_filename(obj, **kwargs) for _ in range(0, 2): size = os.path.getsize(filepath) if size != 0: @@ -472,31 +524,31 @@ class DiskObjectStore(ConcreteObjectStore): else: return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): """Override `ObjectStore`'s stub; delete the file or folder on disk.""" - path = self.get_filename(obj, **kwargs) + path = self._get_filename(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) obj_dir = kwargs.get('obj_dir', False) try: if entire_dir and (extra_dir or obj_dir): shutil.rmtree(path) return True - if self.exists(obj, **kwargs): + if self._exists(obj, **kwargs): os.remove(path) return True except OSError as ex: - log.critical('%s delete error %s' % (self._get_filename(obj, **kwargs), ex)) + log.critical('%s delete error %s' % (self.__get_filename(obj, **kwargs), ex)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): """Override `ObjectStore`'s stub; retrieve data directly from disk.""" - data_file = open(self.get_filename(obj, **kwargs), 'r') # Should be rb? + data_file = open(self._get_filename(obj, **kwargs), 'r') # Should be rb? data_file.seek(start) content = data_file.read(count) data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): """ Override `ObjectStore`'s stub. @@ -514,27 +566,27 @@ class DiskObjectStore(ConcreteObjectStore): raise ObjectNotFound return path - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): """`create` parameter is not used in this implementation.""" preserve_symlinks = kwargs.pop('preserve_symlinks', False) # FIXME: symlinks and the object store model may not play well together # these should be handled better, e.g. registering the symlink'd file # as an object if create: - self.create(obj, **kwargs) - if file_name and self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if file_name and self._exists(obj, **kwargs): try: if preserve_symlinks and os.path.islink(file_name): - force_symlink(os.readlink(file_name), self.get_filename(obj, **kwargs)) + force_symlink(os.readlink(file_name), self._get_filename(obj, **kwargs)) else: - path = self.get_filename(obj, **kwargs) + path = self._get_filename(obj, **kwargs) shutil.copy(file_name, path) umask_fix_perms(path, self.config.umask, 0o666) except IOError as ex: - log.critical('Error copying %s to %s: %s' % (file_name, self._get_filename(obj, **kwargs), ex)) + log.critical('Error copying %s to %s: %s' % (file_name, self.__get_filename(obj, **kwargs), ex)) raise ex - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): """ Override `ObjectStore`'s stub. @@ -542,13 +594,13 @@ class DiskObjectStore(ConcreteObjectStore): """ return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self, **kwargs): """Override `ObjectStore`'s stub by return percent storage used.""" st = os.statvfs(self.file_path) return (float(st.f_blocks - st.f_bavail) / st.f_blocks) * 100 -class NestedObjectStore(ObjectStore): +class NestedObjectStore(BaseObjectStore): """ Base for ObjectStores that use other ObjectStores. @@ -567,51 +619,51 @@ class NestedObjectStore(ObjectStore): store.shutdown() super(NestedObjectStore, self).shutdown() - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Determine if the `obj` exists in any of the backends.""" - return self._call_method('exists', obj, False, False, **kwargs) + return self._call_method('_exists', obj, False, False, **kwargs) def file_ready(self, obj, **kwargs): """Determine if the file for `obj` is ready to be used by any of the backends.""" return self._call_method('file_ready', obj, False, False, **kwargs) - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Create a backing file in a random backend.""" random.choice(list(self.backends.values())).create(obj, **kwargs) - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): """For the first backend that has this `obj`, determine if it is empty.""" - return self._call_method('empty', obj, True, False, **kwargs) + return self._call_method('_empty', obj, True, False, **kwargs) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): """For the first backend that has this `obj`, return its size.""" - return self._call_method('size', obj, 0, False, **kwargs) + return self._call_method('_size', obj, 0, False, **kwargs) - def delete(self, obj, **kwargs): + def _delete(self, obj, **kwargs): """For the first backend that has this `obj`, delete it.""" - return self._call_method('delete', obj, False, False, **kwargs) + return self._call_method('_delete', obj, False, False, **kwargs) - def get_data(self, obj, **kwargs): + def _get_data(self, obj, **kwargs): """For the first backend that has this `obj`, get data from it.""" - return self._call_method('get_data', obj, ObjectNotFound, True, **kwargs) + return self._call_method('_get_data', obj, ObjectNotFound, True, **kwargs) - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): """For the first backend that has this `obj`, get its filename.""" - return self._call_method('get_filename', obj, ObjectNotFound, True, **kwargs) + return self._call_method('_get_filename', obj, ObjectNotFound, True, **kwargs) - def update_from_file(self, obj, **kwargs): + def _update_from_file(self, obj, **kwargs): """For the first backend that has this `obj`, update it from the given file.""" if kwargs.get('create', False): - self.create(obj, **kwargs) + self._create(obj, **kwargs) kwargs['create'] = False - return self._call_method('update_from_file', obj, ObjectNotFound, True, **kwargs) + return self._call_method('_update_from_file', obj, ObjectNotFound, True, **kwargs) - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): """For the first backend that has this `obj`, get its URL.""" - return self._call_method('get_object_url', obj, None, False, **kwargs) + return self._call_method('_get_object_url', obj, None, False, **kwargs) - def get_store_by(self, obj): - return self._call_method('get_store_by', obj, None, False) + def _get_store_by(self, obj): + return self._call_method('_get_store_by', obj, None, False) def _repr_object_for_exception(self, obj): try: @@ -791,9 +843,9 @@ class DistributedObjectStore(NestedObjectStore): self.weighted_backend_ids = new_weighted_backend_ids self.sleeper.sleep(120) # Test free space every 2 minutes - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """The only method in which obj.object_store_id may be None.""" - if obj.object_store_id is None or not self.exists(obj, **kwargs): + if obj.object_store_id is None or not self._exists(obj, **kwargs): if obj.object_store_id is None or obj.object_store_id not in self.backends: try: obj.object_store_id = random.choice(self.weighted_backend_ids) @@ -880,14 +932,14 @@ class HierarchicalObjectStore(NestedObjectStore): as_dict["backends"] = backends return as_dict - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Check all child object stores.""" for store in self.backends.values(): if store.exists(obj, **kwargs): return True return False - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Call the primary object store.""" self.backends[0].create(obj, **kwargs) @@ -913,7 +965,7 @@ def type_to_object_store_class(store, fsmon=False): objectstore_class = HierarchicalObjectStore objectstore_constructor_kwds["fsmon"] = fsmon elif store == 'irods': - from .rods import IRODSObjectStore + from .irods import IRODSObjectStore objectstore_class = IRODSObjectStore elif store == 'azure_blob': from .azure_blob import AzureBlobObjectStore diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index 93c8c335d8c..b47c152edb2 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -310,7 +310,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): # Public Methods # ################## - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): in_cache = in_azure = False rel_path = self._construct_path(obj, **kwargs) @@ -357,9 +357,9 @@ class AzureBlobObjectStore(ConcreteObjectStore): return False - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) @@ -393,25 +393,25 @@ class AzureBlobObjectStore(ConcreteObjectStore): open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') - def empty(self, obj, **kwargs): - if self.exists(obj, **kwargs): - return bool(self.size(obj, **kwargs) > 0) + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) else: raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) if self._in_cache(rel_path): try: return os.path.getsize(self._get_cache_path(rel_path)) except OSError as ex: log.info("Could not get size of file '%s' in local cache, will try Azure. Error: %s", rel_path, ex) - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): return self._get_size_in_azure(rel_path) log.warning("Did not find dataset '%s', returning 0 for size", rel_path) return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): rel_path = self._construct_path(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) base_dir = kwargs.get('base_dir', None) @@ -445,10 +445,10 @@ class AzureBlobObjectStore(ConcreteObjectStore): except AzureHttpError: log.exception("Could not delete blob '%s' from Azure", rel_path) except OSError: - log.exception('%s delete error', self.get_filename(obj, **kwargs)) + log.exception('%s delete error', self._get_filename(obj, **kwargs)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Check cache first and get file if not there if not self._in_cache(rel_path): @@ -460,7 +460,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) @@ -483,7 +483,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): if self._in_cache(rel_path): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if dir_only: # Directories do not get pulled into cache return cache_path else: @@ -495,10 +495,10 @@ class AzureBlobObjectStore(ConcreteObjectStore): # return cache_path raise ObjectNotFound('objectstore.get_filename, no cache_path: %s, kwargs: %s' % (str(obj), str(kwargs))) - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): if create is True: - self.create(obj, **kwargs) - elif self.exists(obj, **kwargs): + self._create(obj, **kwargs) + elif self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Chose whether to use the dataset file itself or an alternate file if file_name: @@ -520,8 +520,8 @@ class AzureBlobObjectStore(ConcreteObjectStore): else: raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def get_object_url(self, obj, **kwargs): - if self.exists(obj, **kwargs): + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: url = self.service.make_blob_url(container_name=self.container_name, blob_name=rel_path) @@ -530,7 +530,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): log.exception("Trouble generating URL for dataset '%s'", rel_path) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 ################## diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 49d1fa778e7..b94c9eba9ea 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -510,7 +510,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_cloud(rel_path)) return False - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): in_cache = False rel_path = self._construct_path(obj, **kwargs) @@ -543,8 +543,8 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): else: return False - def create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + def _create(self, obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) @@ -572,26 +572,26 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') - def empty(self, obj, **kwargs): - if self.exists(obj, **kwargs): - return bool(self.size(obj, **kwargs) > 0) + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) else: raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) if self._in_cache(rel_path): try: return os.path.getsize(self._get_cache_path(rel_path)) except OSError as ex: log.info("Could not get size of file '%s' in local cache, will try cloud. Error: %s", rel_path, ex) - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): return self._get_size_in_cloud(rel_path) log.warning("Did not find dataset '%s', returning 0 for size", rel_path) return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): rel_path = self._construct_path(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) base_dir = kwargs.get('base_dir', None) @@ -626,10 +626,10 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): except Exception: log.exception("Could not delete key '%s' from cloud", rel_path) except OSError: - log.exception('%s delete error', self.get_filename(obj, **kwargs)) + log.exception('%s delete error', self._get_filename(obj, **kwargs)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Check cache first and get file if not there if not self._in_cache(rel_path): @@ -641,7 +641,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) obj_dir = kwargs.get('obj_dir', False) @@ -664,7 +664,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): if self._in_cache(rel_path): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if dir_only: # Directories do not get pulled into cache return cache_path else: @@ -678,10 +678,10 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): % (str(obj), str(kwargs))) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): if create: - self.create(obj, **kwargs) - if self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Chose whether to use the dataset file itself or an alternate file if file_name: @@ -703,8 +703,8 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def get_object_url(self, obj, **kwargs): - if self.exists(obj, **kwargs): + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: key = self.bucket.objects.get(rel_path) @@ -713,5 +713,5 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): log.exception("Trouble generating URL for dataset '%s'", rel_path) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py new file mode 100644 index 00000000000..fed2a2fa184 --- /dev/null +++ b/lib/galaxy/objectstore/irods.py @@ -0,0 +1,653 @@ +""" +Object Store plugin for the Integrated Rule-Oriented Data Store (iRODS) +""" +import logging +import os +import shutil +from datetime import datetime +from functools import partial +try: + from pathlib import Path +except ImportError: + # Use backport on python 2 + from pathlib2 import Path + +try: + import irods + import irods.keywords as kw + from irods.exception import CollectionDoesNotExist + from irods.exception import DataObjectDoesNotExist + from irods.exception import NetworkException + from irods.session import iRODSSession +except ImportError: + irods = None + +from galaxy.exceptions import ObjectInvalid, ObjectNotFound +from galaxy.util import directory_hash_id, umask_fix_perms +from galaxy.util.path import safe_relpath +from ..objectstore import DiskObjectStore + +IRODS_IMPORT_MESSAGE = ('The Python irods package is required to use this feature, please install it') +# 1 MB +CHUNK_SIZE = 2**20 +log = logging.getLogger(__name__) + + +def _config_xml_error(tag): + msg = 'No {tag} element in config XML tree'.format(tag=tag) + raise Exception(msg) + + +def _config_dict_error(key): + msg = 'No {key} key in config dictionary'.forma(key=key) + raise Exception(msg) + + +def parse_config_xml(config_xml): + try: + a_xml = config_xml.findall('auth') + if not a_xml: + _config_xml_error('auth') + username = a_xml[0].get('username') + password = a_xml[0].get('password') + + r_xml = config_xml.findall('resource') + if not r_xml: + _config_xml_error('resource') + resource_name = r_xml[0].get('name') + + z_xml = config_xml.findall('zone') + if not z_xml: + _config_xml_error('zone') + zone_name = z_xml[0].get('name') + + c_xml = config_xml.findall('connection') + if not c_xml: + _config_xml_error('connection') + host = c_xml[0].get('host', None) + port = int(c_xml[0].get('port', 0)) + timeout = int(c_xml[0].get('timeout', 30)) + + c_xml = config_xml.findall('cache') + if not c_xml: + _config_xml_error('cache') + cache_size = float(c_xml[0].get('size', -1)) + staging_path = c_xml[0].get('path', None) + + attrs = ('type', 'path') + e_xml = config_xml.findall('extra_dir') + if not e_xml: + _config_xml_error('extra_dir') + extra_dirs = [dict(((k, e.get(k)) for k in attrs)) for e in e_xml] + + return { + 'auth': { + 'username': username, + 'password': password, + }, + 'resource': { + 'name': resource_name, + }, + 'zone': { + 'name': zone_name, + }, + 'connection': { + 'host': host, + 'port': port, + 'timeout': timeout + }, + 'cache': { + 'size': cache_size, + 'path': staging_path, + }, + 'extra_dirs': extra_dirs, + } + except Exception: + # Toss it back up after logging, we can't continue loading at this point. + log.exception("Malformed iRODS ObjectStore Configuration XML -- unable to continue.") + raise + + +class CloudConfigMixin(object): + + def _config_to_dict(self): + return { + 'auth': { + 'username': self.username, + 'password': self.password, + }, + 'resource': { + 'name': self.resource, + }, + 'zone': { + 'name': self.zone, + }, + 'connection': { + 'host': self.host, + 'port': self.port, + 'timeout': self.timeout, + }, + 'cache': { + 'size': self.cache_size, + 'path': self.staging_path, + } + } + + +class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): + """ + Object store that stores objects as data objects in an iRODS collections. A local cache + exists that is used as an intermediate location for files between Galaxy and iRODS. + """ + + store_type = 'irods' + + def __init__(self, config, config_dict): + super(IRODSObjectStore, self).__init__(config, config_dict) + + auth_dict = config_dict.get('auth') + if auth_dict is None: + _config_dict_error('auth') + + self.username = auth_dict.get('username') + if self.username is None: + _config_dict_error('auth->username') + self.password = auth_dict.get('password') + if self.password is None: + _config_dict_error('auth->password') + + resource_dict = config_dict['resource'] + if resource_dict is None: + _config_dict_error('resource') + self.resource = resource_dict.get('name') + if self.resource is None: + _config_dict_error('resource->name') + + zone_dict = config_dict['zone'] + if zone_dict is None: + _config_dict_error('zone') + self.zone = zone_dict.get('name') + if self.zone is None: + _config_dict_error('zone->name') + + connection_dict = config_dict['connection'] + if connection_dict is None: + _config_dict_error('connection') + self.host = connection_dict.get('host') + if self.host is None: + _config_dict_error('connection->host') + self.port = connection_dict.get('port') + if self.port is None: + _config_dict_error('connection->port') + self.timeout = connection_dict.get('timeout') + if self.timeout is None: + _config_dict_error('connection->timeout') + + cache_dict = config_dict['cache'] + if cache_dict is None: + _config_dict_error('cache') + self.cache_size = cache_dict.get('size', -1) + if self.cache_size is None: + _config_dict_error('cache->size') + self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path + if self.staging_path is None: + _config_dict_error('cache->path') + + extra_dirs = dict((e['type'], e['path']) for e in config_dict.get('extra_dirs', [])) + if not extra_dirs: + _config_dict_error('extra_dirs') + self.extra_dirs.update(extra_dirs) + + self._initialize() + + def __del__(self): + self.session.cleanup() + + def _initialize(self): + if irods is None: + raise Exception(IRODS_IMPORT_MESSAGE) + + self.home = "/" + self.zone + "/home/" + self.username + + self.session = self._configure_connection(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone) + + def _configure_connection(self, host='localhost', port='1247', user='rods', password='rods', zone='tempZone'): + with iRODSSession(host=host, port=port, user=user, password=password, zone=zone) as session: + # Set connection timeout + session.connection_timeout = self.timeout + # Throws NetworkException if connection fails + try: + session.pool.get_connection() + except NetworkException as e: + log.error('Could not create iRODS session: ' + str(e)) + raise + return session + + @classmethod + def parse_xml(cls, config_xml): + return parse_config_xml(config_xml) + + def to_dict(self): + as_dict = super(IRODSObjectStore, self).to_dict() + as_dict.update(self._config_to_dict()) + return as_dict + + def _fix_permissions(self, rel_path): + """ Set permissions on rel_path""" + for basedir, _, files in os.walk(rel_path): + umask_fix_perms(basedir, self.config.umask, 0o777, self.config.gid) + for filename in files: + path = os.path.join(basedir, filename) + # Ignore symlinks + if os.path.islink(path): + continue + umask_fix_perms(path, self.config.umask, 0o666, self.config.gid) + + def _construct_path(self, obj, base_dir=None, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) + rel_path = os.path.join(*directory_hash_id(obj.id)) + if extra_dir is not None: + if extra_dir_at_root: + rel_path = os.path.join(extra_dir, rel_path) + else: + rel_path = os.path.join(rel_path, extra_dir) + + # for JOB_WORK directory + if obj_dir: + rel_path = os.path.join(rel_path, str(obj.id)) + if base_dir: + base = self.extra_dirs.get(base_dir) + return os.path.join(base, rel_path) + + if not dir_only: + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + return rel_path + + def _get_cache_path(self, rel_path): + return os.path.abspath(os.path.join(self.staging_path, rel_path)) + + # rel_path is file or folder? + def _get_size_in_irods(self, rel_path): + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + + try: + data_obj = self.session.data_objects.get(data_object_path) + return data_obj.__sizeof__() + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.warn("Collection or data object (%s) does not exist", data_object_path) + return -1 + + # rel_path is file or folder? + def _data_object_exists(self, rel_path): + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + + try: + self.session.data_objects.get(data_object_path) + return True + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.warn("Collection or data object (%s) does not exist", data_object_path) + return False + + def _in_cache(self, rel_path): + """ Check if the given dataset is in the local cache and return True if so. """ + cache_path = self._get_cache_path(rel_path) + return os.path.exists(cache_path) + + def _pull_into_cache(self, rel_path): + # Ensure the cache directory structure exists (e.g., dataset_#_files/) + rel_path_dir = os.path.dirname(rel_path) + if not os.path.exists(self._get_cache_path(rel_path_dir)): + os.makedirs(self._get_cache_path(rel_path_dir)) + # Now pull in the file + file_ok = self._download(rel_path) + self._fix_permissions(self._get_cache_path(rel_path_dir)) + return file_ok + + def _download(self, rel_path): + log.debug("Pulling data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + data_obj = None + + try: + data_obj = self.session.data_objects.get(data_object_path) + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.warn("Collection or data object (%s) does not exist", data_object_path) + return False + + if self.cache_size > 0 and data_obj.__sizeof__() > self.cache_size: + log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.", + rel_path, data_obj.__sizeof__(), self.cache_size) + return False + + log.debug("Pulled data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + + with data_obj.open('r') as data_obj_fp, open(self._get_cache_path(rel_path), "wb") as cache_fp: + for chunk in iter(partial(data_obj_fp.read, CHUNK_SIZE), b''): + cache_fp.write(chunk) + return True + + def _push_to_irods(self, rel_path, source_file=None, from_string=None): + """ + Push the file pointed to by ``rel_path`` to the iRODS. Extract folder name + from rel_path as iRODS collection name, and extract file name from rel_path + as iRODS data object name. + If ``source_file`` is provided, push that file instead while + still using ``rel_path`` for collection and object store names. + If ``from_string`` is provided, set contents of the file to the value of the string. + """ + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + source_file = source_file if source_file else self._get_cache_path(rel_path) + options = {kw.FORCE_FLAG_KW: ''} + + if os.path.exists(source_file): + # Check if the data object exists in iRODS + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + exists = self.session.data_objects.exists(data_object_path) + if os.path.getsize(source_file) == 0 and exists: + log.debug("Wanted to push file '%s' to iRODS collection '%s' but its size is 0; skipping.", source_file, rel_path) + return True + if from_string: + data_obj = self.session.data_objects.create(data_object_path, self.resource, **options) + with data_obj.open('w') as data_obj_fp: + data_obj_fp.write(from_string) + log.debug("Pushed data from string '%s' to collection '%s'", from_string, data_object_path) + else: + start_time = datetime.now() + log.debug("Pushing cache file '%s' of size %s bytes to collection '%s'", source_file, os.path.getsize(source_file), rel_path) + + # Create sub-collection first + self.session.collections.create(collection_path, recurse=True) + data_obj = self.session.data_objects.create(data_object_path, self.resource, **options) + + # Write to file in subcollection created above + with open(source_file, 'rb') as content_file, data_obj.open('w') as data_obj_fp: + for chunk in iter(partial(content_file.read, CHUNK_SIZE), b''): + data_obj_fp.write(chunk) + + end_time = datetime.now() + log.debug("Pushed cache file '%s' to collection '%s' (%s bytes transfered in %s sec)", + source_file, rel_path, os.path.getsize(source_file), end_time - start_time) + return True + else: + log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", + rel_path, source_file) + return False + + def file_ready(self, obj, **kwargs): + """ + A helper method that checks if a file corresponding to a dataset is + ready and available to be used. Return ``True`` if so, ``False`` otherwise. + """ + rel_path = self._construct_path(obj, **kwargs) + # Make sure the size in cache is available in its entirety + if self._in_cache(rel_path): + if os.path.getsize(self._get_cache_path(rel_path)) == self._get_size_in_irods(rel_path): + return True + log.debug("Waiting for dataset %s to transfer from OS: %s/%s", rel_path, + os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_irods(rel_path)) + return False + + def _exists(self, obj, **kwargs): + in_cache = in_irods = False + rel_path = self._construct_path(obj, **kwargs) + + # Check cache + if self._in_cache(rel_path): + in_cache = True + # Check iRODS + in_irods = self._data_object_exists(rel_path) + + # dir_only does not get synced so shortcut the decision + dir_only = kwargs.get('dir_only', False) + base_dir = kwargs.get('base_dir', None) + if dir_only: + if in_cache or in_irods: + return True + # for JOB_WORK directory + elif base_dir: + if not os.path.exists(rel_path): + os.makedirs(rel_path) + return True + else: + return False + + if in_cache and not in_irods: + return True + elif in_irods: + return True + else: + return False + + def _create(self, obj, **kwargs): + if not self._exists(obj, **kwargs): + # Pull out locally used fields + extra_dir = kwargs.get('extra_dir', None) + extra_dir_at_root = kwargs.get('extra_dir_at_root', False) + dir_only = kwargs.get('dir_only', False) + alt_name = kwargs.get('alt_name', None) + + # Construct hashed path + rel_path = os.path.join(*directory_hash_id(obj.id)) + + # Optionally append extra_dir + if extra_dir is not None: + if extra_dir_at_root: + rel_path = os.path.join(extra_dir, rel_path) + else: + rel_path = os.path.join(rel_path, extra_dir) + + # Create given directory in cache + cache_dir = os.path.join(self.staging_path, rel_path) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + if not dir_only: + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + open(os.path.join(self.staging_path, rel_path), 'w').close() + self._push_to_irods(rel_path, from_string='') + + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) + else: + raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' + % (str(obj), str(kwargs))) + + def _size(self, obj, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + if self._in_cache(rel_path): + try: + return os.path.getsize(self._get_cache_path(rel_path)) + except OSError as ex: + log.info("Could not get size of file '%s' in local cache, will try S3. Error: %s", rel_path, ex) + elif self._exists(obj, **kwargs): + return self._get_size_in_irods(rel_path) + log.warning("Did not find dataset '%s', returning 0 for size", rel_path) + return 0 + + def _delete(self, obj, entire_dir=False, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + extra_dir = kwargs.get('extra_dir', None) + base_dir = kwargs.get('base_dir', None) + dir_only = kwargs.get('dir_only', False) + obj_dir = kwargs.get('obj_dir', False) + try: + # Remove temparory data in JOB_WORK directory + if base_dir and dir_only and obj_dir: + shutil.rmtree(os.path.abspath(rel_path)) + return True + + # For the case of extra_files, because we don't have a reference to + # individual files we need to remove the entire directory structure + # with all the files in it. This is easy for the local file system, + # but requires iterating through each individual key in irods and deleing it. + if entire_dir and extra_dir: + shutil.rmtree(self._get_cache_path(rel_path)) + + col_path = self.home + "/" + str(rel_path) + col = None + try: + col = self.session.collections.get(col_path) + except CollectionDoesNotExist: + log.warn("Collection (%s) does not exist!", col_path) + return False + + cols = col.walk() + # Traverse the tree only one level deep + for _ in range(2): + # get next result + _, _, data_objects = next(cols) + + # Delete data objects + for data_object in data_objects: + data_object.unlink(force=True) + + return True + + else: + # Delete from cache first + os.unlink(self._get_cache_path(rel_path)) + # Delete from irods as well + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + + try: + data_obj = self.session.data_objects.get(data_object_path) + # remove object + data_obj.unlink(force=True) + return True + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.info("Collection or data object (%s) does not exist", data_object_path) + return True + + except OSError: + log.exception('%s delete error', self._get_filename(obj, **kwargs)) + return False + + def _get_data(self, obj, start=0, count=-1, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + # Check cache first and get file if not there + if not self._in_cache(rel_path): + self._pull_into_cache(rel_path) + # Read the file content from cache + data_file = open(self._get_cache_path(rel_path), 'r') + data_file.seek(start) + content = data_file.read(count) + data_file.close() + return content + + def _get_filename(self, obj, **kwargs): + base_dir = kwargs.get('base_dir', None) + dir_only = kwargs.get('dir_only', False) + obj_dir = kwargs.get('obj_dir', False) + rel_path = self._construct_path(obj, **kwargs) + + # for JOB_WORK directory + if base_dir and dir_only and obj_dir: + return os.path.abspath(rel_path) + + cache_path = self._get_cache_path(rel_path) + # iRODS does not recognize directories as files so cannot check if those exist. + # So, if checking dir only, ensure given dir exists in cache and return + # the expected cache path. + # dir_only = kwargs.get('dir_only', False) + # if dir_only: + # if not os.path.exists(cache_path): + # os.makedirs(cache_path) + # return cache_path + # Check if the file exists in the cache first + if self._in_cache(rel_path): + return cache_path + # Check if the file exists in persistent storage and, if it does, pull it into cache + elif self._exists(obj, **kwargs): + if dir_only: # Directories do not get pulled into cache + return cache_path + else: + if self._pull_into_cache(rel_path): + return cache_path + # For the case of retrieving a directory only, return the expected path + # even if it does not exist. + # if dir_only: + # return cache_path + raise ObjectNotFound('objectstore.get_filename, no cache_path: %s, kwargs: %s' + % (str(obj), str(kwargs))) + # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path + + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): + if create: + self._create(obj, **kwargs) + if self._exists(obj, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + # Chose whether to use the dataset file itself or an alternate file + if file_name: + source_file = os.path.abspath(file_name) + # Copy into cache + cache_file = self._get_cache_path(rel_path) + try: + if source_file != cache_file: + # FIXME? Should this be a `move`? + shutil.copy2(source_file, cache_file) + self._fix_permissions(cache_file) + except OSError: + log.exception("Trouble copying source file '%s' to cache '%s'", source_file, cache_file) + else: + source_file = self._get_cache_path(rel_path) + # Update the file on iRODS + self._push_to_irods(rel_path, source_file) + else: + raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' + % (str(obj), str(kwargs))) + + # Unlike S3, url is not really applicable to iRODS + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + + return data_object_path + + def _get_store_usage_percent(self): + return 0.0 diff --git a/lib/galaxy/objectstore/pithos.py b/lib/galaxy/objectstore/pithos.py index 2d658f0133b..63bc8e23e94 100644 --- a/lib/galaxy/objectstore/pithos.py +++ b/lib/galaxy/objectstore/pithos.py @@ -221,7 +221,7 @@ class PithosObjectStore(ConcreteObjectStore): # No need to overwrite "shutdown" - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): """Check if file exists, fix if file in cache and not on Pithos+ :returns: weather the file exists remotely or in cache """ @@ -253,9 +253,9 @@ class PithosObjectStore(ConcreteObjectStore): return True return False - def create(self, obj, **kwargs): + def _create(self, obj, **kwargs): """Touch a file (aka create empty), if it doesn't exist""" - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) extra_dir_at_root = kwargs.get('extra_dir_at_root', False) @@ -288,18 +288,18 @@ class PithosObjectStore(ConcreteObjectStore): open(new_file, 'w').close() self.pithos.upload_from_string(rel_path, '') - def empty(self, obj, **kwargs): + def _empty(self, obj, **kwargs): """ :returns: weather the object has content :raises ObjectNotFound: """ - if not self.exists(obj, **kwargs): + if not self._exists(obj, **kwargs): raise ObjectNotFound( 'objectstore.empty, object does not exist: {obj}, ' 'kwargs: {kwargs}'.format(obj=obj, kwargs=kwargs)) - return bool(self.size(obj, **kwargs)) + return bool(self._size(obj, **kwargs)) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): """ :returns: The size of the object, or 0 if it doesn't exist (sorry for that, not our fault, the ObjectStore interface is like that some @@ -321,7 +321,7 @@ class PithosObjectStore(ConcreteObjectStore): return 0 return int(file['content-length']) - def delete(self, obj, **kwargs): + def _delete(self, obj, **kwargs): """Delete the object :returns: weather the object was deleted """ @@ -347,13 +347,13 @@ class PithosObjectStore(ConcreteObjectStore): self.pithos.del_object(path) except OSError: log.exception( - '{0} delete error'.format(self.get_filename(obj, **kwargs))) + '{0} delete error'.format(self._get_filename(obj, **kwargs))) except ClientError as ce: log.exception('Could not delete {path} from Pithos, {err}'.format( path=path, err=ce)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): """Fetch (e.g., download) data :param start: Chunk of data starts here :param count: Fetch at most as many data, fetch all if negative @@ -369,7 +369,7 @@ class PithosObjectStore(ConcreteObjectStore): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): """Get the expected filename with absolute path""" base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) @@ -386,7 +386,7 @@ class PithosObjectStore(ConcreteObjectStore): return cache_path if self._in_cache(path): return cache_path - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if not dir_only: self._pull_into_cache(path) return cache_path @@ -394,11 +394,11 @@ class PithosObjectStore(ConcreteObjectStore): 'objectstore.get_filename, no cache_path: {obj}, ' 'kwargs: {kwargs}'.format(obj, kwargs)) - def update_from_file(self, obj, **kwargs): + def _update_from_file(self, obj, **kwargs): """Update the store when a file is updated""" if kwargs.get('create'): - self.create(obj, **kwargs) - if not self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if not self._exists(obj, **kwargs): raise ObjectNotFound( 'objectstore.update_from_file, object does not exist: {obj}, ' 'kwargs: {kwargs}'.format(obj, kwargs)) @@ -420,11 +420,11 @@ class PithosObjectStore(ConcreteObjectStore): with open(cache_path) as f: self.pithos.upload_object(obj, f) - def get_object_url(self, obj, **kwargs): + def _get_object_url(self, obj, **kwargs): """ :returns: URL for direct access, None if no object """ - if self.exists(obj, **kwargs): + if self._exists(obj, **kwargs): path = self._construct_path(obj, **kwargs) try: return self.pithos.publish_object(path) @@ -434,7 +434,7 @@ class PithosObjectStore(ConcreteObjectStore): log.exception('Kamaki: {0}'.format(ce)) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): """ :returns: percentage indicating how full the store is """ diff --git a/lib/galaxy/objectstore/pulsar.py b/lib/galaxy/objectstore/pulsar.py index d7d1e8b7cff..c2217311ae4 100644 --- a/lib/galaxy/objectstore/pulsar.py +++ b/lib/galaxy/objectstore/pulsar.py @@ -1,6 +1,6 @@ from __future__ import absolute_import # Need to import pulsar_client absolutely. -from ..objectstore import ObjectStore +from ..objectstore import BaseObjectStore try: from pulsar.client.manager import ObjectStoreClientManager @@ -8,7 +8,7 @@ except ImportError: ObjectStoreClientManager = None -class PulsarObjectStore(ObjectStore): +class PulsarObjectStore(BaseObjectStore): """ Object store implementation that delegates to a remote Pulsar server. @@ -26,38 +26,38 @@ class PulsarObjectStore(ObjectStore): def __init__(self, config, config_xml): self.pulsar_client = self.__build_pulsar_client(config_xml) - def exists(self, obj, **kwds): + def _exists(self, obj, **kwds): return self.pulsar_client.exists(**self.__build_kwds(obj, **kwds)) def file_ready(self, obj, **kwds): return self.pulsar_client.file_ready(**self.__build_kwds(obj, **kwds)) - def create(self, obj, **kwds): + def _create(self, obj, **kwds): return self.pulsar_client.create(**self.__build_kwds(obj, **kwds)) - def empty(self, obj, **kwds): + def _empty(self, obj, **kwds): return self.pulsar_client.empty(**self.__build_kwds(obj, **kwds)) - def size(self, obj, **kwds): + def _size(self, obj, **kwds): return self.pulsar_client.size(**self.__build_kwds(obj, **kwds)) - def delete(self, obj, **kwds): + def _delete(self, obj, **kwds): return self.pulsar_client.delete(**self.__build_kwds(obj, **kwds)) # TODO: Optimize get_data. - def get_data(self, obj, **kwds): + def _get_data(self, obj, **kwds): return self.pulsar_client.get_data(**self.__build_kwds(obj, **kwds)) - def get_filename(self, obj, **kwds): + def _get_filename(self, obj, **kwds): return self.pulsar_client.get_filename(**self.__build_kwds(obj, **kwds)) - def update_from_file(self, obj, **kwds): + def _update_from_file(self, obj, **kwds): return self.pulsar_client.update_from_file(**self.__build_kwds(obj, **kwds)) - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return self.pulsar_client.get_store_usage_percent() - def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None): + def _get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None): return None def __build_kwds(self, obj, **kwds): diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py deleted file mode 100644 index 65114037c16..00000000000 --- a/lib/galaxy/objectstore/rods.py +++ /dev/null @@ -1,355 +0,0 @@ -""" -Object Store plugin for the Integrated Rule-Oriented Data Store (iRODS) - -The module is named rods to avoid conflicting with the PyRods module, irods -""" - -import logging -import os -import time -from posixpath import ( - basename as path_basename, - dirname as path_dirname, - join as path_join -) - -try: - import irods -except ImportError: - irods = None - -from galaxy.exceptions import ( - ObjectInvalid, - ObjectNotFound -) -from galaxy.util.path import safe_relpath -from ..objectstore import ( - DiskObjectStore, - local_extra_dirs -) - -IRODS_IMPORT_MESSAGE = ('The Python irods package is required to use this ' - 'feature, please install it') - -log = logging.getLogger(__name__) - - -class IRODSObjectStore(DiskObjectStore): - """ - Galaxy object store based on iRODS - """ - - def __init__(self, config, file_path=None, extra_dirs=None): - super(IRODSObjectStore, self).__init__(config, file_path=file_path, extra_dirs=extra_dirs) - assert irods is not None, IRODS_IMPORT_MESSAGE - self.cache_path = config.object_store_cache_path - self.default_resource = config.irods_default_resource or None - - # Connect to iRODS (AssertionErrors will be raised if anything goes wrong) - self.rods_env, self.rods_conn = rods_connect() - - # if the root collection path in the config is unset or relative, try to use a sensible default - if config.irods_root_collection_path is None or (config.irods_root_collection_path is not None and not config.irods_root_collection_path.startswith('/')): - rods_home = self.rods_env.rodsHome - assert rods_home != '', "Unable to initialize iRODS Object Store: rodsHome cannot be determined and irods_root_collection_path in Galaxy config is unset or not absolute." - if config.irods_root_collection_path is None: - self.root_collection_path = path_join(rods_home, 'galaxy_data') - else: - self.root_collection_path = path_join(rods_home, config.irods_root_collection_path) - else: - self.root_collection_path = config.irods_root_collection_path - - # will return a collection object regardless of whether it exists - self.root_collection = irods.irodsCollection(self.rods_conn, self.root_collection_path) - - if self.root_collection.getId() == -1: - log.warning("iRODS root collection does not exist, will attempt to create: %s", self.root_collection_path) - self.root_collection.upCollection() - assert self.root_collection.createCollection(os.path.basename(self.root_collection_path)) == 0, "iRODS root collection creation failed: %s" % self.root_collection_path - self.root_collection = irods.irodsCollection(self.rods_conn, self.root_collection_path) - assert self.root_collection.getId() != -1, "iRODS root collection creation claimed success but still does not exist" - - if self.default_resource is None: - self.default_resource = self.rods_env.rodsDefResource - - log.info("iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource) - - def __get_rods_path(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs): - # extra_dir should never be constructed from provided data but just - # make sure there are no shenannigans afoot - if extra_dir and extra_dir != os.path.normpath(extra_dir): - log.warning('extra_dir is not normalized: %s', extra_dir) - raise ObjectInvalid("The requested object is invalid") - # ensure that any parent directory references in alt_name would not - # result in a path not contained in the directory path constructed here - if alt_name: - if not safe_relpath(alt_name): - log.warning('alt_name would locate path outside dir: %s', alt_name) - raise ObjectInvalid("The requested object is invalid") - # alt_name can contain parent directory references, but iRODS will - # not follow them, so if they are valid we normalize them out - alt_name = os.path.normpath(alt_name) - path = "" - if extra_dir is not None: - path = extra_dir - - # extra_dir_at_root is ignored - since the iRODS plugin does not use - # the directory hash, there is only one level of subdirectory. - - if not dir_only: - # the .dat extension is stripped when stored in iRODS - # TODO: is the strip_dat kwarg the best way to implement this? - if strip_dat and alt_name and alt_name.endswith('.dat'): - alt_name = os.path.splitext(alt_name)[0] - default_name = 'dataset_%s' % obj.id - if not strip_dat: - default_name += '.dat' - path = path_join(path, alt_name if alt_name else default_name) - - path = path_join(self.root_collection_path, path) - return path - - def __get_cache_path(self, obj, **kwargs): - # FIXME: does not handle collections - # FIXME: collisions could occur here - return os.path.join(self.cache_path, path_basename(self.__get_rods_path(obj, strip_dat=False, **kwargs))) - - def __clean_cache_entry(self, obj, **kwargs): - # FIXME: does not handle collections - try: - os.unlink(self.__get_cache_path(obj, **kwargs)) - except OSError: - # it is expected that we'll call this method a lot regardless of - # whether we think the cached file exists - pass - - def __get_rods_handle(self, obj, mode='r', **kwargs): - if kwargs.get('dir_only', False): - return irods.irodsCollection(self.rods_conn, self.__get_rods_path(obj, **kwargs)) - else: - return irods.irodsOpen(self.rods_conn, self.__get_rods_path(obj, **kwargs), mode) - - def __mkcolls(self, rods_path): - """ - An os.makedirs() for iRODS collections. `rods_path` is the desired collection to create. - """ - assert rods_path.startswith(self.root_collection_path + '/'), '__mkcolls(): Creating collections outside the root collection is not allowed (requested path was: %s)' % rods_path - mkcolls = [] - c = irods.irodsCollection(self.rods_conn, rods_path) - while c.getId() == -1: - assert c.getCollName().startswith(self.root_collection_path + '/'), '__mkcolls(): Attempted to move above the root collection: %s' % c.getCollName() - mkcolls.append(c.getCollName()) - c.upCollection() - for collname in reversed(mkcolls): - log.debug('Creating collection %s' % collname) - ci = irods.collInp_t() - ci.collName = collname - status = irods.rcCollCreate(self.rods_conn, ci) - assert status == 0, '__mkcolls(): Failed to create collection: %s' % collname - - @local_extra_dirs - def exists(self, obj, **kwargs): - doi = irods.dataObjInp_t() - doi.objPath = self.__get_rods_path(obj, **kwargs) - log.debug('exists(): checking: %s', doi.objPath) - return irods.rcObjStat(self.rods_conn, doi) is not None - - @local_extra_dirs - def create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): - rods_path = self.__get_rods_path(obj, **kwargs) - log.debug('create(): %s', rods_path) - dir_only = kwargs.get('dir_only', False) - # short circuit collection creation since most of the time it will - # be the root collection which already exists - collection_path = rods_path if dir_only else path_dirname(rods_path) - if collection_path != self.root_collection_path: - self.__mkcolls(collection_path) - if not dir_only: - # rcDataObjCreate is used instead of the irodsOpen wrapper so - # that we can prevent overwriting - doi = irods.dataObjInp_t() - doi.objPath = rods_path - doi.createMode = 0o640 - doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable - irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource) - status = irods.rcDataObjCreate(self.rods_conn, doi) - assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % (rods_path, status, irods.strerror(status)) - - @local_extra_dirs - def empty(self, obj, **kwargs): - assert 'dir_only' not in kwargs, 'empty(): `dir_only` parameter is invalid here' - h = self.__get_rods_handle(obj, **kwargs) - try: - return h.getSize() == 0 - except AttributeError: - # h is None - raise ObjectNotFound() - - def size(self, obj, **kwargs): - assert 'dir_only' not in kwargs, 'size(): `dir_only` parameter is invalid here' - h = self.__get_rods_handle(obj, **kwargs) - try: - return h.getSize() - except AttributeError: - # h is None - return 0 - - @local_extra_dirs - def delete(self, obj, entire_dir=False, **kwargs): - assert 'dir_only' not in kwargs, 'delete(): `dir_only` parameter is invalid here' - rods_path = self.__get_rods_path(obj, **kwargs) - # __get_rods_path prepends self.root_collection_path but we are going - # to ensure that it's valid anyway for safety's sake - assert rods_path.startswith(self.root_collection_path + '/'), 'ERROR: attempt to delete object outside root collection (path was: %s)' % rods_path - if entire_dir: - # TODO - raise NotImplementedError() - h = self.__get_rods_handle(obj, **kwargs) - try: - # note: PyRods' irodsFile.delete() does not set force - status = h.delete() - assert status == 0, '%d: %s' % (status, irods.strerror(status)) - return True - except AttributeError: - log.warning('delete(): operation failed: object does not exist: %s', rods_path) - except AssertionError as e: - # delete() does not raise on deletion failure - log.error('delete(): operation failed: %s', e) - finally: - # remove the cached entry (finally is executed even when the try - # contains a return) - self.__clean_cache_entry(self, obj, **kwargs) - return False - - @local_extra_dirs - def get_data(self, obj, start=0, count=-1, **kwargs): - log.debug('get_data(): %s') - h = self.__get_rods_handle(obj, **kwargs) - try: - h.seek(start) - except AttributeError: - raise ObjectNotFound() - if count == -1: - return h.read() - else: - return h.read(count) - # TODO: make sure implicit close is okay, DiskObjectStore actually - # reads data into a var, closes, and returns the var - - @local_extra_dirs - def get_filename(self, obj, **kwargs): - log.debug("get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.", obj.__class__.__name__, obj.id) - cached_path = self.__get_cache_path(obj, **kwargs) - - if not self.exists(obj, **kwargs): - raise ObjectNotFound() - - # TODO: implement or define whether dir_only is valid - if 'dir_only' in kwargs: - raise NotImplementedError() - - # cache hit - if os.path.exists(cached_path): - return os.path.abspath(cached_path) - - # cache miss - # TODO: thread this - incoming_path = os.path.join(os.path.dirname(cached_path), "__incoming_%s" % os.path.basename(cached_path)) - doi = irods.dataObjInp_t() - doi.objPath = self.__get_rods_path(obj, **kwargs) - doi.dataSize = 0 # TODO: does this affect performance? should we get size? - doi.numThreads = 0 - # TODO: might want to VERIFY_CHKSUM_KW - log.debug('get_filename(): caching %s to %s', doi.objPath, incoming_path) - - # do the iget - status = irods.rcDataObjGet(self.rods_conn, doi, incoming_path) - - # if incoming already exists, we'll wait for another process or thread - # to finish caching - if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG: - assert status == 0, 'get_filename(): iget %s failed (%s): %s' % (doi.objPath, status, irods.strerror(status)) - # POSIX rename is atomic - # TODO: rename without clobbering - os.rename(incoming_path, cached_path) - log.debug('get_filename(): cached %s to %s', doi.objPath, cached_path) - - # another process or thread is caching, wait for it - while not os.path.exists(cached_path): - # TODO: force restart after mod time > some configurable, or - # otherwise deal with this potential deadlock and interrupted - # transfers - time.sleep(5) - log.debug("get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id) - - return os.path.abspath(cached_path) - - @local_extra_dirs - def update_from_file(self, obj, file_name=None, create=False, **kwargs): - assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here' - - # do not create if not requested - if create and not self.exists(obj, **kwargs): - raise ObjectNotFound() - - if file_name is None: - file_name = self.__get_cache_path(obj, **kwargs) - - # put will create if necessary - doi = irods.dataObjInp_t() - doi.objPath = self.__get_rods_path(obj, **kwargs) - doi.createMode = 0o640 - doi.dataSize = os.stat(file_name).st_size - doi.numThreads = 0 - irods.addKeyVal(doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource) - irods.addKeyVal(doi.condInput, irods.FORCE_FLAG_KW, '') - # TODO: might want to VERIFY_CHKSUM_KW - log.debug('update_from_file(): updating %s to %s', file_name, doi.objPath) - - # do the iput - status = irods.rcDataObjPut(self.rods_conn, doi, file_name) - assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % (doi.objPath, status, irods.strerror(status)) - - def get_object_url(self, obj, **kwargs): - return None - - def get_store_usage_percent(self): - return 0.0 - - -# monkeypatch an strerror method into the irods module -def _rods_strerror(errno): - """ - The missing `strerror` for iRODS error codes - """ - if not hasattr(irods, '__rods_strerror_map'): - irods.__rods_strerror_map = {} - for name in dir(irods): - v = getattr(irods, name) - if type(v) == int and v < 0: - irods.__rods_strerror_map[v] = name - return irods.__rods_strerror_map.get(errno, 'GALAXY_NO_ERRNO_MAPPING_FOUND') - - -if irods is not None: - irods.strerror = _rods_strerror - - -def rods_connect(): - """ - A basic iRODS connection mechanism that connects using the current iRODS - environment - """ - status, env = irods.getRodsEnv() - assert status == 0, 'connect(): getRodsEnv() failed (%s): %s' % (status, irods.strerror(status)) - conn, err = irods.rcConnect(env.rodsHost, - env.rodsPort, - env.rodsUserName, - env.rodsZone) - assert err.status == 0, 'connect(): rcConnect() failed (%s): %s' % (err.status, err.msg) - status, pw = irods.obfGetPw() - assert status == 0, 'connect(): getting password with obfGetPw() failed (%s): %s' % (status, irods.strerror(status)) - status = irods.clientLoginWithObfPassword(conn, pw) - assert status == 0, 'connect(): logging in with clientLoginWithObfPassword() failed (%s): %s' % (status, irods.strerror(status)) - return env, conn diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index a1729498391..ca0b35a178f 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -504,7 +504,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_s3(rel_path)) return False - def exists(self, obj, **kwargs): + def _exists(self, obj, **kwargs): in_cache = in_s3 = False rel_path = self._construct_path(obj, **kwargs) @@ -537,8 +537,8 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): else: return False - def create(self, obj, **kwargs): - if not self.exists(obj, **kwargs): + def _create(self, obj, **kwargs): + if not self._exists(obj, **kwargs): # Pull out locally used fields extra_dir = kwargs.get('extra_dir', None) @@ -572,26 +572,26 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') - def empty(self, obj, **kwargs): - if self.exists(obj, **kwargs): - return bool(self.size(obj, **kwargs) > 0) + def _empty(self, obj, **kwargs): + if self._exists(obj, **kwargs): + return bool(self._size(obj, **kwargs) > 0) else: raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def size(self, obj, **kwargs): + def _size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) if self._in_cache(rel_path): try: return os.path.getsize(self._get_cache_path(rel_path)) except OSError as ex: log.info("Could not get size of file '%s' in local cache, will try S3. Error: %s", rel_path, ex) - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): return self._get_size_in_s3(rel_path) log.warning("Did not find dataset '%s', returning 0 for size", rel_path) return 0 - def delete(self, obj, entire_dir=False, **kwargs): + def _delete(self, obj, entire_dir=False, **kwargs): rel_path = self._construct_path(obj, **kwargs) extra_dir = kwargs.get('extra_dir', None) base_dir = kwargs.get('base_dir', None) @@ -626,10 +626,10 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): except S3ResponseError: log.exception("Could not delete key '%s' from S3", rel_path) except OSError: - log.exception('%s delete error', self.get_filename(obj, **kwargs)) + log.exception('%s delete error', self._get_filename(obj, **kwargs)) return False - def get_data(self, obj, start=0, count=-1, **kwargs): + def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Check cache first and get file if not there if not self._in_cache(rel_path): @@ -641,7 +641,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): data_file.close() return content - def get_filename(self, obj, **kwargs): + def _get_filename(self, obj, **kwargs): base_dir = kwargs.get('base_dir', None) dir_only = kwargs.get('dir_only', False) obj_dir = kwargs.get('obj_dir', False) @@ -664,7 +664,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): if self._in_cache(rel_path): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache - elif self.exists(obj, **kwargs): + elif self._exists(obj, **kwargs): if dir_only: # Directories do not get pulled into cache return cache_path else: @@ -678,10 +678,10 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): % (str(obj), str(kwargs))) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path - def update_from_file(self, obj, file_name=None, create=False, **kwargs): + def _update_from_file(self, obj, file_name=None, create=False, **kwargs): if create: - self.create(obj, **kwargs) - if self.exists(obj, **kwargs): + self._create(obj, **kwargs) + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) # Chose whether to use the dataset file itself or an alternate file if file_name: @@ -703,8 +703,8 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' % (str(obj), str(kwargs))) - def get_object_url(self, obj, **kwargs): - if self.exists(obj, **kwargs): + def _get_object_url(self, obj, **kwargs): + if self._exists(obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) try: key = Key(self._bucket, rel_path) @@ -713,7 +713,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): log.exception("Trouble generating URL for dataset '%s'", rel_path) return None - def get_store_usage_percent(self): + def _get_store_usage_percent(self): return 0.0 def shutdown(self): diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index 6cc481fbcda..d84922aa153 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -50,6 +50,10 @@ WAIT_TYPES = Bunch( JOB_COMPLETION=WaitType("job_completion", 30), # Wait time for a GIE to spawn. GIE_SPAWN=WaitType("gie_spawn", 30), + # Wait time for toolshed search + SHED_SEARCH=WaitType('shed_search', 30), + # Wait time for repository installation + REPO_INSTALL=WaitType('repo_install', 60), ) # Choose a moderate wait type for operations that don't specify a type. @@ -389,6 +393,12 @@ class NavigatesGalaxy(HasDriver): domain = domain or 'test.test' return self._get_random_name(prefix=username, suffix="@" + domain) + # Creates a random password of length len by creating an array with all ASCII letters and the numbers 0 to 9, + # then using the random number generator to pick one elemenent to concatinate it to the end of the password string until + # we have a password of length len. + def _get_random_password(self, len=6): + return ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(len)) + def submit_login(self, email, password=None, assert_valid=True, retries=0): if password is None: password = self.default_password diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index f35dab23a0b..0180011c451 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -70,6 +70,7 @@ masthead: preferences: selectors: + sign_out: "#edit-preferences-sign-out" change_password: "#edit-preferences-password" manage_information: '#edit-preferences-information' current_email: "#user-preferences-current-email" @@ -88,6 +89,11 @@ change_user_address: type: xpath selector: '//span[contains(text(), "Insert Address")]' +sign_out: + selectors: + cancel_button: '.modal-footer .buttons #button-0' + sign_out_button: '.modal-footer .buttons #button-1' + history_panel: menu: labels: @@ -353,18 +359,31 @@ tour: admin: + toolshed: + selectors: + repo_search: '#toolshed-repo-search' + search_results: '#shed-search-results' + index: selectors: datatypes: '#admin-link-datatypes' + dependencies: '#admin-link-manage-dependencies' data_tables: '#admin-link-data-tables' display_applications: '#admin-link-display-applications' + errors: '#admin-link-error-stack' + forms: '#admin-link-forms' jobs: '#admin-link-jobs' local_data: '#admin-link-local-data' + metadata: '#admin-link-metadata' + migrations: '#admin-link-migrations' + tool_versions: '#admin-link-tool-versions' + toolshed: '#admin-link-toolshed' users: '#admin-link-users' quotas: '#admin-link-quotas' groups: '#admin-link-groups' roles: '#admin-link-roles' impersonate: '#admin-link-impersonate' + whitelist: '#admin-link-whitelist' selectors: # TODO: place betters IDS or something on this in these grids in the DOM diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py index 3e9d26decf6..eff83c4d2d7 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py +++ b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py @@ -1,6 +1,8 @@ import logging import threading +from galaxy.util import unicodify + log = logging.getLogger(__name__) @@ -20,8 +22,8 @@ class AsynchronousReader(threading.Thread): """Read lines and put them on the queue.""" thread_lock = threading.Lock() thread_lock.acquire() - for line in iter(self._fd.readline, ''): - stripped_line = line.rstrip() + for line in iter(self._fd.readline, b''): + stripped_line = unicodify(line).rstrip() self.lines.append(stripped_line) self._queue.put(stripped_line) thread_lock.release() diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py index 3221dad2b84..0f1c4068bd5 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -13,13 +13,16 @@ from fabric.operations import _AttributeString from six.moves import queue from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader -from galaxy.tool_shed.util.basic_util import INSTALLATION_LOG, NO_OUTPUT_TIMEOUT +from galaxy.tool_shed.util.basic_util import ( + INSTALLATION_LOG, + NO_OUTPUT_TIMEOUT, +) from galaxy.tool_shed.util.tool_dependency_util import set_tool_dependency_attributes from galaxy.util import ( DATABASE_MAX_STRING_SIZE, DATABASE_MAX_STRING_SIZE_PRETTY, shrink_string_by_size, - unicodify + unicodify, ) log = logging.getLogger(__name__) @@ -78,13 +81,13 @@ class InstallEnvironment(object): """ stdout_logger = logging.getLogger('install_environment.STDOUT') stderr_logger = logging.getLogger('install_environment.STDERR') - for line in iter(stdout.readline, ''): - output = line.rstrip() + for line in iter(stdout.readline, b''): + output = unicodify(line).rstrip() stdout_logger.debug(output) stdout_queue.put(output) stdout_queue.put(None) - for line in iter(stderr.readline, ''): - output = line.rstrip() + for line in iter(stderr.readline, b''): + output = unicodify(line).rstrip() stderr_logger.debug(output) stderr_queue.put(output) stderr_queue.put(None) @@ -245,19 +248,18 @@ class InstallEnvironment(object): def log_results(self, command, fabric_AttributeString, file_path): """Write attributes of fabric.operations._AttributeString to a specified log file.""" - if os.path.exists(file_path): - logfile = open(file_path, 'ab') - else: - logfile = open(file_path, 'wb') - logfile.write("\n#############################################\n") - logfile.write('%s\nSTDOUT\n' % command) - logfile.write(str(fabric_AttributeString.stdout)) - logfile.write("\n#############################################\n") - logfile.write("\n#############################################\n") - logfile.write('%s\nSTDERR\n' % command) - logfile.write(str(fabric_AttributeString.stderr)) - logfile.write("\n#############################################\n") - logfile.close() + mode = 'a' if os.path.exists(file_path) else 'w' + with open(file_path, mode) as logfile: + logfile.write("\n#############################################\n") + logfile.write(command) + logfile.write('\nSTDOUT\n') + logfile.write(fabric_AttributeString.stdout) + logfile.write("\n#############################################\n") + logfile.write("\n#############################################\n") + logfile.write(command) + logfile.write('\nSTDERR\n') + logfile.write(fabric_AttributeString.stderr) + logfile.write("\n#############################################\n") @contextmanager def use_tmp_dir(self): diff --git a/lib/galaxy/tool_shed/metadata/metadata_generator.py b/lib/galaxy/tool_shed/metadata/metadata_generator.py index 7b2ab03864f..21c5791c047 100644 --- a/lib/galaxy/tool_shed/metadata/metadata_generator.py +++ b/lib/galaxy/tool_shed/metadata/metadata_generator.py @@ -17,10 +17,7 @@ from galaxy.tool_shed.util import ( tool_util, ) from galaxy.tool_shed.util.basic_util import remove_dir, strip_path -from galaxy.tool_shed.util.hg_util import ( - get_config_from_disk, - get_repo_for_repository, -) +from galaxy.tool_shed.util.hg_util import get_config_from_disk from galaxy.tool_shed.util.metadata_util import get_updated_changeset_revisions_from_tool_shed from galaxy.tool_shed.util.repository_util import get_repository_for_dependency_relationship from galaxy.tool_util.loader_directory import looks_like_a_tool @@ -77,7 +74,7 @@ class MetadataGenerator(object): else: # We're in the Tool Shed. if changeset_revision is None and self.repository is not None: - self.changeset_revision = self.repository.tip(self.app) + self.changeset_revision = self.repository.tip() else: self.changeset_revision = changeset_revision if repository_clone_url is None and self.repository is not None: @@ -989,7 +986,7 @@ class MetadataGenerator(object): log.debug(error_message) is_valid = False return repository_dependency_tup, is_valid, error_message - repo = get_repo_for_repository(self.app, repository=repository) + repo = repository.hg_repo # The received changeset_revision may be None since defining it in the dependency definition is optional. # If this is the case, the default will be to set its value to the repository dependency tip revision. @@ -1080,7 +1077,7 @@ class MetadataGenerator(object): if relative_install_dir is None and self.repository is not None: relative_install_dir = repository.repo_path(self.app) if changeset_revision is None and self.repository is not None: - self.set_changeset_revision(self.repository.tip(self.app)) + self.set_changeset_revision(self.repository.tip()) else: self.set_changeset_revision(changeset_revision) self.shed_config_dict = {} diff --git a/lib/galaxy/tool_shed/util/hg_util.py b/lib/galaxy/tool_shed/util/hg_util.py index 86046195b3e..3d6ddd9e731 100644 --- a/lib/galaxy/tool_shed/util/hg_util.py +++ b/lib/galaxy/tool_shed/util/hg_util.py @@ -77,7 +77,7 @@ def get_ctx_file_path_from_manifest(filename, repo, changeset_revision): for changeset in reversed_upper_bounded_changelog(repo, changeset_revision): manifest_ctx = repo[changeset] for ctx_file in manifest_ctx.files(): - ctx_file_name = basic_util.strip_path(ctx_file) + ctx_file_name = basic_util.strip_path(unicodify(ctx_file)) if ctx_file_name == stripped_filename: return manifest_ctx, ctx_file return None, None @@ -93,7 +93,7 @@ def get_file_context_from_ctx(ctx, filename): deleted = False filename = basic_util.strip_path(filename) for ctx_file in ctx.files(): - ctx_file_name = basic_util.strip_path(ctx_file) + ctx_file_name = basic_util.strip_path(unicodify(ctx_file)) if filename == ctx_file_name: try: # If the file was moved, its destination will be returned here. @@ -107,18 +107,6 @@ def get_file_context_from_ctx(ctx, filename): return None -def get_repo_for_repository(app, repository=None, repo_path=None): - # Import from mercurial here to let Galaxy start under Python 3 - from mercurial import ( - hg, - ui - ) - if repository is not None: - return hg.repository(ui.ui(), repository.repo_path(app)) - if repo_path is not None: - return hg.repository(ui.ui(), repo_path) - - def pull_repository(repo_path, repository_clone_url, ctx_rev): """Pull changes from a remote repository to a local one.""" try: @@ -199,7 +187,6 @@ __all__ = ( 'get_config_from_disk', 'get_ctx_file_path_from_manifest', 'get_file_context_from_ctx', - 'get_repo_for_repository', 'pull_repository', 'reversed_lower_upper_bounded_changelog', 'reversed_upper_bounded_changelog', diff --git a/lib/galaxy/tool_shed/util/repository_util.py b/lib/galaxy/tool_shed/util/repository_util.py index 6cc746676b4..16413f77eb7 100644 --- a/lib/galaxy/tool_shed/util/repository_util.py +++ b/lib/galaxy/tool_shed/util/repository_util.py @@ -5,7 +5,12 @@ import shutil from markupsafe import escape from six.moves.urllib.error import HTTPError -from sqlalchemy import and_, false, or_ +from sqlalchemy import ( + and_, + false, + or_, +) +from sqlalchemy.orm import joinedload from galaxy import util from galaxy import web @@ -307,7 +312,7 @@ def get_repository_by_id(app, id): return sa_session.query(app.model.Repository).get(app.security.decode_id(id)) -def get_repository_by_name_and_owner(app, name, owner): +def get_repository_by_name_and_owner(app, name, owner, eagerload_columns=None): """Get a repository from the database via name and owner""" repository_query = get_repository_query(app) if is_tool_shed_client(app): @@ -316,13 +321,16 @@ def get_repository_by_name_and_owner(app, name, owner): app.install_model.ToolShedRepository.table.c.owner == owner)) \ .first() # We're in the tool shed. - user = common_util.get_user_by_username(app, owner) - if user: - return repository_query \ - .filter(and_(app.model.Repository.table.c.name == name, - app.model.Repository.table.c.user_id == user.id)) \ - .first() - return None + q = repository_query.filter( + and_( + app.model.Repository.table.c.name == name, + app.model.User.table.c.username == owner, + app.model.Repository.table.c.user_id == app.model.User.table.c.id + ) + ) + if eagerload_columns: + q = q.options(joinedload(*eagerload_columns)) + return q.first() def get_repository_by_name(app, name): @@ -454,9 +462,12 @@ def get_repository_ids_requiring_prior_import_or_install(app, tsr_ids, repositor return prior_tsr_ids -def get_repository_in_tool_shed(app, id): +def get_repository_in_tool_shed(app, id, eagerload_columns=None): """Get a repository on the tool shed side from the database via id.""" - return get_repository_query(app).get(app.security.decode_id(id)) + q = get_repository_query(app) + if eagerload_columns: + q = q.options(joinedload(*eagerload_columns)) + return q.get(app.security.decode_id(id)) def get_repository_owner(cleaned_repository_url): diff --git a/lib/galaxy/tool_shed/util/tool_util.py b/lib/galaxy/tool_shed/util/tool_util.py index 2ba625c7c4d..aac1609dd2f 100644 --- a/lib/galaxy/tool_shed/util/tool_util.py +++ b/lib/galaxy/tool_shed/util/tool_util.py @@ -97,7 +97,7 @@ def generate_message_for_invalid_tools(app, invalid_file_tups, repository, metad if app.name == 'galaxy': tip_rev = str(repository.changeset_revision) else: - tip_rev = str(repository.tip(app)) + tip_rev = str(repository.tip()) if not displaying_invalid_tool: if metadata_dict: message += "Metadata may have been defined for some items in revision '%s'. " % tip_rev diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 017e9dfad14..9d565192e33 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -123,6 +123,8 @@ def preprocess_volumes(volumes_raw_str, container_type): ['/a/b:rw'] >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:/a:ro,/a/b/c:/a/b:rw", DOCKER_CONTAINER_TYPE) + ['/a/b:/a:ro', '/a/b/c:/a/b:rw'] >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) ['/a/b:ro', '/a/b/c:rw'] >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) @@ -135,8 +137,12 @@ def preprocess_volumes(volumes_raw_str, container_type): for volume_raw_str in volumes_raw_strs: volume_parts = volume_raw_str.split(":") - if len(volume_parts) > 2: + if len(volume_parts) > 3: raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str) + if len(volume_parts) == 3: + volume_parts = ["%s:%s" % (volume_parts[0], volume_parts[1]), volume_parts[2]] + if len(volume_parts) == 2 and volume_parts[1] not in ("rw", "ro", "default_ro"): + volume_parts = ["%s:%s" % (volume_parts[0], volume_parts[1]), "rw"] if len(volume_parts) == 1: volume_parts.append("rw") volumes.append(volume_parts) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index a812c6dc04d..7a0b34cd8b0 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -79,7 +79,7 @@ the tool menu immediately following the hyperlink for the tool (based on the - +For more information, see [planemo documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#macros-reusable-elements) @@ -1291,8 +1291,8 @@ and this can be used instead. @@ -3820,7 +3820,7 @@ Style used for tool provided metadata file (i.e. - this can be either "legacy" or "default". The default of tools with a profile of 17.09 or newer are "default", and "legacy" for older and tools and tools without a specified profile. A discussion of the differences between the styles -can be found at https://github.com/galaxyproject/galaxy/pull/4437 . +can be found [here](https://github.com/galaxyproject/galaxy/pull/4437). ]]> @@ -4174,9 +4174,9 @@ Describe datasets to dynamically collect after the job complete. There are many simple tools with examples of this element distributed with Galaxy, including: -* https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output.xml -* https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_assign_primary.xml -* https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_configured.xml +* [multi_output.xml](https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output.xml) +* [multi_output_assign_primary.xml](https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_assign_primary.xml) +* [multi_output_configured.xml](https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_configured.xml) More information can be found on Planemo's documentation for [multiple output files](https://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files). @@ -4234,9 +4234,9 @@ expressions after the job is complete. There are many simple tools with examples of this element distributed with Galaxy, including: -* https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_split_on_column.xml -* https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_list_of_pairs.xml -* https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_nested.xml +* [collection_split_on_column.xml](https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_split_on_column.xml) +* [collection_creates_dynamic_list_of_pairs.xml](https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_list_of_pairs.xml) +* [collection_creates_dynamic_nested.xml](https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_nested.xml) ]]> @@ -5138,7 +5138,7 @@ Tools may use exit codes to indicate specific execution errors. Many programs us The exit code's range can be any consecutive group of integers. More advanced ranges, such as noncontiguous ranges, are currently not supported. Ranges can be specified in the form "m:n", where m is the start integer and n is the end integer. If ":n" is specified, then the exit code will be compared against all integers less than or equal to n. If "m:" is used, then the exit code will be compared against all integers greater than or equal to m. If the exit code matches, then the error level is applied and the error's description is added to stderr. If a tool's exit code does not match any of the supplied ```` tags' ranges, then no errors are applied to the tool's execution. -Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls out of the range 0 to 255, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found at https://en.wikipedia.org/wiki/Unix_signal as well as man pages on "signal". +Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls outside of this range, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found on [Wikipedia](https://en.wikipedia.org/wiki/Signal_(IPC)) as well as on the man page for "signal" (``man 7 signal``). The ```` tag's supported attributes are as follows: diff --git a/lib/galaxy/tools/recommendations.py b/lib/galaxy/tools/recommendations.py index c8df15f36a7..533d3fb160b 100644 --- a/lib/galaxy/tools/recommendations.py +++ b/lib/galaxy/tools/recommendations.py @@ -23,56 +23,89 @@ class ToolRecommendations(): self.admin_tool_recommendations_path = None self.deprecated_tools = dict() self.admin_recommendations = dict() + self.model_data_dictionary = dict() + self.reverse_dictionary = dict() + self.all_tools = dict() + self.tool_weights_sorted = dict() + self.session = None + self.graph = None + self.loaded_model = None + self.compatible_tools = None + self.max_seq_len = 25 - def set_model(self, trans, remote_model_url): + def get_predictions(self, trans, tool_sequence, remote_model_url): + """ + Compute tool predictions + """ + recommended_tools = dict() + self.__collect_admin_preferences(trans.app.config.admin_tool_recommendations_path) + is_set = self.__set_model(trans, remote_model_url) + if is_set is True: + # get the recommended tools for a tool sequence + recommended_tools = self.__compute_tool_prediction(trans, tool_sequence) + else: + tool_sequence = "" + return tool_sequence, recommended_tools + + def __set_model(self, trans, remote_model_url): """ Create model and associated dictionaries for recommendations """ - if not self.tool_recommendation_model_path: + if not self.graph: # import moves from the top of file: in case the tool recommendation feature is disabled, # keras is not downloaded because of conditional requirement and Galaxy does not build try: from keras.models import model_from_json + import tensorflow as tf except Exception: trans.response.status = 400 return False - self.tool_recommendation_model_path = self.download_model(remote_model_url) - self.all_tools = dict() + # set graph and session only once + if self.graph is None: + self.graph = tf.Graph() + self.session = tf.Session(graph=self.graph) model_weights = list() counter_layer_weights = 0 + self.tool_recommendation_model_path = self.__download_model(remote_model_url) + # read the hdf5 attributes + trained_model = h5py.File(self.tool_recommendation_model_path, 'r') + model_config = json.loads(trained_model['model_config'][()]) + # set tensorflow's graph and session to maintain + # consistency between model load and predict methods + with self.graph.as_default(): + with self.session.as_default(): + try: + # iterate through all the attributes of the model to find weights of neural network layers + for item in trained_model.keys(): + if "weight_" in item: + weight = trained_model["weight_" + str(counter_layer_weights)][()] + model_weights.append(weight) + counter_layer_weights += 1 + self.loaded_model = model_from_json(model_config) + self.loaded_model.set_weights(model_weights) + except Exception as e: + log.exception(e) + trans.response.status = 400 + return False + # set the dictionary of tools + self.model_data_dictionary = json.loads(trained_model['data_dictionary'][()]) + self.reverse_dictionary = dict((v, k) for k, v in self.model_data_dictionary.items()) + # set the list of compatible tools + self.compatible_tools = json.loads(trained_model['compatible_tools'][()]) + tool_weights = json.loads(trained_model['class_weights'][()]) + # sort the tools' usage dictionary + tool_pos_sorted = [int(key) for key in tool_weights.keys()] + for k in tool_pos_sorted: + self.tool_weights_sorted[k] = tool_weights[str(k)] # collect ids and names of all the installed tools for tool_id, tool in trans.app.toolbox.tools(): t_id_renamed = tool_id if t_id_renamed.find("/") > -1: t_id_renamed = t_id_renamed.split("/")[-2] self.all_tools[t_id_renamed] = (tool_id, tool.name) - # read the hdf5 attributes - trained_model = h5py.File(self.tool_recommendation_model_path, 'r') - model_config = json.loads(trained_model['model_config'][()]) - self.loaded_model = model_from_json(model_config) - # set the dictionary of tools - self.model_data_dictionary = json.loads(trained_model['data_dictionary'][()]) - self.reverse_dictionary = dict((v, k) for k, v in self.model_data_dictionary.items()) - # set the list of compatible tools - self.compatible_tools = json.loads(trained_model['compatible_tools'][()]) - self.tool_weights = json.loads(trained_model['class_weights'][()]) - self.tool_weights_sorted = dict() - # sort the tools' usage dictionary - tool_pos_sorted = [int(key) for key in self.tool_weights.keys()] - for k in tool_pos_sorted: - self.tool_weights_sorted[k] = self.tool_weights[str(k)] - # iterate through all the attributes of the model to find weights of neural network layers - for item in trained_model.keys(): - if "weight_" in item: - d_key = "weight_" + str(counter_layer_weights) - weights = trained_model[d_key][()] - model_weights.append(weights) - counter_layer_weights += 1 - # set the model weights - self.loaded_model.set_weights(model_weights) return True - def collect_admin_preferences(self, admin_path): + def __collect_admin_preferences(self, admin_path): """ Collect preferences for recommendations of tools set by admins as dictionaries of deprecated tools and @@ -92,7 +125,7 @@ class ToolRecommendations(): if tool_id not in self.admin_recommendations: self.admin_recommendations[tool_id] = tool_info - def download_model(self, model_url, download_local='database/'): + def __download_model(self, model_url, download_local='database/'): """ Download the model from remote server """ @@ -104,7 +137,7 @@ class ToolRecommendations(): model_file.write(model_binary.content) return local_dir - def get_tool_extensions(self, trans, tool_id): + def __get_tool_extensions(self, trans, tool_id): """ Get the input and output extensions of a tool """ @@ -126,7 +159,7 @@ class ToolRecommendations(): output_extensions.extend(o_ext['extensions']) return input_extensions, output_extensions - def filter_tool_predictions(self, trans, prediction_data, tool_ids, tool_scores, last_tool_name): + def __filter_tool_predictions(self, trans, prediction_data, tool_ids, tool_scores, last_tool_name): """ Filter tool predictions based on datatype compatibility and tool connections. Add admin preferences to recommendations. @@ -143,8 +176,8 @@ class ToolRecommendations(): # select the name and tool id if it is installed in Galaxy if t_id == child and score > 0.0 and child in last_compatible_tools and child not in self.deprecated_tools: full_tool_id = self.all_tools[t_id][0] - pred_input_extensions, _ = self.get_tool_extensions(trans, full_tool_id) - c_dict["name"] = self.all_tools[t_id][1] + " (" + str(score) + "%)" + pred_input_extensions, _ = self.__get_tool_extensions(trans, full_tool_id) + c_dict["name"] = self.all_tools[t_id][1] c_dict["tool_id"] = full_tool_id c_dict["i_extensions"] = list(set(pred_input_extensions)) prediction_data["children"].append(c_dict) @@ -173,13 +206,12 @@ class ToolRecommendations(): break return prediction_data - def compute_tool_prediction(self, trans, tool_sequence): + def __compute_tool_prediction(self, trans, tool_sequence): """ Compute the predicted tools for a tool sequences Return a payload with the tool sequences and recommended tools Return an empty payload with just the tool sequence if anything goes wrong within the try block """ - max_seq_len = 25 topk = trans.app.config.topk_recommendations prediction_data = dict() tool_sequence = tool_sequence.split(",")[::-1] @@ -188,18 +220,29 @@ class ToolRecommendations(): last_tool_name = tool_sequence[-1] # do prediction only if the last is present in the collections of tools if last_tool_name in self.model_data_dictionary: - sample = np.zeros(max_seq_len) + sample = np.zeros(self.max_seq_len) # get the list of datatype extensions of the last tool of the tool sequence - _, last_output_extensions = self.get_tool_extensions(trans, self.all_tools[last_tool_name][0]) + _, last_output_extensions = self.__get_tool_extensions(trans, self.all_tools[last_tool_name][0]) prediction_data["o_extensions"] = list(set(last_output_extensions)) # get tool names without slashes and create a sequence vector for idx, tool_name in enumerate(tool_sequence): if tool_name.find("/") > -1: tool_name = tool_name.split("/")[-2] - sample[idx] = int(self.model_data_dictionary[tool_name]) - sample = np.reshape(sample, (1, max_seq_len)) + try: + sample[idx] = int(self.model_data_dictionary[tool_name]) + except Exception: + log.exception("Failed to find tool %s in model" % (tool_name)) + return prediction_data + sample = np.reshape(sample, (1, self.max_seq_len)) # predict next tools for a test path - prediction = self.loaded_model.predict(sample, verbose=0) + try: + # use the same graph and session to predict + with self.graph.as_default(): + with self.session.as_default(): + prediction = self.loaded_model.predict(sample) + except Exception as e: + log.exception(e) + return prediction_data prediction = np.reshape(prediction, (prediction.shape[1],)) # boost the predicted scores using tools' usage weight_values = list(self.tool_weights_sorted.values()) @@ -215,5 +258,5 @@ class ToolRecommendations(): # read tool names using reverse dictionary pred_tool_ids = [self.reverse_dictionary[int(tool_pos)] for tool_pos in topk_prediction_pos] predicted_scores = [int(prediction[pos] * 100) for pos in topk_prediction_pos] - prediction_data = self.filter_tool_predictions(trans, prediction_data, pred_tool_ids[::-1], predicted_scores[::-1], last_tool_name) + prediction_data = self.__filter_tool_predictions(trans, prediction_data, pred_tool_ids[::-1], predicted_scores[::-1], last_tool_name) return prediction_data diff --git a/lib/galaxy/util/sockets.py b/lib/galaxy/util/sockets.py index ffc246627e1..826e69c1a23 100644 --- a/lib/galaxy/util/sockets.py +++ b/lib/galaxy/util/sockets.py @@ -4,6 +4,19 @@ import socket import subprocess +def get_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(('10.255.255.255', 1)) + ip = s.getsockname()[0] + except Exception: + ip = None + finally: + s.close() + return ip + + def unused_port(range=None): if range: return __unused_port_on_range(range) diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 20448c65260..21714f8dd57 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -15,6 +15,7 @@ from galaxy import util from galaxy.managers.jobs import ( JobManager, JobSearch, + summarize_destination_params, summarize_job_metrics, summarize_job_parameters, ) @@ -283,6 +284,22 @@ class JobController(BaseAPIController, UsesVisualizationMixin): job = self.__get_job(trans, **kwd) return summarize_job_metrics(trans, job) + @require_admin + @expose_api + def destination_params(self, trans, **kwd): + """ + * GET /api/jobs/{job_id}/destination_params + Return destination parameters for specified job. + + :type job_id: string + :param job_id: Encoded job id + + :rtype: list + :returns: list containing job destination parameters + """ + job = self.__get_job(trans, **kwd) + return summarize_destination_params(trans, job) + @expose_api_anonymous def parameters_display(self, trans, **kwd): """ diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 9cfe322b53f..13a59170d2a 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -643,22 +643,11 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn 'tool_sequence' - comma separated sequence of tool ids 'remote_model_url' - (optional) path to the deep learning model """ - remote_model_url = payload.get('remote_model_url', None) - if remote_model_url is None: - remote_model_url = trans.app.config.tool_recommendation_model_path + remote_model_url = payload.get('remote_model_url', trans.app.config.tool_recommendation_model_path) + tool_sequence = payload.get('tool_sequence', "") if 'tool_sequence' not in payload or remote_model_url is None: return - is_model_set = True - recommended_tools = dict() - tool_sequence = "" - # collect tool recommendation preferences if set by admin - self.tool_recommendations.collect_admin_preferences(trans.app.config.admin_tool_recommendations_path) - # recreate the neural network model to be used for prediction - is_model_set = self.tool_recommendations.set_model(trans, remote_model_url) - if is_model_set is True: - # get the recommended tools for a tool sequence - tool_sequence = payload.get('tool_sequence', "") - recommended_tools = self.tool_recommendations.compute_tool_prediction(trans, tool_sequence) + tool_sequence, recommended_tools = self.tool_recommendations.get_predictions(trans, tool_sequence, remote_model_url) return { "current_tool": tool_sequence, "predicted_data": recommended_tools diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 5438ee12b66..9df4eb0ad50 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -946,6 +946,7 @@ def populate_api_routes(webapp, app): webapp.mapper.connect('common_problems', '/api/jobs/{id}/common_problems', controller='jobs', action='common_problems', conditions=dict(method=['GET'])) # Job metrics and parameters by job id or dataset id (for slightly different accessibility checking) webapp.mapper.connect('metrics', '/api/jobs/{job_id}/metrics', controller='jobs', action='metrics', conditions=dict(method=['GET'])) + webapp.mapper.connect('destination_params', '/api/jobs/{job_id}/destination_params', controller='jobs', action='destination_params', conditions=dict(method=['GET'])) webapp.mapper.connect('show_job_lock', '/api/job_lock', controller='jobs', action='show_job_lock', conditions=dict(method=['GET'])) webapp.mapper.connect('update_job_lock', '/api/job_lock', controller='jobs', action='update_job_lock', conditions=dict(method=['PUT'])) webapp.mapper.connect('dataset_metrics', '/api/datasets/{dataset_id}/metrics', controller='jobs', action='metrics', conditions=dict(method=['GET'])) diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 84c37488ec8..788b992ca02 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -1026,6 +1026,19 @@ mapping: desc: | Enable InteractiveTools. + interactivetools_proxy_host: + type: str + desc: | + Proxy host - assumed to just be hosted on the same hostname and port as + Galaxy by default. + + interactivetools_map: + type: str + default: interactivetools_map.sqlite + path_resolves_to: data_dir + desc: | + Map for interactivetool proxy. + visualizations_visible: type: bool default: true diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index eb0126d8b7a..3c5824172b7 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1,12 +1,14 @@ import json import logging import os +from functools import wraps from six import string_types from sqlalchemy import or_ import tool_shed.repository_types.util as rt_util from galaxy import util, web +from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.tool_shed.galaxy_install import install_manager from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager @@ -31,12 +33,24 @@ from .admin import AdminGalaxy log = logging.getLogger(__name__) +def legacy_tool_shed_endpoint(func): + # admin only and only available if running test cases. + @wraps(func) + def wrapper(trans, *args, **kwargs): + if not trans.app.config.config_dict.get("running_functional_tests", False): + raise ConfigDoesNotAllowException("Legacy tool shed endpoint only available during testing.") + return func(trans, *args, **kwargs) + + return wrapper + + class AdminToolshed(AdminGalaxy): installed_repository_grid = admin_toolshed_grids.InstalledRepositoryGrid() @web.expose @web.require_admin + @legacy_tool_shed_endpoint def activate_repository(self, trans, **kwd): """Activate a repository that was deactivated but not uninstalled.""" repository_id = kwd['id'] @@ -59,6 +73,7 @@ class AdminToolshed(AdminGalaxy): @web.legacy_expose_api @web.require_admin + @legacy_tool_shed_endpoint def browse_repositories(self, trans, **kwd): message = kwd.get('message', '') status = kwd.get('status', '') @@ -73,6 +88,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def restore_repository(self, trans, **kwd): repository_id = kwd['id'] repository = repository_util.get_installed_tool_shed_repository(trans.app, repository_id) @@ -157,6 +173,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def view_tool_metadata(self, trans, repository_id, tool_id, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') @@ -193,11 +210,13 @@ class AdminToolshed(AdminGalaxy): @web.json @web.require_admin @web.do_not_cache + @legacy_tool_shed_endpoint def get_file_contents(self, trans, file_path, repository_id): return suc.get_repository_file_contents(trans.app, file_path, repository_id, is_admin=True) @web.expose @web.require_admin + @legacy_tool_shed_endpoint def get_tool_dependencies(self, trans, repository_id, repository_name, repository_owner, changeset_revision): """ Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for @@ -224,6 +243,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def get_updated_repository_information(self, trans, repository_id, repository_name, repository_owner, changeset_revision): """ Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall @@ -246,6 +266,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def initiate_tool_dependency_installation(self, trans, tool_dependencies, **kwd): """ Install specified dependencies for repository tools. The received list of tool_dependencies @@ -285,6 +306,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def install_latest_repository_revision(self, trans, **kwd): """Install the latest installable revision of a repository that has been previously installed.""" repository_id = kwd.get('id', None) @@ -336,6 +358,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def install_tool_dependencies_with_update(self, trans, **kwd): """ Updating an installed tool shed repository where new tool dependencies but no new repository @@ -411,6 +434,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def install_repositories(self, trans, **kwd): reinstalling = util.string_as_bool(kwd.get('reinstalling', False)) encoded_kwd = kwd.get('encoded_kwd') @@ -436,6 +460,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def manage_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') @@ -509,6 +534,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def monitor_repository_installation(self, trans, **kwd): tsr_ids = common_util.get_tool_shed_repository_ids(**kwd) if not tsr_ids: @@ -530,6 +556,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def prepare_for_install(self, trans, **kwd): if not suc.have_shed_tool_conf_for_install(trans.app): message = 'The tool_config_file setting in galaxy.ini must include at least one ' @@ -810,6 +837,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def reinstall_repository(self, trans, **kwd): """ Reinstall a tool shed repository that has been previously uninstalled, making sure to handle all repository @@ -981,6 +1009,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def reselect_tool_panel_section(self, trans, **kwd): """ Select or change the tool panel section to contain the tools included in the tool shed repository @@ -1151,6 +1180,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def uninstall_tool_dependencies(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') @@ -1195,6 +1225,7 @@ class AdminToolshed(AdminGalaxy): @web.expose @web.require_admin + @legacy_tool_shed_endpoint def update_to_changeset_revision(self, trans, **kwd): """Update a cloned repository to the latest revision possible.""" message = escape(kwd.get('message', '')) diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index 817bfb9d6bc..9afea0a9632 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -237,6 +237,7 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin): # Since logging an event requires a session, we'll log prior to ending the session trans.log_event("User logged out") trans.handle_user_logout(logout_all=logout_all) + return {"message": "Success."} @expose_api_anonymous_and_sessionless def create(self, trans, payload={}, **kwd): diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 3b57e15e961..60b20b10de2 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -707,8 +707,8 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi 'version' : version, 'annotation' : self.get_item_annotation_str(trans.sa_session, trans.user, stored), 'toolbox' : trans.app.toolbox.to_dict(trans), - 'module_sections' : module_sections, - 'data_managers' : data_managers, + 'moduleSections' : module_sections, + 'dataManagers' : data_managers, 'workflows' : workflows } diff --git a/lib/galaxy_ext/container_monitor/monitor.py b/lib/galaxy_ext/container_monitor/monitor.py index 0ae66a58a61..1889d45ab7c 100644 --- a/lib/galaxy_ext/container_monitor/monitor.py +++ b/lib/galaxy_ext/container_monitor/monitor.py @@ -1,6 +1,5 @@ import json import os -import socket import subprocess import sys import tempfile @@ -12,6 +11,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa import requests from galaxy.tool_util.deps import docker_util +from galaxy.util.sockets import get_ip def parse_ports(container_name, connection_configuration): @@ -48,11 +48,7 @@ def main(): try: ports_raw = parse_ports(container_name, connection_configuration) if ports_raw is not None: - try: - host_ip = socket.gethostbyname(socket.gethostname()) - except Exception: - # doesn't work on OS X - host_ip = None + host_ip = get_ip() ports = docker_util.parse_port_text(ports_raw) if host_ip is not None: for key in ports: diff --git a/lib/galaxy_test/api/embed_test_1.gxwf.yml b/lib/galaxy_test/api/embed_test_1.gxwf.yml index 76461dff6fd..e43f9426582 100644 --- a/lib/galaxy_test/api/embed_test_1.gxwf.yml +++ b/lib/galaxy_test/api/embed_test_1.gxwf.yml @@ -14,7 +14,7 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: - input2: - $link: embed1#output1 + $link: embed1/output1 diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py index 048f20aa237..6e94c1839d2 100644 --- a/lib/galaxy_test/api/test_workflow_extraction.py +++ b/lib/galaxy_test/api/test_workflow_extraction.py @@ -228,7 +228,7 @@ steps: - tool_id: cat_collection state: input1: - $link: noop#out_file1 + $link: noop/out_file1 test_data: text_input1: type: "list:paired" @@ -274,11 +274,11 @@ steps: tool_id: collection_split_on_column state: input1: - $link: cat_inputs#out_file1 + $link: cat_inputs/out_file1 - tool_id: cat_list state: input1: - $link: split_up#split_output + $link: split_up/split_output test_data: text_input1: "samp1\t10.0\nsamp2\t20.0\n" text_input2: "samp1\t30.0\nsamp2\t40.0\n" @@ -316,16 +316,16 @@ steps: tool_id: collection_creates_pair state: input1: - $link: cat_inputs#out_file1 + $link: cat_inputs/out_file1 - label: cat_pairs tool_id: cat_collection state: input1: - $link: pair_off#paired_output + $link: pair_off/paired_output - tool_id: cat_list state: input1: - $link: cat_pairs#out_file1 + $link: cat_pairs/out_file1 test_data: text_input1: type: list diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index bb0a662896a..d4e2359c8de 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -40,47 +40,6 @@ from galaxy_test.base.workflow_fixtures import ( from ._framework import ApiTestCase -NESTED_WORKFLOW_AUTO_LABELS_LEGACY_SYNTAX = """ -class: GalaxyWorkflow -inputs: - outer_input: data -outputs: - outer_output: - outputSource: second_cat/out_file1 -steps: - first_cat: - tool_id: cat1 - in: - input1: outer_input - nested_workflow: - run: - class: GalaxyWorkflow - inputs: - - id: inner_input - outputs: - - source: 1#out_file1 - steps: - random: - tool_id: random_lines1 - state: - num_lines: 1 - input: - $link: inner_input - seed_source: - seed_source_selector: set_seed - seed: asdf - in: - inner_input: first_cat/out_file1 - second_cat: - tool_id: cat1 - state: - input1: - $link: nested_workflow#1:out_file1 - queries: - - input2: - $link: nested_workflow#1:out_file1 -""" - NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX = """ class: GalaxyWorkflow inputs: @@ -487,10 +446,10 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: input2: - $link: embed1#output1 + $link: embed1/output1 test_data: input1: "hello world" """) @@ -832,12 +791,12 @@ steps: inner_cond: cond_param_inner: true input1: - $link: 0#out_file1 + $link: 0/out_file1 cat: tool_id: cat1 in: - input1: identifier#output1 - queries_0|input2: identifier#output1 + input1: identifier/output1 + queries_0|input2: identifier/output1 """) with self.dataset_populator.test_history() as history_id: invocation_id = self.__invoke_workflow(history_id, workflow_id) @@ -873,11 +832,11 @@ steps: list_in_list_out: tool_id: collection_creates_list in: - input1: job_props#list_output + input1: job_props/list_output identifier: tool_id: identifier_collection in: - input1: list_in_list_out#list_output + input1: list_in_list_out/list_output """) with self.dataset_populator.test_history() as history_id: invocation_id = self.__invoke_workflow(history_id, workflow_id) @@ -1220,8 +1179,7 @@ test_data: "chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", content) - for workflow_text in [NESTED_WORKFLOW_AUTO_LABELS_LEGACY_SYNTAX, NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX]: - run_test(workflow_text) + run_test(NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX) @skip_without_tool("cat1") @skip_without_tool("collection_paired_test") @@ -1274,7 +1232,7 @@ steps: tool_id: '__FLATTEN__' state: input: - $link: nested#list_output + $link: nested/list_output join_identifier: '-' """, test_data={}, history_id=history_id) details = self.dataset_populator.get_history_collection_details(history_id, hid=14) @@ -1400,7 +1358,7 @@ steps: tool_id: "__FILTER_FAILED_DATASETS__" state: input: - $link: mixed_collection#out_file1 + $link: mixed_collection/out_file1 cat: tool_id: cat1 @@ -1870,7 +1828,7 @@ steps: state: num_lines: 2 input: - $link: empty_list#output + $link: empty_list/output seed_source: seed_source_selector: set_seed seed: asdf @@ -1953,7 +1911,7 @@ steps: state: num_lines: 2 input: - $link: empty_list#output + $link: empty_list/output seed_source: seed_source_selector: set_seed seed: asdf @@ -2388,7 +2346,7 @@ steps: state: floattest: 3.14 inttest: - $link: forty_two#out1 + $link: forty_two/out1 test_data: {} """, history_id=history_id) @@ -2417,7 +2375,7 @@ steps: - label: consume_expression_parameter tool_id: validation_default in: - input1: param_out#text_param + input1: param_out/text_param outputs: out_file1: rename: "replaced_param_collection" @@ -3272,7 +3230,7 @@ steps: create_pair: tool_id: collection_creates_pair in: - input1: first_cat#out_file1 + input1: first_cat/out_file1 outputs: paired_output: remove_tags: @@ -4103,10 +4061,10 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: - input2: - $link: embed1#output1 + $link: embed1/output1 test_data: input1: "hello world" """) diff --git a/lib/galaxy_test/api/test_workflows_from_yaml.py b/lib/galaxy_test/api/test_workflows_from_yaml.py index 73e0c101af6..c8dfe57c8b2 100644 --- a/lib/galaxy_test/api/test_workflows_from_yaml.py +++ b/lib/galaxy_test/api/test_workflows_from_yaml.py @@ -296,10 +296,10 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: - input2: - $link: embed1#output1 + $link: embed1/output1 test_data: input1: "hello world" """, history_id=history_id) diff --git a/lib/galaxy_test/base/workflow_fixtures.py b/lib/galaxy_test/base/workflow_fixtures.py index 33651b93ace..f51af2b98e7 100644 --- a/lib/galaxy_test/base/workflow_fixtures.py +++ b/lib/galaxy_test/base/workflow_fixtures.py @@ -143,11 +143,11 @@ steps: - tool_id: collection_paired_test state: f1: - $link: 1#paired_output + $link: 1/paired_output - tool_id: cat_list state: input1: - $link: 2#out1 + $link: 2/out1 """ @@ -175,7 +175,7 @@ steps: state: num_lines: 1 input: - $link: apply#output + $link: apply/output seed_source: seed_source_selector: set_seed seed: asdf @@ -279,7 +279,7 @@ steps: inner_input: data outputs: workflow_output: - outputSource: random_lines#out_file1 + outputSource: random_lines/out_file1 steps: - tool_id: random_lines1 label: random_lines diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index cdd9b3d1f0f..ebc69b195d8 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -66,12 +66,16 @@ uwsgi: http-raw-body: true interactivetools_map: $tempdir/interactivetools_map.sqlite python-raw: scripts/interactivetools/key_type_token_mapping.py + # if interactive tool path, jump to interactive tool, else skip to + # endendend (default uwsgi params). route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)-([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.$test_host:$test_port)$ goto:interactivetool route-run: goto:endendend + route-label: interactivetool route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)-([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.$test_host:$test_port)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $1 $3 $2 $4 $0 5 route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST} route: .* break:404 Not Found + route-label: endendend """) @@ -732,9 +736,14 @@ def launch_uwsgi(kwargs, tempdir, prefix=DEFAULT_CONFIG_PREFIX, config_object=No enable_realtime_mapping = getattr(config_object, "enable_realtime_mapping", False) if enable_realtime_mapping: - config["galaxy"]["interactivetools_prefix"] = "interactivetool" - config["galaxy"]["interactivetools_map"] = os.path.join(tempdir, "interactivetools_map.sqlite") - config['galaxy']['interactivetools_enable'] = True + interactive_tool_defaults = { + "interactivetools_prefix": "interactivetool", + "interactivetools_map": os.path.join(tempdir, "interactivetools_map.sqlite"), + "interactivetools_enable": True + } + for key, value in interactive_tool_defaults.items(): + if key not in config["galaxy"]: + config["galaxy"][key] = value yaml_config_path = os.path.join(tempdir, "galaxy.yml") with open(yaml_config_path, "w") as f: diff --git a/lib/galaxy_test/driver/integration_util.py b/lib/galaxy_test/driver/integration_util.py index 59e3ef17b86..2e402984cc0 100644 --- a/lib/galaxy_test/driver/integration_util.py +++ b/lib/galaxy_test/driver/integration_util.py @@ -2,7 +2,7 @@ Tests that start an actual Galaxy server with a particular configuration in order to test something that cannot be tested with the default functional/api -tessting configuration. +testing configuration. """ import os from unittest import skip, SkipTest, TestCase diff --git a/lib/galaxy_test/selenium/test_admin_app.py b/lib/galaxy_test/selenium/test_admin_app.py index 746b4a4bc5b..0ab96f01e84 100644 --- a/lib/galaxy_test/selenium/test_admin_app.py +++ b/lib/galaxy_test/selenium/test_admin_app.py @@ -1,3 +1,4 @@ +from galaxy_test.base.populators import flakey from .framework import ( selenium_test, SeleniumTestCase, @@ -8,6 +9,46 @@ class AdminAppTestCase(SeleniumTestCase): requires_admin = True + @selenium_test + @flakey # This test relies on an external server that may or may not be available to the testing environment. + def test_admin_toolshed(self): + admin_component = self.components.admin + self.admin_login() + self.admin_open() + self.sleep_for(self.wait_types.UX_RENDER) + self.screenshot('admin_landing') + admin_component.index.toolshed.wait_for_and_click() + self.sleep_for(self.wait_types.UX_RENDER) + self.screenshot('admin_toolshed_landing') + repo_search_input = self.driver.find_element_by_id('toolshed-repo-search') + repo_search_input.clear() + repo_search_input.send_keys('all_fasta') + # If this hasn't succeeded after 30 seconds, the @flakey context should + # allow the test to still pass, since there should definitely be results + # after 30 seconds. + self.sleep_for(self.wait_types.SHED_SEARCH) + self.screenshot('admin_toolshed_search') + admin_component.toolshed.search_results.wait_for_visible() + # It's very unlikely that this data manager will ever stop existing. + repository_row = self.driver.find_element_by_link_text('data_manager_fetch_genome_dbkeys_all_fasta') + repository_row.click() + self.sleep_for(self.wait_types.UX_RENDER) + self.screenshot('admin_toolshed_repo_details') + install_button = self.driver.find_element_by_xpath("(//button[contains(., 'Install')])[1]") + install_button.click() + self.sleep_for(self.wait_types.UX_RENDER) + self.screenshot('admin_toolshed_repo_install_settings') + ok_button = self.driver.find_element_by_xpath("//*[@id='repo-install-settings___BV_modal_footer_']/button[contains(., 'OK')]") + ok_button.click() + self.sleep_for(self.wait_types.REPO_INSTALL) + self.screenshot('admin_toolshed_repo_installed') + # Unfortunately reusing the element isn't feasible, since the div + # containing the button gets replaced with a new div and button. + uninstall_button = self.driver.find_element_by_xpath("(//button[contains(., 'Uninstall')])[1]") + uninstall_button.click() + self.sleep_for(self.wait_types.UX_TRANSITION) + self.screenshot('admin_toolshed_repo_uninstalled') + @selenium_test def test_admin_server_display(self): admin_component = self.components.admin diff --git a/lib/galaxy_test/selenium/test_change_password.py b/lib/galaxy_test/selenium/test_change_password.py index f088b992b6e..b8bc92578b6 100644 --- a/lib/galaxy_test/selenium/test_change_password.py +++ b/lib/galaxy_test/selenium/test_change_password.py @@ -13,32 +13,38 @@ class ChangePasswordTestCase(SeleniumTestCase): self.click_masthead_user() self.components.masthead.preferences.wait_for_and_click() self.components.preferences.change_password.wait_for_and_click() - self.fill_input_fields(self.default_password, "123123", "123123") + new_password = self._get_random_password() + self.fill_input_fields(self.default_password, new_password, new_password) self.logout_if_needed() - self.submit_login(email, "123123") + self.submit_login(email, new_password) @selenium_test def test_new_password_password(self): self.register_and_change_password() - self.fill_input_fields(self.default_password, "", "123123") + self.fill_input_fields(self.default_password, "", self._get_random_password()) self.assert_error_message(contains='Please provide a new password.') @selenium_test def test_no_password_confirmation(self): self.register_and_change_password() - self.fill_input_fields(self.default_password, "123123", "5444484") + password = self._get_random_password() + confirmation = self._get_random_password() + self.fill_input_fields(self.default_password, password, confirmation) self.assert_error_message(contains='Passwords do not match.') @selenium_test def test_currect_password_incorrect(self): self.register_and_change_password() - self.fill_input_fields("4444444", "123123", "123123") + password = self._get_random_password() + assert self.default_password != password + self.fill_input_fields(password, password, password) self.assert_error_message(contains='Invalid current password.') @selenium_test def test_new_password_short(self): self.register_and_change_password() - self.fill_input_fields(self.default_password, "123", "123") + password = self._get_random_password(len=3) + self.fill_input_fields(self.default_password, password, password) self.assert_error_message(contains='Use a password of at least 6 characters.') @selenium_test diff --git a/lib/galaxy_test/selenium/test_sign_out.py b/lib/galaxy_test/selenium/test_sign_out.py new file mode 100644 index 00000000000..9a9162aef83 --- /dev/null +++ b/lib/galaxy_test/selenium/test_sign_out.py @@ -0,0 +1,21 @@ +from .framework import ( + selenium_test, + SeleniumTestCase +) + + +class SignOutTestCase(SeleniumTestCase): + @selenium_test + def test_sign_out(self): + email = self._get_random_email() + self.register(email) + self.click_masthead_user() + self.components.masthead.preferences.wait_for_and_click() + self.components.preferences.sign_out.wait_for_and_click() + self.components.sign_out.cancel_button.wait_for_and_click() + assert self.is_logged_in() + new_email = self.driver.find_element_by_id("user-preferences-current-email").text + self.assertTrue(email == new_email) + self.components.preferences.sign_out.wait_for_and_click() + self.components.sign_out.sign_out_button.wait_for_and_click() + assert not self.is_logged_in() diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index 5c212d61448..39015c30946 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -85,7 +85,7 @@ class RepositoryGrid(grids.Grid): def get_value(self, trans, grid, repository): """Display the current repository heads.""" - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo heads = hg_util.get_repository_heads(repo) multiple_heads = len(heads) > 1 if multiple_heads: @@ -140,7 +140,7 @@ class RepositoryGrid(grids.Grid): def get_value(self, trans, grid, repository): """Display the repository tip revision label.""" - return escape_html(repository.revision(trans.app)) + return escape_html(repository.revision()) class DescriptionColumn(grids.TextColumn): @@ -418,7 +418,7 @@ class MyWritableRepositoriesGrid(RepositoryGrid): for repository in trans.sa_session.query(model.Repository) \ .filter(and_(model.Repository.table.c.deprecated == false(), model.Repository.table.c.deleted == false())): - allow_push = repository.allow_push(trans.app) + allow_push = repository.allow_push() if allow_push: allow_push_usernames = allow_push.split(',') if username in allow_push_usernames: @@ -710,7 +710,7 @@ class MyWritableRepositoriesMissingToolTestComponentsGrid(RepositoriesMissingToo for repository in trans.sa_session.query(model.Repository) \ .filter(and_(model.Repository.table.c.deprecated == false(), model.Repository.table.c.deleted == false())): - allow_push = repository.allow_push(trans.app) + allow_push = repository.allow_push() if allow_push: allow_push_usernames = allow_push.split(',') if username in allow_push_usernames: @@ -849,7 +849,7 @@ class MyWritableRepositoriesWithInvalidToolsGrid(RepositoriesWithInvalidToolsGri for repository in trans.sa_session.query(model.Repository) \ .filter(and_(model.Repository.table.c.deprecated == false(), model.Repository.table.c.deleted == false())): - allow_push = repository.allow_push(trans.app) + allow_push = repository.allow_push() if allow_push: allow_push_usernames = allow_push.split(',') if username in allow_push_usernames: diff --git a/lib/tool_shed/grids/repository_review_grids.py b/lib/tool_shed/grids/repository_review_grids.py index 15c037ea7ee..9ff94b87911 100644 --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -56,7 +56,7 @@ class RepositoriesWithReviewsGrid(RepositoryGrid): # Restrict to revisions that have been reviewed. if repository.reviews: rval = '' - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo for review in repository.reviews: changeset_revision = review.changeset_revision rev, label = hg_util.get_rev_label_from_changeset_revision(repo, changeset_revision) diff --git a/lib/tool_shed/grids/util.py b/lib/tool_shed/grids/util.py index bfe6cd87e65..39b54089472 100644 --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -112,7 +112,7 @@ def get_latest_downloadable_repository_metadata(trans, repository): tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id(repository.id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo tip_ctx = str(repo[repo.changelog.tip()]) repository_metadata = None try: @@ -151,7 +151,7 @@ def get_latest_repository_metadata(trans, repository): tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id(repository.id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo tip_ctx = str(repo[repo.changelog.tip()]) try: repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, encoded_repository_id, tip_ctx) diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py index 599983bd078..85b8dca45d3 100644 --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -438,7 +438,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): if repository.type in [rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION]: clause_list.append(self.app.model.Repository.table.c.id == repository.id) else: - allow_push = repository.allow_push(self.app) + allow_push = repository.allow_push() if allow_push: # Include all repositories that are writable by the current user. allow_push_usernames = allow_push.split(',') @@ -735,7 +735,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): def reset_all_metadata_on_repository_in_tool_shed(self): """Reset all metadata on a single repository in a tool shed.""" log.debug("Resetting all metadata on repository: %s" % self.repository.name) - repo = hg_util.get_repo_for_repository(self.app, repository=self.repository) + repo = self.repository.hg_repo # The list of changeset_revisions refers to repository_metadata records that have been created # or updated. When the following loop completes, we'll delete all repository_metadata records # for this repository that do not have a changeset_revision value in this list. @@ -752,7 +752,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): log.debug("Cloning repository changeset revision: %s", str(ctx.rev())) cloned_ok, error_message = hg_util.clone_repository(self.repository_clone_url, work_dir, str(ctx.rev())) if cloned_ok: - log.debug("Generating metadata for changset revision: %s", str(ctx.rev())) + log.debug("Generating metadata for changeset revision: %s", str(ctx.rev())) self.set_changeset_revision(str(ctx)) self.set_repository_files_dir(work_dir) self.generate_metadata_for_changeset_revision() @@ -904,8 +904,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): message = '' status = 'done' encoded_id = self.app.security.encode_id(self.repository.id) - repo_dir = self.repository.repo_path(self.app) - repo = hg_util.get_repo_for_repository(self.app, repo_path=repo_dir) + repo = self.repository.hg_repo self.generate_metadata_for_changeset_revision() if self.metadata_dict: repository_metadata = None @@ -913,7 +912,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): tip_only = isinstance(repository_type_class, TipOnly) if not tip_only and self.new_metadata_required_for_utilities(): # Create a new repository_metadata table row. - repository_metadata = self.create_or_update_repository_metadata(self.repository.tip(self.app), + repository_metadata = self.create_or_update_repository_metadata(self.repository.tip(), self.metadata_dict) # If this is the first record stored for this repository, see if we need to send any email alerts. if len(self.repository.downloadable_revisions) == 1: @@ -931,7 +930,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): if repository_metadata: downloadable = metadata_util.is_downloadable(self.metadata_dict) # Update the last saved repository_metadata table row. - repository_metadata.changeset_revision = self.repository.tip(self.app) + repository_metadata.changeset_revision = self.repository.tip() repository_metadata.metadata = self.metadata_dict repository_metadata.downloadable = downloadable if 'datatypes' in self.metadata_dict: @@ -962,7 +961,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): self.sa_session.flush() else: # There are no metadata records associated with the repository. - repository_metadata = self.create_or_update_repository_metadata(self.repository.tip(self.app), + repository_metadata = self.create_or_update_repository_metadata(self.repository.tip(), self.metadata_dict) if 'tools' in self.metadata_dict and repository_metadata and status != 'error': # Set tool versions on the new downloadable change set. The order of the list of changesets is @@ -975,7 +974,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator): self.add_tool_versions(encoded_id, repository_metadata, changeset_revisions) elif len(repo) == 1 and not self.invalid_file_tups: message = "Revision %s includes no Galaxy utilities for which metadata can " % \ - str(self.repository.tip(self.app)) + str(self.repository.tip()) message += "be defined so this revision cannot be automatically installed into a local Galaxy instance." status = "error" if self.invalid_file_tups: diff --git a/lib/tool_shed/repository_registry.py b/lib/tool_shed/repository_registry.py index 35fcbfb92a1..d599233cb18 100644 --- a/lib/tool_shed/repository_registry.py +++ b/lib/tool_shed/repository_registry.py @@ -196,7 +196,7 @@ class Registry(object): # The received repository has been determined to be level one certified. name = str(repository.name) owner = str(repository.user.username) - tip_changeset_hash = repository.tip(self.app) + tip_changeset_hash = repository.tip() if tip_changeset_hash != hg_util.INITIAL_CHANGELOG_HASH: certified_level_one_tuple = (name, owner, tip_changeset_hash) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: @@ -261,7 +261,7 @@ class Registry(object): if not repository.deleted and not repository.deprecated: is_valid = self.is_valid(repository) encoded_repository_id = self.app.security.encode_id(repository.id) - tip_changeset_hash = repository.tip(self.app) + tip_changeset_hash = repository.tip() repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(self.app, encoded_repository_id, tip_changeset_hash) @@ -353,7 +353,7 @@ class Registry(object): # The received repository has been determined to be level one certified. name = str(repository.name) owner = str(repository.user.username) - tip_changeset_hash = repository.tip(self.app) + tip_changeset_hash = repository.tip() if tip_changeset_hash != hg_util.INITIAL_CHANGELOG_HASH: certified_level_one_tuple = (name, owner, tip_changeset_hash) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: diff --git a/lib/tool_shed/repository_types/metadata.py b/lib/tool_shed/repository_types/metadata.py index f09c152441e..58d1135cd39 100644 --- a/lib/tool_shed/repository_types/metadata.py +++ b/lib/tool_shed/repository_types/metadata.py @@ -1,7 +1,5 @@ import logging -from tool_shed.util import hg_util - log = logging.getLogger(__name__) @@ -11,10 +9,10 @@ class Metadata(object): self.type = None def get_changesets_for_setting_metadata(self, app, repository): - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo return repo.changelog - def is_valid_for_type(self, app, repository, revisions_to_check=None): + def is_valid_for_type(self, repository, revisions_to_check=None): raise Exception("Unimplemented Method") @@ -24,5 +22,5 @@ class TipOnly(Metadata): self.type = None def get_changesets_for_setting_metadata(self, app, repository): - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo return [repo.changelog.tip()] diff --git a/lib/tool_shed/repository_types/repository_suite_definition.py b/lib/tool_shed/repository_types/repository_suite_definition.py index 9d82f7db6f7..a891d4605b4 100644 --- a/lib/tool_shed/repository_types/repository_suite_definition.py +++ b/lib/tool_shed/repository_types/repository_suite_definition.py @@ -1,11 +1,9 @@ import logging import tool_shed.repository_types.util as rt_util +from galaxy.util import unicodify from tool_shed.repository_types.metadata import TipOnly -from tool_shed.util import ( - basic_util, - hg_util -) +from tool_shed.util import basic_util log = logging.getLogger(__name__) @@ -17,13 +15,13 @@ class RepositorySuiteDefinition(TipOnly): self.label = 'Repository suite definition' self.valid_file_names = ['repository_dependencies.xml'] - def is_valid_for_type(self, app, repository, revisions_to_check=None): + def is_valid_for_type(self, repository, revisions_to_check=None): """ Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type. If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions in the list. """ - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo if revisions_to_check: changeset_revisions = revisions_to_check else: @@ -34,7 +32,7 @@ class RepositorySuiteDefinition(TipOnly): # is named repository_dependencies.xml. files_changed_in_changeset = ctx.files() for file_path in files_changed_in_changeset: - file_name = basic_util.strip_path(file_path) + file_name = basic_util.strip_path(unicodify(file_path)) if file_name not in self.valid_file_names: return False return True diff --git a/lib/tool_shed/repository_types/tool_dependency_definition.py b/lib/tool_shed/repository_types/tool_dependency_definition.py index 13db77ac12d..52e30e30252 100644 --- a/lib/tool_shed/repository_types/tool_dependency_definition.py +++ b/lib/tool_shed/repository_types/tool_dependency_definition.py @@ -1,11 +1,9 @@ import logging import tool_shed.repository_types.util as rt_util +from galaxy.util import unicodify from tool_shed.repository_types.metadata import TipOnly -from tool_shed.util import ( - basic_util, - hg_util -) +from tool_shed.util import basic_util log = logging.getLogger(__name__) @@ -17,13 +15,13 @@ class ToolDependencyDefinition(TipOnly): self.label = 'Tool dependency definition' self.valid_file_names = ['tool_dependencies.xml'] - def is_valid_for_type(self, app, repository, revisions_to_check=None): + def is_valid_for_type(self, repository, revisions_to_check=None): """ Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type. If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions in the list. """ - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo if revisions_to_check: changeset_revisions = revisions_to_check else: @@ -33,7 +31,7 @@ class ToolDependencyDefinition(TipOnly): # Inspect all files in the changeset (in sorted order) to make sure there is only one and it is named tool_dependencies.xml. files_changed_in_changeset = ctx.files() for file_path in files_changed_in_changeset: - file_name = basic_util.strip_path(file_path) + file_name = basic_util.strip_path(unicodify(file_path)) if file_name not in self.valid_file_names: return False return True diff --git a/lib/tool_shed/repository_types/unrestricted.py b/lib/tool_shed/repository_types/unrestricted.py index 87496cecd90..9ccb55568cc 100644 --- a/lib/tool_shed/repository_types/unrestricted.py +++ b/lib/tool_shed/repository_types/unrestricted.py @@ -12,9 +12,9 @@ class Unrestricted(Metadata): self.type = rt_util.UNRESTRICTED self.label = 'Unrestricted' - def is_valid_for_type(self, app, repository, revisions_to_check=None): + def is_valid_for_type(self, repository, revisions_to_check=None): """A repository's type can only be changed to the unrestricted type if it is new or has never been installed.""" - if repository.is_new(app): + if repository.is_new(): return True if repository.times_downloaded == 0: return True diff --git a/lib/tool_shed/repository_types/util.py b/lib/tool_shed/repository_types/util.py index 974141d496c..2cd7fccc9f5 100644 --- a/lib/tool_shed/repository_types/util.py +++ b/lib/tool_shed/repository_types/util.py @@ -30,7 +30,7 @@ def build_repository_type_select_field(trans, repository=None, name='repository_ if repository: if repository.type == option_value: repository_type_select_field.add_option(option_label, option_value, selected=selected) - elif type_class.is_valid_for_type(trans.app, repository): + elif type_class.is_valid_for_type(repository): repository_type_select_field.add_option(option_label, option_value, selected=selected) else: repository_type_select_field.add_option(option_label, option_value, selected=selected) diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index e678d6ae21d..3786eb2dfa3 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -7,44 +7,37 @@ import string import tempfile import time from json import loads -from xml.etree import ElementTree -# Be sure to use Galaxy's vanilla pyparsing instead of the older version -# imported by twill. -import pyparsing # noqa: F401 import requests -import twill import twill.commands as tc from mercurial import commands, hg, ui -from six import string_types, StringIO from six.moves.urllib.parse import ( quote_plus, urlencode, urlparse ) -from twill.other_packages._mechanize_dist import ClientForm +from twill.utils import ResultWrapper import galaxy.model.tool_shed_install as galaxy_model import galaxy.util -import tool_shed.webapp.util.hgweb_config from galaxy.security import idencoding from galaxy.util import smart_str, unicodify from galaxy_test.base.api_util import get_master_api_key from galaxy_test.driver.testcase import FunctionalTestCase -from tool_shed.util import hg_util, xml_util +from tool_shed.util import ( + hg_util, + hgweb_config, + xml_util, +) from . import common, test_db_util # Set a 10 minute timeout for repository installation. repository_installation_timeout = 600 -# Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? -buffer = StringIO() -twill.set_output(buffer) -tc.config('use_tidy', 0) - # Dial ClientCookie logging down (very noisy) logging.getLogger("ClientCookie.cookies").setLevel(logging.WARNING) log = logging.getLogger(__name__) +tc.options['equiv_refresh_interval'] = 0 class ShedTwillTestCase(FunctionalTestCase): @@ -54,7 +47,7 @@ class ShedTwillTestCase(FunctionalTestCase): self.security = idencoding.IdEncodingHelper(id_secret='changethisinproductiontoo') self.history_id = None self.hgweb_config_dir = os.environ.get('TEST_HG_WEB_CONFIG_DIR') - self.hgweb_config_manager = tool_shed.webapp.util.hgweb_config.HgWebConfigManager() + self.hgweb_config_manager = hgweb_config.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.hgweb_config_dir self.tool_shed_test_tmp_dir = os.environ.get('TOOL_SHED_TEST_TMP_DIR', None) self.host = os.environ.get('TOOL_SHED_TEST_HOST') @@ -124,111 +117,33 @@ class ShedTwillTestCase(FunctionalTestCase): invalid_username = False try: self.check_page_for_string("Created new user account") - except Exception: + except AssertionError: try: # May have created the account in a previous test run... self.check_page_for_string("User with email '%s' already exists." % email) previously_created = True - except Exception: + except AssertionError: try: self.check_page_for_string('Public name is taken; please choose another') username_taken = True - except Exception: + except AssertionError: + # Note that we're only checking if the usr name is >< 4 chars here... try: - # Note that we're only checking if the usr name is >< 4 chars here... self.check_page_for_string('Public name must be at least 4 characters in length') invalid_username = True - except Exception: + except AssertionError: pass return previously_created, username_taken, invalid_username - def get_all_history_ids_from_api(self): - return [history['id'] for history in self.json_from_url('/api/histories')] - - def get_form_controls(self, form): - formcontrols = [] - for i, control in enumerate(form.controls): - formcontrols.append("control %d: %s" % (i, str(control))) - return formcontrols - - def get_hids_in_history(self, history_id): - """Returns the list of hid values for items in a history""" - hids = [] - api_url = '/api/histories/%s/contents' % history_id - hids = [history_item['hid'] for history_item in self.json_from_url(api_url)] - return hids - - def get_history_as_data_list(self, show_deleted=False): - """Returns the data elements of a history""" - tree = self.history_as_xml_tree(show_deleted=show_deleted) - data_list = [elem for elem in tree.findall("data")] - return data_list - - def get_history_from_api(self, encoded_history_id=None, show_deleted=None, show_details=False): - if encoded_history_id is None: - history = self.get_latest_history() - encoded_history_id = history['id'] - params = dict() - if show_deleted is not None: - params['deleted'] = show_deleted - api_url = '/api/histories/%s/contents' % encoded_history_id - json_data = self.json_from_url(api_url, params=params) - if show_deleted is not None: - hdas = [] - for hda in json_data: - if show_deleted: - hdas.append(hda) - else: - if not hda['deleted']: - hdas.append(hda) - json_data = hdas - if show_details: - params['details'] = ','.join(hda['id'] for hda in json_data) - api_url = '/api/histories/%s/contents' % encoded_history_id - json_data = self.json_from_url(api_url, params=params) - return json_data - - def get_latest_history(self): - return self.json_from_url('/api/histories')[0] - - def get_running_datasets(self): - self.visit_url('/api/histories') - history_id = loads(self.last_page())[0]['id'] - self.visit_url('/api/histories/%s' % history_id) - jsondata = loads(self.last_page()) - return jsondata['state'] in ['queued', 'running'] - - def history_as_xml_tree(self, show_deleted=False): - """Returns a parsed xml object of a history""" - params = { - 'as_xml': True, - 'show_deleted': show_deleted - } - self.visit_url('/history', params=params) - xml = self.last_page() - tree = ElementTree.fromstring(xml) - return tree - - def is_history_empty(self): - """ - Uses history page JSON to determine whether this history is empty - (i.e. has no undeleted datasets). - """ - return len(self.get_history_from_api()) == 0 - - def json_from_url(self, url, params={}): - self.visit_url(url, params) - return loads(self.last_page()) - def last_page(self): """ Return the last visited page (usually HTML, but can binary data as well). """ - return tc.browser.get_html() + return tc.browser.html def last_url(self): - return tc.browser.get_url() + return tc.browser.url def login(self, email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='', logout_first=True): # Clear cookies. @@ -248,137 +163,35 @@ class ShedTwillTestCase(FunctionalTestCase): self.submit_form('login', 'login_button', login=email, redirect=redirect, password=password) def logout(self): - self.visit_url("%s/user/logout" % self.url) + self.visit_url("/user/logout") self.check_page_for_string("You have been logged out") - tc.browser.cj.clear() def showforms(self): """Shows form, helpful for debugging new tests""" - return tc.showforms() + return tc.browser.forms - def submit_form(self, form_no=0, button="runtool_btn", **kwd): + def submit_form(self, form_no=0, button="runtool_btn", form=None, **kwd): """Populates and submits a form from the keyword arguments.""" # An HTMLForm contains a sequence of Controls. Supported control classes are: # TextControl, FileControl, ListControl, RadioControl, CheckboxControl, SelectControl, # SubmitControl, ImageControl - for i, f in enumerate(self.showforms()): - if i == form_no: - break - # To help with debugging a tool, print out the form controls when the test fails - print("form '%s' contains the following controls ( note the values )" % f.name) - controls = {} - formcontrols = self.get_form_controls(f) - hc_prefix = ' 30 check below. - control.get(label=elem).selected = True - else: # control.is_of_kind( "singlelist" ) - for elem in control_value: - try: - tc.fv(f.name, control.name, str(elem)) - except Exception: - try: - # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py - if len(elem) > 30: - elem_name = '%s..%s' % (elem[:17], elem[-11:]) - tc.fv(f.name, control.name, str(elem_name)) - pass - else: - raise - except Exception: - raise - except Exception: - for formcontrol in formcontrols: - log.debug(formcontrol) - log.exception("Attempting to set control '%s' to value '%s' (also tried '%s') threw exception.", control.name, elem, elem_name) - pass - except Exception as exc: - for formcontrol in formcontrols: - log.debug(formcontrol) - errmsg = "Attempting to set field '%s' to value '%s' in form '%s' threw exception: %s\n" % (control_name, str(control_value), f.name, str(exc)) - errmsg += "control: %s\n" % str(control) - errmsg += "If the above control is a DataToolparameter whose data type class does not include a sniff() method,\n" - errmsg += "make sure to include a proper 'ftype' attribute to the tag for the control within the tag set.\n" - raise AssertionError(errmsg) + control_type = getattr(control, "type", None) + if control_type in ("text", "textfield", "submit", "password", "TextareaElement", "checkbox", "radio", None): + for cv in control_value: + tc.fv(form_name, control.name, cv) else: # Add conditions for other control types here when necessary. pass @@ -399,15 +212,11 @@ class ShedTwillTestCase(FunctionalTestCase): if params: url += '?%s' % urlencode(params, doseq=doseq) new_url = tc.go(url) - return_code = tc.browser.get_code() + return_code = tc.browser.code assert return_code in allowed_codes, 'Invalid HTTP return code %s, allowed codes: %s' % \ (return_code, ', '.join(str(code) for code in allowed_codes)) return new_url - def wait(self, **kwds): - """Waits for the tools to finish""" - return self.wait_for(lambda: self.get_running_datasets(), **kwds) - def write_temp_file(self, content, suffix='.html'): with tempfile.NamedTemporaryFile(suffix=suffix, prefix='twilltestcase-', delete=False) as fh: fh.write(smart_str(content)) @@ -730,9 +539,6 @@ class ShedTwillTestCase(FunctionalTestCase): if filename in filenames: files_to_delete.append(os.path.join(basepath, filename)) self.browse_repository(repository) - # Twill sets hidden form fields to read-only by default. We need to write to this field. - form = tc.browser.get_form('select_files_to_delete') - form.find_control("selected_files_to_delete").readonly = False tc.fv("2", "selected_files_to_delete", ','.join(files_to_delete)) tc.submit('select_files_to_delete_button') self.check_for_strings(strings_displayed, strings_not_displayed) @@ -941,7 +747,6 @@ class ShedTwillTestCase(FunctionalTestCase): def galaxy_logout(self): self.visit_galaxy_url("/user/logout", params=dict(session_csrf_token=self.galaxy_token())) - tc.browser.cj.clear() def generate_complex_dependency_xml(self, filename, filepath, repository_tuples, package, version): file_path = os.path.join(filepath, filename) @@ -1021,7 +826,7 @@ class ShedTwillTestCase(FunctionalTestCase): return os.path.abspath(os.path.join(self.file_dir, filename)) def get_hg_repo(self, path): - return hg.repository(ui.ui(), path) + return hg.repository(ui.ui(), path.encode('utf-8')) def get_last_reviewed_revision_by_user(self, user, repository): changelog_tuples = self.get_repository_changelog_tuples(repository) @@ -1253,21 +1058,34 @@ class ShedTwillTestCase(FunctionalTestCase): } self.visit_url('/repository/install_repositories_by_revision', params=params) self.check_for_strings(strings_displayed, strings_not_displayed) - # This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to - # be hacked in through tc.browser, putting the form field in kwd doesn't work. - form = tc.browser.get_form('select_tool_panel_section') + form = tc.browser.form('select_tool_panel_section') if form is None: - form = tc.browser.get_form('select_shed_tool_panel_config') + form = tc.browser.form('select_shed_tool_panel_config') assert form is not None, 'Could not find form select_shed_tool_panel_config or select_tool_panel_section.' - kwd = self.set_form_value(form, kwd, 'install_tool_dependencies', install_tool_dependencies) - kwd = self.set_form_value(form, kwd, 'install_repository_dependencies', install_repository_dependencies) - kwd = self.set_form_value(form, kwd, 'install_resolver_dependencies', install_resolver_dependencies) - kwd = self.set_form_value(form, kwd, 'shed_tool_conf', self.shed_tool_conf) + kwds = { + 'install_tool_dependencies': install_tool_dependencies, + 'install_repository_dependencies': install_repository_dependencies, + 'install_resolver_dependencies': install_resolver_dependencies, + 'shed_tool_conf': self.shed_tool_conf, + } if new_tool_panel_section_label is not None: - kwd = self.set_form_value(form, kwd, 'new_tool_panel_section_label', new_tool_panel_section_label) - submit_button_control = form.find_control(type='submit') - assert submit_button_control is not None, 'No submit button found for form %s.' % form.attrs.get('id') - self.submit_form(form.attrs.get('id'), str(submit_button_control.name), **kwd) + kwds['new_tool_panel_section_label'] = new_tool_panel_section_label + submit_button = [inp.name for inp in form.inputs if getattr(inp, 'type', None) == 'submit'] + if len(submit_button) == 0: + # TODO: refactor, use regular TS install API + submit_kwargs = {inp.name: inp.value for inp in tc.browser.forms[0].inputs if getattr(inp, 'type', None) == 'submit'} + payload = {_: form.inputs[_].value for _ in form.fields.keys()} + payload.update(kwds) + payload.update(submit_kwargs) + r = tc.browser._session.post( + self.galaxy_url + form.action, + data=payload, + ) + tc.browser.result = ResultWrapper(r) + else: + assert len(submit_button) == 1, 'Expected to find a single submit button, found %s (%s)' % (len(submit_button), ','.join(submit_button)) + submit_button = submit_button[0] + self.submit_form(form=form, button=submit_button, **kwds) self.check_for_strings(post_submit_strings_displayed, strings_not_displayed) repository_ids = self.initiate_installation_process(new_tool_panel_section_label=new_tool_panel_section_label) log.debug('Waiting for the installation of repository IDs: %s' % str(repository_ids)) @@ -1502,8 +1320,8 @@ class ShedTwillTestCase(FunctionalTestCase): Set the form field field_name to field_value if it exists, and return the provided dict containing that value. If the field does not exist in the provided form, return a dict without that index. ''' - form_id = form.attrs.get('id') - controls = [control for control in form.controls if str(control.name) == field_name] + form_id = form.attrib.get('id') + controls = [control for control in form.inputs if str(control.name) == field_name] if len(controls) > 0: log.debug('Setting field %s of form %s to %s.' % (field_name, form_id, str(field_value))) tc.formvalue(form_id, field_name, str(field_value)) @@ -1532,9 +1350,9 @@ class ShedTwillTestCase(FunctionalTestCase): if changeset_revision is None: changeset_revision = self.get_repository_tip(repository) self.display_manage_repository_page(repository, changeset_revision=changeset_revision) - form = tc.browser.get_form('skip_tool_tests') + form = tc.browser.form('skip_tool_tests') assert form is not None, 'Could not find form skip_tool_tests.' - for control in form.controls: + for control in form.inputs: control_name = str(control.name) if control_name == 'skip_tool_tests' and control.type == 'checkbox': checkbox = control.get() diff --git a/lib/tool_shed/test/functional/test_1400_review_migration_stages.py b/lib/tool_shed/test/functional/test_1400_review_migration_stages.py deleted file mode 100644 index 8413101d43b..00000000000 --- a/lib/tool_shed/test/functional/test_1400_review_migration_stages.py +++ /dev/null @@ -1,41 +0,0 @@ -from ..base.twilltestcase import common, ShedTwillTestCase - - -class TestToolMigrationStages(ShedTwillTestCase): - '''Verify that the migration stages display correctly.''' - - def test_0000_initiate_users(self): - """Create necessary user accounts and login as an admin user.""" - self.login(email=common.admin_email, username=common.admin_username) - admin_user = self.test_db_util.get_user(common.admin_email) - assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - self.test_db_util.get_private_role(admin_user) - self.galaxy_login(email=common.admin_email, username=common.admin_username) - admin_user = self.test_db_util.get_user(common.admin_email) - assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email - self.test_db_util.get_private_role(admin_user) - - def test_0005_load_migration_stages_page(self): - '''Load the migration page and check for the appropriate migration stages.''' - stages = [] - migration_message_strings = ['The Emboss 5.0.0 tools have been eliminated', - 'The freebayes tool has been eliminated', - 'The NCBI BLAST+ tools', - 'The tools "Map with BWA for Illumina"', - 'FASTQ to BAM, SAM to FASTQ, BAM ', - 'Map with Bowtie for Illumina, ', - 'BAM-to-SAM converts BAM format'] - migrated_repository_names = ['emboss_5', 'emboss_datatypes', 'freebayes', 'ncbi_blast_plus', - 'blast_datatypes', 'bwa_wrappers', 'picard', 'lastz', - 'lastz_paired_reads', 'bowtie_color_wrappers', 'bowtie_wrappers', - 'xy_plot', 'bam_to_sam'] - migrated_tool_dependencies = ['emboss', '5.0.0', 'freebayes', '0.9.4_a46483351fd0196637614121868fb5c386612b55', - 'samtools', '0.1.18', 'blast+', '2.2.26+', 'bwa', '0.5.9', 'picard', '1.56.0', - 'lastz', '1.02.00', 'bowtie', '0.12.7', 'FreeBayes requires g++', 'ncurses', 'zlib', - 'blast.ncbi.nlm.nih.gov', 'fastx_toolkit', '0.0.13', 'samtools', '0.1.16', 'cufflinks', - '2.1.1', 'R', '2.11.0'] - migration_scripts = ['0002_tools.sh', '0003_tools.sh', '0004_tools.sh', '0005_tools.sh', '0006_tools.sh', - '0007_tools.sh', '0008_tools.sh'] - stages.extend(migration_scripts + migrated_tool_dependencies + migrated_repository_names) - stages.extend(migration_message_strings) - self.load_galaxy_tool_migrations_page(strings_displayed=stages) diff --git a/lib/tool_shed/test/functional_tests.py b/lib/tool_shed/test/functional_tests.py index 2356a15c469..e77f0465ac9 100644 --- a/lib/tool_shed/test/functional_tests.py +++ b/lib/tool_shed/test/functional_tests.py @@ -95,9 +95,11 @@ class ToolShedTestDriver(driver_util.TestDriver): use_heartbeat=False) kwargs.update(toolshed_database_conf) # Generate the tool_data_table_conf.xml file. - open(default_tool_data_table_config_path, 'w').write(tool_data_table_conf_xml_template) + with open(default_tool_data_table_config_path, 'w') as fh: + fh.write(tool_data_table_conf_xml_template) # Generate the shed_tool_data_table_conf.xml file. - open(shed_tool_data_table_conf_file, 'w').write(tool_data_table_conf_xml_template) + with open(shed_tool_data_table_conf_file, 'w') as fh: + fh.write(tool_data_table_conf_xml_template) os.environ['TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF'] = shed_tool_data_table_conf_file # ---- Build Tool Shed Application -------------------------------------------------- toolshedapp = driver_util.build_shed_app(kwargs) @@ -121,18 +123,22 @@ class ToolShedTestDriver(driver_util.TestDriver): # Generate the shed_tool_conf.xml file. tool_sheds_conf_template_parser = string.Template(tool_sheds_conf_xml_template) tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute(shed_url=tool_shed_test_host, shed_port=tool_shed_test_port) - open(galaxy_tool_sheds_conf_file, 'w').write(tool_sheds_conf_xml) + with open(galaxy_tool_sheds_conf_file, 'w') as fh: + fh.write(tool_sheds_conf_xml) # Generate the tool_sheds_conf.xml file. shed_tool_conf_template_parser = string.Template(shed_tool_conf_xml_template) shed_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute(shed_tool_path=galaxy_shed_tool_path) - open(galaxy_shed_tool_conf_file, 'w').write(shed_tool_conf_xml) + with open(galaxy_shed_tool_conf_file, 'w') as fh: + fh.write(shed_tool_conf_xml) # Generate the migrated_tool_conf.xml file. migrated_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute(shed_tool_path=galaxy_migrated_tool_path) - open(galaxy_migrated_tool_conf_file, 'w').write(migrated_tool_conf_xml) + with open(galaxy_migrated_tool_conf_file, 'w') as fh: + fh.write(migrated_tool_conf_xml) os.environ['GALAXY_TEST_SHED_TOOL_CONF'] = galaxy_shed_tool_conf_file # Generate shed_data_manager_conf.xml if not os.environ.get('GALAXY_SHED_DATA_MANAGER_CONF'): - open(galaxy_shed_data_manager_conf_file, 'wb').write(shed_data_manager_conf_xml_template) + with open(galaxy_shed_data_manager_conf_file, 'w') as fh: + fh.write(shed_data_manager_conf_xml_template) kwargs = dict(migrated_tools_config=galaxy_migrated_tool_conf_file, shed_data_manager_config_file=galaxy_shed_data_manager_conf_file, shed_tool_path=galaxy_shed_tool_path, diff --git a/lib/tool_shed/test/test_data/filtering/readme.txt b/lib/tool_shed/test/test_data/filtering/readme.txt index 291b190d703..ce3ff7330fe 100644 --- a/lib/tool_shed/test/test_data/filtering/readme.txt +++ b/lib/tool_shed/test/test_data/filtering/readme.txt @@ -1,2 +1,2 @@ -These characters should not result in a unicode decoding error: ®ĂââčĂ -.... \ No newline at end of file +These characters should not result in a unicode decoding error: cĂłmplÇcá¸tĂ«d strĩñg +.... diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py index cdef3885b14..f3ed34b80de 100644 --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -5,6 +5,7 @@ import tempfile from galaxy.tool_shed.tools.tool_validator import ToolValidator as GalaxyToolValidator from galaxy.tools import Tool +from galaxy.util import unicodify from tool_shed.util import ( basic_util, hg_util, @@ -29,7 +30,7 @@ class ToolValidator(GalaxyToolValidator): # The file no longer exists on disk, so it must have been deleted at some previous # point in the change log. return False - if changeset_revision == repository.tip(self.app): + if changeset_revision == repository.tip(): return True file_name = basic_util.strip_path(file_path) latest_version_of_file = \ @@ -79,7 +80,7 @@ class ToolValidator(GalaxyToolValidator): for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): manifest_ctx = repo[changeset] for ctx_file in manifest_ctx.files(): - ctx_file_name = basic_util.strip_path(ctx_file) + ctx_file_name = basic_util.strip_path(unicodify(ctx_file)) if ctx_file_name == stripped_filename: try: fctx = manifest_ctx[ctx_file] @@ -111,7 +112,7 @@ class ToolValidator(GalaxyToolValidator): for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): changeset_ctx = repo[changeset] for ctx_file in changeset_ctx.files(): - ctx_file_name = basic_util.strip_path(ctx_file) + ctx_file_name = basic_util.strip_path(unicodify(ctx_file)) # If we decide in the future that files deleted later in the changelog should # not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' ) # and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: @@ -183,7 +184,7 @@ class ToolValidator(GalaxyToolValidator): """ repository = repository_util.get_repository_in_tool_shed(self.app, repository_id) repo_files_dir = repository.repo_path(self.app) - repo = hg_util.get_repo_for_repository(self.app, repo_path=repo_files_dir) + repo = repository.hg_repo tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository(repo_files_dir, tool_config_filename) work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr") can_use_disk_file = self.can_use_tool_config_disk_file(repository, diff --git a/lib/tool_shed/tools/tool_version_manager.py b/lib/tool_shed/tools/tool_version_manager.py index ccb05364533..6abab5dfc80 100644 --- a/lib/tool_shed/tools/tool_version_manager.py +++ b/lib/tool_shed/tools/tool_version_manager.py @@ -38,7 +38,7 @@ class ToolVersionManager(object): is called only from the Tool Shed. """ repository = repository_util.get_repository_by_id(self.app, repository_id) - repo = hg_util.get_repo_for_repository(self.app, repository=repository) + repo = repository.hg_repo # Initialize the tool lineage version_lineage = [guid] # Get all ancestor guids of the received guid. @@ -55,7 +55,7 @@ class ToolVersionManager(object): current_parent_guid = guid for changeset in hg_util.reversed_lower_upper_bounded_changelog(repo, repository_metadata.changeset_revision, - repository.tip(self.app)): + repository.tip()): ctx = repo[changeset] rm = metadata_util.get_repository_metadata_by_changeset_revision(self.app, repository_id, str(ctx)) if rm: diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 64277418093..2f287a9f8eb 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -157,11 +157,11 @@ def handle_bz2(repository, uploaded_file_name): def handle_directory_changes(app, host, username, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed): repo_path = repository.repo_path(app) - repo = hg_util.get_repo_for_repository(app, repo_path=repo_path) content_alert_str = '' files_to_remove = [] filenames_in_archive = [os.path.join(full_path, name) for name in filenames_in_archive] - if remove_repo_files_not_in_tar and not repository.is_new(app): + repo = repository.hg_repo + if remove_repo_files_not_in_tar and not repository.is_new(): # We have a repository that is not new (it contains files), so discover those files that are in the # repository, but not in the uploaded archive. for root, dirs, files in os.walk(full_path): diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index 686f51d1a42..4108c9aab96 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -13,7 +13,6 @@ from galaxy.tool_shed.util.hg_util import ( get_config_from_disk, get_ctx_file_path_from_manifest, get_file_context_from_ctx, - get_repo_for_repository, pull_repository, reversed_lower_upper_bounded_changelog, reversed_upper_bounded_changelog, @@ -93,7 +92,7 @@ def get_named_tmpfile_from_ctx(ctx, filename, dir): """ filename = basic_util.strip_path(filename) for ctx_file in ctx.files(): - ctx_file_name = basic_util.strip_path(ctx_file) + ctx_file_name = basic_util.strip_path(unicodify(ctx_file)) if filename == ctx_file_name: try: # If the file was moved, its destination file contents will be returned here. @@ -140,7 +139,7 @@ def get_revision_label(app, repository, changeset_revision, include_date=True, i Return a string consisting of the human readable changeset rev and the changeset revision string which includes the revision date if the receive include_date is True. """ - repo = get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo ctx = get_changectx_for_changeset(repo, changeset_revision) if ctx: return get_revision_label_from_ctx(ctx, include_date=include_date, include_hash=include_hash) @@ -155,7 +154,7 @@ def get_rev_label_changeset_revision_from_repository_metadata(app, repository_me include_date=True, include_hash=True): if repository is None: repository = repository_metadata.repository - repo = get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo changeset_revision = repository_metadata.changeset_revision ctx = get_changectx_for_changeset(repo, changeset_revision) if ctx: @@ -264,7 +263,6 @@ __all__ = ( 'get_file_context_from_ctx', 'get_named_tmpfile_from_ctx', 'get_readable_ctx_date', - 'get_repo_for_repository', 'get_repository_heads', 'get_reversed_changelog_changesets', 'get_revision_label', diff --git a/lib/tool_shed/webapp/util/hgweb_config.py b/lib/tool_shed/util/hgweb_config.py similarity index 93% rename from lib/tool_shed/webapp/util/hgweb_config.py rename to lib/tool_shed/util/hgweb_config.py index b38a72ff0dc..dcc5ca2b92b 100644 --- a/lib/tool_shed/webapp/util/hgweb_config.py +++ b/lib/tool_shed/util/hgweb_config.py @@ -80,9 +80,8 @@ class HgWebConfigManager(object): # raise Exception( "Required file %s does not exist - check config setting for hgweb_config_dir." % hgweb_config ) # ...but now we just log the missing file and create a new empty one. log.debug("Required file %s does not exist, so creating a new, empty file. Check your config setting for hgweb_config_dir." % hgweb_config) - hgweb_config_file = open(hgweb_config, 'wb') - hgweb_config_file.write(new_hgweb_config_template) - hgweb_config_file.close() + with open(hgweb_config, 'w') as hgweb_config_file: + hgweb_config_file.write(new_hgweb_config_template) return os.path.abspath(hgweb_config) def make_backup(self): @@ -101,6 +100,8 @@ class HgWebConfigManager(object): def write_config(self): """Writing the in-memory configuration to the hgweb.config file on disk.""" - config_file = open(self.hgweb_config, 'wb') - self.in_memory_config.write(config_file) - config_file.close + with open(self.hgweb_config, 'w') as config_file: + self.in_memory_config.write(config_file) + + +hgweb_config_manager = HgWebConfigManager() diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index 7be1eab8373..91a3b0b2881 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -4,7 +4,6 @@ from operator import itemgetter from sqlalchemy import and_ from galaxy.tool_shed.util.hg_util import ( - get_repo_for_repository, INITIAL_CHANGELOG_HASH, reversed_lower_upper_bounded_changelog, ) @@ -77,7 +76,7 @@ def get_dependencies_for_metadata_revision(app, metadata): def get_latest_changeset_revision(app, repository): - repository_tip = repository.tip(app) + repository_tip = repository.tip() repository_metadata = get_repository_metadata_by_changeset_revision(app, app.security.encode_id(repository.id), repository_tip) @@ -90,7 +89,7 @@ def get_latest_changeset_revision(app, repository): def get_latest_downloadable_changeset_revision(app, repository): - repository_tip = repository.tip(app) + repository_tip = repository.tip() repository_metadata = get_repository_metadata_by_changeset_revision(app, app.security.encode_id(repository.id), repository_tip) if repository_metadata and repository_metadata.downloadable: return repository_tip @@ -153,7 +152,7 @@ def get_next_downloadable_changeset_revision(app, repository, after_changeset_re if changeset_revision == after_changeset_revision: return after_changeset_revision found_after_changeset_revision = False - repo = get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo for changeset in repo.changelog: changeset_revision = str(repo[changeset]) if found_after_changeset_revision: @@ -288,7 +287,7 @@ def get_updated_changeset_revisions(app, name, owner, changeset_revision): upper_bound_changeset_revision = get_next_downloadable_changeset_revision(app, repository, changeset_revision) # Build the list of changeset revision hashes defining each available update up to, but excluding # upper_bound_changeset_revision. - repo = get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo changeset_hashes = [] for changeset in reversed_lower_upper_bounded_changelog(repo, changeset_revision, upper_bound_changeset_revision): # Make sure to exclude upper_bound_changeset_revision. diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index 477e33e6de3..927ee061b3c 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -1,3 +1,4 @@ +import io import json import logging import os @@ -6,12 +7,18 @@ from mako.template import Template import tool_shed.util.shed_util_common as suc from galaxy import web -from galaxy.util import rst_to_html, unicodify, url_get -from tool_shed.util import basic_util -from tool_shed.util import common_util -from tool_shed.util import hg_util -from tool_shed.util import metadata_util -from tool_shed.util import repository_util +from galaxy.util import ( + rst_to_html, + unicodify, + url_get, +) +from tool_shed.util import ( + basic_util, + common_util, + hg_util, + metadata_util, + repository_util, +) log = logging.getLogger(__name__) @@ -39,9 +46,8 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ full_path_to_readme_file = os.path.abspath(relative_path_to_readme_file) text = None try: - f = open(full_path_to_readme_file, 'r') - text = unicodify(f.read()) - f.close() + with io.open(full_path_to_readme_file, 'r', encoding='utf-8') as f: + text = f.read() except Exception: log.exception("Error reading README file '%s' from disk", relative_path_to_readme_file) text = None @@ -68,7 +74,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ readme_files_dict[readme_file_name] = text_of_reasonable_length else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) if ctx: fctx = hg_util.get_file_context_from_ctx(ctx, readme_file_name) diff --git a/lib/tool_shed/util/repository_content_util.py b/lib/tool_shed/util/repository_content_util.py index 0050a09fd49..b062ea8e7d0 100644 --- a/lib/tool_shed/util/repository_content_util.py +++ b/lib/tool_shed/util/repository_content_util.py @@ -2,14 +2,15 @@ import os import shutil import tool_shed.repository_types.util as rt_util -from tool_shed.util import commit_util, hg_util, xml_util +from tool_shed.util import ( + commit_util, + xml_util, +) def upload_tar(trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert): # Upload a tar archive of files. - repo_dir = repository.repo_path(trans.app) - hg_util.get_repo_for_repository(trans.app, repo_path=repo_dir) undesirable_dirs_removed = 0 undesirable_files_removed = 0 check_results = commit_util.check_archive(repository, tar) @@ -20,6 +21,8 @@ def upload_tar(trans, rdah, tdah, repository, tar, uploaded_file, upload_point, ' '.join(check_results.errors), ', '.join(check_results.invalid)) return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: + repository.hg_repo + repo_dir = repository.repo_path(trans.app) if upload_point is not None: full_path = os.path.abspath(os.path.join(repo_dir, upload_point)) else: diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 6f6533c2653..091423e223b 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -406,7 +406,7 @@ def change_repository_name_in_hgrc_file(hgrc_file, new_name): config = configparser.ConfigParser() config.read(hgrc_file) config.set('web', 'name', new_name) - with open(hgrc_file, 'wb') as fh: + with open(hgrc_file, 'w') as fh: config.write(fh) diff --git a/lib/tool_shed/util/review_util.py b/lib/tool_shed/util/review_util.py index 992fe09ba7f..876c34f7929 100644 --- a/lib/tool_shed/util/review_util.py +++ b/lib/tool_shed/util/review_util.py @@ -72,7 +72,7 @@ def get_previous_repository_reviews(app, repository, changeset_revision): Return an ordered dictionary of repository reviews up to and including the received changeset revision. """ - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews] previous_reviews_dict = OrderedDict() for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): @@ -122,7 +122,7 @@ def has_previous_repository_reviews(app, repository, changeset_revision): Determine if a repository has a changeset revision review prior to the received changeset revision. """ - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews] for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): previous_changeset_revision = str(repo[changeset]) diff --git a/lib/tool_shed/util/shed_index.py b/lib/tool_shed/util/shed_index.py index 206be3a3ce9..5e537304b61 100644 --- a/lib/tool_shed/util/shed_index.py +++ b/lib/tool_shed/util/shed_index.py @@ -13,10 +13,10 @@ from galaxy.util import ( pretty_print_time_interval, unicodify ) +from tool_shed.util.hgweb_config import hgweb_config_manager from tool_shed.webapp import model from tool_shed.webapp.search.repo_search import schema as repo_schema from tool_shed.webapp.search.tool_search import schema as tool_schema -from tool_shed.webapp.util.hgweb_config import HgWebConfigManager log = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def _get_or_create_index(index_dir, schema): if index.exists_in(index_dir): idx = index.open_dir(index_dir) try: - assert idx.schema == repo_schema + assert idx.schema == schema return idx except AssertionError: log.warning("Index at '%s' uses outdated schema, creating new index", index_dir) @@ -92,7 +92,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): """ Load repos from DB and included tools from .xml configs. """ - hgwcm = HgWebConfigManager() + hgwcm = hgweb_config_manager hgwcm.hgweb_config_dir = hgweb_config_dir # Do not index deleted, deprecated, or "tool_dependency_definition" type repositories. q = sa_session.query(model.Repository).filter_by(deleted=False).filter_by(deprecated=False).order_by(model.Repository.update_time.desc()) @@ -147,20 +147,20 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): tools_in_dir = load_one_dir(os.path.join(root, dirname)) tools_list.extend(tools_in_dir) - yield (dict(id=repo_id, - name=name, - description=description, - long_description=long_description, - homepage_url=homepage_url, - remote_repository_url=remote_repository_url, - repo_owner_username=repo_owner_username, - times_downloaded=times_downloaded, - approved=approved, - last_updated=last_updated, - full_last_updated=full_last_updated, + yield (dict(id=unicodify(repo_id), + name=unicodify(name), + description=unicodify(description), + long_description=unicodify(long_description), + homepage_url=unicodify(homepage_url), + remote_repository_url=unicodify(remote_repository_url), + repo_owner_username=unicodify(repo_owner_username), + times_downloaded=unicodify(times_downloaded), + approved=unicodify(approved), + last_updated=unicodify(last_updated), + full_last_updated=unicodify(full_last_updated), tools_list=tools_list, - repo_lineage=repo_lineage, - categories=categories)) + repo_lineage=unicodify(repo_lineage), + categories=unicodify(categories))) def debug_handler(path, exc_info): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index d67f87ab57d..b9f3f3f1d44 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -250,7 +250,7 @@ def get_repository_from_refresh_on_change(app, **kwd): if k.startswith(changeset_revision_str): repository_id = app.security.encode_id(int(k.lstrip(changeset_revision_str))) repository = repository_util.get_repository_in_tool_shed(app, repository_id) - if repository.tip(app) != v: + if repository.tip() != v: return v, repository # This should never be reached - raise an exception? return v, None @@ -321,7 +321,7 @@ def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_al that was included in the change set. """ sa_session = app.model.context.current - repo = hg_util.get_repo_for_repository(app, repository=repository) + repo = repository.hg_repo sharable_link = repository_util.generate_sharable_link_for_repository_in_tool_shed(repository, changeset_revision=None) smtp_server = app.config.smtp_server if smtp_server and (new_repo_alert or repository.email_alerts): diff --git a/lib/tool_shed/webapp/api/repositories.py b/lib/tool_shed/webapp/api/repositories.py index 396db8b041e..d1bd8291866 100644 --- a/lib/tool_shed/webapp/api/repositories.py +++ b/lib/tool_shed/webapp/api/repositories.py @@ -111,14 +111,15 @@ class RepositoriesController(BaseAPIController): if owner is None: owner = kwd.get('owner', None) tsr_id = kwd.get('tsr_id', None) + eagerload_columns = ['downloadable_revisions'] if None not in [name, owner]: # Get the repository information. - repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner) + repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner, eagerload_columns=eagerload_columns) if repository is None: trans.response.status = 404 return {'status': 'error', 'message': 'No repository named %s found with owner %s' % (name, owner)} elif tsr_id is not None: - repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id) + repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id, eagerload_columns=eagerload_columns) else: error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " error_message += "invalid parameters received." @@ -197,7 +198,7 @@ class RepositoriesController(BaseAPIController): # http:///api/repositories/get_repository_revision_install_info?name=&owner=&changeset_revision= if name and owner and changeset_revision: # Get the repository information. - repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner) + repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner, eagerload_columns=['downloadable_revisions']) if repository is None: log.debug('Cannot locate repository %s owned by %s' % (str(name), str(owner))) return {}, {}, {} @@ -263,7 +264,7 @@ class RepositoriesController(BaseAPIController): # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?tsr_id=9d37e53072ff9fa4 tsr_id = kwd.get('tsr_id', None) if tsr_id is not None: - repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id) + repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id, eagerload_columns=['downloadable_revisions']) else: error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " error_message += "missing or invalid parameter received." @@ -713,16 +714,17 @@ class RepositoriesController(BaseAPIController): owner = kwd.get('owner', None) changeset_revision = kwd.get('changeset_revision', None) hexlify_this = util.asbool(kwd.get('hexlify', True)) - repository = repository_util.get_repository_by_name_and_owner(trans.app, name, owner) - if repository: + repository = repository_util.get_repository_by_name_and_owner(trans.app, name, owner, eagerload_columns=['downloadable_revisions']) + if repository and repository.downloadable_revisions: repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, trans.security.encode_id(repository.id), changeset_revision) tool_shed_status_dict = {} # Handle repository deprecation. tool_shed_status_dict['repository_deprecated'] = str(repository.deprecated) + tip_revision = repository.downloadable_revisions[0] # Handle latest installable revision. - if changeset_revision == repository.tip(trans.app): + if changeset_revision == tip_revision: tool_shed_status_dict['latest_installable_revision'] = 'True' else: next_installable_revision = metadata_util.get_next_downloadable_changeset_revision(trans.app, repository, changeset_revision) @@ -737,7 +739,7 @@ class RepositoriesController(BaseAPIController): else: tool_shed_status_dict['latest_installable_revision'] = 'True' # Handle revision updates. - if changeset_revision == repository.tip(trans.app): + if changeset_revision == tip_revision: tool_shed_status_dict['revision_update'] = 'False' else: if repository_metadata is None: @@ -801,7 +803,7 @@ class RepositoriesController(BaseAPIController): raise MalformedId('The given id is invalid.') recursive = util.asbool(kwd.get('recursive', 'True')) all_metadata = {} - repository = repository_util.get_repository_in_tool_shed(self.app, id) + repository = repository_util.get_repository_in_tool_shed(self.app, id, eagerload_columns=['downloadable_revisions']) for changeset, changehash in repository.installable_revisions(self.app): metadata = metadata_util.get_current_repository_metadata_for_changeset_revision(self.app, repository, changehash) if metadata is None: @@ -975,7 +977,7 @@ class RepositoriesController(BaseAPIController): repo_dir = repository.repo_path(self.app) upload_point = commit_util.get_upload_point(repository, **kwd) - tip = repository.tip(self.app) + tip = repository.tip() file_data = payload.get('file') # Code stolen from gx's upload_common.py @@ -1025,7 +1027,7 @@ class RepositoriesController(BaseAPIController): # Update the repository files for browsing. hg_util.update_repository(repo_dir) # Get the new repository tip. - if tip == repository.tip(self.app): + if tip == repository.tip(): trans.response.status = 400 message = 'No changes to repository.' ok = False diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index c98ff24dc55..77db4e3e236 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -15,6 +15,7 @@ from galaxy.security import idencoding from galaxy.util.dbkeys import GenomeBuilds from galaxy.web_stack import application_stack_instance from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager +from tool_shed.util.hgweb_config import hgweb_config_manager from . import config log = logging.getLogger(__name__) @@ -73,7 +74,7 @@ class UniverseApplication(object): # Initialize the baseline Tool Shed statistics component. self.shed_counter = self.model.shed_counter # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. - self.hgweb_config_manager = self.model.hgweb_config_manager + self.hgweb_config_manager = hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) diff --git a/lib/tool_shed/webapp/config.py b/lib/tool_shed/webapp/config.py index dc4292c0a7e..40572c13d18 100644 --- a/lib/tool_shed/webapp/config.py +++ b/lib/tool_shed/webapp/config.py @@ -4,12 +4,15 @@ Universe configuration builder. import logging import logging.config import os -import re from datetime import timedelta from six.moves import configparser -from galaxy.config import BaseAppConfiguration +from galaxy.config import ( + BaseAppConfiguration, + CommonConfigurationMixin, + get_database_engine_options, +) from galaxy.config.schema import AppSchema from galaxy.util import string_as_bool from galaxy.version import VERSION, VERSION_MAJOR @@ -24,37 +27,18 @@ TOOLSHED_APP_NAME = 'tool_shed' TOOLSHED_CONFIG_SCHEMA_PATH = 'lib/tool_shed/webapp/config_schema.yml' -def resolve_path(path, root): - """If 'path' is relative make absolute by prepending 'root'""" - if not(os.path.isabs(path)): - path = os.path.join(root, path) - return path - - class ConfigurationError(Exception): pass -class ToolShedAppConfiguration(BaseAppConfiguration): +class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): default_config_file_name = 'tool_shed.yml' def _load_schema(self): return AppSchema(TOOLSHED_CONFIG_SCHEMA_PATH, TOOLSHED_APP_NAME) - def _update_raw_config_from_kwargs(self, kwargs): - pass # Intentionally does nothing. To enable, remove + call base __init__(). - - def _create_attributes_from_raw_config(self): - pass # Intentionally does nothing. To enable, remove + call base __init__(). - - def _resolve_paths(self, paths_to_resolve): - pass # Intentionally does nothing. To enable, remove + call base __init__(). - def __init__(self, **kwargs): - self.config_dict = kwargs - self.root = kwargs.get('root_dir', '.') - - self._set_config_base(kwargs) + super(ToolShedAppConfiguration, self).__init__(**kwargs) # Resolve paths of other config files self.parse_config_file_options(kwargs) @@ -66,50 +50,29 @@ class ToolShedAppConfiguration(BaseAppConfiguration): self.version_major = VERSION_MAJOR self.version = VERSION # Database related configuration - self.database_connection = kwargs.get("database_connection", False) - self.database_connection = kwargs.get("database_connection", - "sqlite:///%s?isolation_level=IMMEDIATE" % resolve_path("community.sqlite", self.data_dir)) + if not self.database_connection: # Provide default if not supplied by user + self.database_connection = 'sqlite:///%s?isolation_level=IMMEDIATE' % self._in_data_dir('community.sqlite') self.database_engine_options = get_database_engine_options(kwargs) self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True")) - # Repository and Tool search API - self.toolshed_search_on = string_as_bool(kwargs.get("toolshed_search_on", True)) - self.whoosh_index_dir = kwargs.get("whoosh_index_dir", 'database/toolshed_whoosh_indexes') - self.repo_name_boost = kwargs.get("repo_name_boost", 0.9) - self.repo_description_boost = kwargs.get("repo_description_boost", 0.6) - self.repo_long_description_boost = kwargs.get("repo_long_description_boost", 0.5) - self.repo_homepage_url_boost = kwargs.get("repo_homepage_url_boost", 0.3) - self.repo_remote_repository_url_boost = kwargs.get("repo_remote_repository_url_boost", 0.2) - self.repo_owner_username_boost = kwargs.get("repo_owner_username_boost", 0.3) - self.tool_name_boost = kwargs.get("tool_name_boost", 1.2) - self.tool_description_boost = kwargs.get("tool_description_boost", 0.6) - self.tool_help_boost = kwargs.get("tool_help_boost", 0.4) - self.tool_repo_owner_username = kwargs.get("tool_repo_owner_username", 0.3) - # Analytics - self.ga_code = kwargs.get("ga_code", None) - self.session_duration = int(kwargs.get('session_duration', 0)) # Where dataset files are stored - self.file_path = resolve_path(kwargs.get("file_path", "database/community_files"), self.root) - self.new_file_path = resolve_path(kwargs.get("new_file_path", "database/tmp"), self.root) + self.file_path = self._in_root_dir(self.file_path) + self.new_file_path = self._in_root_dir(self.new_file_path) self.cookie_path = kwargs.get("cookie_path", None) self.cookie_domain = kwargs.get("cookie_domain", None) self.enable_quotas = string_as_bool(kwargs.get('enable_quotas', False)) - self.id_secret = kwargs.get("id_secret", "changethisinproductiontoo") # Tool stuff - self.tool_path = resolve_path(kwargs.get("tool_path", "tools"), self.root) + self.tool_path = self._in_root_dir(kwargs.get("tool_path", "tools")) self.tool_secret = kwargs.get("tool_secret", "") - self.tool_data_path = resolve_path(kwargs.get("tool_data_path", "shed-tool-data"), os.getcwd()) + self.tool_data_path = os.path.join(os.getcwd(), kwargs.get("tool_data_path", "shed-tool-data")) self.tool_data_table_config_path = None - self.integrated_tool_panel_config = resolve_path(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.root) - self.builds_file_path = resolve_path(kwargs.get("builds_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'builds.txt')), self.root) - self.len_file_path = resolve_path(kwargs.get("len_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'chrom')), self.root) + self.integrated_tool_panel_config = self._in_root_dir(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml')) + self.builds_file_path = self._in_root_dir(kwargs.get("builds_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'builds.txt'))) + self.len_file_path = self._in_root_dir(kwargs.get("len_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'chrom'))) self.ftp_upload_dir = kwargs.get('ftp_upload_dir', None) self.update_integrated_tool_panel = False # Galaxy flavor Docker Image - self.enable_galaxy_flavor_docker_image = string_as_bool(kwargs.get("enable_galaxy_flavor_docker_image", "False")) - self.use_remote_user = string_as_bool(kwargs.get("use_remote_user", "False")) self.user_activation_on = None self.registration_warning_message = kwargs.get('registration_warning_message', None) - self.terms_url = kwargs.get('terms_url', None) self.blacklist_location = kwargs.get('blacklist_file', None) self.blacklist_content = None self.whitelist_location = kwargs.get('whitelist_file', None) @@ -118,41 +81,25 @@ class ToolShedAppConfiguration(BaseAppConfiguration): self.remote_user_header = kwargs.get("remote_user_header", 'HTTP_REMOTE_USER') self.remote_user_logout_href = kwargs.get("remote_user_logout_href", None) self.remote_user_secret = kwargs.get("remote_user_secret", None) - self.require_login = string_as_bool(kwargs.get("require_login", "False")) - self.allow_user_creation = string_as_bool(kwargs.get("allow_user_creation", "True")) - self.allow_user_deletion = string_as_bool(kwargs.get("allow_user_deletion", "False")) self.template_path = templates_path - self.template_cache_path = resolve_path(kwargs.get("template_cache_path", "database/compiled_templates/community"), self.root) - self.admin_users = kwargs.get("admin_users", "") - self.admin_users_list = [u.strip() for u in self.admin_users.split(',') if u] - self.mailing_join_addr = kwargs.get('mailing_join_addr', "galaxy-announce-join@bx.psu.edu") + self.template_cache_path = self._in_root_dir(kwargs.get("template_cache_path", "database/compiled_templates/community")) self.error_email_to = kwargs.get('error_email_to', None) self.smtp_server = kwargs.get('smtp_server', None) - self.smtp_username = kwargs.get('smtp_username', None) - self.smtp_password = kwargs.get('smtp_password', None) self.smtp_ssl = kwargs.get('smtp_ssl', None) self.email_from = kwargs.get('email_from', None) self.nginx_upload_path = kwargs.get('nginx_upload_path', False) self.log_actions = string_as_bool(kwargs.get('log_actions', 'False')) - self.brand = kwargs.get('brand', None) - self.pretty_datetime_format = expand_pretty_datetime_format(kwargs.get('pretty_datetime_format', '$locale (UTC)')) + self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format) # Configuration for the message box directly below the masthead. - self.message_box_visible = string_as_bool(kwargs.get('message_box_visible', False)) - self.message_box_content = kwargs.get('message_box_content', None) - self.message_box_class = kwargs.get('message_box_class', 'info') - self.support_url = kwargs.get('support_url', 'https://galaxyproject.org/support') self.wiki_url = kwargs.get('wiki_url', 'https://galaxyproject.org/') self.blog_url = kwargs.get('blog_url', None) self.screencasts_url = kwargs.get('screencasts_url', None) self.log_events = False self.cloud_controller_instance = False self.server_name = '' - # Error logging with sentry - self.sentry_dsn = kwargs.get('sentry_dsn', None) # Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory. - self.hgweb_config_dir = resolve_path(kwargs.get('hgweb_config_dir', ''), self.root) + self.hgweb_config_dir = self._in_root_dir(self.hgweb_config_dir) or self.root # Proxy features - self.apache_xsendfile = kwargs.get('apache_xsendfile', False) self.nginx_x_accel_redirect_base = kwargs.get('nginx_x_accel_redirect_base', False) self.drmaa_external_runjob_script = kwargs.get('drmaa_external_runjob_script', None) # Parse global_conf and save the parser @@ -163,8 +110,8 @@ class ToolShedAppConfiguration(BaseAppConfiguration): global_conf_parser.read(global_conf['__file__']) self.running_functional_tests = string_as_bool(kwargs.get('running_functional_tests', False)) self.citation_cache_type = kwargs.get("citation_cache_type", "file") - self.citation_cache_data_dir = resolve_path(kwargs.get("citation_cache_data_dir", "database/tool_shed_citations/data"), self.root) - self.citation_cache_lock_dir = resolve_path(kwargs.get("citation_cache_lock_dir", "database/tool_shed_citations/locks"), self.root) + self.citation_cache_data_dir = self._in_root_dir(kwargs.get("citation_cache_data_dir", "database/tool_shed_citations/data")) + self.citation_cache_lock_dir = self._in_root_dir(kwargs.get("citation_cache_lock_dir", "database/tool_shed_citations/locks")) self.password_expiration_period = timedelta(days=int(kwargs.get("password_expiration_period", 0))) # Security/Policy Compliance @@ -182,19 +129,6 @@ class ToolShedAppConfiguration(BaseAppConfiguration): def shed_tool_data_path(self): return self.tool_data_path - @property - def sentry_dsn_public(self): - """ - Sentry URL with private key removed for use in client side scripts, - sentry server will need to be configured to accept events - """ - # TODO refactor this to a common place between toolshed/galaxy config, along - # with other duplicated methods. - if self.sentry_dsn: - return re.sub(r"^([^:/?#]+:)?//(\w+):(\w+)", r"\1//\2", self.sentry_dsn) - else: - return None - def parse_config_file_options(self, kwargs): defaults = dict( auth_config_file=[self._in_config_dir('config/auth_conf.xml')], @@ -207,18 +141,9 @@ class ToolShedAppConfiguration(BaseAppConfiguration): # Backwards compatibility for names used in too many places to fix self.datatypes_config = self.datatypes_config_file - def get(self, key, default=None): - return self.config_dict.get(key, default) - - def get_bool(self, key, default): - if key in self.config_dict: - return string_as_bool(self.config_dict[key]) - else: - return default - def check(self): # Check that required directories exist. - paths_to_check = [self.root, self.file_path, self.hgweb_config_dir, self.tool_data_path, self.template_path] + paths_to_check = [self.file_path, self.hgweb_config_dir, self.tool_data_path, self.template_path] for path in paths_to_check: if path not in [None, False] and not os.path.isdir(path): try: @@ -238,40 +163,5 @@ class ToolShedAppConfiguration(BaseAppConfiguration): if not os.path.isfile(self.datatypes_config): raise ConfigurationError("File not found: %s" % self.datatypes_config) - def is_admin_user(self, user): - """ - Determine if the provided user is listed in `admin_users`. - """ - admin_users = self.get("admin_users", "").split(",") - return user is not None and user.email in admin_users - Configuration = ToolShedAppConfiguration - - -def get_database_engine_options(kwargs): - """ - Allow options for the SQLAlchemy database engine to be passed by using - the prefix "database_engine_option". - """ - conversions = { - 'convert_unicode': string_as_bool, - 'pool_timeout': int, - 'echo': string_as_bool, - 'echo_pool': string_as_bool, - 'pool_recycle': int, - 'pool_size': int, - 'max_overflow': int, - 'pool_threadlocal': string_as_bool, - 'server_side_cursors': string_as_bool - } - prefix = "database_engine_option_" - prefix_len = len(prefix) - rval = {} - for key, value in kwargs.items(): - if key.startswith(prefix): - key = key[prefix_len:] - if key in conversions: - value = conversions[key](value) - rval[key] = value - return rval diff --git a/lib/tool_shed/webapp/controllers/admin.py b/lib/tool_shed/webapp/controllers/admin.py index a5e2834d092..0bb900a4aea 100644 --- a/lib/tool_shed/webapp/controllers/admin.py +++ b/lib/tool_shed/webapp/controllers/admin.py @@ -95,7 +95,7 @@ class AdminController(BaseUIController, Admin): if k.startswith(changeset_revision_str): repository_id = trans.security.encode_id(int(k.lstrip(changeset_revision_str))) repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) - if repository.tip(trans.app) != v: + if repository.tip() != v: return trans.response.send_redirect(web.url_for(controller='repository', action='browse_repositories', operation='view_or_manage_repository', diff --git a/lib/tool_shed/webapp/controllers/hg.py b/lib/tool_shed/webapp/controllers/hg.py index 1501550e138..89abf667786 100644 --- a/lib/tool_shed/webapp/controllers/hg.py +++ b/lib/tool_shed/webapp/controllers/hg.py @@ -19,7 +19,7 @@ class HgController(BaseUIController): cmd = kwd.get('cmd', None) def make_web_app(): - hgwebapp = hgwebdir(hgweb_config) + hgwebapp = hgwebdir(hgweb_config.encode('utf-8')) return hgwebapp wsgi_app = wsgiapplication(make_web_app) if cmd == 'getbundle': diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index a24d866d413..58f940c272c 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -61,7 +61,7 @@ def get_mercurial_default_options_dict(command): possible = possible[0] if len(possible) != 1: raise Exception('unable to find mercurial command "%s"' % command) - return dict((r[1].replace('-', '_'), r[2]) for r in next(iter(possible.values()))[1][1]) + return dict((r[1].replace(b'-', b'_'), r[2]) for r in next(iter(possible.values()))[1][1]) class RepositoryController(BaseUIController, ratings_util.ItemRatings): @@ -436,7 +436,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): repo_path = repository.repo_path(trans.app) # Update repository files for browsing. hg_util.update_repository(repo_path) - changeset_revision = repository.tip(trans.app) + changeset_revision = repository.tip() metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision(trans.app, id, changeset_revision, @@ -606,7 +606,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): owner = kwd.get('owner', None) changeset_revision = kwd.get('changeset_revision', None) repository = repository_util.get_repository_by_name_and_owner(trans.app, name, owner) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo # Default to the current changeset revision. update_to_ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) latest_changeset_revision = changeset_revision @@ -625,7 +625,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = 'Unable to check for updates due to an invalid Galaxy URL: %s. ' % galaxy_url message += 'You may need to enable third-party cookies in your browser. ' return trans.show_error_message(message) - if changeset_revision == repository.tip(trans.app): + if changeset_revision == repository.tip(): # If changeset_revision is the repository tip, there are no additional updates. if from_update_manager: return no_update @@ -649,7 +649,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): changeset_hash = str(repo[changeset]) hg_util.get_changectx_for_changeset(repo, changeset_hash) if update_to_changeset_hash: - if changeset_hash == repository.tip(trans.app): + if changeset_hash == repository.tip(): update_to_ctx = hg_util.get_changectx_for_changeset(repo, changeset_hash) latest_changeset_revision = changeset_hash break @@ -684,7 +684,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): repository = repository_util.get_repository_in_tool_shed(trans.app, id) metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision(trans.app, id, - repository.tip(trans.app), + repository.tip(), metadata_only=True) if trans.user and trans.user.email: return trans.fill_template("/webapps/tool_shed/repository/contact_owner.mako", @@ -1084,7 +1084,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): has_repository_dependencies_only_if_compiling_contained_td = \ has_galaxy_utilities_dict['has_repository_dependencies_only_if_compiling_contained_td'] includes_workflows = has_galaxy_utilities_dict['includes_workflows'] - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo # Default to the received changeset revision and ctx_rev. update_to_ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) ctx_rev = str(update_to_ctx.rev()) @@ -1099,7 +1099,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): has_repository_dependencies=has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td=has_repository_dependencies_only_if_compiling_contained_td, includes_workflows=includes_workflows) - if changeset_revision == repository.tip(trans.app): + if changeset_revision == repository.tip(): # If changeset_revision is the repository tip, there are no additional updates. return encoding_util.tool_shed_encode(update_dict) else: @@ -1130,7 +1130,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): has_repository_dependencies_only_if_compiling_contained_td = has_galaxy_utilities_dict['has_repository_dependencies_only_if_compiling_contained_td'] includes_workflows = has_galaxy_utilities_dict['includes_workflows'] # We found a RepositoryMetadata record. - if changeset_hash == repository.tip(trans.app): + if changeset_hash == repository.tip(): # The current ctx is the repository tip, so use it. update_to_ctx = hg_util.get_changectx_for_changeset(repo, changeset_hash) latest_changeset_revision = changeset_hash @@ -1160,7 +1160,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): repository_owner = kwd['owner'] changeset_revision = kwd['changeset_revision'] repository = repository_util.get_repository_by_name_and_owner(trans.app, repository_name, repository_owner) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) if ctx: return str(ctx.rev()) @@ -1354,7 +1354,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): repository_owner = kwd['owner'] repository = repository_util.get_repository_by_name_and_owner(trans.app, repository_name, repository_owner) encoded_id = trans.app.security.encode_id(repository.id) - repository_tip = repository.tip(trans.app) + repository_tip = repository.tip() repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, encoded_id, repository_tip) @@ -1370,7 +1370,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): owner = kwd['owner'] changeset_revision = kwd['changeset_revision'] repository = repository_util.get_repository_by_name_and_owner(trans.app, name, owner) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo tool_version_dicts = [] for changeset in repo.changelog: current_changeset_revision = str(repo[changeset]) @@ -1643,9 +1643,9 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, id) repository_type = kwd.get('repository_type', str(repository.type)) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo repo_name = kwd.get('repo_name', repository.name) - changeset_revision = kwd.get('changeset_revision', repository.tip(trans.app)) + changeset_revision = kwd.get('changeset_revision', repository.tip()) repository.share_url = repository_util.generate_sharable_link_for_repository_in_tool_shed(repository, changeset_revision=changeset_revision) repository.clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(trans.user, repository) remote_repository_url = kwd.get('remote_repository_url', repository.remote_repository_url) @@ -1708,7 +1708,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): user = trans.sa_session.query(trans.model.User).get(trans.security.decode_id(user_id)) usernames.append(user.username) usernames = ','.join(usernames) - repository.set_allow_push(trans.app, usernames, remove_auth=remove_auth) + repository.set_allow_push(usernames, remove_auth=remove_auth) message = "The repository information has been updated." elif kwd.get('receive_email_alerts_button', False): flush_needed = False @@ -1730,7 +1730,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): status = 'error' allow_push_select_field = SelectField(name='allow_push', multiple=True) - current_allow_push = repository.allow_push(trans.app) + current_allow_push = repository.allow_push() if current_allow_push: current_allow_push_list = current_allow_push.split(',') else: @@ -1749,7 +1749,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): selected_value=changeset_revision, add_id_to_name=False, downloadable=False) - revision_label = hg_util.get_revision_label(trans.app, repository, repository.tip(trans.app), include_date=False) + revision_label = hg_util.get_revision_label(trans.app, repository, repository.tip(), include_date=False) repository_metadata = None metadata = None is_malicious = False @@ -1854,7 +1854,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, id) - changeset_revision = kwd.get('changeset_revision', repository.tip(trans.app)) + changeset_revision = kwd.get('changeset_revision', repository.tip()) metadata = None if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, id, changeset_revision) @@ -1953,7 +1953,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) - changeset_revision = kwd.get('changeset_revision', repository.tip(trans.app)) + changeset_revision = kwd.get('changeset_revision', repository.tip()) self.validate_changeset_revision(trans, changeset_revision, repository_id) repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_id, changeset_revision) if repository_metadata: @@ -2032,11 +2032,11 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): repository = repository_util.get_repository_by_name_and_owner(trans.app, name, owner) from_tip = util.string_as_bool(from_tip) if from_tip: - changeset_revision = repository.tip(trans.app) + changeset_revision = repository.tip() else: changeset_revision = kwd.get('changeset_revision', None) if changeset_revision is not None: - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo # Get the lower bound changeset revision. lower_bound_changeset_revision = \ metadata_util.get_previous_metadata_changeset_revision(trans.app, repository, changeset_revision, downloadable=True) @@ -2064,7 +2064,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message='Select a repository to rate', status='error')) repository = repository_util.get_repository_in_tool_shed(trans.app, id) - changeset_revision = repository.tip(trans.app) + changeset_revision = repository.tip() if repository.user == trans.user: return trans.response.send_redirect(web.url_for(controller='repository', action='browse_repositories', @@ -2146,13 +2146,13 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): commit_message = escape(kwd.get('commit_message', 'Deleted selected files')) repository = repository_util.get_repository_in_tool_shed(trans.app, id) repo_dir = repository.repo_path(trans.app) - repo = hg_util.get_repo_for_repository(trans.app, repo_path=repo_dir) + repo = repository.hg_repo selected_files_to_delete = kwd.get('selected_files_to_delete', '') if kwd.get('select_files_to_delete_button', False): if selected_files_to_delete: selected_files_to_delete = selected_files_to_delete.split(',') # Get the current repository tip. - tip = repository.tip(trans.app) + tip = repository.tip() for selected_file in selected_files_to_delete: try: hg_util.remove_file(repo_dir, selected_file, force=True) @@ -2189,7 +2189,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): # Update the repository files for browsing. hg_util.update_repository(repo_dir) # Get the new repository tip. - if tip == repository.tip(trans.app): + if tip == repository.tip(): message += 'No changes to repository. ' else: rmm = repository_metadata_manager.RepositoryMetadataManager(app=trans.app, @@ -2204,7 +2204,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = "Select at least 1 file to delete from the repository before clicking Delete selected files." status = "error" repository_type_select_field = rt_util.build_repository_type_select_field(trans, repository=repository) - changeset_revision = repository.tip(trans.app) + changeset_revision = repository.tip() metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision(trans.app, id, changeset_revision, @@ -2429,7 +2429,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo changesets = [] for changeset in repo.changelog: ctx = repo[changeset] @@ -2450,7 +2450,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): changesets.insert(0, change_dict) metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision(trans.app, id, - repository.tip(trans.app), + repository.tip(), metadata_only=True) return trans.fill_template('/webapps/tool_shed/repository/view_changelog.mako', repository=repository, @@ -2464,7 +2464,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo ctx = hg_util.get_changectx_for_changeset(repo, ctx_str) if ctx is None: message = "Repository does not include changeset revision '%s'." % str(ctx_str) @@ -2480,9 +2480,10 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): else: ctx_child = None diffs = [] - options_dict = get_mercurial_default_options_dict('diff') + options_dict = get_mercurial_default_options_dict(b"diff") # Not quite sure if the following settings make any difference, but with a combination of them and the size check on each # diff, we don't run out of memory when viewing the changelog of the cisortho2 repository on the test tool shed. + options_dict = {util.unicodify(k): util.unicodify(v) for k, v in options_dict.items()} options_dict['maxfile'] = basic_util.MAXDIFFSIZE options_dict['maxtotal'] = basic_util.MAXDIFFSIZE diffopts = mdiff.diffopts(**options_dict) @@ -2492,6 +2493,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): diffs.append(basic_util.to_html_string(diff)) modified, added, removed, deleted, unknown, ignored, clean = repo.status(node1=ctx_parent.node(), node2=ctx.node()) anchors = modified + added + removed + deleted + unknown + ignored + clean + anchors = util.unicodify(anchors) metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision(trans.app, id, ctx_str, @@ -2556,9 +2558,9 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo avg_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, repository, webapp_model=trans.model) - changeset_revision = kwd.get('changeset_revision', repository.tip(trans.app)) + changeset_revision = kwd.get('changeset_revision', repository.tip()) self.validate_changeset_revision(trans, changeset_revision, id) repository.share_url = repository_util.generate_sharable_link_for_repository_in_tool_shed(repository, changeset_revision=changeset_revision) repository.clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(trans.user, repository) @@ -2610,7 +2612,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): status = 'warning' else: metadata = None - is_malicious = metadata_util.is_malicious(trans.app, id, repository.tip(trans.app)) + is_malicious = metadata_util.is_malicious(trans.app, id, repository.tip()) if is_malicious: if trans.app.security_agent.can_push(trans.app, trans.user, repository): message += malicious_error_can_push @@ -2649,7 +2651,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed') repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) repo_files_dir = repository.repo_path(trans.app) - repo = hg_util.get_repo_for_repository(trans.app, repo_path=repo_files_dir) + repo = repository.hg_repo tool_metadata_dict = {} tool_lineage = [] tool = None @@ -2725,7 +2727,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): """In case changeset revision is invalid send them to the repository page""" if changeset_revision: repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo if not hg_util.get_changectx_for_changeset(repo, changeset_revision): message = 'Invalid changeset revision' return trans.response.send_redirect(web.url_for(controller='repository', diff --git a/lib/tool_shed/webapp/controllers/repository_review.py b/lib/tool_shed/webapp/controllers/repository_review.py index 779c1f31353..11769086b93 100644 --- a/lib/tool_shed/webapp/controllers/repository_review.py +++ b/lib/tool_shed/webapp/controllers/repository_review.py @@ -82,7 +82,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): status = kwd.get('status', 'done') review = review_util.get_review(trans.app, kwd['id']) repository = review.repository - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(repo, review.changeset_revision) return trans.fill_template('/webapps/tool_shed/repository_review/browse_review.mako', repository=repository, @@ -244,7 +244,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): for component in review_util.get_components(trans.app): components_dict[component.name] = dict(component=component, component_review=None) repository = review.repository - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo for component_review in review.component_reviews: if component_review and component_review.component: component_name = component_review.component.name @@ -483,7 +483,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): repository_id = kwd.get('id', None) if repository_id: repository = suc.get_repository_in_tool_shed(trans.app, repository_id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo metadata_revision_hashes = [metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions] reviewed_revision_hashes = [review.changeset_revision for review in repository.reviews] reviews_dict = OrderedDict() @@ -531,7 +531,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): repository_id = kwd.get('id', None) changeset_revision = kwd.get('changeset_revision', None) repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo installable = changeset_revision in [metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions] rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(repo, changeset_revision) reviews = review_util.get_reviews_by_repository_id_changeset_revision(trans.app, @@ -594,7 +594,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings): status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, kwd['id']) changeset_revision = kwd.get('changeset_revision', None) - repo = hg_util.get_repo_for_repository(trans.app, repository=repository) + repo = repository.hg_repo previous_reviews_dict = review_util.get_previous_repository_reviews(trans.app, repository, changeset_revision) diff --git a/lib/tool_shed/webapp/controllers/upload.py b/lib/tool_shed/webapp/controllers/upload.py index 8043b12a983..0a972b62e5b 100644 --- a/lib/tool_shed/webapp/controllers/upload.py +++ b/lib/tool_shed/webapp/controllers/upload.py @@ -46,13 +46,13 @@ class UploadController(BaseUIController): remove_repo_files_not_in_tar = util.string_as_bool(kwd.get('remove_repo_files_not_in_tar', 'true')) uploaded_file = None upload_point = commit_util.get_upload_point(repository, **kwd) - tip = repository.tip(trans.app) + tip = repository.tip() file_data = kwd.get('file_data', '') url = kwd.get('url', '') # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. - new_repo_alert = repository.is_new(trans.app) + new_repo_alert = repository.is_new() uploaded_directory = None if kwd.get('upload_button', False): if file_data == '' and url == '': @@ -231,7 +231,7 @@ class UploadController(BaseUIController): # Update the repository files for browsing. hg_util.update_repository(repo_dir) # Get the new repository tip. - if tip == repository.tip(trans.app): + if tip == repository.tip(): message = 'No changes to repository. ' status = 'warning' else: diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index 78461817067..d1981ed661a 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -1,10 +1,16 @@ import logging import os +import weakref from datetime import ( datetime, timedelta ) +from mercurial import ( + hg, + ui +) + import tool_shed.repository_types.util as rt_util from galaxy import util from galaxy.model.orm.now import now @@ -17,9 +23,12 @@ from tool_shed.util import ( hg_util, metadata_util ) +from tool_shed.util.hgweb_config import hgweb_config_manager log = logging.getLogger(__name__) +WEAK_HG_REPO_CACHE = weakref.WeakKeyDictionary() + class APIKeys(object): pass @@ -191,6 +200,12 @@ class Repository(Dictifiable): self.deprecated = deprecated self.create_time = create_time + @property + def hg_repo(self): + if not WEAK_HG_REPO_CACHE.get(self): + WEAK_HG_REPO_CACHE[self] = hg.cachedlocalrepo(hg.repository(ui.ui(), self.repo_path().encode('utf-8'))) + return WEAK_HG_REPO_CACHE[self].fetch()[0] + @property def admin_role(self): admin_role_name = '%s_%s_admin' % (str(self.name), str(self.user.username)) @@ -201,14 +216,18 @@ class Repository(Dictifiable): raise Exception('Repository %s owned by %s is not associated with a required administrative role.' % (str(self.name), str(self.user.username))) - def allow_push(self, app): - repo = hg_util.get_repo_for_repository(app, repository=self) - return repo.ui.config('web', 'allow_push') + def allow_push(self): + hgrc_file = hg_util.get_hgrc_path(self.repo_path()) + with open(hgrc_file, 'r') as fh: + for line in fh.read().splitlines(): + if line.startswith('allow_push = '): + return line[len('allow_push = '):] + return '' - def can_change_type(self, app): + def can_change_type(self): # Allow changing the type only if the repository has no contents, has never been installed, or has # never been changed from the default type. - if self.is_new(app): + if self.is_new(): return True if self.times_downloaded == 0: return True @@ -219,9 +238,9 @@ class Repository(Dictifiable): def can_change_type_to(self, app, new_type_label): if self.type == new_type_label: return False - if self.can_change_type(app): + if self.can_change_type(): new_type = app.repository_types_registry.get_class_by_label(new_type_label) - if new_type.is_valid_for_type(app, self): + if new_type.is_valid_for_type(self): return True return False @@ -260,21 +279,21 @@ class Repository(Dictifiable): def installable_revisions(self, app, sort_revisions=True): return metadata_util.get_metadata_revisions(app, self, sort_revisions=sort_revisions) - def is_new(self, app): - repo = hg_util.get_repo_for_repository(app, repository=self) - tip_rev = repo.changelog.tiprev() + def is_new(self): + tip_rev = self.hg_repo.changelog.tiprev() return tip_rev < 0 - def repo_path(self, app): - return app.hgweb_config_manager.get_entry(os.path.join("repos", self.user.username, self.name)) + def repo_path(self, app=None): + # Keep app argument for compatibility with tool_shed_install Repository model + return hgweb_config_manager.get_entry(os.path.join("repos", self.user.username, self.name)) - def revision(self, app): - repo = hg_util.get_repo_for_repository(app, repository=self) + def revision(self): + repo = self.hg_repo tip_ctx = repo[repo.changelog.tip()] return "%s:%s" % (str(tip_ctx.rev()), str(tip_ctx)) - def set_allow_push(self, app, usernames, remove_auth=''): - allow_push = util.listify(self.allow_push(app)) + def set_allow_push(self, usernames, remove_auth=''): + allow_push = util.listify(self.allow_push()) if remove_auth: allow_push.remove(remove_auth) else: @@ -284,19 +303,19 @@ class Repository(Dictifiable): allow_push = '%s\n' % ','.join(allow_push) # Why doesn't the following work? # repo.ui.setconfig('web', 'allow_push', allow_push) - repo_dir = self.repo_path(app) + repo_dir = self.repo_path() hgrc_file = hg_util.get_hgrc_path(repo_dir) - with open(hgrc_file, 'rb') as fh: + with open(hgrc_file, 'r') as fh: lines = fh.readlines() - with open(hgrc_file, 'wb') as fh: + with open(hgrc_file, 'w') as fh: for line in lines: if line.startswith('allow_push'): fh.write('allow_push = %s' % allow_push) else: fh.write(line) - def tip(self, app): - repo = hg_util.get_repo_for_repository(app, repository=self) + def tip(self): + repo = self.hg_repo return str(repo[repo.changelog.tip()]) def to_dict(self, view='collection', value_mapper=None): diff --git a/lib/tool_shed/webapp/model/mapping.py b/lib/tool_shed/webapp/model/mapping.py index 599b259ccb8..98bf309db38 100644 --- a/lib/tool_shed/webapp/model/mapping.py +++ b/lib/tool_shed/webapp/model/mapping.py @@ -8,7 +8,6 @@ from sqlalchemy import Boolean, Column, DateTime, desc, false, ForeignKey, Integ from sqlalchemy.orm import backref, mapper, relation import tool_shed.webapp.model -import tool_shed.webapp.util.hgweb_config import tool_shed.webapp.util.shed_statistics as shed_statistics from galaxy.model.base import ModelMapping from galaxy.model.custom_types import JSONType, TrimmedString @@ -328,5 +327,4 @@ def init(file_path, url, engine_options={}, create_tables=False): # Load local tool shed security policy result.security_agent = CommunityRBACAgent(result) result.shed_counter = shed_statistics.ShedCounter(result) - result.hgweb_config_manager = tool_shed.webapp.util.hgweb_config.HgWebConfigManager() return result diff --git a/lib/tool_shed/webapp/model/migrate/check.py b/lib/tool_shed/webapp/model/migrate/check.py index 95780f995df..dbd97242bf1 100644 --- a/lib/tool_shed/webapp/model/migrate/check.py +++ b/lib/tool_shed/webapp/model/migrate/check.py @@ -5,6 +5,10 @@ import sys from migrate.versioning import repository, schema from sqlalchemy import create_engine, MetaData, Table from sqlalchemy.exc import NoSuchTableError +from sqlalchemy_utils import ( + create_database, + database_exists, +) log = logging.getLogger(__name__) @@ -26,6 +30,11 @@ def create_or_verify_database(url, engine_options={}): """ # Create engine and metadata + if not database_exists(url): + message = "Creating database for URI [%s]" % url + log.info(message) + create_database(url) + engine = create_engine(url, **engine_options) meta = MetaData(bind=engine) # Try to load dataset table diff --git a/lib/tool_shed/webapp/security/__init__.py b/lib/tool_shed/webapp/security/__init__.py index b212e674e6e..29b88fc835b 100644 --- a/lib/tool_shed/webapp/security/__init__.py +++ b/lib/tool_shed/webapp/security/__init__.py @@ -214,7 +214,7 @@ class CommunityRBACAgent(RBACAgent): def can_push(self, app, user, repository): if user: - return user.username in listify(repository.allow_push(app)) + return user.username in listify(repository.allow_push()) return False def user_can_administer_repository(self, user, repository): diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/common/common.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/common/common.mako index 5603b65e688..37b90a5b303 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/common/common.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/common/common.mako @@ -109,6 +109,7 @@ else: disabled_str = '' html = '' + rating = rating or 0 for index in range( 1, 6 ): html += ' ( index - 0.5 ) and rating < ( index + 0.5 ): diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/common/repository_actions_menu.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/common/repository_actions_menu.mako index 33e35125db4..008e9103a55 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/common/repository_actions_menu.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/common/repository_actions_menu.mako @@ -22,12 +22,12 @@ else: is_deprecated = False - if repository.is_new( trans.app ): + if repository.is_new(): is_new = True else: is_new = False - if is_malicious( trans.app, trans.security.encode_id( repository.id ), repository.tip( trans.app ) ): + if is_malicious( trans.app, trans.security.encode_id( repository.id ), repository.tip() ): changeset_is_malicious = True else: changeset_is_malicious = False @@ -102,7 +102,7 @@ can_set_metadata = False if changeset_revision is not None: - if changeset_revision == repository.tip( trans.app ): + if changeset_revision == repository.tip(): changeset_revision_is_repository_tip = True else: changeset_revision_is_repository_tip = False @@ -148,9 +148,9 @@ Upload files to repository %endif %if can_administer: - Manage repository + Manage repository %else: - View repository + View repository %endif %if can_view_change_log: View change log @@ -180,9 +180,9 @@ %if metadata is not None and changeset_revision is not None: Export this revision %endif - Download as a .tar.gz file - Download as a .tar.bz2 file - Download as a zip file + Download as a .tar.gz file + Download as a .tar.bz2 file + Download as a zip file %endif %endif diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/browse_repository.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/browse_repository.mako index 12e9308673f..6eaf1455d46 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/browse_repository.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/browse_repository.mako @@ -24,7 +24,7 @@ <% - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() can_push = trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_new and ( not is_malicious or can_push ) can_browse_contents = not is_new @@ -38,7 +38,7 @@ ${render_tool_shed_repository_actions( repository, metadata=metadata, changeset_ %if can_browse_contents:
-
Repository '${repository.name | h}' revision ${repository.tip( trans.app ) | h} (repository tip)
+
Repository '${repository.name | h}' revision ${repository.tip() | h} (repository tip)
%if can_download:
diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/manage_repository.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/manage_repository.mako index 556456b2062..66e19e6c2b6 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/manage_repository.mako @@ -16,14 +16,14 @@ has_metadata = False is_admin = trans.user_is_admin - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() if repository.deprecated: is_deprecated = True else: is_deprecated = False - if is_malicious( trans.app, trans.security.encode_id( repository.id ), repository.tip( trans.app ) ): + if is_malicious( trans.app, trans.security.encode_id( repository.id ), repository.tip() ): changeset_is_malicious = True else: changeset_is_malicious = False @@ -48,7 +48,7 @@ else: can_set_metadata = False - if changeset_revision == repository.tip( trans.app ): + if changeset_revision == repository.tip(): changeset_revision_is_repository_tip = True else: changeset_revision_is_repository_tip = False diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako index 015bb91f293..a83be362d10 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako @@ -36,7 +36,7 @@ ${render_galaxy_repository_actions( repository=repository )}
<% - if changeset_revision == repository.tip( trans.app ): + if changeset_revision == repository.tip(): tip_str = 'repository tip' else: tip_str = '' diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/rate_repository.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/rate_repository.mako index 68b926150c7..72e49ca9535 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/rate_repository.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/rate_repository.mako @@ -5,7 +5,7 @@ <% from galaxy.web.framework.helpers import time_ago - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() can_push = trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_new and ( not is_malicious or can_push ) %> diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako index fa93af64552..e2fc5dd4306 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako @@ -3,7 +3,7 @@ <%namespace file="/webapps/tool_shed/common/repository_actions_menu.mako" import="render_tool_shed_repository_actions" /> <% - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() %> <%! @@ -138,7 +138,7 @@ ${render_tool_shed_repository_actions( repository=repository)}
- %if not repository.is_new( trans.app ): + %if not repository.is_new():
diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changelog.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changelog.mako index bcbe001c113..2b9e4f23c44 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changelog.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changelog.mako @@ -5,7 +5,7 @@ <%namespace file="/webapps/tool_shed/common/repository_actions_menu.mako" import="render_tool_shed_repository_actions" /> <% - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() can_push = trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_new and ( not is_malicious or can_push ) %> diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changeset.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changeset.mako index d5080052f28..15f34e1c889 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changeset.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_changeset.mako @@ -5,7 +5,7 @@ <%namespace file="/webapps/tool_shed/common/repository_actions_menu.mako" import="render_tool_shed_repository_actions" /> <% - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() can_push = trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_new and ( not is_malicious or can_push ) %> diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_repository.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_repository.mako index 8063a03e74e..d07644ff0fa 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_repository.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_repository.mako @@ -8,14 +8,14 @@ from galaxy.web.framework.helpers import time_ago from tool_shed.util.basic_util import to_html_string - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() is_deprecated = repository.deprecated can_browse_contents = trans.webapp.name == 'tool_shed' and not is_new can_push = not is_deprecated and trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_deprecated and not is_new and ( not is_malicious or can_push ) can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new - changeset_revision_is_repository_tip = changeset_revision == repository.tip( trans.app ) + changeset_revision_is_repository_tip = changeset_revision == repository.tip() if changeset_revision_is_repository_tip: tip_str = 'repository tip' diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_tool_metadata.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_tool_metadata.mako index a84993a49c6..fad4e51e036 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_tool_metadata.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/view_tool_metadata.mako @@ -5,7 +5,7 @@ <%namespace file="/webapps/tool_shed/common/repository_actions_menu.mako" import="*" /> <% - is_new = repository.is_new( trans.app ) + is_new = repository.is_new() can_push = trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_new and ( not is_malicious or can_push ) diff --git a/run_tests.sh b/run_tests.sh index 7e6bdb798f2..1a1336346a7 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -269,7 +269,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='galaxy/testing-base:19.05.0' +DOCKER_DEFAULT_IMAGE='galaxy/testing-base:20.05.00' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" @@ -312,7 +312,7 @@ then echo "Docker version:" docker --version echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." - name=$(python -c 'import re; import uuid; print re.sub("-", "", str(uuid.uuid4()))') + name=$(python -c 'import re; import uuid; print(re.sub("-", "", str(uuid.uuid4())))') # Create a cache dir for pip, so it has the right owner DOCKER_PIP_CACHE_DIR="$HOME"/.cache/docker_galaxy_pip mkdir -p "$DOCKER_PIP_CACHE_DIR" diff --git a/templates/show_params.mako b/templates/show_params.mako index 2a4e8f3549c..17e02f87ce1 100644 --- a/templates/show_params.mako +++ b/templates/show_params.mako @@ -119,26 +119,8 @@ ${ job.command_line | h } %if trans.user_is_admin:

Destination Parameters

- - - - - - %if job.destination_params: - %for (k, v) in job.destination_params.items(): - - - - %endfor - %endif - -
Runner${ job.job_runner_name }
Runner Job ID${ job.job_runner_external_id }
Handler${ job.handler }
${ k | h } - %if str(k) in ('nativeSpecification', 'rank', 'requirements'): - 
${ v | h }
- %else: - ${ v | h } - %endif -
+
+
%endif %if job and job.dependencies: @@ -193,5 +175,6 @@ $(function(){ }) window.bundleEntries.mountJobMetrics(); window.bundleEntries.mountJobParameters(); + window.bundleEntries.mountDestinationParams(); }); diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index af8c07fc030..758ccad7c9b 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="John Chilton " ARG CHROME_VERSION="google-chrome-beta" @@ -11,8 +11,6 @@ ENV DEBIAN_FRONTEND=noninteractive \ POSTGRES_MAJOR=9.5 \ GALAXY_ROOT=/galaxy \ GALAXY_VIRTUAL_ENV=/galaxy_venv \ - GALAXY_VIRTUAL_ENV_2=/galaxy_venv \ - GALAXY_VIRTUAL_ENV_3=/galaxy_venv3 \ LC_ALL=C.UTF-8 # Pre-install a bunch of packages to speed up ansible steps. @@ -20,13 +18,10 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans apt-add-repository -y ppa:ansible/ansible && \ curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ - curl -s http://neuro.debian.net/lists/xenial.us-ca.full > /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \ apt-get update -y && \ apt-get install -y --no-install-recommends postgresql postgresql-client ansible wget \ python3-dev \ git-core \ - python-prettytable python-virtualenv python-pip \ rsync swig sysstat unzip \ openssl \ bzip2 \ @@ -37,9 +32,6 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans locales \ xvfb \ ${CHROME_VERSION:-google-chrome-stable} \ - ffmpeg \ - bcftools \ - singularity-container \ libnss3 libgconf-2-4 && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -69,24 +61,11 @@ RUN mkdir /etc/galaxy && cd /tmp/ansible && mkdir roles && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN cd $GALAXY_ROOT && \ - virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV_3 && \ - for VENV in $GALAXY_VIRTUAL_ENV_2 $GALAXY_VIRTUAL_ENV_3; do \ - GALAXY_VIRTUAL_ENV=$VENV ./scripts/common_startup.sh --dev-wheels && \ - $VENV/bin/pip install psycopg2-binary; done && \ + virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV && \ + ./scripts/common_startup.sh --dev-wheels && \ + $GALAXY_VIRTUAL_ENV/bin/pip install psycopg2-binary && \ rm -rf ~/.cache/pip -RUN cd $GALAXY_ROOT && \ - echo "Prepopulating postgres database" && \ - su -c '/usr/lib/postgresql/${POSTGRES_MAJOR}/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ - sleep 3 && \ - GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" bash create_db.sh && \ - echo "Prepopulating sqlite database" && \ - GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" bash create_db.sh && \ - echo "Prepopulating toolshed postgres database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" bash create_db.sh tool_shed && \ - echo "Prepopulating toolshed sqlite database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed - #======================================== # Add Selenium user with passwordless sudo #======================================== diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml index d92d87f6f41..027365d086e 100644 --- a/test/docker/base/ansible_vars.yml +++ b/test/docker/base/ansible_vars.yml @@ -17,7 +17,7 @@ galaxy_db_log: /tmp/pSQL.log psql_galaxyftp_password: ignoredthis galaxy_config_file: "{{ galaxy_server_dir }}/config/galaxy.ini" configure_docker: no -postgresql_bin_dir: /usr/lib/postgresql/9.5/bin +postgresql_bin_dir: /usr/lib/postgresql/10/bin galaxy_manage_database: no # avoid some packages unneeded packages install by ansible-galaxy-os install_maintainance_packages: false @@ -25,6 +25,7 @@ galaxy_user_name: "root" galaxy_extras_install_packages: true galaxy_job_conf_path: "/etc/galaxy/job_conf.xml" galaxy_build_client: no +galaxy_virtualenv_python: python3 galaxy_web_processes: 1 galaxy_handler_processes: 2 diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index 46295549ef0..4ac43222424 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -4,7 +4,7 @@ set -e if [ ! -z "$USE_SELENIUM" ]; then # Start Selenium. - sudo -H -E -u seluser /opt/bin/entry_point.sh & + sudo -E -H -u seluser /opt/bin/entry_point.sh & fi echo "Deleting galaxy user - it may not exist and this is fine." @@ -16,16 +16,15 @@ echo "Creating galaxy group with gid $GALAXY_TEST_UID - it may already exist and groupadd -r galaxy -g "$GALAXY_TEST_UID" | true echo "Creating galaxy user with uid $GALAXY_TEST_UID - it may already exist and this is fine." useradd -u $GALAXY_TEST_UID -r -g galaxy -d /home/galaxy -c "Galaxy User" galaxy -s /bin/bash | true -mkdir -p /home/galaxy/.singularity echo "Setting galaxy user password - the operation may fail." echo "galaxy:galaxy" | chpasswd | true -chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" /home/galaxy/ +chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" : ${GALAXY_TEST_DATABASE_TYPE:-"postgres"} if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; then echo "Starting postgres and then sleeping for 3 seconds" - su -c '/usr/lib/postgresql/9.5/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres + su -c '/usr/lib/postgresql/10/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres sleep 3 GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="postgresql://root@localhost:5930/galaxy?client_encoding=utf8" @@ -49,15 +48,8 @@ cd /galaxy : ${GALAXY_VIRTUAL_ENV:=.venv} -HOME=/home/galaxy - echo "Running common startup for updated dependencies (if any)" -sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } - -echo "Upgrading test database..." -sudo -E -u "#${GALAXY_TEST_UID}" GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" sh manage_db.sh upgrade -echo "Upgrading tool shed database... $TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION" -sudo -E -u "#${GALAXY_TEST_UID}" TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION="$TOOL_SHED_TEST_DBURI" sh manage_db.sh upgrade tool_shed +sudo -E -H -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } # Ensure Selenium is running before starting tests. if [ ! -z "$USE_SELENIUM" ]; @@ -78,7 +70,7 @@ fi if [ -z "$GALAXY_NO_TESTS" ]; then - sudo -E -u "#${GALAXY_TEST_UID}" sh run_tests.sh --skip-common-startup $@ + sudo -E -H -u "#${GALAXY_TEST_UID}" sh run_tests.sh --skip-common-startup $@ else GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION="$TOOL_SHED_TEST_DBURI" diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 57000428c9e..35317ac3d38 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,3 +1,8 @@ +import tempfile + +import pytest + + def pytest_unconfigure(config): try: # This needs to be run if no test were run. @@ -6,3 +11,9 @@ def pytest_unconfigure(config): print("Galaxy test driver shutdown succesfull") except Exception: pass + + +@pytest.fixture +def temp_file(): + with tempfile.NamedTemporaryFile(delete=True, mode='wb') as fh: + yield fh diff --git a/test/integration/test_datatype_upload.py b/test/integration/test_datatype_upload.py index 83ac60f82c2..0250d3d3a6d 100644 --- a/test/integration/test_datatype_upload.py +++ b/test/integration/test_datatype_upload.py @@ -1,6 +1,5 @@ import collections import os -import tempfile import pytest @@ -48,12 +47,6 @@ class UploadTestDatatypeDataTestCase(BaseUploadContentConfigurationInstance): instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCase) -@pytest.fixture -def temp_file(): - with tempfile.NamedTemporaryFile(delete=True, mode='wb') as fh: - yield fh - - registry = Registry() registry.load_datatypes(root_dir=GALAXY_ROOT, config=DATATYPES_CONFIG) TEST_CASES = collect_test_data(registry) @@ -61,6 +54,10 @@ TEST_CASES = collect_test_data(registry) @pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) def test_upload_datatype_auto(instance, test_data, temp_file): + upload_datatype_helper(instance, test_data, temp_file) + + +def upload_datatype_helper(instance, test_data, temp_file): is_compressed = False for is_method in (is_bz2, is_gzip, is_zip): is_compressed = is_method(test_data.path) diff --git a/test/integration/test_datatype_upload_irods.py b/test/integration/test_datatype_upload_irods.py new file mode 100644 index 00000000000..3001da5d3a1 --- /dev/null +++ b/test/integration/test_datatype_upload_irods.py @@ -0,0 +1,104 @@ + +import os +import string +import subprocess +import time + +import pytest + +from galaxy_test.driver import integration_util +from .test_datatype_upload import ( + TEST_CASES, + upload_datatype_helper, + UploadTestDatatypeDataTestCase +) + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +# Run test for only the first 10 test files +IRODS_TEST_CASES = dict(list(TEST_CASES.items())[0:10]) +OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_HOST', 'localhost') +OBJECT_STORE_PORT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PORT', 1247)) +OBJECT_STORE_TIMEOUT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_TIMEOUT', 30)) +OBJECT_STORE_USERNAME = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_USERNAME', 'rods') +OBJECT_STORE_PASSWORD = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PASSWORD', 'rods') +OBJECT_STORE_RESOURCE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_RESOURCE', 'demoResc') +OBJECT_STORE_ZONE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_ZONE', 'tempZone') +OBJECT_STORE_CONFIG = string.Template(""" + + + + + + + + + +""") + + +def start_irods(container_name): + irods_start_args = [ + 'docker', + 'run', + '-p', + '1247:1247', + '-d', + '--name', + container_name, + 'kxk302/irods-server:0.1'] + subprocess.check_call(irods_start_args) + + # Sleep so integration test's iRODS instance is up and running + time.sleep(20) + + +def stop_irods(container_name): + subprocess.check_call(['docker', 'rm', '-f', container_name]) + + +class UploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): + + @classmethod + def setUpClass(cls): + cls.container_name = "%s_container" % cls.__name__ + start_irods(cls.container_name) + super(UploadTestDatatypeDataTestCase, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + stop_irods(cls.container_name) + super(UploadTestDatatypeDataTestCase, cls).tearDownClass() + + @classmethod + def handle_galaxy_config_kwds(cls, config): + temp_directory = cls._test_driver.mkdtemp() + cls.object_stores_parent = temp_directory + config_path = os.path.join(temp_directory, "object_store_conf.xml") + # This doesn't quite work yet, fails with extra_files_path + # config["metadata_strategy"] = "extended" + config["outpus_to_working_dir"] = True + config["retry_metadata_internally"] = False + with open(config_path, "w") as f: + f.write( + OBJECT_STORE_CONFIG.safe_substitute( + { + "temp_directory": temp_directory, + "host": OBJECT_STORE_HOST, + "port": OBJECT_STORE_PORT, + "timeout": OBJECT_STORE_TIMEOUT, + "username": OBJECT_STORE_USERNAME, + "password": OBJECT_STORE_PASSWORD, + "resource": OBJECT_STORE_RESOURCE, + "zone": OBJECT_STORE_ZONE, + } + ) + ) + config["object_store_config_file"] = config_path + + +instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCase) + + +@pytest.mark.parametrize('test_data', IRODS_TEST_CASES.values(), ids=list(IRODS_TEST_CASES.keys())) +def test_upload_datatype_irods(instance, test_data, temp_file): + upload_datatype_helper(instance, test_data, temp_file) diff --git a/test/integration/test_fail_job_tool_unavailable.py b/test/integration/test_fail_job_tool_unavailable.py index 1b53845441a..da638369e25 100644 --- a/test/integration/test_fail_job_tool_unavailable.py +++ b/test/integration/test_fail_job_tool_unavailable.py @@ -27,20 +27,21 @@ class FailJobWhenToolUnavailableTestCase(integration_util.IntegrationTestCase): self.workflow_populator.run_workflow(""" class: GalaxyWorkflow steps: - - label: sleep + sleep: run: class: GalaxyTool command: sleep 20s && echo 'hello world 2' > '$output1' outputs: output1: format: txt - - tool_id: cat1 + cat: + tool_id: cat1 state: input1: - $link: sleep#output1 + $link: sleep/output1 queries: input2: - $link: sleep#output1 + $link: sleep/output1 """, history_id=self.history_id, assert_ok=False, wait=False) # Wait until workflow is fully scheduled, otherwise can't test effect of removing tool from queued job time.sleep(10) diff --git a/test/integration/test_interactivetools_api.py b/test/integration/test_interactivetools_api.py index bdccee5391d..762ecd458a0 100644 --- a/test/integration/test_interactivetools_api.py +++ b/test/integration/test_interactivetools_api.py @@ -2,6 +2,7 @@ import os +import pytest import requests from galaxy_test.base import api_asserts @@ -12,6 +13,7 @@ from galaxy_test.base.populators import ( from .test_containerized_jobs import ( ContainerizedIntegrationTestCase, disable_dependency_resolution, + DOCKERIZED_JOB_CONFIG_FILE, ) SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) @@ -120,3 +122,23 @@ class InteractiveToolsPulsarIntegrationTestCase(BaseInteractiveToolsIntegrationT config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE_DOCKER config["galaxy_infrastructure_url"] = 'http://localhost:$UWSGI_PORT' disable_dependency_resolution(config) + + +class InteractiveToolsRemoteProxyIntegrationTestCase(BaseInteractiveToolsIntegrationTestCase, RunsInterativeToolTests): + """ + $ cd gx-it-proxy + $ ./lib/createdb.js --sessions $HOME/gxitexproxy.sqlite + $ ./lib/main.js --port 9001 --ip 0.0.0.0 --verbose --sessions $HOME/gxitexproxy.sqlite + $ # Need to create new DB for each test I think, duplicate IDs are the problem I think because each test starts at 1 + $ GALAXY_TEST_EXTERNAL_PROXY_HOST="localhost:9001" GALAXY_TEST_EXTERNAL_PROXY_MAP="$HOME/gxitexproxy.sqlite" pytest -s test/integration/test_interactivetools_api.py::InteractiveToolsRemoteProxyIntegrationTestCase + """ + @classmethod + def handle_galaxy_config_kwds(cls, config): + interactivetools_map = os.environ.get("GALAXY_TEST_EXTERNAL_PROXY_MAP") + interactivetools_proxy_host = os.environ.get("GALAXY_TEST_EXTERNAL_PROXY_HOST") + if not interactivetools_map or not interactivetools_proxy_host: + pytest.skip("External proxy not configured for test [map=%s,host=%s]" % (interactivetools_map, interactivetools_proxy_host)) + config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE + config["interactivetools_proxy_host"] = interactivetools_proxy_host + config["interactivetools_map"] = interactivetools_map + disable_dependency_resolution(config) diff --git a/test/integration/test_kubernetes_runner.py b/test/integration/test_kubernetes_runner.py index 000b1029737..09b325b92b0 100644 --- a/test/integration/test_kubernetes_runner.py +++ b/test/integration/test_kubernetes_runner.py @@ -12,7 +12,10 @@ import time import pytest from galaxy.util import unicodify -from galaxy_test.base.populators import skip_without_tool +from galaxy_test.base.populators import ( + DatasetPopulator, + skip_without_tool, +) from galaxy_test.driver import integration_util from .test_containerized_jobs import MulledJobTestCases from .test_job_environments import BaseJobEnvironmentIntegrationTestCase @@ -151,11 +154,22 @@ def job_config(jobs_directory): return Config(job_conf.name) +class KubernetesDatasetPopulator(DatasetPopulator): + + def wait_for_history(self, *args, **kwargs): + try: + super(KubernetesDatasetPopulator, self).wait_for_history(*args, **kwargs) + except AssertionError: + print("Kubernetes status:\n %s" % unicodify(subprocess.check_output(['kubectl', 'describe', 'nodes']))) + raise + + @integration_util.skip_unless_kubernetes() class BaseKubernetesIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase, MulledJobTestCases): def setUp(self): super(BaseKubernetesIntegrationTestCase, self).setUp() + self.dataset_populator = KubernetesDatasetPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() @classmethod diff --git a/test/integration/test_repository_operations.py b/test/integration/test_repository_operations.py index ea41c27ae22..13a8c010372 100644 --- a/test/integration/test_repository_operations.py +++ b/test/integration/test_repository_operations.py @@ -41,6 +41,21 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC self._install_repository() self._uninstall_repository() + def test_datatype_uninstall(self): + repo = ('devteam', 'blast_datatypes', '01b38f20197e') + self.install_repository(*repo) + self.uninstall_repository(*repo) + + def test_package_uninstall(self): + repo = ('iuc', 'package_fastqc_0_11_4', 'a8f485b2efd9') + self.install_repository(*repo) + self.uninstall_repository(*repo) + + def test_tool_with_package_dependency_uninstall(self): + repo = ('iuc', 'fastqc', 'e7b2202befea') + self.install_repository(*repo) + self.uninstall_repository(*repo) + def test_repository_update(self): response = self._install_repository(revision=REVISION_4, version="0.0.3")[0] assert response['ctx_rev'] == '4' diff --git a/test/unit/objectstore/irods_object_store_conf.xml b/test/unit/objectstore/irods_object_store_conf.xml new file mode 100644 index 00000000000..275ef682ad5 --- /dev/null +++ b/test/unit/objectstore/irods_object_store_conf.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/test/unit/objectstore/irods_object_store_conf_no_auth.xml b/test/unit/objectstore/irods_object_store_conf_no_auth.xml new file mode 100644 index 00000000000..787f6161a38 --- /dev/null +++ b/test/unit/objectstore/irods_object_store_conf_no_auth.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/test/unit/objectstore/irods_object_store_conf_no_extra_dir.xml b/test/unit/objectstore/irods_object_store_conf_no_extra_dir.xml new file mode 100644 index 00000000000..54bc0159279 --- /dev/null +++ b/test/unit/objectstore/irods_object_store_conf_no_extra_dir.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/test/unit/objectstore/test_irods.py b/test/unit/objectstore/test_irods.py new file mode 100644 index 00000000000..b721f3ac947 --- /dev/null +++ b/test/unit/objectstore/test_irods.py @@ -0,0 +1,51 @@ +import os +import xml.etree.ElementTree as ET + +import pytest + +from galaxy.objectstore.irods import parse_config_xml + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) + +CONFIG_FILE_NAME = 'irods_object_store_conf.xml' +CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, CONFIG_FILE_NAME) + +CONFIG_FILE_NAME_NO_EXTRA_DIR = 'irods_object_store_conf_no_extra_dir.xml' +CONFIG_FILE_NO_EXTRA_DIR = os.path.join(SCRIPT_DIRECTORY, CONFIG_FILE_NAME_NO_EXTRA_DIR) + +CONFIG_FILE_NAME_NO_AUTH = 'irods_object_store_conf_no_auth.xml' +CONFIG_FILE_NO_AUTH = os.path.join(SCRIPT_DIRECTORY, CONFIG_FILE_NAME_NO_AUTH) + + +def test_parse_valid_config_xml(): + tree = ET.parse(CONFIG_FILE) + root = tree.getroot() + config = parse_config_xml(root) + + assert config['auth']['username'] == 'rods' + assert config['auth']['password'] == 'rods' + assert config['resource']['name'] == 'demoResc' + assert config['zone']['name'] == 'tempZone' + assert config['connection']['host'] == 'localhost' + assert config['connection']['port'] == 1247 + assert config['connection']['timeout'] == 30 + assert config['cache']['path'] == 'database/object_store_cache' + assert config['cache']['size'] == 1000 + assert config['extra_dirs'][0]['type'] == 'job_work' + assert config['extra_dirs'][0]['path'] == 'database/job_working_directory_irods' + assert config['extra_dirs'][1]['type'] == 'temp' + assert config['extra_dirs'][1]['path'] == 'database/tmp_irods' + + +def test_parse_config_xml_no_extra_dir(): + tree = ET.parse(CONFIG_FILE_NO_EXTRA_DIR) + root = tree.getroot() + with pytest.raises(Exception, match='No extra_dir element in config XML tree'): + parse_config_xml(root) + + +def test_parse_config_xml_no_auth(): + tree = ET.parse(CONFIG_FILE_NO_AUTH) + root = tree.getroot() + with pytest.raises(Exception, match='No auth element in config XML tree'): + parse_config_xml(root) diff --git a/tools/data_source/sra.xml b/tools/data_source/sra.xml new file mode 100644 index 00000000000..f0b791ef166 --- /dev/null +++ b/tools/data_source/sra.xml @@ -0,0 +1,16 @@ + + server + python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_size_limit + + go to SRA server $GALAXY_URL + + + + + + + + 10.1093/database/bar011 + 10.1101/gr.229102 + + diff --git a/tools/interactive/interactivetool_climate_notebook.xml b/tools/interactive/interactivetool_climate_notebook.xml new file mode 100644 index 00000000000..2f7c989bdb1 --- /dev/null +++ b/tools/interactive/interactivetool_climate_notebook.xml @@ -0,0 +1,116 @@ + + + nordicesmhub/docker-climate-notebook:1.2 + + + + 8888 + ipython/lab + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The Climate Notebook is based on Jupyter an open-source web application that allows you to create and share documents that contain live code, equations, + visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, + machine learning, and much more. + + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. + + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. + If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. + + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. + + The Climate version of the Jupyter Notebook offers a lot of preinstalled tools for climate science. The list of packages is based on what is available + on the [Pangeo](http://pangeo.io/) platform and [Pangeo stacks](https://pangeo-data.github.io/pangeo-stacks/). + + - **Core scipy packages**: numpy, scipy, matplotlib, pandas, xarray, sparse and sympy + - **Data science**: scikit-image, scikit-learn, dask-ml, tensorflow, keras, pytorch-cpu, dask_labextension + - **Visualization**: holoviews, panel, geoviews, hvplot, geoviews, datashader, seaborn, altair, descartes, folium, vega, + vega_datasets, palettable, cmocean,plotly, psy-maps, psy-reg, psyplot, psyplot-gui, psy-maps, psy-reg, + geopy, branca + - **Geospatial**: iris, cartopy, basemap, basemap-data-hires, geopandas, rasterio, netcdf4, erddapy, pydap, h5py, h5netcdf, regionmask and rio-cogeo + - **Geoscience related**: climlab, metpy, satpy, gsw, eofs, esmpy, xesmf, windspharm, rasterstats, geojsoncontour + - **Climate related**: pyaerocom, cdo, cdsapi, cfgrib, cis. esmvaltool, nc-time-axis, nco + - **Intake related**: intake, intake-xarray, intake-esm, fsspec and intake-stac + - **zarr related**: zarr, numcodecs, python-blosc, lz4, gcsfs, s3fs, tiledb-py + - **jupyter related**: ipyleaflet, papermill, jupytext, ipydatawidgets, sidecar + - **xarray related**: xgcm, xrft, xhistogram, xlrd, xrviz, climpred, pytide, pyinterp + - **misc**: python-wget, prefect, requests, pillow, pip, nbgitpuller, pysplit, biopython, bioblend and galaxy-ie-helpers + + + diff --git a/tools/interactive/interactivetool_geoexplorer.xml b/tools/interactive/interactivetool_geoexplorer.xml new file mode 100644 index 00000000000..3c8e1f5a536 --- /dev/null +++ b/tools/interactive/interactivetool_geoexplorer.xml @@ -0,0 +1,74 @@ + + An interactive spatial analysis platform using ggvis and Leaflet + + ylebras/geoexplorer-docker + + + + 3838 + /sample-apps/SIG/ + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + > /var/log/shiny-server.log 2>&1 + + ]]> + + + + + + + + + + +`_ is An interactive spatial analysis platform using ggvis and Leaflet. + +Author: David Stephens + +App: http://www.davesteps.com/geoExploreR/ + +.. class:: infomark + +**Input data file MUST have as a uniq ID per row on first column, longitude column on column 2, latitude column on column 3 and quantitative values on 4th column ** + +Example input file (csv):: + +"ID" "x" "y" "test" +01 -60.291838 46.328137 2 +02 -114.58927 35.022485 3 +03 -93.37406 30.00586 4 +04 -79.336288 43.682218 5 +05 -109.156024 31.904185 2 +06 -71.098031 42.297408 9 +07 -110.927215 32.18203 12 + +]]> + + + @misc{githubsurvey2018, + author = {davesteps}, + title = {{dashboard to visualise geographic data}}, + publisher = {Github}, + url = {https://github.com/davesteps/geoExploreR} + } + } + + diff --git a/tools/interactive/interactivetool_guacamole_desktop.xml b/tools/interactive/interactivetool_guacamole_desktop.xml new file mode 100644 index 00000000000..665f4d33eb8 --- /dev/null +++ b/tools/interactive/interactivetool_guacamole_desktop.xml @@ -0,0 +1,27 @@ + + + quay.io/bgruening/guacamole-desktop + + + + 8080 + + + + + + + + + + + + + + Simple Ubuntu XFCE all-in-one desktop. The Username is "user" and the Password is "password". + This image is based on the awesome work from CyVerse. + + diff --git a/tools/interactive/interactivetool_higlass.xml b/tools/interactive/interactivetool_higlass.xml new file mode 100644 index 00000000000..6d0510925da --- /dev/null +++ b/tools/interactive/interactivetool_higlass.xml @@ -0,0 +1,39 @@ + + an interactive Hi-C data visualizer + + quay.io/bgruening/galaxy-higlass + + + + 80 + + + + + + + + + + + + + + Interactive tool for visualising Hi-C data, works only for multi-cooler files which store multiple resolutions. + For a detailed documentaition please visit https://docs.higlass.io/. + + + + 10.1186/s13059-018-1486-1 + + + + diff --git a/tools/interactive/interactivetool_openrefine.xml b/tools/interactive/interactivetool_openrefine.xml new file mode 100644 index 00000000000..11a0b01ffc3 --- /dev/null +++ b/tools/interactive/interactivetool_openrefine.xml @@ -0,0 +1,77 @@ + + Working with messy data + + ylebras/openrefine-docker + + + + 3333 + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + + &1) +while [[ \${STATUS} =~ "refused" ]] +do + echo "Waiting for openrefine: \$STATUS \n" + STATUS=\$(curl --include 'http://127.0.0.1:3333' 2>&1) + sleep 4 +done +]]> + + + + + + + + + + + + + + + `_ (previously Google Refine) is a powerful tool for working with messy data: +cleaning it; transforming it from one format into another; and extending it with web services and external data. + +.. class:: infomark + +Example input file (TAB separated):: + + "name" "longitude" "latitude" "countryCode" + Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA + Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US + Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US + Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA + Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US + Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US + Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US + +]]> + + diff --git a/tools/interactive/interactivetool_panoply.xml b/tools/interactive/interactivetool_panoply.xml new file mode 100644 index 00000000000..d663ce89b61 --- /dev/null +++ b/tools/interactive/interactivetool_panoply.xml @@ -0,0 +1,48 @@ + + interative plotting tool for geo-referenced data + + 4.5.1 + + + quay.io/nordicesmhub/docker-panoply:@VERSION@ + + + + 5800 + + + + output/version.txt && + cp /config/home/output/* output/ | true && + cd output && + sleep 2 && + for file in *; do mv "\$file" "\${file// /_}"; done && + for file in *; do mv "\$file" "\$file.\${file\#\#*.}"; done + ]]> + + + + + + + + + + + + `_ plots geo-referenced and other arrays from netCDF, HDF, GRIB, and other datasets. + ]]> + + diff --git a/tools/interactive/interactivetool_paraview.xml b/tools/interactive/interactivetool_paraview.xml new file mode 100644 index 00000000000..c801600ef0e --- /dev/null +++ b/tools/interactive/interactivetool_paraview.xml @@ -0,0 +1,50 @@ + + + + bmcv/galaxy-paraviewweb:latest + + + + 8777 + + + + + localhost:8080 + wss + -dr,--mesa-swr + + + + + + + + + + + + + ParaView is an open-source, multi-platform application designed to visualize data sets of varying sizes from small to very large. + + diff --git a/tools/interactive/interactivetool_pyiron.xml b/tools/interactive/interactivetool_pyiron.xml new file mode 100644 index 00000000000..69caecdf71b --- /dev/null +++ b/tools/interactive/interactivetool_pyiron.xml @@ -0,0 +1,100 @@ + + + quay.io/bgruening/docker-jupyter-notebook:pyiron + + + + 8888 + ipython/lab + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + /home/jovyan/resources + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + pyiron - an integrated development environment (IDE) for computational materials science. It combines several tools in a common platform: + + The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, + visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, + machine learning, and much more. + + Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to + do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter. + + You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset. + If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history. + + You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function. + + diff --git a/tools/interactive/interactivetool_radiant.xml b/tools/interactive/interactivetool_radiant.xml new file mode 100644 index 00000000000..05b623d0ffc --- /dev/null +++ b/tools/interactive/interactivetool_radiant.xml @@ -0,0 +1,75 @@ + + Data analytics using Radiant R Shiny app + + ylebras/radiant-docker + + + + 3838 + /sample-apps/STAT/inst/app + + + + $__history_id__ + $__galaxy_url__ + 8080 + $__galaxy_url__ + + + > /var/log/shiny-server.log 2>&1 + ]]> + + + + + + + + + + +`_ is an open-source platform-independent browser-based interface for business analytics in R. The application is based on the Shiny package and can be run locally or on a server. Radiant was developed by Vincent Nijs. Please use the issue tracker on GitHub to suggest enhancements or report problems: https://github.com/radiant-rstats/radiant/issues. For other questions and comments please use radiant@rady.ucsd.edu. +Key features + +- Explore: Quickly and easily summarize, visualize, and analyze your data +- Cross-platform: It runs in a browser on Windows, Mac, and Linux +- Reproducible: Recreate results and share work with others as a state file or an Rmarkdown report +- Programming: Integrate Radiant’s analysis functions with your own R-code +- Context: Data and examples focus on business applications + +.. class:: infomark + +Example input file (TAB separated):: + + "name" "longitude" "latitude" "countryCode" + Accipiter striatus Vieillot, 1808 -60.291838 46.328137 CA + Accipiter striatus Vieillot, 1808 -114.58927 35.022485 US + Accipiter striatus Vieillot, 1808 -93.37406 30.00586 US + Accipiter striatus Vieillot, 1808 -79.336288 43.682218 CA + Accipiter striatus Vieillot, 1808 -109.156024 31.904185 US + Accipiter striatus Vieillot, 1808 -71.098031 42.297408 US + Accipiter striatus Vieillot, 1808 -110.927215 32.18203 US + +]]> + + + @misc{githubsurvey2018, + author = {vnijs}, + title = {{Radiant - Business analytics using R and Shiny}}, + publisher = {Github}, + url = {https://github.com/vnijs/radiant} + } + } + + diff --git a/tools/interactive/interactivetool_vcf_iobio.xml b/tools/interactive/interactivetool_vcf_iobio.xml new file mode 100644 index 00000000000..3be0266c593 --- /dev/null +++ b/tools/interactive/interactivetool_vcf_iobio.xml @@ -0,0 +1,50 @@ + + + qiaoy/iobio-bundle.vcf-iobio:dev-ondemand + + + + 80 + + + + /tmp/app.conf && + mv /tmp/app.conf /etc/supervisor.d/app.conf && + /usr/bin/supervisord -c /etc/supervisord.conf + + ]]> + + + + + + + + + + `_. + +This visualization is using Galaxy Interactive Tool and utilizes an all-in-one Docker container from http://iobio.io. + +Make sure your VCF file is compressed to the vcf_bgzip datatype to load it into the Visualization. + ]]> + + + diff --git a/tox.ini b/tox.ini index 65f75359293..0757e8e992a 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ passenv = CI CONDA_EXE setenv = first_startup: GALAXY_PYTHON=python first_startup: GALAXY_CONFIG_DATABASE_AUTO_MIGRATE=true - py{35,36,37}-first_startup: GALAXY_VIRTUAL_ENV=.venv3 + py{35,36,37,38}-first_startup: GALAXY_VIRTUAL_ENV=.venv3 unit: GALAXY_VIRTUAL_ENV={envdir} unit: GALAXY_ENABLE_BETA_COMPRESSED_GENBANK_SNIFFING=1 mulled: GALAXY_TEST_INCLUDE_SLOW=1