diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e970a2d9b66..6599981196f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,9 @@ repository](https://github.com/galaxyproject/galaxy). For information on contributing more broadly to the Galaxy ecosystem and a deeper discussion of some of these points - please see the [Develop](https://galaxyproject.org/develop) section of the [Galaxy Community -Hub](https://galaxyproject.org). +Hub](https://galaxyproject.org) and the [Galaxy Code +Architecture](https://training.galaxyproject.org/training-material/topics/dev/tutorials/architecture/slides.html#1) slides that are part of the [Galaxy Training +Materials](https://training.galaxyproject.org/). ## Before you Begin diff --git a/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue b/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue new file mode 100644 index 00000000000..3847b70fa31 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/DatasetAsImage/DatasetAsImage.vue @@ -0,0 +1,41 @@ + + + diff --git a/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue new file mode 100644 index 00000000000..762e014b574 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/DatasetIndex/DatasetIndex.vue @@ -0,0 +1,88 @@ + + + diff --git a/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue new file mode 100644 index 00000000000..c3a4411734d --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/DatasetLink/DatasetLink.vue @@ -0,0 +1,67 @@ + + + diff --git a/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js b/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js new file mode 100644 index 00000000000..67a0bdf5f27 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/compositeDatasetUtils.js @@ -0,0 +1,42 @@ +import { getAppRoot } from "onload/loadConfig"; +import { Services } from "components/Dataset/services"; +import store from "../../store/index"; + +export const getPathDestination = async (history_dataset_id, path) => { + const services = new Services({ root: getAppRoot() }); + + const computePathDestination = (pathDestination) => { + if (pathDestination.datasetContent[0].class === "Directory") + pathDestination.datasetRootDir = datasetContent[0].path; + else return; + + if (path === undefined || path === "undefined") { + return pathDestination; + } + + const filepath = `${pathDestination.datasetRootDir}/${path}`; + + const datasetEntry = datasetContent.find((datasetEntry) => { + return filepath === datasetEntry.path; + }); + + if (datasetEntry) { + if (datasetEntry.class === "Directory") { + pathDestination.isDirectory = true; + pathDestination.filepath = filepath; + return pathDestination; + } + pathDestination.fileLink = services.getCompositeDatasetLink(history_dataset_id, datasetEntry.path); + } + return pathDestination; + }; + + let datasetContent = store.getters.getDatasetExtFiles(history_dataset_id); + + if (datasetContent == null) { + await store.dispatch("fetchDatasetExtFiles", history_dataset_id); + datasetContent = store.getters.getDatasetExtFiles(history_dataset_id); + } + + return computePathDestination({ datasetContent: datasetContent }); +}; diff --git a/client/galaxy/scripts/components/Dataset/mount.js b/client/galaxy/scripts/components/Dataset/mount.js new file mode 100644 index 00000000000..a577a4374e4 --- /dev/null +++ b/client/galaxy/scripts/components/Dataset/mount.js @@ -0,0 +1,32 @@ +/** + * Endpoint for mounting dataset components from non-Vue environment. + */ +import $ from "jquery"; +import DatasetLink from "./DatasetLink/DatasetLink"; +import { mountVueComponent } from "utils/mountVueComponent"; +import DatasetIndex from "./DatasetIndex/DatasetIndex"; +import DatasetAsImage from "./DatasetAsImage/DatasetAsImage"; + +export const mountDatasetAsImage = () => { + return mountDatasetComponent(".dataset-as-image", ["history_dataset_id", "path"], DatasetAsImage); +}; +export const mountDatasetIndex = () => { + return mountDatasetComponent(".dataset-index", ["history_dataset_id", "path"], DatasetIndex); +}; +export const mountDatasetLink = () => { + return mountDatasetComponent(".dataset-link", ["history_dataset_id", "path", "label"], DatasetLink); +}; + +export const mountDatasetComponents = () => { + const all = [mountDatasetAsImage, mountDatasetIndex, mountDatasetLink]; + all.forEach((func) => func()); +}; + +const mountDatasetComponent = (selector, props, component, propsData = {}) => { + $(selector).each((index, el) => { + props.forEach((prop) => { + propsData[prop] = $(el).attr(prop); + }); + mountVueComponent(component)(propsData, el); + }); +}; diff --git a/client/galaxy/scripts/components/Dataset/services.js b/client/galaxy/scripts/components/Dataset/services.js index 4d836372d5f..f276215d467 100644 --- a/client/galaxy/scripts/components/Dataset/services.js +++ b/client/galaxy/scripts/components/Dataset/services.js @@ -1,4 +1,5 @@ import axios from "axios"; +import { rethrowSimple } from "utils/simple-error"; /** Datasets request helper **/ export class Services { @@ -68,6 +69,30 @@ export class Services { } } + async getCompositeDatasetContentFiles(id) { + const url = `${this.root}api/histories/${id}/contents/${id}/extra_files`; + try { + const response = await axios.get(url); + return response.data; + } catch (e) { + rethrowSimple(e); + } + } + + getCompositeDatasetLink(history_dataset_id, path) { + return `${this.root}api/histories/${history_dataset_id}/contents/${history_dataset_id}/display?filename=${path}`; + } + + async getCompositeDatasetInfo(id) { + const url = `${this.root}api/histories/${id}/contents/${id}`; + try { + const response = await axios.get(url); + return response.data; + } catch (e) { + rethrowSimple(e); + } + } + _errorMessage(e) { let message = "Request failed."; if (e.response) { diff --git a/client/galaxy/scripts/components/Datatypes/factory.js b/client/galaxy/scripts/components/Datatypes/factory.js new file mode 100644 index 00000000000..19637145afb --- /dev/null +++ b/client/galaxy/scripts/components/Datatypes/factory.js @@ -0,0 +1,7 @@ +import { DatatypesMapperModel } from "./model.js"; +import { getDatatypes } from "./services"; + +export async function getDatatypesMapper() { + const typesAndMapping = await getDatatypes(); + return new DatatypesMapperModel(typesAndMapping); +} diff --git a/client/galaxy/scripts/components/Datatypes/index.js b/client/galaxy/scripts/components/Datatypes/index.js new file mode 100644 index 00000000000..3350500fd3b --- /dev/null +++ b/client/galaxy/scripts/components/Datatypes/index.js @@ -0,0 +1 @@ +export { getDatatypesMapper } from "./factory.js"; diff --git a/client/galaxy/scripts/components/Datatypes/index.test.js b/client/galaxy/scripts/components/Datatypes/index.test.js new file mode 100644 index 00000000000..bf895148209 --- /dev/null +++ b/client/galaxy/scripts/components/Datatypes/index.test.js @@ -0,0 +1,27 @@ +import axios from "axios"; +import MockAdapter from "axios-mock-adapter"; + +import { getDatatypesMapper } from "./index"; +import { typesAndMappingResponse } from "./test_fixtures"; + +describe("Datatypes/index.js", () => { + let axiosMock; + + beforeEach(() => { + axiosMock = new MockAdapter(axios); + }); + + afterEach(() => { + axiosMock.restore(); + }); + + describe("getDatatypesMapper", () => { + it("should fetch logic from API for comparing datatypes in a hierarchy", async () => { + axiosMock.onGet(`/api/datatypes/types_and_mapping`).reply(200, typesAndMappingResponse); + await getDatatypesMapper().then((mapper) => { + expect(mapper.isSubType("txt", "data")).to.equals(true); + expect(mapper.isSubType("data", "txt")).to.equals(false); + }); + }); + }); +}); diff --git a/client/galaxy/scripts/components/Datatypes/model.js b/client/galaxy/scripts/components/Datatypes/model.js new file mode 100644 index 00000000000..1039dd96d20 --- /dev/null +++ b/client/galaxy/scripts/components/Datatypes/model.js @@ -0,0 +1,13 @@ +export class DatatypesMapperModel { + constructor(typesAndMapping) { + this.datatypes = typesAndMapping.datatypes; + this.datatypesMapping = typesAndMapping.datatypes_mapping; + } + + isSubType(child, parent) { + const mapping = this.datatypesMapping; + child = mapping.ext_to_class_name[child]; + parent = mapping.ext_to_class_name[parent]; + return mapping.class_to_classes[child] && parent in mapping.class_to_classes[child]; + } +} diff --git a/client/galaxy/scripts/components/Datatypes/services.js b/client/galaxy/scripts/components/Datatypes/services.js new file mode 100644 index 00000000000..f6ba04aa71f --- /dev/null +++ b/client/galaxy/scripts/components/Datatypes/services.js @@ -0,0 +1,12 @@ +import axios from "axios"; +import { rethrowSimple } from "utils/simple-error"; +import { getAppRoot } from "onload/loadConfig"; + +export async function getDatatypes() { + try { + const request = await axios.get(`${getAppRoot()}api/datatypes/types_and_mapping`); + return request.data; + } catch (e) { + rethrowSimple(e); + } +} diff --git a/client/galaxy/scripts/components/Datatypes/test_fixtures.js b/client/galaxy/scripts/components/Datatypes/test_fixtures.js new file mode 100644 index 00000000000..39897c7480a --- /dev/null +++ b/client/galaxy/scripts/components/Datatypes/test_fixtures.js @@ -0,0 +1,10 @@ +import DatatypesMappingJson from "qunit/test-data/json/datatypes.mapping.json"; +import DatatypesJson from "qunit/test-data/json/datatypes.json"; +import { DatatypesMapperModel } from "./model.js"; + +export const typesAndMappingResponse = { + datatypes: DatatypesJson, + datatypes_mapping: DatatypesMappingJson, +}; + +export const testDatatypesMapper = new DatatypesMapperModel(typesAndMappingResponse); diff --git a/client/galaxy/scripts/components/JobStates/mixin.js b/client/galaxy/scripts/components/JobStates/mixin.js index 9d9652bd372..320d08e08ca 100644 --- a/client/galaxy/scripts/components/JobStates/mixin.js +++ b/client/galaxy/scripts/components/JobStates/mixin.js @@ -26,17 +26,8 @@ export default { errorCount() { return this.countStates(["error"]); }, - runningPercent() { - return this.runningCount / (this.jobCount * 1.0); - }, - okPercent() { - return this.okCount / (this.jobCount * 1.0); - }, - errorPercent() { - return this.errorCount / (this.jobCount * 1.0); - }, - otherPercent() { - return 1.0 - this.okPercent - this.runningPercent - this.errorPercent; + newCount() { + return this.jobCount - this.okCount - this.runningCount - this.errorCount; }, }, methods: { diff --git a/client/galaxy/scripts/components/Markdown/Markdown.vue b/client/galaxy/scripts/components/Markdown/Markdown.vue index 9ac141bcaea..70b8538491f 100644 --- a/client/galaxy/scripts/components/Markdown/Markdown.vue +++ b/client/galaxy/scripts/components/Markdown/Markdown.vue @@ -14,6 +14,7 @@ import { getGalaxyInstance } from "app"; import { render_embedded_items } from "mvc/embedded-objects"; import { mountJobMetrics } from "components/JobMetrics"; import { mountJobParameters } from "components/JobParameters"; +import { mountDatasetComponents } from "components/Dataset/mount"; import MarkdownIt from "markdown-it"; import JOB_STATES_MODEL from "mvc/history/job-states-model"; @@ -22,7 +23,7 @@ import HDCAListItemEdit from "mvc/history/hdca-li-edit"; import HDCAListItem from "mvc/history/hdca-li"; const FUNCTION_VALUE_REGEX = `\\s*(?:[\\w_\\-]+|\\"[^\\"]+\\"|\\'[^\\']+\\')\\s*`; const FUNCTION_CALL = `\\s*\\w+\\s*=` + FUNCTION_VALUE_REGEX; -const FUNCTION_CALL_LINE = `\\s*(\\w+)\\s*\\((` + FUNCTION_CALL + `)(,` + FUNCTION_CALL + `)*\\)\\s*`; +const FUNCTION_CALL_LINE = `\\s*(\\w+)\\s*\\(\\s*(?:(${FUNCTION_CALL})(,${FUNCTION_CALL})*)?\\s*\\)\\s*`; const FUNCTION_CALL_LINE_TEMPLATE = new RegExp(FUNCTION_CALL_LINE, "m"); const md = MarkdownIt(); @@ -61,6 +62,17 @@ const RENDER_FUNCTIONS = { `; }, + history_dataset_link: (action, args, content) => { + const history_dataset_id = args.history_dataset_id; + const path = args.path; + const label = args.label; + return `
`; + }, + history_dataset_index: (action, args, content) => { + const history_dataset_id = args.history_dataset_id; + const path = args.path; + return `
`; + }, history_dataset_collection_display: (action, args, content) => { const history_dataset_collection_id = args.history_dataset_collection_id; return `
`; @@ -90,7 +102,11 @@ const RENDER_FUNCTIONS = { }, history_dataset_as_image: (action, args, content) => { const history_dataset_id = args.history_dataset_id; - return ``; + const path = args.path; + if (path) { + `${getAppRoot}api/histories/${history_dataset_id}/contents/${history_dataset_id}/display?filename=${path}`; + } + return `
`; }, history_dataset_peek: (action, args, content) => { const history_dataset_id = args.history_dataset_id; @@ -100,6 +116,14 @@ const RENDER_FUNCTIONS = { const history_dataset_id = args.history_dataset_id; return `
`; }, + history_dataset_name: (action, args, contents) => { + const history_dataset_id = args.history_dataset_id; + return `
`; + }, + history_dataset_type: (actions, args, contents) => { + const history_dataset_id = args.history_dataset_id; + return `
`; + }, tool_stdout: (action, args, content) => { const jobId = args.job_id; return `
`; @@ -117,27 +141,42 @@ const RENDER_FUNCTIONS = { const param = args.param; return `
`; }, + generate_galaxy_version: (actions, args, content) => { + return `
`; + }, + generate_time: (actions, args, content) => { + return `
`; + }, + invocation_time: (actions, args, content) => { + const invocationId = args.invocation_id; + return `
`; + }, }; md.renderer.rules.fence = function (tokens, idx, options, env, slf) { const token = tokens[idx]; const info = token.info ? token.info.trim() : ""; const content = token.content; + if (info == "galaxy") { - const arr = FUNCTION_CALL_LINE_TEMPLATE.exec(content); + const galaxy_function = FUNCTION_CALL_LINE_TEMPLATE.exec(content); const args = {}; - const action = arr[1]; - for (let i = 2; i < arr.length; i++) { - if (arr[i] === undefined) continue; - const arguments_str = arr[i].replace(/,/g, "").trim(); + const function_name = galaxy_function[1]; + // we need [... ] to return empty string, if regex doesn't match + const function_arguments = [...content.matchAll(new RegExp(FUNCTION_CALL, "g"))]; + + for (let i = 0; i < function_arguments.length; i++) { + if (function_arguments[i] === undefined) continue; + const arguments_str = function_arguments[i].toString().replace(/,/g, "").trim(); if (arguments_str) { const [key, val] = arguments_str.split("="); args[key.trim()] = val.replace(/['"]+/g, "").trim(); } } - return RENDER_FUNCTIONS[action](action, args, content); + + return RENDER_FUNCTIONS[function_name](function_name, args, content); } else { return default_fence(tokens, idx, options, env, slf); } @@ -172,6 +211,9 @@ export default { historyDatasetCollections: {}, workflows: {}, jobs: {}, + invocations: {}, + generateTime: null, + generateGalaxyVersion: null, }; }, computed: { @@ -188,11 +230,16 @@ export default { this.historyDatasetCollections = mConfig.history_dataset_collections || {}; this.workflows = mConfig.workflows || {}; this.jobs = mConfig.jobs || {}; + this.invocations = mConfig.invocations || {}; + this.generateGalaxyVersion = mConfig.generate_version || "Unknown Galaxy Version"; + this.generateTime = mConfig.generate_time; this.$nextTick(() => { render_embedded_items(); mountJobMetrics({ includeTitle: false }); mountJobParameters({ includeTitle: false }); + mountDatasetComponents(); + $("span.render-name").each((i, el) => { const historyDatasetId = $(el).attr("history_dataset_id"); if (historyDatasetId) { @@ -208,6 +255,13 @@ export default { render_fenced_output("tool-stderr", this.jobs, "job_id", "tool_stderr"); render_fenced_output("dataset-peek", this.historyDatasets, "history_dataset_id", "peek"); render_fenced_output("dataset-info", this.historyDatasets, "history_dataset_id", "info"); + render_fenced_output("dataset-name", this.historyDatasets, "history_dataset_id", "name"); + render_fenced_output("dataset-type", this.historyDatasets, "history_dataset_id", "ext"); + + $(".galaxy-version code").text(this.generateGalaxyVersion); + $(".generate-time code").text(this.generateTime); + + render_fenced_output("invocation-time", this.invocations, "invocation_id", "create_time"); $(".dataset-collection").each((i, el) => { const Galaxy = getGalaxyInstance(); diff --git a/client/galaxy/scripts/components/ProgressBar.vue b/client/galaxy/scripts/components/ProgressBar.vue index 5a9b7a58c2d..13fd7f5544b 100644 --- a/client/galaxy/scripts/components/ProgressBar.vue +++ b/client/galaxy/scripts/components/ProgressBar.vue @@ -1,35 +1,47 @@ diff --git a/client/galaxy/scripts/components/Workflow/Editor/modules/services.js b/client/galaxy/scripts/components/Workflow/Editor/modules/services.js index 3696e07290c..07c4098e144 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/modules/services.js +++ b/client/galaxy/scripts/components/Workflow/Editor/modules/services.js @@ -13,18 +13,6 @@ export async function getVersions(id) { } } -export async function getDatatypes() { - try { - const datatypesRequest = await axios.get(`${getAppRoot()}api/datatypes`); - const datatypes = datatypesRequest.data; - const mappingRequest = await axios.get(`${getAppRoot()}api/datatypes/mapping`); - const datatypes_mapping = mappingRequest.data; - return { datatypes, datatypes_mapping }; - } catch (e) { - rethrowSimple(e); - } -} - export async function getModule(request_data) { try { const { data } = await axios.post(`${getAppRoot()}api/workflows/build_module`, request_data); diff --git a/client/galaxy/scripts/components/Workflow/Editor/modules/terminals.js b/client/galaxy/scripts/components/Workflow/Editor/modules/terminals.js index eeb5444b96f..df022d214e0 100644 --- a/client/galaxy/scripts/components/Workflow/Editor/modules/terminals.js +++ b/client/galaxy/scripts/components/Workflow/Editor/modules/terminals.js @@ -219,7 +219,7 @@ class Terminal extends EventEmitter { class BaseInputTerminal extends Terminal { constructor(attr) { super(attr); - this.datatypesMapping = attr.datatypesMapping; + this.datatypesMapper = attr.datatypesMapper; this.update(attr.input); // subclasses should implement this... } canAccept(other) { @@ -348,10 +348,7 @@ class BaseInputTerminal extends Terminal { ); } _isSubType(child, parent) { - const mapping = this.datatypesMapping; - child = mapping.ext_to_class_name[child]; - parent = mapping.ext_to_class_name[parent]; - return mapping.class_to_classes[child] && parent in mapping.class_to_classes[child]; + return this.datatypesMapper.isSubType(child, parent); } _producesAcceptableDatatypeAndOptionalness(other) { if (!this.optional && !this.multiple && other.optional) { diff --git a/client/galaxy/scripts/components/Workflow/Run/WorkflowRun.test.js b/client/galaxy/scripts/components/Workflow/Run/WorkflowRun.test.js index 218650ac4f9..9bb7018ffb2 100644 --- a/client/galaxy/scripts/components/Workflow/Run/WorkflowRun.test.js +++ b/client/galaxy/scripts/components/Workflow/Run/WorkflowRun.test.js @@ -3,6 +3,7 @@ import axios from "axios"; import { __RewireAPI__ as rewire } from "./WorkflowRun"; import WorkflowRun from "./WorkflowRun.vue"; import { mount, createLocalVue } from "@vue/test-utils"; +import flushPromises from "flush-promises"; import sampleRunData1 from "./testdata/run1.json"; @@ -34,10 +35,7 @@ describe("WorkflowRun.vue", () => { expect(wrapper.vm.loading).to.equal(true); expect(wrapper.vm.error).to.equal(null); expect(wrapper.vm.model).to.equal(null); - await localVue.nextTick(); - await localVue.nextTick(); - await localVue.nextTick(); - await localVue.nextTick(); + await flushPromises(); expect(wrapper.vm.error).to.equal(null); expect(wrapper.vm.loading).to.equal(false); const model = wrapper.vm.model; diff --git a/client/galaxy/scripts/components/Workflow/Run/WorkflowRunSuccess.vue b/client/galaxy/scripts/components/Workflow/Run/WorkflowRunSuccess.vue index 68a810febcf..8b11155a155 100644 --- a/client/galaxy/scripts/components/Workflow/Run/WorkflowRunSuccess.vue +++ b/client/galaxy/scripts/components/Workflow/Run/WorkflowRunSuccess.vue @@ -19,11 +19,11 @@

- + />
diff --git a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue index 17542e426e0..6474a022013 100644 --- a/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue +++ b/client/galaxy/scripts/components/WorkflowInvocationState/WorkflowInvocationState.vue @@ -1,65 +1,46 @@ - diff --git a/client/galaxy/scripts/layout/page.js b/client/galaxy/scripts/layout/page.js index 5c29c54022e..634a48f67a7 100644 --- a/client/galaxy/scripts/layout/page.js +++ b/client/galaxy/scripts/layout/page.js @@ -114,10 +114,13 @@ const View = Backbone.View.extend({ if (this.config.message_box_visible) { const content = this.config.message_box_content || ""; const level = this.config.message_box_class || "info"; - this.$el.addClass("has-message-box"); - this.$messagebox.attr("class", `panel-${level}-message`).html(content).toggle(!!content).show(); + this.$messagebox + .attr("class", `alert alert-${level} rounded-0 m-0 p-2`) + .append($("
").attr("class", "fa fa-fw mr-1 fa-exclamation")) + .append(content) + .toggle(!!content) + .show(); } else { - this.$el.removeClass("has-message-box"); this.$messagebox.hide(); } return this; @@ -128,12 +131,16 @@ const View = Backbone.View.extend({ if (this.config.show_inactivity_warning) { const content = this.config.inactivity_box_content || ""; const verificationLink = $("") + .attr("class", "ml-1") .attr("href", `${getAppRoot()}user/resend_verification`) .text("Resend verification"); - this.$el.addClass("has-inactivity-box"); - this.$inactivebox.html(`${content} `).append(verificationLink).toggle(!!content).show(); + this.$inactivebox + .append($("
").attr("class", "fa fa-fw mr-1 fa-exclamation-triangle")) + .append(content) + .append(verificationLink) + .toggle(!!content) + .show(); } else { - this.$el.removeClass("has-inactivity-box"); this.$inactivebox.hide(); } return this; @@ -159,8 +166,8 @@ const View = Backbone.View.extend({ `
-
-
+ +
diff --git a/client/galaxy/scripts/qunit/tests/workflow_editor_tests.js b/client/galaxy/scripts/qunit/tests/workflow_editor_tests.js index 19c0d7a0c8e..040e2982de2 100644 --- a/client/galaxy/scripts/qunit/tests/workflow_editor_tests.js +++ b/client/galaxy/scripts/qunit/tests/workflow_editor_tests.js @@ -1,6 +1,6 @@ /* global QUnit */ import $ from "jquery"; -import DatatypesMapping from "qunit/test-data/json/datatypes.mapping.json"; +import { testDatatypesMapper } from "components/Datatypes/test_fixtures"; import sinon from "sinon"; import WorkflowNode from "components/Workflow/Editor/Node"; import Terminals from "components/Workflow/Editor/modules/terminals"; @@ -49,7 +49,7 @@ function buildNode(propsData) { propsData.getCanvasManager = () => { return {}; }; - propsData.datatypesMapping = DatatypesMapping; + propsData.datatypesMapper = testDatatypesMapper; return new component({ propsData: propsData, el: "#canvas-container", @@ -63,7 +63,7 @@ QUnit.module("Input terminal model test", { this.input = { extensions: ["txt"], multiple: false, optional: false }; const inputEl = $("
")[0]; this.input_terminal = new Terminals.InputTerminal({ - datatypesMapping: DatatypesMapping, + datatypesMapper: testDatatypesMapper, element: inputEl, input: this.input, node: this.node, @@ -309,7 +309,7 @@ QUnit.module("Input collection terminal model test", { this.input = { extensions: ["txt"], collection_types: ["list"] }; const inputEl = $("
")[0]; this.input_terminal = new Terminals.InputCollectionTerminal({ - datatypesMapping: DatatypesMapping, + datatypesMapper: testDatatypesMapper, element: inputEl, input: this.input, node: this.node, @@ -869,7 +869,7 @@ QUnit.module("terminal mapping logic", { } const inputEl = $("
")[0]; const inputTerminal = new Terminals.InputTerminal({ - datatypesMapping: DatatypesMapping, + datatypesMapper: testDatatypesMapper, element: inputEl, input: input, }); @@ -887,7 +887,7 @@ QUnit.module("terminal mapping logic", { } const inputEl = $("
")[0]; const inputTerminal = new Terminals.InputCollectionTerminal({ - datatypesMapping: DatatypesMapping, + datatypesMapper: testDatatypesMapper, element: inputEl, input: input, node: node, diff --git a/client/galaxy/scripts/store/datasetExtFilesStore.js b/client/galaxy/scripts/store/datasetExtFilesStore.js new file mode 100644 index 00000000000..c07cecb1d85 --- /dev/null +++ b/client/galaxy/scripts/store/datasetExtFilesStore.js @@ -0,0 +1,35 @@ +export const state = { + datasetExtFilesById: {}, +}; + +import Vue from "vue"; +import { getAppRoot } from "onload/loadConfig"; +import axios from "axios"; + +const getters = { + getDatasetExtFiles: (state) => (history_dataset_id) => { + return state.datasetExtFilesById[history_dataset_id] || null; + }, +}; + +const actions = { + fetchDatasetExtFiles: async ({ commit }, history_dataset_id) => { + const { data } = await axios.get( + `${getAppRoot()}api/histories/${history_dataset_id}/contents/${history_dataset_id}/extra_files` + ); + commit("saveDatasetExtFiles", { history_dataset_id, datasetExtFiles: data }); + }, +}; + +const mutations = { + saveDatasetExtFiles: (state, { history_dataset_id, datasetExtFiles }) => { + Vue.set(state.datasetExtFilesById, history_dataset_id, datasetExtFiles); + }, +}; + +export const datasetExtFilesStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/galaxy/scripts/store/datasetPathDestinationStore.js b/client/galaxy/scripts/store/datasetPathDestinationStore.js new file mode 100644 index 00000000000..b687a3aa1ce --- /dev/null +++ b/client/galaxy/scripts/store/datasetPathDestinationStore.js @@ -0,0 +1,35 @@ +export const state = { + datasetPathDestination: {}, +}; + +import { getPathDestination } from "components/Dataset/compositeDatasetUtils"; + +const getters = { + pathDestination: (state) => (history_dataset_id, path) => { + if (state.datasetPathDestination[history_dataset_id]) { + return state.datasetPathDestination[history_dataset_id][path]; + } else { + return; + } + }, +}; + +const actions = { + fetchPathDestination: async ({ commit }, { history_dataset_id, path }) => { + const data = await getPathDestination(history_dataset_id, path); + commit("savePathDestination", { history_dataset_id, path, pathDestination: data }); + }, +}; + +const mutations = { + savePathDestination: (state, { history_dataset_id, path, pathDestination }) => { + state.datasetPathDestination[history_dataset_id] = { [path]: pathDestination }; + }, +}; + +export const datasetPathDestinationStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/galaxy/scripts/store/index.js b/client/galaxy/scripts/store/index.js index 0f89419a996..9554dd86af8 100644 --- a/client/galaxy/scripts/store/index.js +++ b/client/galaxy/scripts/store/index.js @@ -15,6 +15,8 @@ import { historyStore } from "./historyStore"; import { userStore } from "./userStore"; import { configStore } from "./configStore"; import { workflowStore } from "./workflowStore"; +import { datasetPathDestinationStore } from "./datasetPathDestinationStore"; +import { datasetExtFilesStore } from "./datasetExtFilesStore"; Vue.use(Vuex); @@ -33,6 +35,8 @@ export function createStore() { tags: tagStore, jobMetrics: jobMetricsStore, destinationParameters: jobDestinationParametersStore, + datasetPathDestination: datasetPathDestinationStore, + datasetExtFiles: datasetExtFilesStore, invocations: invocationStore, user: userStore, config: configStore, diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index 44471795d57..a14fbf59619 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -316,43 +316,7 @@ body { } } -// Messages for message box, slightly different style - -.panel-error-message, -.panel-warning-message, -.panel-done-message, -.panel-info-message { - height: $panel-message-height; - line-height: $panel-message-height; - padding: 0px; - padding-left: 2.5rem; - background-color: $state-danger-bg; - background-image: url(../../images/error_small.png); - background-repeat: no-repeat; - background-position: 0.5rem 50%; -} - -.panel-warning-message { - background-image: url(../../images/warn_small.png); - background-color: $state-warning-bg; -} - -.panel-done-message { - background-image: url(../../images/ok_small.png); - background-color: $state-success-bg; -} - -.panel-info-message { - background-image: url(../../images/info_icon.svg); - background-size: 1.5rem; - background-color: $state-info-bg; - svg:path { - fill: black; - } -} - // ==== Masthead ==== - #masthead { @extend .p-0; @extend .mb-0; @@ -961,34 +925,7 @@ ul.manage-table-actions li { margin-left: 0.5em; } -.state-progress { - @extend .position-relative; - @extend .m-1; - border: $border-default; - .info { - color: $text-color; - background: $body-bg; - } - .new { - background: $state-default-bg; - } - .running { - background: $state-running-bg; - } - .ok { - background: $state-success-bg; - } - .error { - background: $state-danger-bg; - } - .note { - @extend .position-absolute; - @extend .ml-2; - } -} - // State colors - .state-color-new { border-color: $state-default-border; background: $state-default-bg; diff --git a/client/galaxy/style/scss/overrides.scss b/client/galaxy/style/scss/overrides.scss index ca716be3741..69b0aa6727c 100644 --- a/client/galaxy/style/scss/overrides.scss +++ b/client/galaxy/style/scss/overrides.scss @@ -147,3 +147,14 @@ pre.code { padding: 0.5rem 1.5rem 0.5rem 1rem; font-weight: bold; } + +// progress bar +.progress-bar.bg-success { + background: $state-success-bg !important; +} +.progress-bar.bg-danger { + background: $state-danger-bg !important; +} +.progress-bar.bg-warning { + background: $state-warning-bg !important; +} diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 1a3199e173f..c30c71da4e5 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1261,8 +1261,9 @@ :Description: E-mail domains blocklist is used for filtering out users that are using disposable email addresses at registration. If their - address domain matches any domain on the list, they are refused - registration. + address's base domain matches any domain on the list, they are + refused registration. Address subdomains are ignored (both + 'name@spam.com' and 'name@foo.spam.com' will match 'spam.com'). Example value 'email_blocklist.conf' The value of this option will be resolved with respect to . @@ -1278,8 +1279,11 @@ E-mail domains allowlist is used to specify allowed email address domains. If the list is non-empty and a user attempts registration using an email address belonging to a domain that is not on the - list, registration will be denied. This is a more restrictive - option than , and therefore, in case + list, registration will be denied. Unlike + which matches the address's base + domain, here email addresses are matched against the full domain + (base + subdomain). This is a more restrictive option than + , and therefore, in case is set and is not empty, will be ignored. Example value 'email_allowlist.conf' @@ -3569,6 +3573,22 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~~~ +``metadata_strategy`` +~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Determines how metadata will be set. Valid values are `directory`, + `extended` and `legacy`. In extended mode jobs will decide if a + tool run failed, the object stores configuration is serialized and + made available to the job and is used for writing output datasets + to the object store as part of the job and dynamic output + discovery (e.g. discovered datasets , + unpopulated collections, etc) happens as part of the job. +:Default: ``directory`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``retry_metadata_internally`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 0a459a371de..dfc9f93430a 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -13,3 +13,4 @@ Development Documentation faq debugging_tests debugging_galaxy + starting_galaxy diff --git a/doc/source/dev/starting_galaxy.md b/doc/source/dev/starting_galaxy.md new file mode 100644 index 00000000000..2f8c483f45e --- /dev/null +++ b/doc/source/dev/starting_galaxy.md @@ -0,0 +1,95 @@ +# Starting Galaxy + +## What happens when you start Galaxy? + +Galaxy is started by running ./run.sh script. The following call graph documents what happens during Galaxy startup process. Please note, Galaxy is a WSGI app, so in principle all Python WSGI servers can run the Galaxy app (Paste, Gunicorn and uWSGI are known to work and have some level of support). This document separates web server specific section (e.g. Paste) from Galaxy specific section (FYI, Galaxy uses uWSGI by default). + +## Paste + +* ./galaxy/scripts/paster.py + * Add ./galaxy/lib to system path + * Check Python version + * version >= 3.5 -> OK + * version = 2.7 -> Deprecated + * No other version is supported + * serve.run() + +* ./galaxy/lib/galaxy/util/pastescript/serve.py (The Python Paste package contains Python modules that help in implementing WSGI middleware) + * serve.run() + * Parse Python command line arguments + * Extract the command name from command line arguments (e.g., 'serve') + * Get the Command object for command name, then invoke it (call Command's run() method) + * Command.run() + * Setup default options, validate command line arguments, then call ServeCommand's command() method + * ServeCommand.command() + * Conditionally, stop the daemon + * If a config file is required, make sure a config file is provided on the command line arguments. + * Possible subcommands: start, stop, restart, status (or None) + * Call loadserver() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py + * Calls loadobj() -> calls LoaderContext's create() -> calls _Server's invoke() + * Call loadapp() method in ./galaxy/lib/galaxy/util/pastescript/loadwsgi.py + * Calls loadobj() -> calls LoaderContext's create() -> calls _App's invoke() -> calls fix_call() on app_factory (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + +## Galaxy + +* app_factory: + * Create UniverseApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * Create GalaxyAppConfiguration instance from config file specified on the command line, then check for errors + * Configure logging (in ./galaxy/lib/galaxy/config/__init__.py) + * Create ExecutionTimerFactory + * Which has a GalaxyStatsdClient field + * Configure FluentTraceLogger + * Create the application stack (uWSGi, Paste, or Webless) + * App registers post fork functions to start application stack threads + * Create GalaxyQueueWorker instance (a worker for Galaxy's queues, used for dispatching control tasks. Sqlite instance) + * Configure tool shed registry + * Configure object store. build_object_store_from_config() (in ./galaxy/lib/galaxy/objectstore/__init__.py) calls objectstore_class.from_xml(), calls clazz.parse_xml() + * Configure models in _configure_models() + * Calls create_or_verify_database() in ./galaxy/libl/galaxy/model/migrate/check.py + * If a new database, calls migrate_from_scratch() in ./galaxy/libl/galaxy/model/migrate/check.py + * Calls mapping.init() in ./galaxy/libl/galaxy/model/mapping.py (Details of how data objects are mapped to relational DB tables). + * Calls init_models_from_config in in ./galaxy/libl/galaxy/model/mapping.py + * Configure security + * Set GalaxyTagHandler (in ./galaxy/lib/galaxy/model/tags.py) + * Handles CRUD operations for tagging datasets, workflows, visualizations, etc + * Set DatasetCollectionManager (in ./galaxy/lib/galaxy/managers/collections.py) + * Configure HistoryManager + * Configure HistoryDatasetAssociation (HDA) Manager + * Create WorkflowsManager + * Handles CRUD operations related to workflows + * Configure DependencyResolverView (a RESTful interface to galaxy.tool_util.deps.DependencyResolver) + * Configure TestDataResolver + * Configure LibraryManager (Data libraries under Shared Data) + * Configure FolderManager (Folders are folders in a data library) + * Configure DynamicToolManager (Tools that are loaded from DB instead of disk files) + * Configure DataProviderRegistry + * Configure JobMetrics manager + * Configure ErrorReport plugins + * Configure job management configuration + * Setup a tool cache + * Watch various config files for immediate reload + * Configure Data Manager + * Configure UpdateRespositoyManager + * Load proprietary datatype convertors and display applications + * Configure VisualizationRegistry, ToursRegistry, and WebHookRegistry + * Load security policy and quota management + * Configure heartbeat for thread profiling + * If configured, create Sentry client + * App registers post fork functions to start Sentry client thread + * Create JobManager + * App registers post fork functions to start application job manager thread + * Job manager thread starts monitor threads for JobHandlerQueue and JobHandlerStopQueue (2 threads) + * JobManager has a JobHandler field, which manages the preparation, running, tracking, and finishing of jobs + * JobHandler has a job dispatcher, which launches underlying job runners + * DefaultJobDispatcher calls app's JobConfiguration's get_job_runner_plugins() to create LocalJobRunner instances + * The default number of LocalJobRuner instances is 4 (4 threads) + * JobHandler also has queues for starting and stopping jobs (unless track_jobs_in_database is True) + * Create WorkflowSchedulingManager + * Starts workflow request monitor thread in __start_request_monitor() method (1 thread) + * Configure InteractiveToolManager + * Create database heartbeat instance + * App registers post fork functions to start application DB heartbeat thread (1 thread) + * Create CommunityWebApplication (in ./galaxy/lib/galaxy/webapps/galaxy/buildapp.py) + * CommunityWebApplication: A WSGI application that maps requests to objects using routes and to methods on those objects + * Map all requests to controllers/actions + * Wrap web app in some middleware and return webapp diff --git a/doc/source/releases/20.05.rst b/doc/source/releases/20.05.rst index 9a2031412d6..226acc2097e 100644 --- a/doc/source/releases/20.05.rst +++ b/doc/source/releases/20.05.rst @@ -62,7 +62,7 @@ Enhancements (thanks to `@OlegZharkov `__). `Pull Request 9562`_ * Datasource tool for SRA manifest - `Pull Request 9613`_ + `Pull Request 9613`_, `Pull Request 9892`_ .. enhancement_tag_viz * Skip visualization build, if no changes were registered @@ -175,7 +175,7 @@ Enhancements `Pull Request 9186`_ * Full Python dependencies update (thanks to `@nsoranzo `__). - `Pull Request 9188`_ + `Pull Request 9188`_, `Pull Request 9402`_, `Pull Request 9529`_, `Pull Request 9697`_ * Allow mixed store_by objectstore configurations. `Pull Request 9197`_ * implement workflow bookmark selenium test @@ -217,9 +217,6 @@ Enhancements `Pull Request 9390`_ * Replace editor grid image with css, add editor client unit tests `Pull Request 9392`_ -* Full Python dependencies update - (thanks to `@nsoranzo `__). - `Pull Request 9402`_ * Vuefy interactive tools (thanks to `@OlegZharkov `__). `Pull Request 9409`_ @@ -254,9 +251,6 @@ Enhancements `Pull Request 9521`_ * Rename and modify current_history_contents method `Pull Request 9526`_ -* Full Python dependencies update - (thanks to `@nsoranzo `__). - `Pull Request 9529`_ * Improvement to the selenium test to change password from user preferences (thanks to `@Realjuan `__). `Pull Request 9545`_ @@ -306,14 +300,11 @@ Enhancements * Feature/3937 advanced search tooltip (thanks to `@assuntad23 `__). `Pull Request 9696`_ -* Full Python dependencies update - (thanks to `@nsoranzo `__). - `Pull Request 9697`_ .. small_enhancement * Rename function, params in ConfigSerializer `Pull Request 9247`_ -* Make `str.join()`s more efficient +* Make ``str.join()``s more efficient (thanks to `@nsoranzo `__). `Pull Request 9258`_ * Faster and cleaner set creation @@ -338,7 +329,7 @@ Enhancements `Pull Request 9443`_ * Move sanitize_whitelist into mutable_config_dir `Pull Request 9455`_ -* Use auto_create functionality of `get_private_user_role()` +* Use auto_create functionality of ``get_private_user_role()`` (thanks to `@nsoranzo `__). `Pull Request 9509`_ * Combine selenium tests for creating and switching histories @@ -401,6 +392,9 @@ Fixes * extract_genomic_dna.py: import Comment, Header from bx.tabular.io instead of bx.intervals.io (thanks to `@cat-bro `__). `Pull Request 9748`_ +* Move some tools that no longer need galaxy on the PATH + (thanks to `@bernt-matthias `__). + `Pull Request 9850`_ .. bug_tag_workflows * Fix workflow list filtering dropdown @@ -411,6 +405,8 @@ Fixes `Pull Request 9676`_ * Trigger onchange event, fixes tool version changes being lost `Pull Request 9695`_ +* Input dataset module format filtering fix; + `Pull Request 9884`_ .. bug_tag_ui * fix for python3 and GEDA, was getting "Connection closed at byte 0" @@ -451,6 +447,8 @@ Fixes * Migrate condor runner to jwd for job-files (thanks to `@bgruening `__). `Pull Request 9833`_ +* Fix incorrect variable assignment + `Pull Request 9907`_ .. bug_tag_admin * Fix vue helper in admin router, fixes toolshed installation, avoids console errors @@ -514,7 +512,7 @@ Fixes * Use a Galaxy root path with a space for converters and mulled tests (thanks to `@nsoranzo `__). `Pull Request 9429`_ -* Fix `test_run_simple()` test broken on Python 2 by virtualenv 20.0.5 +* Fix ``test_run_simple()`` test broken on Python 2 by virtualenv 20.0.5 (thanks to `@nsoranzo `__). `Pull Request 9458`_ * Use browser locale over global default @@ -575,8 +573,8 @@ Fixes * Fix indexing script `Pull Request 9565`_ * Assign random password in user manager if none specified - (thanks to `@nuwang `__). - `Pull Request 9587`_ + (thanks to `@nuwang `__, `@nsoranzo `__). + `Pull Request 9587`_, `Pull Request 9851`_ * Fix removal of workflow outputs from workflow nodes in editor `Pull Request 9606`_ * Fix ELIXIR AAI @@ -587,7 +585,7 @@ Fixes `Pull Request 9622`_ * Change session timeout redirect URL to login entrypoint. `Pull Request 9626`_ -* Changed CSS overflow property from scroll to hidden to make table scr… +* Changed CSS overflow property from scroll to hidden to make table scrollable (thanks to `@assuntad23 `__). `Pull Request 9627`_ * fix links in Galaxy xsd docs @@ -601,7 +599,7 @@ Fixes * Hide "python: not found" error when Python 2 is not installed (thanks to `@nsoranzo `__). `Pull Request 9657`_ -* Fix for `time.clock()` removal in Python 3.8 +* Fix for ``time.clock()`` removal in Python 3.8 (thanks to `@nsoranzo `__). `Pull Request 9666`_ * Fixed an issue in object store integration tests @@ -622,6 +620,8 @@ Fixes `Pull Request 9724`_ * Fix custos credential fetching in python3 `Pull Request 9726`_ +* oidc-based authnz provider overhaul; breaking changes for keycloak + `Pull Request 9730`_ * Sort Workflow list by last updated time instead of created time `Pull Request 9740`_ * Remove dummy tool sequence @@ -648,6 +648,31 @@ Fixes `Pull Request 9821`_ * Tolerate exception in container resolution `Pull Request 9824`_ +* Invalidate tool cache when tool modified outside of Galaxy + `Pull Request 9866`_ +* Custos spec fixes + `Pull Request 9889`_ +* Fixes for flake8 3.8 and bugbear + (thanks to `@nsoranzo `__). + `Pull Request 9896`_ +* Fix webpack import proxy prefix + `Pull Request 9898`_ +* Fix extra preference values leaking between requests + (thanks to `@nuwang `__). + `Pull Request 9913`_ +* Rewrite ``check_html()`` and ``Text.count_data_lines()`` to read a chunk of each line + (thanks to `@nsoranzo `__). + `Pull Request 9920`_ +* Make cookie handling more robust + `Pull Request 9921`_ +* Enable regex validator to validate lists of values. + `Pull Request 9925`_ +* oidc: fix dict_keys does not support indexing + (thanks to `@almahmoud `__). + `Pull Request 9931`_ +* Move the ``--debug`` flag after the conda subcommand + (thanks to `@nsoranzo `__). + `Pull Request 9932`_ .. include:: 20.05_prs.rst diff --git a/doc/source/releases/20.05_prs.rst b/doc/source/releases/20.05_prs.rst index da0105d63ed..eee1092d8cc 100644 --- a/doc/source/releases/20.05_prs.rst +++ b/doc/source/releases/20.05_prs.rst @@ -226,6 +226,7 @@ .. _Pull Request 9719: https://github.com/galaxyproject/galaxy/pull/9719 .. _Pull Request 9724: https://github.com/galaxyproject/galaxy/pull/9724 .. _Pull Request 9726: https://github.com/galaxyproject/galaxy/pull/9726 +.. _Pull Request 9730: https://github.com/galaxyproject/galaxy/pull/9730 .. _Pull Request 9731: https://github.com/galaxyproject/galaxy/pull/9731 .. _Pull Request 9733: https://github.com/galaxyproject/galaxy/pull/9733 .. _Pull Request 9734: https://github.com/galaxyproject/galaxy/pull/9734 @@ -250,3 +251,22 @@ .. _Pull Request 9824: https://github.com/galaxyproject/galaxy/pull/9824 .. _Pull Request 9833: https://github.com/galaxyproject/galaxy/pull/9833 .. _Pull Request 9835: https://github.com/galaxyproject/galaxy/pull/9835 +.. _Pull Request 9838: https://github.com/galaxyproject/galaxy/pull/9838 +.. _Pull Request 9850: https://github.com/galaxyproject/galaxy/pull/9850 +.. _Pull Request 9851: https://github.com/galaxyproject/galaxy/pull/9851 +.. _Pull Request 9866: https://github.com/galaxyproject/galaxy/pull/9866 +.. _Pull Request 9870: https://github.com/galaxyproject/galaxy/pull/9870 +.. _Pull Request 9878: https://github.com/galaxyproject/galaxy/pull/9878 +.. _Pull Request 9884: https://github.com/galaxyproject/galaxy/pull/9884 +.. _Pull Request 9889: https://github.com/galaxyproject/galaxy/pull/9889 +.. _Pull Request 9892: https://github.com/galaxyproject/galaxy/pull/9892 +.. _Pull Request 9894: https://github.com/galaxyproject/galaxy/pull/9894 +.. _Pull Request 9896: https://github.com/galaxyproject/galaxy/pull/9896 +.. _Pull Request 9898: https://github.com/galaxyproject/galaxy/pull/9898 +.. _Pull Request 9907: https://github.com/galaxyproject/galaxy/pull/9907 +.. _Pull Request 9913: https://github.com/galaxyproject/galaxy/pull/9913 +.. _Pull Request 9920: https://github.com/galaxyproject/galaxy/pull/9920 +.. _Pull Request 9921: https://github.com/galaxyproject/galaxy/pull/9921 +.. _Pull Request 9925: https://github.com/galaxyproject/galaxy/pull/9925 +.. _Pull Request 9931: https://github.com/galaxyproject/galaxy/pull/9931 +.. _Pull Request 9932: https://github.com/galaxyproject/galaxy/pull/9932 diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 11f6e43a52a..18f907a03f7 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -17,8 +17,8 @@ from galaxy.model import CustosAuthnzToken, User from ..authnz import IdentityProvider log = logging.getLogger(__name__) -STATE_COOKIE_NAME = 'custos-state' -NONCE_COOKIE_NAME = 'custos-nonce' +STATE_COOKIE_NAME = 'galaxy-oidc-state' +NONCE_COOKIE_NAME = 'galaxy-oidc-nonce' KEYCLOAK_BACKENDS = {'custos', 'cilogon', 'keycloak'} diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index be52449ec4e..e994605f08c 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -240,7 +240,7 @@ class AuthnzManager: ID does not exist, or the configuration is defined for a another user than trans.user. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type authz_id: int @@ -292,7 +292,7 @@ class AuthnzManager: raise exceptions.AuthenticationFailed(e.message) except Exception as e: msg = 'The following error occurred when handling callback from `{}` identity provider: ' \ - '{}'.format(provider, e.message) + '{}'.format(provider, str(e)) log.exception(msg) return False, msg, (None, None) @@ -321,7 +321,7 @@ class AuthnzManager: return True, message, backend.logout(trans, post_logout_redirect_url) except Exception as e: msg = 'The following error occurred when logging out from `{}` identity provider: ' \ - '{}'.format(provider, e.message) + '{}'.format(provider, str(e)) log.exception(msg) return False, msg, None diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index accfb7ac66c..7182026b332 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -1029,7 +1029,7 @@ class ConfiguresGalaxyMixin: self._toolbox_lock = threading.RLock() self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) galaxy_root_dir = os.path.abspath(self.config.root) - file_path = os.path.abspath(getattr(self.config, "file_path")) + file_path = os.path.abspath(self.config.file_path) app_info = AppInfo( galaxy_root_dir=galaxy_root_dir, default_file_path=file_path, diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index d6aa6df4953..5eda281b6aa 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -700,9 +700,10 @@ galaxy: #instance_resource_url: null # E-mail domains blocklist is used for filtering out users that are - # using disposable email addresses at registration. If their address - # domain matches any domain on the list, they are refused - # registration. + # using disposable email addresses at registration. If their + # address's base domain matches any domain on the list, they are + # refused registration. Address subdomains are ignored (both + # 'name@spam.com' and 'name@foo.spam.com' will match 'spam.com'). # Example value 'email_blocklist.conf' # The value of this option will be resolved with respect to # . @@ -711,8 +712,11 @@ galaxy: # E-mail domains allowlist is used to specify allowed email address # domains. If the list is non-empty and a user attempts registration # using an email address belonging to a domain that is not on the - # list, registration will be denied. This is a more restrictive option - # than , and therefore, in case + # list, registration will be denied. Unlike + # which matches the address's base + # domain, here email addresses are matched against the full domain + # (base + subdomain). This is a more restrictive option than + # , and therefore, in case # is set and is not empty, # will be ignored. # Example value 'email_allowlist.conf' @@ -1747,6 +1751,15 @@ galaxy: # database. #enable_job_recovery: true + # Determines how metadata will be set. Valid values are `directory`, + # `extended` and `legacy`. In extended mode jobs will decide if a tool + # run failed, the object stores configuration is serialized and made + # available to the job and is used for writing output datasets to the + # object store as part of the job and dynamic output discovery (e.g. + # discovered datasets , unpopulated collections, + # etc) happens as part of the job. + #metadata_strategy: directory + # Although it is fairly reliable, setting metadata can occasionally # fail. In these instances, you can choose to retry setting it # internally or leave it in a failed state (since retrying internally diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced index 53254618eba..beb5feb3d13 100644 --- a/lib/galaxy/config/sample/job_conf.xml.sample_advanced +++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced @@ -401,6 +401,17 @@ try it early - it will slightly speed up local jobs by embedding metadata calculation in job script itself. --> + directory + @@ -194,7 +214,7 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): if name is None: name = "unnamed output" - element_datasets = [] + element_datasets = {'element_identifiers': [], 'datasets': [], 'tag_lists': [], 'paths': []} for filename, discovered_file in filenames.items(): create_dataset_timer = ExecutionTimer() fields_match = discovered_file.match @@ -213,7 +233,6 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): dataset_name = fields_match.name or designation link_data = discovered_file.match.link_data - tag_list = discovered_file.match.tag_list sources = discovered_file.match.sources hashes = discovered_file.match.hashes @@ -225,10 +244,8 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): visible=visible, dbkey=dbkey, name=dataset_name, - filename=filename, metadata_source_name=metadata_source_name, link_data=link_data, - tag_list=tag_list, sources=sources, hashes=hashes, created_from_basename=created_from_basename, @@ -242,18 +259,13 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): name, create_dataset_timer, ) - element_datasets.append((element_identifiers, dataset)) + element_datasets['element_identifiers'].append(element_identifiers) + element_datasets['datasets'].append(dataset) + element_datasets['tag_lists'].append(discovered_file.match.tag_list) + element_datasets['paths'].append(filename) - add_datasets_timer = ExecutionTimer() - self.add_datasets_to_history([d for (ei, d) in element_datasets]) - log.debug( - "(%s) Add dynamic collection datasets to history for output [%s] %s", - self.job_id(), - name, - add_datasets_timer, - ) - - for (element_identifiers, dataset) in element_datasets: + self.add_tags_to_datasets(datasets=element_datasets['datasets'], tag_lists=element_datasets['tag_lists']) + for (element_identifiers, dataset) in zip(element_datasets['element_identifiers'], element_datasets['datasets']): current_builder = root_collection_builder for element_identifier in element_identifiers[:-1]: current_builder = current_builder.get_level(element_identifier) @@ -265,6 +277,31 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): self.add_output_dataset_association(association_name, dataset) self.flush() + self.update_object_store_with_datasets(datasets=element_datasets['datasets'], paths=element_datasets['paths']) + add_datasets_timer = ExecutionTimer() + self.add_datasets_to_history(element_datasets['datasets']) + log.debug( + "(%s) Add dynamic collection datasets to history for output [%s] %s", + self.job_id(), + name, + add_datasets_timer, + ) + self.set_datasets_metadata(datasets=element_datasets['datasets']) + + def add_tags_to_datasets(self, datasets, tag_lists): + if any(tag_lists): + # This works around SessionlessModelPersistenceContext not implementing a tag handler ... + # that's not better or worse than what we previously did in create_datasets + # TDOD: implement that or figure out why it is not implemented and find a better solution. + # Could it be that SessionlessModelPersistenceContext doesn't support tags? + tag_session = self.tag_handler.create_tag_handler_session() + for dataset, tags in zip(datasets, tag_lists): + tag_session.add_tags_from_list(self.job.user, dataset, tags, flush=False) + + def update_object_store_with_datasets(self, datasets, paths): + for dataset, path in zip(datasets, paths): + self.object_store.update_from_file(dataset.dataset, file_name=path, create=True) + dataset.set_size(no_extra_files=True) @abc.abstractproperty def tag_handler(self): diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 1d23d3a6580..627c27c25a2 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -41,26 +41,31 @@ class TagHandler: # Initialize with known classes - add to this in subclasses. self.item_tag_assoc_info = {} - def add_tags_from_list(self, user, item, new_tags_list): + def create_tag_handler_session(self): + # Creates a transient tag handler that avoids repeated flushes + return GalaxyTagHandlerSession(self.sa_session) + + def add_tags_from_list(self, user, item, new_tags_list, flush=True): new_tags_set = set(new_tags_list) if item.tags: new_tags_set.update(self.get_tags_str(item.tags).split(',')) - return self.set_tags_from_list(user, item, new_tags_set) + return self.set_tags_from_list(user, item, new_tags_set, flush=flush) - def remove_tags_from_list(self, user, item, tag_to_remove_list): + def remove_tags_from_list(self, user, item, tag_to_remove_list, flush=True): tag_to_remove_set = set(tag_to_remove_list) tags_set = {_.strip() for _ in self.get_tags_str(item.tags).split(',')} if item.tags: tags_set -= tag_to_remove_set - return self.set_tags_from_list(user, item, tags_set) + return self.set_tags_from_list(user, item, tags_set, flush=flush) - def set_tags_from_list(self, user, item, new_tags_list): + def set_tags_from_list(self, user, item, new_tags_list, flush=True): # precondition: item is already security checked against user # precondition: incoming tags is a list of sanitized/formatted strings self.delete_item_tags(user, item) new_tags_str = ','.join(new_tags_list) - self.apply_item_tags(user, item, unicodify(new_tags_str, 'utf-8')) - self.sa_session.flush() + self.apply_item_tags(user, item, unicodify(new_tags_str, 'utf-8'), flush=flush) + if flush: + self.sa_session.flush() return item.tags def get_tag_assoc_class(self, item_class): @@ -126,7 +131,9 @@ class TagHandler: """Delete tags from an item.""" # Delete item-tag associations. for tag in item.tags: - self.sa_session.delete(tag) + if tag.id: + # Only can and need to delete tag if tag is persisted + self.sa_session.delete(tag) # Delete tags from item. del item.tags[:] @@ -145,7 +152,7 @@ class TagHandler: return True return False - def apply_item_tag(self, user, item, name, value=None): + def apply_item_tag(self, user, item, name, value=None, flush=True): # Use lowercase name for searching/creating tag. lc_name = name.lower() # Get or create item-tag association. @@ -173,17 +180,17 @@ class TagHandler: item_tag_assoc.user_tname = name item_tag_assoc.user_value = value item_tag_assoc.value = lc_value - # Need to flush to get an ID. We need an ID to apply multiple tags with the same tname to an object. - self.sa_session.flush() + if flush: + self.sa_session.flush() return item_tag_assoc - def apply_item_tags(self, user, item, tags_str): + def apply_item_tags(self, user, item, tags_str, flush=True): """Apply tags to an item.""" # Parse tags. parsed_tags = self.parse_tags(tags_str) # Apply each tag. for name, value in parsed_tags: - self.apply_item_tag(user, item, name, value) + self.apply_item_tag(user, item, name, value, flush=flush) def get_tags_str(self, tags): """Build a string from an item's tags.""" @@ -217,9 +224,9 @@ class TagHandler: for sub_tag in tag_hierarchy: # Get or create subtag. tag_name = tag_prefix + self._scrub_tag_name(sub_tag) - tag = self.sa_session.query(galaxy.model.Tag).filter_by(name=tag_name).first() + tag = self._get_tag(tag_name) if not tag: - tag = galaxy.model.Tag(type=0, name=tag_name) + tag = self._create_tag_instance(tag_name) # Set tag parent. tag.parent = parent_tag # Update parent and tag prefix. @@ -227,6 +234,12 @@ class TagHandler: tag_prefix = tag.name + self.hierarchy_separator return tag + def _get_tag(self, tag_name): + return self.sa_session.query(galaxy.model.Tag).filter_by(name=tag_name).first() + + def _create_tag_instance(self, tag_name): + return galaxy.model.Tag(type=0, name=tag_name) + def _get_or_create_tag(self, tag_str): """Get or create a Tag object from a tag string.""" # Scrub tag; if tag is None after being scrubbed, return None. @@ -364,6 +377,24 @@ class GalaxyTagHandler(TagHandler): model.VisualizationTagAssociation.table.c.visualization_id) +class GalaxyTagHandlerSession(GalaxyTagHandler): + """Like GalaxyTagHandler, but avoids one flush per created tag.""" + def __init__(self, sa_session): + super().__init__(sa_session) + self.created_tags = {} + + def _get_tag(self, tag_name): + """Get tag from cache or database.""" + # Avoids creating multiple new tags with the same tag_name, which violates unique key constraint + return self.created_tags.get(tag_name) or super(GalaxyTagHandler, self)._get_tag(tag_name) + + def _create_tag_instance(self, tag_name): + """Create tag and and store in cache.""" + tag = super()._create_tag_instance(tag_name) + self.created_tags[tag_name] = tag + return tag + + class CommunityTagHandler(TagHandler): def __init__(self, sa_session): TagHandler.__init__(self, sa_session) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 4a1d16cd464..6c34673a580 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -4,6 +4,7 @@ Object Store plugin for the Integrated Rule-Oriented Data Store (iRODS) import logging import os import shutil +from contextlib import contextmanager from datetime import datetime from functools import partial try: @@ -110,7 +111,32 @@ def parse_config_xml(config_xml): raise -class CloudConfigMixin: +def acquire_session(host='localhost', port='1247', user='rods', password='rods', zone='tempZone', timeout='30'): + session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone) + # Set connection timeout + session.connection_timeout = timeout + return session + + +def release_session(session): + # This call will cleanup all the connections in the connection pool + # OSError sometimes happens on GitHub Actions, after the test has successfully completed. Ignore it if it happens. + try: + session.cleanup() + except OSError: + pass + + +@contextmanager +def managed_session(host='localhost', port='1247', user='rods', password='rods', zone='tempZone', timeout='30'): + session = acquire_session(host=host, port=port, user=user, password=password, zone=zone, timeout=timeout) + try: + yield session + finally: + release_session(session) + + +class CloudConfigMixin(object): def _config_to_dict(self): return { @@ -205,48 +231,11 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): _config_dict_error('extra_dirs') self.extra_dirs.update(extra_dirs) - self._initialize() - log.debug("irods __init__ %s", reload_timer) - - def shutdown(self): - # This call will cleanup all the connections in the connection pool - log.debug("In __shutdown__ Number of active connections: %s, Number of idle connections: %s", len(self.session.pool.active), len(self.session.pool.idle)) - # OSError sometimes happens on GitHub Actions, after the test has successfully completed. Ignore it if it happens. - try: - self.session.cleanup() - except OSError: - pass - - def _initialize(self): - reload_timer = ExecutionTimer() if irods is None: raise Exception(IRODS_IMPORT_MESSAGE) self.home = "/" + self.zone + "/home/" + self.username - - self.session = self._configure_connection(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone) - log.debug("irods _initialize %s", reload_timer) - - def _configure_connection(self, host='localhost', port='1247', user='rods', password='rods', zone='tempZone'): - self.connections = [] - reload_timer_1 = ExecutionTimer() - reload_timer_2 = ExecutionTimer() - session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone) - log.debug("irods _configure_connection, iRODSSession call %s", reload_timer_2) - # Set connection timeout - session.connection_timeout = self.timeout - # Throws NetworkException if connection fails - try: - # We will create as many conections as poolsize. - for idx in range(self.poolsize): - reload_timer_2 = ExecutionTimer() - self.connections.append(session.pool.get_connection()) - log.debug("irods _configure_connection, session.pool.get_connection() %s", reload_timer_2) - except NetworkException as e: - log.error('Could not create iRODS session: ' + str(e)) - raise - log.debug("irods _configure_connection %s", reload_timer_1) - return session + log.debug("irods __init__ %s", reload_timer) @classmethod def parse_xml(cls, config_xml): @@ -306,40 +295,43 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): # rel_path is file or folder? def _get_size_in_irods(self, rel_path): - p = Path(rel_path) - data_object_name = p.stem + p.suffix - subcollection_name = p.parent + with managed_session(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone, timeout=self.timeout) as session: + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent - collection_path = self.home + "/" + str(subcollection_name) - data_object_path = collection_path + "/" + str(data_object_name) + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) - log.debug("In _get_size_in_irods(): Number of active connections: %s, Number of idle connections: %s", len(self.session.pool.active), len(self.session.pool.idle)) - try: - data_obj = self.session.data_objects.get(data_object_path) - return data_obj.__sizeof__() - except (DataObjectDoesNotExist, CollectionDoesNotExist): - log.warn("Collection or data object (%s) does not exist", data_object_path) - return -1 + try: + data_obj = session.data_objects.get(data_object_path) + return data_obj.__sizeof__() + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.warn("Collection or data object (%s) does not exist", data_object_path) + return -1 + except NetworkException as e: + log.exception(e) + return -1 # rel_path is file or folder? def _data_object_exists(self, rel_path): - p = Path(rel_path) - data_object_name = p.stem + p.suffix - subcollection_name = p.parent + with managed_session(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone, timeout=self.timeout) as session: + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent - collection_path = self.home + "/" + str(subcollection_name) - data_object_path = collection_path + "/" + str(data_object_name) + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) - log.debug("In _data_object_exists(): Number of active connections: %s, Number of idle connections: %s", len(self.session.pool.active), len(self.session.pool.idle)) - try: - self.session.data_objects.get(data_object_path) - return True - except (DataObjectDoesNotExist, CollectionDoesNotExist): - log.debug("Collection or data object (%s) does not exist", data_object_path) - return False - except Exception as e: - log.exception(e) - return False + try: + session.data_objects.get(data_object_path) + return True + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.debug("Collection or data object (%s) does not exist", data_object_path) + return False + except NetworkException as e: + log.exception(e) + return False def _in_cache(self, rel_path): """ Check if the given dataset is in the local cache and return True if so. """ @@ -357,34 +349,37 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): return file_ok def _download(self, rel_path): - log.debug("Pulling data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + with managed_session(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone, timeout=self.timeout) as session: + log.debug("Pulling data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) - p = Path(rel_path) - data_object_name = p.stem + p.suffix - subcollection_name = p.parent + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent - collection_path = self.home + "/" + str(subcollection_name) - data_object_path = collection_path + "/" + str(data_object_name) - data_obj = None + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + data_obj = None - log.debug("In _download(): Number of active connections: %s, Number of idle connections: %s", len(self.session.pool.active), len(self.session.pool.idle)) - try: - data_obj = self.session.data_objects.get(data_object_path) - except (DataObjectDoesNotExist, CollectionDoesNotExist): - log.warn("Collection or data object (%s) does not exist", data_object_path) - return False + try: + data_obj = session.data_objects.get(data_object_path) + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.warn("Collection or data object (%s) does not exist", data_object_path) + return False + except NetworkException as e: + log.exception(e) + return False - if self.cache_size > 0 and data_obj.__sizeof__() > self.cache_size: - log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.", - rel_path, data_obj.__sizeof__(), self.cache_size) - return False + if self.cache_size > 0 and data_obj.__sizeof__() > self.cache_size: + log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.", + rel_path, data_obj.__sizeof__(), self.cache_size) + return False - log.debug("Pulled data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + log.debug("Pulled data object '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) - with data_obj.open('r') as data_obj_fp, open(self._get_cache_path(rel_path), "wb") as cache_fp: - for chunk in iter(partial(data_obj_fp.read, CHUNK_SIZE), b''): - cache_fp.write(chunk) - return True + with data_obj.open('r') as data_obj_fp, open(self._get_cache_path(rel_path), "wb") as cache_fp: + for chunk in iter(partial(data_obj_fp.read, CHUNK_SIZE), b''): + cache_fp.write(chunk) + return True def _push_to_irods(self, rel_path, source_file=None, from_string=None): """ @@ -395,52 +390,54 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): still using ``rel_path`` for collection and object store names. If ``from_string`` is provided, set contents of the file to the value of the string. """ - p = Path(rel_path) - data_object_name = p.stem + p.suffix - subcollection_name = p.parent + with managed_session(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone, timeout=self.timeout) as session: + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent - source_file = source_file if source_file else self._get_cache_path(rel_path) - options = {kw.FORCE_FLAG_KW: ''} + source_file = source_file if source_file else self._get_cache_path(rel_path) + options = {kw.FORCE_FLAG_KW: ''} - log.debug("In _push_to_irods(): Number of active connections: %s, Number of idle connections: %s", len(self.session.pool.active), len(self.session.pool.idle)) - if os.path.exists(source_file): - # Check if the data object exists in iRODS - collection_path = self.home + "/" + str(subcollection_name) - data_object_path = collection_path + "/" + str(data_object_name) - exists = False - try: - exists = self.session.data_objects.exists(data_object_path) - except NetworkException as e: - log.exception(e) - if os.path.getsize(source_file) == 0 and exists: - log.debug("Wanted to push file '%s' to iRODS collection '%s' but its size is 0; skipping.", source_file, rel_path) - return True - if from_string: - data_obj = self.session.data_objects.create(data_object_path, self.resource, **options) - with data_obj.open('w') as data_obj_fp: - data_obj_fp.write(from_string) - log.debug("Pushed data from string '%s' to collection '%s'", from_string, data_object_path) - else: - start_time = datetime.now() - log.debug("Pushing cache file '%s' of size %s bytes to collection '%s'", source_file, os.path.getsize(source_file), rel_path) + if os.path.exists(source_file): + # Check if the data object exists in iRODS + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + exists = False + try: + exists = session.data_objects.exists(data_object_path) - # Create sub-collection first - self.session.collections.create(collection_path, recurse=True) - data_obj = self.session.data_objects.create(data_object_path, self.resource, **options) + if os.path.getsize(source_file) == 0 and exists: + log.debug("Wanted to push file '%s' to iRODS collection '%s' but its size is 0; skipping.", source_file, rel_path) + return True - # Write to file in subcollection created above - with open(source_file, 'rb') as content_file, data_obj.open('w') as data_obj_fp: - for chunk in iter(partial(content_file.read, CHUNK_SIZE), b''): - data_obj_fp.write(chunk) + if from_string: + data_obj = session.data_objects.create(data_object_path, self.resource, **options) + with data_obj.open('w') as data_obj_fp: + data_obj_fp.write(from_string) + log.debug("Pushed data from string '%s' to collection '%s'", from_string, data_object_path) + else: + start_time = datetime.now() + log.debug("Pushing cache file '%s' of size %s bytes to collection '%s'", source_file, os.path.getsize(source_file), rel_path) - end_time = datetime.now() - log.debug("Pushed cache file '%s' to collection '%s' (%s bytes transfered in %s sec)", - source_file, rel_path, os.path.getsize(source_file), end_time - start_time) - return True - else: - log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", - rel_path, source_file) - return False + # Create sub-collection first + session.collections.create(collection_path, recurse=True) + data_obj = session.data_objects.create(data_object_path, self.resource, **options) + + # Write to file in subcollection created above + with open(source_file, 'rb') as content_file, data_obj.open('w') as data_obj_fp: + for chunk in iter(partial(content_file.read, CHUNK_SIZE), b''): + data_obj_fp.write(chunk) + + end_time = datetime.now() + log.debug("Pushed cache file '%s' to collection '%s' (%s bytes transfered in %s sec)", + source_file, rel_path, os.path.getsize(source_file), end_time - start_time) + return True + except NetworkException as e: + log.exception(e) + return False + + log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", rel_path, source_file) + return False def file_ready(self, obj, **kwargs): """ @@ -457,35 +454,21 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): return False def _exists(self, obj, **kwargs): - in_cache = in_irods = False rel_path = self._construct_path(obj, **kwargs) - # Check cache - if self._in_cache(rel_path): - in_cache = True - # Check iRODS - in_irods = self._data_object_exists(rel_path) + # Check cache and irods + if self._in_cache(rel_path) or self._data_object_exists(rel_path): + return True # dir_only does not get synced so shortcut the decision dir_only = kwargs.get('dir_only', False) base_dir = kwargs.get('base_dir', None) - if dir_only: - if in_cache or in_irods: - return True + if dir_only and base_dir: # for JOB_WORK directory - elif base_dir: - if not os.path.exists(rel_path): - os.makedirs(rel_path) - return True - else: - return False - - if in_cache and not in_irods: + if not os.path.exists(rel_path): + os.makedirs(rel_path) return True - elif in_irods: - return True - else: - return False + return False def _create(self, obj, **kwargs): if not self._exists(obj, **kwargs): @@ -535,69 +518,76 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): return 0 def _delete(self, obj, entire_dir=False, **kwargs): - rel_path = self._construct_path(obj, **kwargs) - extra_dir = kwargs.get('extra_dir', None) - base_dir = kwargs.get('base_dir', None) - dir_only = kwargs.get('dir_only', False) - obj_dir = kwargs.get('obj_dir', False) + with managed_session(host=self.host, port=self.port, user=self.username, password=self.password, zone=self.zone, timeout=self.timeout) as session: + rel_path = self._construct_path(obj, **kwargs) + extra_dir = kwargs.get('extra_dir', None) + base_dir = kwargs.get('base_dir', None) + dir_only = kwargs.get('dir_only', False) + obj_dir = kwargs.get('obj_dir', False) - log.debug("In _delete(): Number of active connections: %s, Number of idle connections: %s", len(self.session.pool.active), len(self.session.pool.idle)) - try: - # Remove temparory data in JOB_WORK directory - if base_dir and dir_only and obj_dir: - shutil.rmtree(os.path.abspath(rel_path)) - return True - - # For the case of extra_files, because we don't have a reference to - # individual files we need to remove the entire directory structure - # with all the files in it. This is easy for the local file system, - # but requires iterating through each individual key in irods and deleing it. - if entire_dir and extra_dir: - shutil.rmtree(self._get_cache_path(rel_path)) - - col_path = self.home + "/" + str(rel_path) - col = None - try: - col = self.session.collections.get(col_path) - except CollectionDoesNotExist: - log.warn("Collection (%s) does not exist!", col_path) - return False - - cols = col.walk() - # Traverse the tree only one level deep - for _ in range(2): - # get next result - _, _, data_objects = next(cols) - - # Delete data objects - for data_object in data_objects: - data_object.unlink(force=True) - - return True - - else: - # Delete from cache first - os.unlink(self._get_cache_path(rel_path)) - # Delete from irods as well - p = Path(rel_path) - data_object_name = p.stem + p.suffix - subcollection_name = p.parent - - collection_path = self.home + "/" + str(subcollection_name) - data_object_path = collection_path + "/" + str(data_object_name) - - try: - data_obj = self.session.data_objects.get(data_object_path) - # remove object - data_obj.unlink(force=True) - return True - except (DataObjectDoesNotExist, CollectionDoesNotExist): - log.info("Collection or data object (%s) does not exist", data_object_path) + try: + # Remove temparory data in JOB_WORK directory + if base_dir and dir_only and obj_dir: + shutil.rmtree(os.path.abspath(rel_path)) return True - except OSError: - log.exception('%s delete error', self._get_filename(obj, **kwargs)) - return False + # For the case of extra_files, because we don't have a reference to + # individual files we need to remove the entire directory structure + # with all the files in it. This is easy for the local file system, + # but requires iterating through each individual key in irods and deleing it. + if entire_dir and extra_dir: + shutil.rmtree(self._get_cache_path(rel_path)) + + col_path = self.home + "/" + str(rel_path) + col = None + try: + col = session.collections.get(col_path) + except CollectionDoesNotExist: + log.warn("Collection (%s) does not exist!", col_path) + return False + except NetworkException as e: + log.exception(e) + return False + + cols = col.walk() + # Traverse the tree only one level deep + for _ in range(2): + # get next result + _, _, data_objects = next(cols) + + # Delete data objects + for data_object in data_objects: + data_object.unlink(force=True) + + return True + + else: + # Delete from cache first + os.unlink(self._get_cache_path(rel_path)) + # Delete from irods as well + p = Path(rel_path) + data_object_name = p.stem + p.suffix + subcollection_name = p.parent + + collection_path = self.home + "/" + str(subcollection_name) + data_object_path = collection_path + "/" + str(data_object_name) + + try: + data_obj = session.data_objects.get(data_object_path) + # remove object + data_obj.unlink(force=True) + return True + except (DataObjectDoesNotExist, CollectionDoesNotExist): + log.info("Collection or data object (%s) does not exist", data_object_path) + return True + except NetworkException as e: + log.exception(e) + return False + except OSError: + log.exception('%s delete error', self._get_filename(obj, **kwargs)) + except NetworkException as e: + log.exception(e) + return False def _get_data(self, obj, start=0, count=-1, **kwargs): rel_path = self._construct_path(obj, **kwargs) diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py index 7f907508893..cece83e23d7 100644 --- a/lib/galaxy/quota/__init__.py +++ b/lib/galaxy/quota/__init__.py @@ -156,7 +156,13 @@ class QuotaAgent(NoQuotaAgent): except ZeroDivisionError: return 100 - def set_entity_quota_associations(self, quotas=[], users=[], groups=[], delete_existing_assocs=True): + def set_entity_quota_associations(self, quotas=None, users=None, groups=None, delete_existing_assocs=True): + if quotas is None: + quotas = [] + if users is None: + users = [] + if groups is None: + groups = [] for quota in quotas: if delete_existing_assocs: flush_needed = False diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index d55cc58a81d..ef016330032 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -66,21 +66,25 @@ def validate_email(trans, email, user=None, check_dup=True, allow_empty=False): message = "User with email '%s' already exists." % email # If the allowlist is not empty filter out any domain not in the list and ignore blocklist. elif trans.app.config.email_domain_allowlist_content is not None: - domain = email.split('@')[1] - if len(domain.split('.')) > 2: - domain = ('.').join(domain.split('.')[-2:]) + domain = extract_domain(email) if domain not in trans.app.config.email_domain_allowlist_content: message = "Please enter an allowed domain email address for this server." # If the blocklist is not empty filter out the disposable domains. elif trans.app.config.email_domain_blocklist_content is not None: - domain = email.split('@')[1] - if len(domain.split('.')) > 2: - domain = ('.').join(domain.split('.')[-2:]) + domain = extract_domain(email, base_only=True) if domain in trans.app.config.email_domain_blocklist_content: message = "Please enter your permanent email address." return message +def extract_domain(email, base_only=False): + domain = email.split('@')[1] + parts = domain.split('.') + if len(parts) > 2 and base_only: + return ('.').join(parts[-2:]) + return domain + + def validate_publicname(trans, publicname, user=None): """ Check that publicname respects the minimum and maximum string length, the diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index 7b1ee99aceb..53c093849b6 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -1413,13 +1413,15 @@ class NavigatesGalaxy(HasDriver): assert text == expected, "Tooltip text [{}] was not expected text [{}].".format(text, expected) def assert_error_message(self, contains=None): - return self._assert_message("error", contains=contains) + element = self.components._.messages["error"] + return self.assert_message(element, contains=contains) def assert_warning_message(self, contains=None): - return self._assert_message("warning", contains=contains) + element = self.components._.messages["warning"] + return self.assert_message(element, contains=contains) - def _assert_message(self, message_type, contains=None): - element = self.components._.messages[message_type].wait_for_visible() + def assert_message(self, element, contains=None): + element = element.wait_for_visible() assert element, "No error message found, one expected." if contains is not None: text = element.text diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index e85d8c5962f..2182533ddc4 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -326,6 +326,7 @@ workflows: workflow_run: selectors: + warning: ".ui-form-composite .alert-warning" input_div: "[step-label='${label}']" input_data_div: "[step-label='${label}'] .select2-container" # TODO: put step labels in the DOM ideally @@ -415,6 +416,7 @@ admin: allowlist: '#admin-link-allowlist' selectors: + warning: '#center .alert-warning' # TODO: place betters IDS or something on this in these grids in the DOM datatypes_grid: '#data-types-grid' data_tables_grid: '#data-tables-grid' diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index ce4836e7319..b043c543bcf 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -107,11 +107,13 @@ class DependencyManager: """ cached = False - def __init__(self, default_base_path, conf_file=None, app_config={}): + def __init__(self, default_base_path, conf_file=None, app_config=None): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. """ + if app_config is None: + app_config = {} if not os.path.exists(default_base_path): log.warning("Path '%s' does not exist, ignoring", default_base_path) if not os.path.isdir(default_base_path): @@ -149,9 +151,11 @@ class DependencyManager: """Returns the union of enabled container types.""" return self._enabled_container_types - def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + def get_resolver_option(self, resolver, key, explicit_resolver_options=None): """Look in resolver-specific settings for option and then fallback to global settings. """ + if explicit_resolver_options is None: + explicit_resolver_options = {} default = resolver.config_options.get(key) config_prefix = resolver.resolver_type global_key = "{}_{}".format(config_prefix, key) @@ -408,7 +412,9 @@ class CachedDependencyManager(DependencyManager): class NullDependencyManager(DependencyManager): cached = False - def __init__(self, default_base_path=None, conf_file=None, app_config={}): + def __init__(self, default_base_path=None, conf_file=None, app_config=None): + if app_config is None: + app_config = {} self.__app_config = app_config self.resolver_classes = set() self.dependency_resolvers = [] diff --git a/lib/galaxy/tool_util/deps/conda_util.py b/lib/galaxy/tool_util/deps/conda_util.py index 469301d2d83..c538e9f58ee 100644 --- a/lib/galaxy/tool_util/deps/conda_util.py +++ b/lib/galaxy/tool_util/deps/conda_util.py @@ -219,9 +219,9 @@ class CondaContext(installable.InstallableContext): Return the process exit code (i.e. 0 in case of success). """ cmd = [self.conda_exec] + cmd.extend(operation.split()) if self.debug: cmd.append("--debug") - cmd.append(operation) cmd.extend(args) env = {} if self.condarc_override: @@ -248,15 +248,15 @@ class CondaContext(installable.InstallableContext): """ Return the process exit code (i.e. 0 in case of success). """ - create_base_args = [ + create_args = [ "-y", "--quiet" ] if allow_local and self.use_local: - create_base_args.extend(["--use-local"]) - create_base_args.extend(self._override_channels_args) - create_base_args.extend(args) - return self.exec_command("create", create_base_args, stdout_path=stdout_path) + create_args.extend(["--use-local"]) + create_args.extend(self._override_channels_args) + create_args.extend(args) + return self.exec_command("create", create_args, stdout_path=stdout_path) def exec_remove(self, args): """ @@ -264,38 +264,38 @@ class CondaContext(installable.InstallableContext): Return the process exit code (i.e. 0 in case of success). """ - remove_base_args = [ - "remove", + remove_args = [ "-y", "--name" ] - remove_base_args.extend(args) - return self.exec_command("env", remove_base_args) + remove_args.extend(args) + return self.exec_command("env remove", remove_args) def exec_install(self, args, allow_local=True, stdout_path=None): """ Return the process exit code (i.e. 0 in case of success). """ - install_base_args = [ + install_args = [ "-y" ] if allow_local and self.use_local: - install_base_args.append("--use-local") - install_base_args.extend(self._override_channels_args) - install_base_args.extend(args) - return self.exec_command("install", install_base_args, stdout_path=stdout_path) + install_args.append("--use-local") + install_args.extend(self._override_channels_args) + install_args.extend(args) + return self.exec_command("install", install_args, stdout_path=stdout_path) - def exec_clean(self, args=[], quiet=False): + def exec_clean(self, args=None, quiet=False): """ Clean up after conda installation. Return the process exit code (i.e. 0 in case of success). """ - clean_base_args = [ + clean_args = [ "--tarballs", "-y" ] - clean_args = clean_base_args + args + if args: + clean_args.extend(args) stdout_path = None if quiet: stdout_path = "/dev/null" diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index f9d2fd03f11..899cf01d933 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -1,7 +1,5 @@ import logging import re -import sys -import traceback import uuid from collections import OrderedDict from math import isinf @@ -13,7 +11,11 @@ from galaxy.tool_util.parser.util import ( DEFAULT_DELTA, DEFAULT_DELTA_FRAC ) -from galaxy.util import string_as_bool, xml_text, xml_to_string +from galaxy.util import ( + string_as_bool, + xml_text, + xml_to_string +) from .interface import ( InputSource, PageSource, @@ -488,12 +490,14 @@ class XmlToolSource(ToolSource): def parse_strict_shell(self): command_el = self._command_el - if command_el is not None: - return string_as_bool(command_el.get("strict", "False")) - elif self.legacy_defaults: - return False + if packaging.version.parse(self.parse_profile()) < packaging.version.parse('20.09'): + default = "False" else: - return True + default = "True" + if command_el is not None: + return string_as_bool(command_el.get("strict", default)) + else: + return string_as_bool(default) def parse_help(self): help_elem = self.root.find('help') @@ -829,7 +833,7 @@ class StdioParser: self.parse_stdio_exit_codes(stdio_elem) self.parse_stdio_regexes(stdio_elem) except Exception: - log.error("Exception in parse_stdio! " + str(sys.exc_info())) + log.exception("Exception in parse_stdio!") def parse_stdio_exit_codes(self, stdio_elem): """ @@ -857,8 +861,7 @@ class StdioParser: if code_range is None: code_range = exit_code_elem.get("value", "") if code_range is None: - log.warning("Tool stdio exit codes must have " + - "a range or value") + log.warning("Tool stdio exit codes must have a range or value") continue # Parse the range. We look for: # :Y @@ -899,18 +902,11 @@ class StdioParser: # isn't bogus. If we have two infinite values, then # the start must be -inf and the end must be +inf. # So at least warn about this situation: - if (isinf(exit_code.range_start) and - isinf(exit_code.range_end)): - log.warning("Tool exit_code range %s will match on " + - "all exit codes" % code_range) + if isinf(exit_code.range_start) and isinf(exit_code.range_end): + log.warning("Tool exit_code range %s will match on all exit codes" % code_range) self.stdio_exit_codes.append(exit_code) except Exception: - log.error("Exception in parse_stdio_exit_codes! " + - str(sys.exc_info())) - trace = sys.exc_info()[2] - if trace is not None: - trace_msg = repr(traceback.format_tb(trace)) - log.error("Traceback: %s" % trace_msg) + log.exception("Exception in parse_stdio_exit_codes!") def parse_stdio_regexes(self, stdio_elem): """ @@ -973,12 +969,7 @@ class StdioParser: regex.stderr_match = True self.stdio_regexes.append(regex) except Exception: - log.error("Exception in parse_stdio_exit_codes! " + - str(sys.exc_info())) - trace = sys.exc_info()[2] - if trace is not None: - trace_msg = repr(traceback.format_tb(trace)) - log.error("Traceback: %s" % trace_msg) + log.exception("Exception in parse_stdio_exit_codes!") # TODO: This method doesn't have to be part of the Tool class. def parse_error_level(self, err_level): @@ -1003,12 +994,7 @@ class StdioParser: log.debug("Tool %s: error level %s did not match log/warning/fatal" % (self.id, err_level)) except Exception: - log.error("Exception in parse_error_level " + - str(sys.exc_info())) - trace = sys.exc_info()[2] - if trace is not None: - trace_msg = repr(traceback.format_tb(trace)) - log.error("Traceback: %s" % trace_msg) + log.exception("Exception in parse_error_level") return return_level @@ -1085,7 +1071,7 @@ class XmlInputSource(InputSource): def parse_static_options(self): static_options = list() elem = self.input_elem - for index, option in enumerate(elem.findall("option")): + for option in elem.findall("option"): value = option.get("value") selected = string_as_bool(option.get("selected", False)) static_options.append((option.text or value, value, selected)) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index d0ee3d4682b..6bfd45cf079 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2889,7 +2889,7 @@ Prior to Galaxy release 19.01 the stdio block has only been used for non-legacy - This boolean forces the ``#set -e`` directive on in shell scripts - so that in a multi-part command if any part fails the job exits with a non-zero exit code. This is enabled by default for tools with ``profile>=16.04`` and disabled on legacy tools. + This boolean forces the ``#set -e`` directive on in shell scripts - so that in a multi-part command if any part fails the job exits with a non-zero exit code. This is enabled by default for tools with ``profile>=20.09`` and disabled on legacy tools. diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index ab10eb5fb1c..4082d499f17 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1583,13 +1583,13 @@ class Tool(Dictifiable): output_collections=execution_tracker.output_collections, implicit_collections=execution_tracker.implicit_collections) - def handle_single_execution(self, trans, rerun_remap_job_id, execution_slice, history, execution_cache=None, completed_job=None, collection_info=None): + def handle_single_execution(self, trans, rerun_remap_job_id, execution_slice, history, execution_cache=None, completed_job=None, collection_info=None, flush_job=True): """ Return a pair with whether execution is successful as well as either resulting output data or an error message indicating the problem. """ try: - job, out_data = self.execute( + rval = self.execute( trans, incoming=execution_slice.param_combination, history=history, @@ -1598,7 +1598,13 @@ class Tool(Dictifiable): dataset_collection_elements=execution_slice.dataset_collection_elements, completed_job=completed_job, collection_info=collection_info, + flush_job=flush_job, ) + job = rval[0] + out_data = rval[1] + if len(rval) == 4: + execution_slice.datasets_to_persist = rval[2] + execution_slice.history = rval[3] except (webob.exc.HTTPFound, exceptions.MessageException) as e: # if it's a webob redirect exception, pass it up the stack raise e diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 1e13fc4e8fc..027deabd1a9 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -32,6 +32,7 @@ class ToolExecutionCache: self.trans = trans self.current_user_roles = trans.get_current_user_roles() self.chrom_info = {} + self.cached_collection_elements = {} def get_chrom_info(self, tool_id, input_dbkey): genome_builds = self.trans.app.genome_builds @@ -275,7 +276,7 @@ class DefaultToolAction: preserved_tags[tag.value] = tag return history, inp_data, inp_dataset_collections, preserved_tags, all_permissions - def execute(self, tool, trans, incoming=None, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, execution_cache=None, dataset_collection_elements=None, completed_job=None, collection_info=None): + def execute(self, tool, trans, incoming=None, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, execution_cache=None, dataset_collection_elements=None, completed_job=None, collection_info=None, flush_job=True): """ Executes a tool, creating job and tool outputs, associating them, and submitting the job to the job queue. If history is not specified, use @@ -380,6 +381,7 @@ class DefaultToolAction: inp_dataset_collections, input_ext, python_template_version=tool.python_template_version, + execution_cache=execution_cache, ) create_datasets = True dataset = None @@ -407,7 +409,7 @@ class DefaultToolAction: dataset_collection_elements[name].hda = data trans.sa_session.add(data) if not completed_job: - trans.app.security_agent.set_all_dataset_permissions(data.dataset, output_permissions, new=True) + trans.app.security_agent.set_all_dataset_permissions(data.dataset, output_permissions, new=True, flush=False) data.copy_tags_to(preserved_tags) if not completed_job and trans.app.config.legacy_eager_objectstore_initialization: @@ -453,6 +455,7 @@ class DefaultToolAction: # Flush all datasets at once. return data + datasets_to_persist = [] for name, output in tool.outputs.items(): if not filter_output(output, incoming): handle_output_timer = ExecutionTimer() @@ -460,7 +463,6 @@ class DefaultToolAction: collections_manager = app.dataset_collections_service element_identifiers = [] known_outputs = output.known_outputs(input_collections, collections_manager.type_registry) - created_element_datasets = [] # Just to echo TODO elsewhere - this should be restructured to allow # nested collections. for output_part_def in known_outputs: @@ -487,7 +489,7 @@ class DefaultToolAction: effective_output_name = output_part_def.effective_output_name element = handle_output(effective_output_name, output_part_def.output_def, hidden=True) - created_element_datasets.append(element) + datasets_to_persist.append(element) # TODO: this shouldn't exist in the top-level of the history at all # but for now we are still working around that by hiding the contents # there. @@ -499,18 +501,21 @@ class DefaultToolAction: "name": output_part_def.element_identifier, }) - history.add_datasets(trans.sa_session, created_element_datasets, set_hid=set_output_hid, quota=False, flush=True) if output.dynamic_structure: assert not element_identifiers # known_outputs must have been empty element_kwds = dict(elements=collections_manager.ELEMENTS_UNINITIALIZED) else: element_kwds = dict(element_identifiers=element_identifiers) - output_collections.create_collection( + hdca = output_collections.create_collection( output=output, name=name, + set_hid=True if flush_job else False, + flush=flush_job, **element_kwds ) - log.info("Handled collection output named {} for tool {} {}".format(name, tool.id, handle_output_timer)) + if hdca: + datasets_to_persist.append(hdca) + log.info("Handled collection output named %s for tool %s %s" % (name, tool.id, handle_output_timer)) else: handle_output(name, output) log.info("Handled output named {} for tool {} {}".format(name, tool.id, handle_output_timer)) @@ -520,13 +525,9 @@ class DefaultToolAction: 'Added output datasets to history', ) # Add all the top-level (non-child) datasets to the history unless otherwise specified - datasets_to_persist = [] for name, data in out_data.items(): if name not in child_dataset_names and name not in incoming: # don't add children; or already existing datasets, i.e. async created datasets_to_persist.append(data) - # Set HID and add to history. - # This is brand new and certainly empty so don't worry about quota. - history.add_datasets(trans.sa_session, datasets_to_persist, set_hid=set_output_hid, quota=False, flush=False) # Add all the children to their parents for parent_name, child_name in parent_to_child_pairs: @@ -549,12 +550,17 @@ class DefaultToolAction: # Now that we have a job id, we can remap any outputs if this is a rerun and the user chose to continue dependent jobs # This functionality requires tracking jobs in the database. if app.config.track_jobs_in_database and rerun_remap_job_id is not None: + # We need a flush here and get hids in order to rewrite jobs parameter, + # but remapping jobs should only affect single jobs anyway, so this is not too costly. + trans.sa_session.flush() + history.add_datasets(trans.sa_session, datasets_to_persist, set_hid=set_output_hid, quota=False, flush=False) self._remap_job_on_rerun(trans=trans, galaxy_session=galaxy_session, rerun_remap_job_id=rerun_remap_job_id, current_job=job, out_data=out_data) - log.info("Setup for job {} complete, ready to be enqueued {}".format(job.log_str(), job_setup_timer)) + datasets_to_persist = [] + log.info("Setup for job %s complete, ready to be enqueued %s" % (job.log_str(), job_setup_timer)) # Some tools are not really executable, but jobs are still created for them ( for record keeping ). # Examples include tools that redirect to other applications ( epigraph ). These special tools must @@ -577,10 +583,18 @@ class DefaultToolAction: trans.sa_session.flush() trans.response.send_redirect(url_for(controller='tool_runner', action='redirect', redirect_url=redirect_url)) else: - # Dispatch to a job handler. enqueue() is responsible for flushing the job - app.job_manager.enqueue(job, tool=tool) - trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) - return job, out_data + if flush_job: + # Set HID and add to history. + # This is brand new and certainly empty so don't worry about quota. + history.add_datasets(trans.sa_session, datasets_to_persist, set_hid=set_output_hid, quota=False, flush=False) + job_flush_timer = ExecutionTimer() + trans.sa_session.flush() + log.info("Flushed transaction for job %s %s" % (job.log_str(), job_flush_timer)) + + # Dispatch to a job handler. enqueue() is responsible for flushing the job + app.job_manager.enqueue(job, tool=tool) + trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) + return job, out_data, datasets_to_persist, history def _remap_job_on_rerun(self, trans, galaxy_session, rerun_remap_job_id, current_job, out_data): """ @@ -803,7 +817,7 @@ class OutputCollections: self.out_collection_instances = {} self.tags = tags - def create_collection(self, output, name, collection_type=None, **element_kwds): + def create_collection(self, output, name, collection_type=None, set_hid=True, flush=True, **element_kwds): input_collections = self.input_collections collections_manager = self.trans.app.dataset_collections_service collection_type = collection_type or output.structure.collection_type @@ -882,11 +896,14 @@ class OutputCollections: collection_type=collection_type, trusted_identifiers=True, tags=self.tags, + set_hid=set_hid, + flush=flush, **element_kwds ) # name here is name of the output element - not name # of the hdca. self.out_collection_instances[name] = hdca + return hdca def on_text_for_names(input_names): @@ -933,7 +950,7 @@ def get_ext_or_implicit_ext(hda): return hda.ext -def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3'): +def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3', execution_cache=None): """ Determines the output format for a dataset based on an abstract description of the output (galaxy.tool_util.parser.ToolOutput), the parameter wrappers, a map of the input datasets (name => HDA), and the last input @@ -974,7 +991,13 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat try: input_element = input_collection_collection[element_index] except KeyError: - for element in input_collection_collection.dataset_elements: + if execution_cache: + dataset_elements = execution_cache.cached_collection_elements.get(input_collection_collection.id) + if dataset_elements is None: + dataset_elements = execution_cache.cached_collection_elements[input_collection_collection.id] = input_collection_collection.dataset_elements + else: + dataset_elements = input_collection_collection.dataset_elements + for element in dataset_elements: if element.element_identifier == element_index: input_element = element break diff --git a/lib/galaxy/tools/actions/data_manager.py b/lib/galaxy/tools/actions/data_manager.py index ac3795fc7d1..6b0a761d212 100644 --- a/lib/galaxy/tools/actions/data_manager.py +++ b/lib/galaxy/tools/actions/data_manager.py @@ -9,8 +9,8 @@ class DataManagerToolAction(DefaultToolAction): """Tool action used for Data Manager Tools""" def execute(self, tool, trans, **kwds): - rval = super().execute(tool, trans, **kwds) - if isinstance(rval, tuple) and len(rval) == 2 and isinstance(rval[0], trans.app.model.Job): + rval = super(DataManagerToolAction, self).execute(tool, trans, **kwds) + if isinstance(rval, tuple) and len(rval) >= 2 and isinstance(rval[0], trans.app.model.Job): assoc = trans.app.model.DataManagerJobAssociation(job=rval[0], data_manager_id=tool.data_manager_id) trans.sa_session.add(assoc) trans.sa_session.flush() diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 058955f347c..b01195e2afe 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -6,9 +6,9 @@ import socket import tempfile from collections import OrderedDict from json import dump, dumps +from urllib.parse import urlparse from six import StringIO -from six.moves.urllib.parse import urlparse from sqlalchemy.orm import eagerload_all from webob.compat import cgi_FieldStorage @@ -37,6 +37,8 @@ def validate_url(url, ip_allowlist): if not (url.lstrip().startswith('http://') or url.lstrip().startswith('https://')): return url + # Strip leading whitespace before passing url to urlparse() + url = url.lstrip() # Extract hostname component parsed_url = urlparse(url).netloc # If credentials are in this URL, we need to strip those. diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index d60719c7c38..6ffa6002c59 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -68,7 +68,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle del params['__workflow_resource_params__'] if validate_outputs: params['__validate_outputs__'] = True - job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache, completed_job, collection_info) + job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache, completed_job, collection_info, flush_job=False) if job: log.debug(job_timer.to_str(tool_id=tool.id, job_id=job.id)) execution_tracker.record_success(execution_slice, job, result) @@ -93,6 +93,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle jobs_executed = 0 has_remaining_jobs = False + datasets_to_persist = [] for i, execution_slice in enumerate(execution_tracker.new_execution_slices()): if max_num_jobs and jobs_executed >= max_num_jobs: @@ -100,6 +101,19 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle break else: execute_single_job(execution_slice, completed_jobs[i]) + if execution_slice.datasets_to_persist: + datasets_to_persist.extend(execution_slice.datasets_to_persist) + + if datasets_to_persist: + execution_slice.history.add_datasets(trans.sa_session, datasets_to_persist, set_hid=True, quota=False, flush=False) + # a side effect of history.add_datasets is a commit within db_next_hid (even with flush=False). + else: + # Make sure collections, implicit jobs etc are flushed even if there are no precreated output datasets + trans.sa_session.flush() + for job in execution_tracker.successful_jobs: + # Put the job in the queue if tracking in memory + tool.app.job_manager.enqueue(job, tool=tool) + trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) if has_remaining_jobs: raise PartialJobExecution(execution_tracker) @@ -116,6 +130,8 @@ class ExecutionSlice: self.job_index = job_index self.param_combination = param_combination self.dataset_collection_elements = dataset_collection_elements + self.datasets_to_persist = None + self.history = None class ExecutionTracker: @@ -360,7 +376,7 @@ class ExecutionTracker: job_assoc = model.ImplicitCollectionJobsJobAssociation() job_assoc.order_index = execution_slice.job_index job_assoc.implicit_collection_jobs = implicit_collection_jobs - job_assoc.job_id = job.id + job_assoc.job = job self.trans.sa_session.add(job_assoc) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index da9bb2c3901..f6d03c8d08c 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -65,6 +65,11 @@ class RegexValidator(Validator): Traceback (most recent call last): ... ValueError: Not gonna happen + >>> t = p.validate(["Foo", "foo"]) + >>> t = p.validate(["Foo", "Fop"]) + Traceback (most recent call last): + ... + ValueError: Not gonna happen """ @classmethod @@ -78,8 +83,11 @@ class RegexValidator(Validator): self.expression = expression def validate(self, value, trans=None): - if re.match(self.expression, value or '') is None: - raise ValueError(self.message) + if not isinstance(value, list): + value = [value] + for val in value: + if re.match(self.expression, val or '') is None: + raise ValueError(self.message) class ExpressionValidator(Validator): diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 972dcf8a214..78800ad1500 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -17,6 +17,7 @@ import stat import string import sys import tempfile +import textwrap import threading import time import unicodedata @@ -815,10 +816,9 @@ class Params: # name. Anything relying on NEVER_SANITIZE should be # changed to not require this and NEVER_SANITIZE should be # removed. - if (value is not None and - key not in self.NEVER_SANITIZE and - True not in [key.endswith("|%s" % nonsanitize_parameter) for - nonsanitize_parameter in self.NEVER_SANITIZE]): + if (value is not None and key not in self.NEVER_SANITIZE + and True not in [key.endswith("|%s" % nonsanitize_parameter) for + nonsanitize_parameter in self.NEVER_SANITIZE]): self.__dict__[key] = sanitize_param(value) else: self.__dict__[key] = value @@ -1042,11 +1042,10 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals value = str(value) # Now in Python 2, value is an instance of basestring, but may be not unicode # Now in Python 3, value is an instance of bytes or str - if not isinstance(value, str): - value = str(value, encoding, error) - except Exception: - msg = "Value '%s' could not be coerced to Unicode" % value - log.exception(msg) + if not isinstance(value, text_type): + value = text_type(value, encoding, error) + except Exception as e: + msg = "Value '%s' could not be coerced to Unicode: %s('%s')" % (value, type(e).__name__, e) raise Exception(msg) if strip_null: return value.replace('\0', '') @@ -1114,6 +1113,19 @@ def string_to_object(s): return binascii.unhexlify(s) +def clean_multiline_string(multiline_string, sep='\n'): + """ + Dedent, split, remove first and last empty lines, rejoin. + """ + multiline_string = textwrap.dedent(multiline_string) + string_list = multiline_string.split(sep) + if not string_list[0]: + string_list = string_list[1:] + if not string_list[-1]: + string_list = string_list[:-1] + return '\n'.join(string_list) + '\n' + + class ParamsWithSpecs(collections.defaultdict): """ """ @@ -1246,7 +1258,7 @@ def read_build_sites(filename, check_builds=True): def relativize_symlinks(path, start=None, followlinks=False): - for root, dirs, files in os.walk(path, followlinks=followlinks): + for root, _, files in os.walk(path, followlinks=followlinks): rel_start = None for file_name in files: symlink_file_name = os.path.join(root, file_name) @@ -1280,7 +1292,7 @@ def mkstemp_ln(src, prefix='mkstemp_ln_'): """ dir = os.path.dirname(src) names = tempfile._get_candidate_names() - for seq in xrange(tempfile.TMP_MAX): + for _ in xrange(tempfile.TMP_MAX): name = next(names) file = os.path.join(dir, prefix + name) try: @@ -1682,6 +1694,15 @@ def download_to_file(url, dest_file_path, timeout=30, chunk_size=2 ** 20): f.write(chunk) +class classproperty(object): + + def __init__(self, f): + self.f = f + + def __get__(self, obj, owner): + return self.f(owner) + + def get_executable(): exe = sys.executable if exe.endswith('uwsgi'): diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index e4822b98c6c..25453a37f75 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -1,43 +1,49 @@ import bz2 import gzip +import io import re import tarfile import zipfile -from six import BytesIO +from six import ( + BytesIO, + StringIO +) +from six.moves import filter from galaxy import util from galaxy.util.image_util import image_type HTML_CHECK_LINES = 100 +CHUNK_SIZE = 2 ** 15 # 32Kb +HTML_REGEXPS = ( + re.compile(r"]*HREF[^>]+>", re.I), + re.compile(r"]*>", re.I), + re.compile(r"]*>", re.I), + re.compile(r"]*>", re.I), + re.compile(r"]*>", re.I), +) -def check_html(file_path, chunk=None): - if chunk is None: - temp = open(file_path, mode='rb') - elif hasattr(chunk, "splitlines"): - temp = chunk.splitlines() +def check_html(name, file_path=True): + """ + Returns True if the file/string contains HTML code. + """ + # Handles files if file_path is True or text if file_path is False + if file_path: + temp = io.open(name, "r", encoding='utf-8') else: - temp = chunk - regexp1 = re.compile(r"]*HREF[^>]+>", re.I) - regexp2 = re.compile(r"]*>", re.I) - regexp3 = re.compile(r"]*>", re.I) - regexp4 = re.compile(r"]*>", re.I) - regexp5 = re.compile(r"]*>", re.I) - lineno = 0 - # TODO: Potentially reading huge lines into string here, this should be - # reworked. - for line in temp: - line = util.unicodify(line) - lineno += 1 - matches = regexp1.search(line) or regexp2.search(line) or regexp3.search(line) or regexp4.search(line) or regexp5.search(line) - if matches: - if chunk is None: - temp.close() - return True - if HTML_CHECK_LINES and (lineno > HTML_CHECK_LINES): - break - if chunk is None: + temp = StringIO(util.unicodify(name)) + try: + for _ in range(HTML_CHECK_LINES): + line = temp.readline(CHUNK_SIZE) + if not line: + break + if any(regexp.search(line) for regexp in HTML_REGEXPS): + return True + except UnicodeDecodeError: + return False + finally: temp.close() return False @@ -78,12 +84,10 @@ def check_gzip(file_path, check_content=True): if not check_content: return (True, True) - CHUNK_SIZE = 2 ** 15 # 32Kb - gzipped_file = gzip.GzipFile(file_path, mode='rb') - chunk = gzipped_file.read(CHUNK_SIZE) - gzipped_file.close() + with gzip.open(file_path, mode='rb') as gzipped_file: + chunk = gzipped_file.read(CHUNK_SIZE) # See if we have a compressed HTML file - if check_html(file_path, chunk=chunk): + if check_html(chunk, file_path=False): return (True, False) return (True, True) @@ -100,12 +104,10 @@ def check_bz2(file_path, check_content=True): if not check_content: return (True, True) - CHUNK_SIZE = 2 ** 15 # reKb - bzipped_file = bz2.BZ2File(file_path, mode='rb') - chunk = bzipped_file.read(CHUNK_SIZE) - bzipped_file.close() + with bz2.BZ2File(file_path, mode='rb') as bzipped_file: + chunk = bzipped_file.read(CHUNK_SIZE) # See if we have a compressed HTML file - if check_html(file_path, chunk=chunk): + if check_html(chunk, file_path=False): return (True, False) return (True, True) @@ -117,12 +119,11 @@ def check_zip(file_path, check_content=True, files=1): if not check_content: return (True, True) - CHUNK_SIZE = 2 ** 15 # 32Kb chunk = None for filect, member in enumerate(iter_zip(file_path)): handle, name = member chunk = handle.read(CHUNK_SIZE) - if chunk and check_html(file_path, chunk): + if chunk and check_html(chunk, file_path=False): return (True, False) if filect >= files: break @@ -145,7 +146,7 @@ def is_zip(file_path): def is_single_file_zip(file_path): - for i, member in enumerate(iter_zip(file_path)): + for i, _ in enumerate(iter_zip(file_path)): if i > 1: return False return True diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index f0806ae30e1..9393468d6e7 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -20,13 +20,7 @@ def plugins_dict(module, plugin_type_identifier): plugin_dict = {} for plugin_module in import_submodules(module, ordered=True): - # FIXME: this is not how one is suppose to use __all__ why did you do - # this past John? - for clazz in getattr(plugin_module, "__all__", []): - try: - clazz = getattr(plugin_module, clazz) - except TypeError: - clazz = clazz + for clazz in __plugin_classes_in_module(plugin_module): plugin_type = getattr(clazz, plugin_type_identifier, None) if plugin_type: plugin_dict[plugin_type] = clazz @@ -43,6 +37,15 @@ def load_plugins(plugins_dict, plugin_source, extra_kwds=None, plugin_type_keys= return __load_plugins_from_dicts(plugins_dict, plugin_source.source, extra_kwds, plugin_type_keys=plugin_type_keys) +def __plugin_classes_in_module(plugin_module): + for clazz in getattr(plugin_module, "__all__", []): + try: + clazz = getattr(plugin_module, clazz) + except TypeError: + clazz = clazz + yield clazz + + def __load_plugins_from_element(plugins_dict, plugins_element, extra_kwds): plugins = [] diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 37d793de2f2..e424bf9f9aa 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -9,14 +9,13 @@ import tarfile import tempfile import time import types -from http.cookies import SimpleCookie +from http.cookies import CookieError, SimpleCookie import routes import webob.compat import webob.exc import webob.exc as httpexceptions # noqa: F401 # We will use some very basic HTTP/wsgi utilities from the paste library -from paste.request import get_cookies from paste.response import HeaderDict from galaxy.util import smart_str @@ -385,7 +384,16 @@ class Request(webob.Request): @lazy_property def cookies(self): - return get_cookies(self.environ) + cookies = SimpleCookie() + cookie_header = self.environ.get("HTTP_COOKIE") + if cookie_header: + galaxy_cookies = "; ".join(x.strip() for x in cookie_header.split('; ') if x.startswith('galaxy')) + if galaxy_cookies: + try: + cookies.load(galaxy_cookies) + except CookieError: + pass + return cookies @lazy_property def base(self): diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index 9d2a2ce1916..b082cbdb36e 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -3,7 +3,7 @@ Galaxy web framework helpers The functions in this module should be considered part of the API used by visualizations in their mako files through the `$h` object, see -GalaxyWebTransaction in galaxy/web/framework/webapp.py +GalaxyWebTransaction in galaxy/webapps/base/webapp.py """ from datetime import datetime, timedelta diff --git a/lib/galaxy/web_stack/__init__.py b/lib/galaxy/web_stack/__init__.py index 88430b0230f..0d1be0ea540 100644 --- a/lib/galaxy/web_stack/__init__.py +++ b/lib/galaxy/web_stack/__init__.py @@ -535,7 +535,7 @@ class WeblessApplicationStack(ApplicationStack): or (dialect.name == 'mysql' and dialect.server_version_info >= (8, 0, 1))): add_method = HANDLER_ASSIGNMENT_METHODS.DB_SKIP_LOCKED else: - HANDLER_ASSIGNMENT_METHODS.DB_TRANSACTION_ISOLATION + add_method = HANDLER_ASSIGNMENT_METHODS.DB_TRANSACTION_ISOLATION if add_method in job_config.UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS: remove_methods.append(add_method) add_method = HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/webapps/base/webapp.py similarity index 100% rename from lib/galaxy/web/framework/webapp.py rename to lib/galaxy/webapps/base/webapp.py diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index b14931f7a81..e01945ec6a8 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -44,7 +44,7 @@ class CloudController(BaseAPIController): """ * POST /api/cloud/storage/get gets given objects from a given cloud-based bucket to a Galaxy history. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type payload: dict @@ -142,7 +142,7 @@ class CloudController(BaseAPIController): using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type payload: dictionary diff --git a/lib/galaxy/webapps/galaxy/api/cloudauthz.py b/lib/galaxy/webapps/galaxy/api/cloudauthz.py index 56a24049be5..7b084ea128c 100644 --- a/lib/galaxy/webapps/galaxy/api/cloudauthz.py +++ b/lib/galaxy/webapps/galaxy/api/cloudauthz.py @@ -44,7 +44,7 @@ class CloudAuthzController(BaseAPIController): * GET /api/cloud/authz Lists all the cloud authorizations user has defined. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :param kwargs: empty dict @@ -64,7 +64,7 @@ class CloudAuthzController(BaseAPIController): * POST /api/cloud/authz Request to store the payload as a cloudauthz (cloud authorization) configuration for a user. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type payload: dict @@ -160,7 +160,7 @@ class CloudAuthzController(BaseAPIController): * DELETE /api/cloud/authz/{encoded_authz_id} Deletes the CloudAuthz record with the given ``encoded_authz_id`` from database. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type encoded_authz_id: string @@ -200,7 +200,7 @@ class CloudAuthzController(BaseAPIController): With this API only the following attributes of a cloudauthz configuration can be updated: `authn_id`, `provider`, `config`, `deleted`. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type encoded_authz_id: string diff --git a/lib/galaxy/webapps/galaxy/api/dataset_collections.py b/lib/galaxy/webapps/galaxy/api/dataset_collections.py index 1d99c6554d6..28baf26c510 100644 --- a/lib/galaxy/webapps/galaxy/api/dataset_collections.py +++ b/lib/galaxy/webapps/galaxy/api/dataset_collections.py @@ -4,6 +4,7 @@ import routes from galaxy import managers from galaxy.exceptions import ObjectNotFound +from galaxy.managers.base import decode_id from galaxy.managers.collections_util import ( api_payload_to_create_params, dictify_dataset_collection_instance, @@ -52,7 +53,7 @@ class DatasetCollectionsController( instance_type = payload.pop("instance_type", "history") if instance_type == "history": history_id = payload.get('history_id') - history_id = self.decode_id(history_id) + history_id = decode_id(self.app, history_id) history = self.history_manager.get_owned(history_id, trans.user, current_history=trans.history) create_params["parent"] = history elif instance_type == "library": @@ -115,7 +116,7 @@ class DatasetCollectionsController( instance_type=instance_type) # check to make sure the dsc is part of the validated hdca - decoded_parent_id = trans.app.security.decode_id(parent_id) + decoded_parent_id = decode_id(self.app, parent_id) if not hdca.contains_collection(decoded_parent_id): errmsg = 'Requested dataset collection is not contained within indicated history content' raise ObjectNotFound(errmsg) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index efaea4ac91a..cf4b859a836 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -3,12 +3,8 @@ API operations allowing clients to determine datatype supported by Galaxy. """ import logging -from galaxy import exceptions from galaxy.datatypes.data import Data -from galaxy.util import ( - asbool, - unicodify -) +from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.webapps.base.controller import BaseAPIController @@ -23,28 +19,28 @@ class DatatypesController(BaseAPIController): GET /api/datatypes Return an object containing upload datatypes. """ - datatypes_registry = self._datatypes_registry try: - extension_only = asbool(kwd.get('extension_only', True)) - upload_only = asbool(kwd.get('upload_only', True)) - if extension_only: - if upload_only: - return datatypes_registry.upload_file_formats - else: - return [ext for ext in datatypes_registry.datatypes_by_extension] - else: - rval = [] - for datatype_info_dict in datatypes_registry.datatype_info_dicts: - if not datatype_info_dict.get('display_in_upload') and upload_only: - continue - rval.append(datatype_info_dict) - return rval + return self._index(trans, **kwd) except Exception as e: log.exception('Could not get datatypes') - if not isinstance(e, exceptions.MessageException): - raise exceptions.InternalServerError(unicodify(e)) + raise e + + def _index(self, trans, **kwd): + datatypes_registry = self._datatypes_registry + extension_only = asbool(kwd.get('extension_only', True)) + upload_only = asbool(kwd.get('upload_only', True)) + if extension_only: + if upload_only: + return datatypes_registry.upload_file_formats else: - raise + return [ext for ext in datatypes_registry.datatypes_by_extension] + else: + rval = [] + for datatype_info_dict in datatypes_registry.datatype_info_dicts: + if not datatype_info_dict.get('display_in_upload') and upload_only: + continue + rval.append(datatype_info_dict) + return rval @expose_api_anonymous_and_sessionless def mapping(self, trans, **kwd): @@ -53,32 +49,45 @@ class DatatypesController(BaseAPIController): Return a dictionary of class to class mappings. ''' try: - ext_to_class_name = dict() - classes = [] - for k, v in self._datatypes_registry.datatypes_by_extension.items(): - c = v.__class__ - ext_to_class_name[k] = c.__module__ + "." + c.__name__ - classes.append(c) - class_to_classes = dict() - - def visit_bases(types, cls): - for base in cls.__bases__: - if issubclass(base, Data): - types.add(base.__module__ + "." + base.__name__) - visit_bases(types, base) - for c in classes: - n = c.__module__ + "." + c.__name__ - types = {n} - visit_bases(types, c) - class_to_classes[n] = {t: True for t in types} - return dict(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) - + return self._mapping() except Exception as e: log.exception('Could not get datatype mapping') - if not isinstance(e, exceptions.MessageException): - raise exceptions.InternalServerError(unicodify(e)) - else: - raise + raise e + + def _mapping(self): + ext_to_class_name = dict() + classes = [] + for k, v in self._datatypes_registry.datatypes_by_extension.items(): + c = v.__class__ + ext_to_class_name[k] = c.__module__ + "." + c.__name__ + classes.append(c) + class_to_classes = dict() + + def visit_bases(types, cls): + for base in cls.__bases__: + if issubclass(base, Data): + types.add(base.__module__ + "." + base.__name__) + visit_bases(types, base) + for c in classes: + n = c.__module__ + "." + c.__name__ + types = {n} + visit_bases(types, c) + class_to_classes[n] = dict((t, True) for t in types) + return dict(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + + @expose_api_anonymous_and_sessionless + def types_and_mapping(self, trans, **kwd): + """ + GET /api/datatypes/types_and_mapping + + Combine the datatype information from (/api/datatypes) and the + mapping information from (/api/datatypes/mapping) into a single + response. + """ + return { + "datatypes": self._index(trans, **kwd), + "datatypes_mapping": self._mapping(), + } @expose_api_anonymous_and_sessionless def sniffers(self, trans, **kwd): @@ -95,10 +104,7 @@ class DatatypesController(BaseAPIController): return rval except Exception as e: log.exception('Could not get datatypes') - if not isinstance(e, exceptions.MessageException): - raise exceptions.InternalServerError(unicodify(e)) - else: - raise + raise e @expose_api_anonymous_and_sessionless def converters(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index b0d6a1d43ad..34a79f3ee7c 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -14,7 +14,7 @@ import galaxy.datatypes.registry import galaxy.model import galaxy.model.mapping import galaxy.web.framework -import galaxy.web.framework.webapp +import galaxy.webapps.base.webapp from galaxy import util from galaxy.util import asbool from galaxy.util.properties import load_app_properties @@ -27,7 +27,7 @@ from galaxy.webapps.util import wrap_if_allowed log = logging.getLogger(__name__) -class GalaxyWebApplication(galaxy.web.framework.webapp.WebApplication): +class GalaxyWebApplication(galaxy.webapps.base.webapp.WebApplication): pass @@ -196,7 +196,7 @@ def app_factory(global_conf, load_app_kwds={}, **kwargs): def uwsgi_app(): - return galaxy.web.framework.webapp.build_native_uwsgi_app(app_factory, "galaxy") + return galaxy.webapps.base.webapp.build_native_uwsgi_app(app_factory, "galaxy") # For backwards compatibility @@ -459,7 +459,7 @@ def populate_api_routes(webapp, app): webapp.mapper.resource('datatype', 'datatypes', path_prefix='/api', - collection={'sniffers': 'GET', 'mapping': 'GET', 'converters': 'GET', 'edam_data': 'GET', 'edam_formats': 'GET'}, + collection={'sniffers': 'GET', 'mapping': 'GET', 'converters': 'GET', 'edam_data': 'GET', 'edam_formats': 'GET', 'types_and_mapping': 'GET'}, parent_resources=dict(member_name='datatype', collection_name='datatypes')) webapp.mapper.resource('search', 'search', path_prefix='/api') webapp.mapper.connect('/api/pages/{id}.pdf', action='show_pdf', controller="pages", conditions=dict(method=["GET"])) @@ -1249,5 +1249,5 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): def wrap_in_static(app, global_conf, plugin_frameworks=None, **local_conf): - urlmap, cache_time = galaxy.web.framework.webapp.build_url_map(app, global_conf, local_conf) + urlmap, cache_time = galaxy.webapps.base.webapp.build_url_map(app, global_conf, local_conf) return urlmap diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index e75292a776b..67d1ef8494f 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -930,8 +930,9 @@ mapping: required: false desc: | E-mail domains blocklist is used for filtering out users that are using - disposable email addresses at registration. If their address domain matches any - domain on the list, they are refused registration. + disposable email addresses at registration. If their address's base domain + matches any domain on the list, they are refused registration. Address subdomains + are ignored (both 'name@spam.com' and 'name@foo.spam.com' will match 'spam.com'). Example value 'email_blocklist.conf' @@ -944,7 +945,9 @@ mapping: E-mail domains allowlist is used to specify allowed email address domains. If the list is non-empty and a user attempts registration using an email address belonging to a domain that is not on the list, registration will be - denied. This is a more restrictive option than , and + denied. Unlike which matches the address's + base domain, here email addresses are matched against the full domain (base + subdomain). + This is a more restrictive option than , and therefore, in case is set and is not empty, will be ignored. @@ -2642,6 +2645,18 @@ mapping: it can "recover" them when it starts). This is not safe to use if you are running more than one Galaxy server using the same database. + metadata_strategy: + type: str + required: false + default: directory + desc: | + Determines how metadata will be set. Valid values are `directory`, `extended` and `legacy`. + In extended mode jobs will decide if a tool run failed, the object stores + configuration is serialized and made available to the job and is used for + writing output datasets to the object store as part of the job and dynamic + output discovery (e.g. discovered datasets , unpopulated collections, + etc) happens as part of the job. + retry_metadata_internally: type: bool default: true diff --git a/lib/galaxy/webapps/galaxy/controllers/authnz.py b/lib/galaxy/webapps/galaxy/controllers/authnz.py index 93abe3883be..1bc76272182 100644 --- a/lib/galaxy/webapps/galaxy/controllers/authnz.py +++ b/lib/galaxy/webapps/galaxy/controllers/authnz.py @@ -29,7 +29,7 @@ class OIDC(JSAppLauncher): GET /authnz/ returns a list of third-party identities associated with the user. - :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction + :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction. :param kwargs: empty dict diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 21f68373619..da6c4237850 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -88,7 +88,7 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): len(trans.app.config.oidc) == 1 and len(trans.app.auth_manager.authenticators) == 0 and is_logout_redirect is False): - provider = trans.app.config.oidc.keys()[0] + provider = next(iter(trans.app.config.oidc.keys())) success, message, redirect_uri = trans.app.authnz_manager.authenticate(provider, trans) if success: return trans.response.send_redirect(redirect_uri) diff --git a/lib/galaxy/webapps/reports/buildapp.py b/lib/galaxy/webapps/reports/buildapp.py index 0b2a951c97e..0c220d9ccc8 100644 --- a/lib/galaxy/webapps/reports/buildapp.py +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -10,7 +10,7 @@ from paste import httpexceptions import galaxy.model import galaxy.model.mapping -import galaxy.web.framework.webapp +import galaxy.webapps.base.webapp from galaxy.util import asbool from galaxy.util.properties import load_app_properties from galaxy.webapps.util import wrap_if_allowed @@ -18,7 +18,7 @@ from galaxy.webapps.util import wrap_if_allowed log = logging.getLogger(__name__) -class ReportsWebApplication(galaxy.web.framework.webapp.WebApplication): +class ReportsWebApplication(galaxy.webapps.base.webapp.WebApplication): pass @@ -127,9 +127,9 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): def wrap_in_static(app, global_conf, **local_conf): - urlmap, _ = galaxy.web.framework.webapp.build_url_map(app, global_conf, local_conf) + urlmap, _ = galaxy.webapps.base.webapp.build_url_map(app, global_conf, local_conf) return urlmap def uwsgi_app(): - return galaxy.web.framework.webapp.build_native_uwsgi_app(app_factory, "reports") + return galaxy.webapps.base.webapp.build_native_uwsgi_app(app_factory, "reports") diff --git a/lib/galaxy/webapps/util.py b/lib/galaxy/webapps/util.py index 5e1f7a88f47..7d82610aed5 100644 --- a/lib/galaxy/webapps/util.py +++ b/lib/galaxy/webapps/util.py @@ -53,7 +53,7 @@ def wrap_if_allowed(app, stack, wrap, name=None, args=None, kwargs=None): """ Wrap the application with the given method if the application stack allows for it. - :type app: :class:`galaxy.web.framework.webapp.WebApplication` subclass + :type app: :class:`galaxy.webapps.base.webapp.WebApplication` subclass :param app: application to wrap :type stack: :class:`galaxy.web_stack.ApplicationStack` subclass :param stack: instance of application stack implementing `allowed_middleware()` method diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index a47b9521ecc..f893f4f74c5 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -1800,17 +1800,16 @@ class ToolModule(WorkflowModule): # Combine workflow and runtime post job actions into the effective post # job actions for this execution. - flush_required = False effective_post_job_actions = self._effective_post_job_actions(step) for pja in effective_post_job_actions: if pja.action_type in ActionBox.immediate_actions or isinstance(self.tool, DatabaseOperationTool): ActionBox.execute(self.trans.app, self.trans.sa_session, pja, job, replacement_dict) else: - pjaa = model.PostJobActionAssociation(pja, job_id=job.id) + if job.id: + pjaa = model.PostJobActionAssociation(pja, job_id=job.id) + else: + pjaa = model.PostJobActionAssociation(pja, job=job) self.trans.sa_session.add(pjaa) - flush_required = True - if flush_required: - self.trans.sa_session.flush() def __restore_step_meta_runtime_state(self, step_runtime_state): if RUNTIME_POST_JOB_ACTIONS_KEY in step_runtime_state: diff --git a/lib/galaxy_test/api/_framework.py b/lib/galaxy_test/api/_framework.py index 264851d707a..f642666c5d1 100644 --- a/lib/galaxy_test/api/_framework.py +++ b/lib/galaxy_test/api/_framework.py @@ -1,5 +1,19 @@ -# TODO: implement a remote-only version of ApiTestCase if galaxy_test.driver isn't -# available. Most of these tests do not depend on Galaxy internals. -from galaxy_test.driver.api import ApiTestCase +from galaxy_test.base.api import UsesApiTestCaseMixin +from galaxy_test.base.testcase import FunctionalTestCase +try: + from galaxy_test.driver.driver_util import GalaxyTestDriver +except ImportError: + # Galaxy libraries and galaxy test driver not available, just assume we're + # targetting a remote Galaxy. + GalaxyTestDriver = None + + +class ApiTestCase(FunctionalTestCase, UsesApiTestCaseMixin): + galaxy_driver_class = GalaxyTestDriver + + def setUp(self): + super(ApiTestCase, self).setUp() + self._setup_interactor() + __all__ = ('ApiTestCase', ) diff --git a/lib/galaxy_test/api/test_configuration.py b/lib/galaxy_test/api/test_configuration.py index 19e15204fb1..58a93cb9a77 100644 --- a/lib/galaxy_test/api/test_configuration.py +++ b/lib/galaxy_test/api/test_configuration.py @@ -2,9 +2,6 @@ from galaxy_test.base.api_asserts import ( assert_has_keys, assert_not_has_keys, ) -from galaxy_test.base.populators import ( - LibraryPopulator -) from ._framework import ApiTestCase TEST_KEYS_FOR_ALL_USERS = [ @@ -26,10 +23,6 @@ TEST_KEYS_FOR_ADMIN_ONLY = [ class ConfigurationApiTestCase(ApiTestCase): - def setUp(self): - super().setUp() - self.library_populator = LibraryPopulator(self.galaxy_interactor) - def test_normal_user_configuration(self): config = self._get_configuration() assert_has_keys(config, *TEST_KEYS_FOR_ALL_USERS) @@ -40,18 +33,6 @@ class ConfigurationApiTestCase(ApiTestCase): assert_has_keys(config, *TEST_KEYS_FOR_ALL_USERS) assert_has_keys(config, *TEST_KEYS_FOR_ADMIN_ONLY) - def test_admin_decode_id(self): - new_lib = self.library_populator.new_library('DecodeTestLibrary') - decode_response = self._get("configuration/decode/" + new_lib["id"], admin=True) - response_id = decode_response.json()["decoded_id"] - decoded_library_id = self.security.decode_id(new_lib["id"]) - assert decoded_library_id == response_id - # fake valid folder id by prepending F - valid_encoded_folder_id = 'F' + new_lib["id"] - folder_decode_response = self._get("configuration/decode/" + valid_encoded_folder_id, admin=True) - folder_response_id = folder_decode_response.json()["decoded_id"] - assert decoded_library_id == folder_response_id - def test_normal_user_decode_id(self): decode_response = self._get("configuration/decode/badhombre", admin=False) self._assert_status_code_is(decode_response, 403) diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py index 0c2bdf207f3..fffd4f8913b 100644 --- a/lib/galaxy_test/api/test_dataset_collections.py +++ b/lib/galaxy_test/api/test_dataset_collections.py @@ -3,6 +3,7 @@ import tarfile from six import BytesIO +from galaxy_test.base.api_asserts import assert_object_id_error from galaxy_test.base.populators import DatasetCollectionPopulator, DatasetPopulator, skip_if_github_down from ._framework import ApiTestCase @@ -287,10 +288,10 @@ class DatasetCollectionApiTestCase(ApiTestCase): hdca, contents_url = self._create_collection_contents_pair() response = self._get(contents_url) self._assert_status_code_is(response, 200) - fake_collection_id = self.security.encode_id(1000) - fake_contents_url = '/api/dataset_collections/{}/contents/{}'.format(hdca['id'], fake_collection_id) + fake_collection_id = '5d7db0757a2eb7ef' + fake_contents_url = '/api/dataset_collections/%s/contents/%s' % (hdca['id'], fake_collection_id) error_response = self._get(fake_contents_url) - self._assert_status_code_is(error_response, 404) + assert_object_id_error(error_response) def test_show_dataset_collection_contents(self): # Get contents_url from history contents, use it to show the first level diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index 4c7763f62a3..9557ff3ca88 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -38,6 +38,18 @@ class DatatypesApiTestCase(ApiTestCase): mapping_dict = response.json() self._assert_has_keys(mapping_dict, "ext_to_class_name", "class_to_classes") + def test_types_and_mapping(self): + response = self._get("datatypes/types_and_mapping") + self._assert_status_code_is(response, 200) + response_dict = response.json() + self._assert_has_keys(response_dict, "datatypes", "datatypes_mapping") + mapping_dict = response_dict["datatypes_mapping"] + self._assert_has_keys(mapping_dict, "ext_to_class_name", "class_to_classes") + + datatypes = response_dict["datatypes"] + for common_type in ["tabular", "fasta"]: + assert common_type in datatypes, "%s not in %s" % (common_type, datatypes) + def test_sniffers(self): response = self._get("datatypes/sniffers") self._assert_status_code_is(response, 200) diff --git a/lib/galaxy_test/api/test_users.py b/lib/galaxy_test/api/test_users.py index 05affbc43db..f413eeed504 100644 --- a/lib/galaxy_test/api/test_users.py +++ b/lib/galaxy_test/api/test_users.py @@ -6,6 +6,7 @@ from requests import ( put ) +from galaxy_test.base.api_asserts import assert_object_id_error from galaxy_test.base.populators import skip_without_tool from ._framework import ApiTestCase @@ -63,10 +64,10 @@ class UsersApiTestCase(ApiTestCase): self._assert_status_code_is(update_response, 403) # non-existent - no_user_id = self.security.encode_id(100) + no_user_id = '5d7db0757a2eb7ef' update_url = self._api_url("users/%s" % (no_user_id), use_key=True) update_response = put(update_url, data=json.dumps(dict(username=new_name))) - self._assert_status_code_is(update_response, 404) + assert_object_id_error(update_response) def test_admin_update(self): new_name = 'flexo' diff --git a/lib/galaxy_test/driver/api.py b/lib/galaxy_test/base/api.py similarity index 93% rename from lib/galaxy_test/driver/api.py rename to lib/galaxy_test/base/api.py index 6275916b39b..4fa37b1d80e 100644 --- a/lib/galaxy_test/driver/api.py +++ b/lib/galaxy_test/base/api.py @@ -3,22 +3,21 @@ from contextlib import contextmanager from six.moves.urllib.parse import urlencode -from galaxy_test.base.api_asserts import ( +from .api_asserts import ( assert_error_code_is, assert_has_keys, assert_not_has_keys, assert_status_code_is, assert_status_code_is_ok, ) -from galaxy_test.base.api_util import ( +from .api_util import ( ADMIN_TEST_USER, get_master_api_key, get_user_api_key, OTHER_USER, TEST_USER, ) -from galaxy_test.base.interactor import TestCaseGalaxyInteractor as BaseInteractor -from .testcase import FunctionalTestCase +from .interactor import TestCaseGalaxyInteractor as BaseInteractor class UsesApiTestCaseMixin: @@ -111,13 +110,6 @@ class UsesApiTestCaseMixin: _assert_has_key = _assert_has_keys -class ApiTestCase(FunctionalTestCase, UsesApiTestCaseMixin): - - def setUp(self): - super().setUp() - self._setup_interactor() - - class ApiTestInteractor(BaseInteractor): """ Specialized variant of the API interactor (originally developed for tool functional tests) for testing the API generally. diff --git a/lib/galaxy_test/base/api_asserts.py b/lib/galaxy_test/base/api_asserts.py index 8286bf29632..f4efec5f729 100644 --- a/lib/galaxy_test/base/api_asserts.py +++ b/lib/galaxy_test/base/api_asserts.py @@ -44,4 +44,16 @@ def assert_error_code_is(response, error_code): assert err_code == int(error_code), ASSERT_FAIL_ERROR_CODE % (err_code, int(error_code)) +def assert_object_id_error(response): + # for tests that use fake object IDs - API might throw MalformedId (400) or + # or ObjectNotFound (404) - depending if the ID happens to be parseable with + # servers API key. + error_code = response.status_code + assert error_code in [400, 404] + if error_code == 400: + assert_error_code_is(response, 400009) + else: + assert_error_code_is(response, 404001) + + assert_has_key = assert_has_keys diff --git a/lib/galaxy_test/base/env.py b/lib/galaxy_test/base/env.py new file mode 100644 index 00000000000..fc89bc1ab8a --- /dev/null +++ b/lib/galaxy_test/base/env.py @@ -0,0 +1,35 @@ +"""Base utilities for working Galaxy test environments. +""" +import fcntl +import os +import socket +import struct + +DEFAULT_WEB_HOST = socket.gethostbyname('localhost') + + +def setup_keep_outdir(): + keep_outdir = os.environ.get('GALAXY_TEST_SAVE', '') + if keep_outdir > '': + try: + os.makedirs(keep_outdir) + except Exception: + pass + return keep_outdir + + +def target_url_parts(): + host = socket.gethostbyname(os.environ.get('GALAXY_TEST_HOST', DEFAULT_WEB_HOST)) + port = os.environ.get('GALAXY_TEST_PORT') + default_url = "http://%s:%s" % (host, port) + url = os.environ.get('GALAXY_TEST_EXTERNAL', default_url) + return host, port, url + + +def get_ip_address(ifname): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + return socket.inet_ntoa(fcntl.ioctl( + s.fileno(), + 0x8915, # SIOCGIFADDR + struct.pack('256s', ifname[:15].encode('utf-8')) + )[20:24]) diff --git a/lib/galaxy_test/base/instrument.py b/lib/galaxy_test/base/instrument.py index 8403398f8eb..9ba2be9043d 100644 --- a/lib/galaxy_test/base/instrument.py +++ b/lib/galaxy_test/base/instrument.py @@ -1,6 +1,6 @@ """ Utilities to help instrument tool tests. -Including structed data nose plugin that allows storing arbitrary structured +Including structured data nose plugin that allows storing arbitrary structured data on a per test case basis - used by tool test to store inputs, output problems, job tests, etc... but could easily by used by other test types in a different way. diff --git a/lib/galaxy_test/base/testcase.py b/lib/galaxy_test/base/testcase.py new file mode 100644 index 00000000000..880d38e1347 --- /dev/null +++ b/lib/galaxy_test/base/testcase.py @@ -0,0 +1,45 @@ +from __future__ import print_function + +import logging +import os +import unittest + +from galaxy.tool_util.verify.test_data import TestDataResolver +from galaxy_test.base.env import setup_keep_outdir, target_url_parts + +log = logging.getLogger(__name__) + + +class FunctionalTestCase(unittest.TestCase): + """Base class for tests targetting actual Galaxy servers. + + Subclass should override galaxy_driver_class if a Galaxy server + needs to be launched to run the test, this base class assumes a + server is already running. + """ + galaxy_driver_class = None + + def setUp(self): + self.history_id = os.environ.get('GALAXY_TEST_HISTORY_ID', None) + self.host, self.port, self.url = target_url_parts() + self.test_data_resolver = TestDataResolver() + self.keepOutdir = setup_keep_outdir() + + @classmethod + def setUpClass(cls): + """Configure and start Galaxy for a test.""" + cls._test_driver = None + + if cls.galaxy_driver_class is not None and not os.environ.get("GALAXY_TEST_ENVIRONMENT_CONFIGURED"): + cls._test_driver = cls.galaxy_driver_class() + cls._test_driver.setup(config_object=cls) + + @classmethod + def tearDownClass(cls): + """Shutdown Galaxy server and cleanup temp directory.""" + if cls._test_driver: + cls._test_driver.tear_down() + + def get_filename(self, filename): + # No longer used by tool tests - drop if isn't used else where. + return self.test_data_resolver.get_filename(filename) diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index db8b10d32d2..fde18182780 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -1,6 +1,5 @@ """Scripts for drivers of Galaxy functional tests.""" -import fcntl import logging import os import random @@ -9,7 +8,6 @@ import shutil import signal import socket import string -import struct import subprocess import sys import tempfile @@ -41,13 +39,16 @@ from galaxy.util import asbool, download_to_file, galaxy_directory from galaxy.util.properties import load_app_properties from galaxy.web import buildapp from galaxy_test.base.api_util import get_master_api_key, get_user_api_key +from galaxy_test.base.env import ( + DEFAULT_WEB_HOST, + target_url_parts, +) from galaxy_test.base.instrument import StructuredTestDataPlugin from galaxy_test.base.nose_util import run from tool_shed.webapp.app import UniverseApplication as ToolshedUniverseApplication from .test_logging import logging_config_file galaxy_root = galaxy_directory() -DEFAULT_WEB_HOST = socket.gethostbyname('localhost') DEFAULT_CONFIG_PREFIX = "GALAXY" GALAXY_TEST_DIRECTORY = os.path.join(galaxy_root, "test") GALAXY_TEST_FILE_DIR = "test-data,https://github.com/galaxyproject/galaxy-test-data.git" @@ -614,24 +615,6 @@ def build_shed_app(simple_kwargs): return app -class classproperty: - - def __init__(self, f): - self.f = f - - def __get__(self, obj, owner): - return self.f(owner) - - -def get_ip_address(ifname): - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - return socket.inet_ntoa(fcntl.ioctl( - s.fileno(), - 0x8915, # SIOCGIFADDR - struct.pack('256s', ifname[:15].encode('utf-8')) - )[20:24]) - - def explicitly_configured_host_and_port(prefix, config_object): host_env_key = "%s_TEST_HOST" % prefix port_env_key = "%s_TEST_PORT" % prefix @@ -1078,24 +1061,6 @@ def drive_test(test_driver_class): sys.exit(test_driver.run()) -def setup_keep_outdir(): - keep_outdir = os.environ.get('GALAXY_TEST_SAVE', '') - if keep_outdir > '': - try: - os.makedirs(keep_outdir) - except Exception: - pass - return keep_outdir - - -def target_url_parts(): - host = socket.gethostbyname(os.environ.get('GALAXY_TEST_HOST', DEFAULT_WEB_HOST)) - port = os.environ.get('GALAXY_TEST_PORT') - default_url = "http://{}:{}".format(host, port) - url = os.environ.get('GALAXY_TEST_EXTERNAL', default_url) - return host, port, url - - __all__ = ( "copy_database_template", "build_logger", @@ -1106,9 +1071,7 @@ __all__ = ( "database_conf", "get_webapp_global_conf", "nose_config_and_run", - "setup_keep_outdir", "setup_galaxy_config", - "target_url_parts", "TestDriver", "wait_for_http_server", ) diff --git a/lib/galaxy_test/driver/integration_util.py b/lib/galaxy_test/driver/integration_util.py index 6792fa93c33..c7f30fbdb22 100644 --- a/lib/galaxy_test/driver/integration_util.py +++ b/lib/galaxy_test/driver/integration_util.py @@ -11,7 +11,7 @@ import pytest from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.util.commands import which -from .api import UsesApiTestCaseMixin +from galaxy_test.base.api import UsesApiTestCaseMixin from .driver_util import GalaxyTestDriver NO_APP_MESSAGE = "test_case._app called though no Galaxy has been configured." diff --git a/lib/galaxy_test/driver/testcase.py b/lib/galaxy_test/driver/testcase.py index 3c66f607163..4de25244ac9 100644 --- a/lib/galaxy_test/driver/testcase.py +++ b/lib/galaxy_test/driver/testcase.py @@ -1,38 +1,8 @@ -import logging -import os -import unittest - -from galaxy.security import idencoding -from galaxy.tool_util.verify.test_data import TestDataResolver -from .driver_util import GalaxyTestDriver, setup_keep_outdir, target_url_parts - -log = logging.getLogger(__name__) +from galaxy_test.base.testcase import FunctionalTestCase +from .driver_util import GalaxyTestDriver -class FunctionalTestCase(unittest.TestCase): - - def setUp(self): - self.security = idencoding.IdEncodingHelper(id_secret='changethisinproductiontoo') - self.history_id = os.environ.get('GALAXY_TEST_HISTORY_ID', None) - self.host, self.port, self.url = target_url_parts() - self.test_data_resolver = TestDataResolver() - self.keepOutdir = setup_keep_outdir() - - @classmethod - def setUpClass(cls): - """Configure and start Galaxy for a test.""" - cls._test_driver = None - - if not os.environ.get("GALAXY_TEST_ENVIRONMENT_CONFIGURED"): - cls._test_driver = GalaxyTestDriver() - cls._test_driver.setup(config_object=cls) - - @classmethod - def tearDownClass(cls): - """Shutdown Galaxy server and cleanup temp directory.""" - if cls._test_driver: - cls._test_driver.tear_down() - - def get_filename(self, filename): - # No longer used by tool tests - drop if isn't used else where. - return self.test_data_resolver.get_filename(filename) +class DrivenFunctionalTestCase(FunctionalTestCase): + """Variant of FunctionalTestCase that can launch Galaxy instances. + """ + galaxy_driver_class = GalaxyTestDriver diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py index 60f20ae904a..16d07d27a32 100644 --- a/lib/galaxy_test/selenium/framework.py +++ b/lib/galaxy_test/selenium/framework.py @@ -25,11 +25,15 @@ from galaxy.selenium.navigates_galaxy import ( NavigatesGalaxy, retry_during_transitions ) -from galaxy.util import asbool +from galaxy.util import asbool, classproperty from galaxy_test.base import populators -from galaxy_test.driver.api import UsesApiTestCaseMixin -from galaxy_test.driver.driver_util import classproperty, DEFAULT_WEB_HOST, get_ip_address -from galaxy_test.driver.testcase import FunctionalTestCase +from galaxy_test.base.api import UsesApiTestCaseMixin +from galaxy_test.base.env import DEFAULT_WEB_HOST, get_ip_address +from galaxy_test.base.testcase import FunctionalTestCase +try: + from galaxy_test.driver.driver_util import GalaxyTestDriver +except ImportError: + GalaxyTestDriver = None DEFAULT_TIMEOUT_MULTIPLIER = 1 DEFAULT_TEST_ERRORS_DIRECTORY = os.path.abspath("database/test_errors") @@ -396,6 +400,7 @@ class TestWithSeleniumMixin(NavigatesGalaxy, UsesApiTestCaseMixin): class SeleniumTestCase(FunctionalTestCase, TestWithSeleniumMixin): + galaxy_driver_class = GalaxyTestDriver def setUp(self): super().setUp() diff --git a/lib/galaxy_test/selenium/test_admin_app.py b/lib/galaxy_test/selenium/test_admin_app.py index 28b54fd82a8..530845235ed 100644 --- a/lib/galaxy_test/selenium/test_admin_app.py +++ b/lib/galaxy_test/selenium/test_admin_app.py @@ -84,7 +84,7 @@ class AdminAppTestCase(SeleniumTestCase): self.screenshot("admin_data_tables") admin_component.index.display_applications.wait_for_and_click() - self.assert_warning_message("No display applications available.") + self.assert_message(self.components.admin.warning, "No display applications available.") self.screenshot("admin_display_applications") admin_component.index.jobs.wait_for_and_click() diff --git a/lib/galaxy_test/selenium/test_workflow_run.py b/lib/galaxy_test/selenium/test_workflow_run.py index 005b11a8a0e..bb149d62101 100644 --- a/lib/galaxy_test/selenium/test_workflow_run.py +++ b/lib/galaxy_test/selenium/test_workflow_run.py @@ -153,7 +153,7 @@ steps: self.workflow_run_with_name(name) self.sleep_for(self.wait_types.UX_TRANSITION) # Check that this tool form contains a warning about different versions. - self.assert_warning_message(contains="different versions") + self.assert_message(self.components.workflow_run.warning, contains="different versions") self.screenshot("workflow_run_tool_upgrade") @selenium_test diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index ccce9b9b112..74764fff5a8 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -21,7 +21,7 @@ import galaxy.util from galaxy.security import idencoding from galaxy.util import smart_str, unicodify from galaxy_test.base.api_util import get_master_api_key -from galaxy_test.driver.testcase import FunctionalTestCase +from galaxy_test.driver.testcase import DrivenFunctionalTestCase from tool_shed.util import ( hg_util, hgweb_config, @@ -38,7 +38,7 @@ log = logging.getLogger(__name__) tc.options['equiv_refresh_interval'] = 0 -class ShedTwillTestCase(FunctionalTestCase): +class ShedTwillTestCase(DrivenFunctionalTestCase): def setUp(self): # Security helper diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 67fdecb429f..1f22df35e2f 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -41,13 +41,14 @@ def check_archive(repository, archive): undesirable_files.append(member) continue head = tail = member.name - try: - while tail: - head, tail = os.path.split(head) - if tail in UNDESIRABLE_DIRS: - undesirable_dirs.append(member) - assert False - except AssertionError: + found_undesirable_dir = False + while tail: + head, tail = os.path.split(head) + if tail in UNDESIRABLE_DIRS: + undesirable_dirs.append(member) + found_undesirable_dir = True + break + if found_undesirable_dir: continue if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: errors.append('Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.') diff --git a/lib/tool_shed/webapp/buildapp.py b/lib/tool_shed/webapp/buildapp.py index e20c5d58344..d15ed084ffa 100644 --- a/lib/tool_shed/webapp/buildapp.py +++ b/lib/tool_shed/webapp/buildapp.py @@ -11,7 +11,7 @@ from paste import httpexceptions from routes.middleware import RoutesMiddleware from six.moves.urllib.parse import parse_qs -import galaxy.web.framework.webapp +import galaxy.webapps.base.webapp import tool_shed.webapp.model import tool_shed.webapp.model.mapping from galaxy import util @@ -22,7 +22,7 @@ from galaxy.webapps.util import wrap_if_allowed log = logging.getLogger(__name__) -class CommunityWebApplication(galaxy.web.framework.webapp.WebApplication): +class CommunityWebApplication(galaxy.webapps.base.webapp.WebApplication): pass @@ -271,7 +271,7 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf): def wrap_in_static(app, global_conf, **local_conf): - urlmap, _ = galaxy.web.framework.webapp.build_url_map(app, global_conf, local_conf) + urlmap, _ = galaxy.webapps.base.webapp.build_url_map(app, global_conf, local_conf) return urlmap @@ -292,4 +292,4 @@ def _map_redirects(mapper): def uwsgi_app(): - return galaxy.web.framework.webapp.build_native_uwsgi_app(app_factory, "tool_shed") + return galaxy.webapps.base.webapp.build_native_uwsgi_app(app_factory, "tool_shed") diff --git a/packages/app/HISTORY.rst b/packages/app/HISTORY.rst index ec2f0e746e9..72090ac4c88 100644 --- a/packages/app/HISTORY.rst +++ b/packages/app/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/app/galaxy/project_galaxy_app.py b/packages/app/galaxy/project_galaxy_app.py index 95cac65f339..61a6ae88785 100644 --- a/packages/app/galaxy/project_galaxy_app.py +++ b/packages/app/galaxy/project_galaxy_app.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-app" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/app/requirements.txt b/packages/app/requirements.txt index da8a0e43f0c..e51c3b80fbc 100644 --- a/packages/app/requirements.txt +++ b/packages/app/requirements.txt @@ -1,4 +1,6 @@ galaxy-auth galaxy-job-execution +galaxy-job-metrics +galaxy-tool-util galaxy-web-framework galaxy-web-stack diff --git a/packages/app/setup.py b/packages/app/setup.py index 3287cc2a5f4..47ebf20445f 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -116,9 +116,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/app/test-requirements.txt b/packages/app/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/app/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/auth/HISTORY.rst b/packages/auth/HISTORY.rst index ec2f0e746e9..72090ac4c88 100644 --- a/packages/auth/HISTORY.rst +++ b/packages/auth/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/auth/galaxy/project_galaxy_auth.py b/packages/auth/galaxy/project_galaxy_auth.py index dbd94f1ced5..a15658b18cb 100644 --- a/packages/auth/galaxy/project_galaxy_auth.py +++ b/packages/auth/galaxy/project_galaxy_auth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-auth" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/auth/setup.py b/packages/auth/setup.py index 63367f57581..c6f71e467a7 100644 --- a/packages/auth/setup.py +++ b/packages/auth/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/auth/test-requirements.txt b/packages/auth/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/auth/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/test/unit/datatypes/__init__.py b/packages/auth/tests/__init__.py similarity index 100% rename from test/unit/datatypes/__init__.py rename to packages/auth/tests/__init__.py diff --git a/packages/auth/tests/auth b/packages/auth/tests/auth new file mode 120000 index 00000000000..46631295e63 --- /dev/null +++ b/packages/auth/tests/auth @@ -0,0 +1 @@ +../../../test/unit/auth \ No newline at end of file diff --git a/packages/auth/tests/test_auth.py b/packages/auth/tests/test_auth.py deleted file mode 120000 index 58ff6f86bf1..00000000000 --- a/packages/auth/tests/test_auth.py +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/test_auth.py \ No newline at end of file diff --git a/packages/containers/HISTORY.rst b/packages/containers/HISTORY.rst index 52c2df7066e..7879816905a 100644 --- a/packages/containers/HISTORY.rst +++ b/packages/containers/HISTORY.rst @@ -6,11 +6,17 @@ History .. to_doc --------------------- -19.9.1.dev0 +20.5.1.dev0 --------------------- +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. + --------------------- 19.9.0 (2019-12-16) --------------------- diff --git a/packages/containers/galaxy/project_galaxy_containers.py b/packages/containers/galaxy/project_galaxy_containers.py index 54af538bb38..8d1702ac5a9 100644 --- a/packages/containers/galaxy/project_galaxy_containers.py +++ b/packages/containers/galaxy/project_galaxy_containers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.1.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-containers" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/containers/setup.py b/packages/containers/setup.py index 4d4fb9caec1..5e6ad0baf30 100644 --- a/packages/containers/setup.py +++ b/packages/containers/setup.py @@ -55,9 +55,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/containers/test-requirements.txt b/packages/containers/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/containers/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/test/unit/datatypes/converters/__init__.py b/packages/containers/tests/__init__.py similarity index 100% rename from test/unit/datatypes/converters/__init__.py rename to packages/containers/tests/__init__.py diff --git a/packages/containers/tests/containers b/packages/containers/tests/containers new file mode 120000 index 00000000000..5efd00dbb90 --- /dev/null +++ b/packages/containers/tests/containers @@ -0,0 +1 @@ +../../../test/unit/containers \ No newline at end of file diff --git a/packages/containers/tests/test_containers.py b/packages/containers/tests/test_containers.py deleted file mode 120000 index 2fb68299fa1..00000000000 --- a/packages/containers/tests/test_containers.py +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/test_containers.py \ No newline at end of file diff --git a/packages/data/HISTORY.rst b/packages/data/HISTORY.rst index ec2f0e746e9..4f7ba0c7b9d 100644 --- a/packages/data/HISTORY.rst +++ b/packages/data/HISTORY.rst @@ -6,7 +6,14 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/data/galaxy/project_galaxy_data.py b/packages/data/galaxy/project_galaxy_data.py index 1a2efe784b6..25e11b314df 100644 --- a/packages/data/galaxy/project_galaxy_data.py +++ b/packages/data/galaxy/project_galaxy_data.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev1' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-data" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/data/setup.py b/packages/data/setup.py index 28faa87c3d9..4c8506104bd 100644 --- a/packages/data/setup.py +++ b/packages/data/setup.py @@ -70,9 +70,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/data/test-requirements.txt b/packages/data/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/data/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/data/tests/data b/packages/data/tests/data new file mode 120000 index 00000000000..2b74d984e14 --- /dev/null +++ b/packages/data/tests/data @@ -0,0 +1 @@ +../../../test/unit/data \ No newline at end of file diff --git a/packages/data/tests/datatypes b/packages/data/tests/datatypes deleted file mode 120000 index d01dc2e3776..00000000000 --- a/packages/data/tests/datatypes +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/datatypes \ No newline at end of file diff --git a/packages/data/tests/test_galaxy_mapping.py b/packages/data/tests/test_galaxy_mapping.py deleted file mode 120000 index 0cc477d90ee..00000000000 --- a/packages/data/tests/test_galaxy_mapping.py +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/test_galaxy_mapping.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/objectstore_helpers.py b/packages/data/tests/unittest_utils/objectstore_helpers.py new file mode 120000 index 00000000000..c1eefa086c4 --- /dev/null +++ b/packages/data/tests/unittest_utils/objectstore_helpers.py @@ -0,0 +1 @@ +../../../../test/unit/unittest_utils/objectstore_helpers.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/tempfilecache.py b/packages/data/tests/unittest_utils/tempfilecache.py deleted file mode 120000 index 588f6eec7d8..00000000000 --- a/packages/data/tests/unittest_utils/tempfilecache.py +++ /dev/null @@ -1 +0,0 @@ -../../../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/utility.py b/packages/data/tests/unittest_utils/utility.py deleted file mode 120000 index 3c69a8fd67e..00000000000 --- a/packages/data/tests/unittest_utils/utility.py +++ /dev/null @@ -1 +0,0 @@ -../../../../test/unit/unittest_utils/utility.py \ No newline at end of file diff --git a/packages/job_execution/HISTORY.rst b/packages/job_execution/HISTORY.rst index ec2f0e746e9..72090ac4c88 100644 --- a/packages/job_execution/HISTORY.rst +++ b/packages/job_execution/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/job_execution/galaxy/project_galaxy_job_execution.py b/packages/job_execution/galaxy/project_galaxy_job_execution.py index f2c11c58e0e..c88e1f230ac 100644 --- a/packages/job_execution/galaxy/project_galaxy_job_execution.py +++ b/packages/job_execution/galaxy/project_galaxy_job_execution.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-job-execution" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/job_execution/setup.py b/packages/job_execution/setup.py index e0d5cb5c879..4f0c6e9e1c8 100644 --- a/packages/job_execution/setup.py +++ b/packages/job_execution/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/job_execution/test-requirements.txt b/packages/job_execution/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/job_execution/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/job_metrics/HISTORY.rst b/packages/job_metrics/HISTORY.rst index 52c2df7066e..7879816905a 100644 --- a/packages/job_metrics/HISTORY.rst +++ b/packages/job_metrics/HISTORY.rst @@ -6,11 +6,17 @@ History .. to_doc --------------------- -19.9.1.dev0 +20.5.1.dev0 --------------------- +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. + --------------------- 19.9.0 (2019-12-16) --------------------- diff --git a/packages/job_metrics/galaxy/project_galaxy_job_metrics.py b/packages/job_metrics/galaxy/project_galaxy_job_metrics.py index 1a710fcca5a..1b7831b5354 100644 --- a/packages/job_metrics/galaxy/project_galaxy_job_metrics.py +++ b/packages/job_metrics/galaxy/project_galaxy_job_metrics.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.1.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-job-metrics" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/job_metrics/setup.py b/packages/job_metrics/setup.py index 8970bc45aec..22ca9eb8ba7 100644 --- a/packages/job_metrics/setup.py +++ b/packages/job_metrics/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/job_metrics/test-requirements.txt b/packages/job_metrics/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/job_metrics/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/meta/setup.py b/packages/meta/setup.py index 07e0d7a2d4f..bbae1d39c75 100644 --- a/packages/meta/setup.py +++ b/packages/meta/setup.py @@ -50,9 +50,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") # TODO: diff --git a/packages/meta/test-requirements.txt b/packages/meta/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/meta/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/objectstore/HISTORY.rst b/packages/objectstore/HISTORY.rst index 52c2df7066e..7879816905a 100644 --- a/packages/objectstore/HISTORY.rst +++ b/packages/objectstore/HISTORY.rst @@ -6,11 +6,17 @@ History .. to_doc --------------------- -19.9.1.dev0 +20.5.1.dev0 --------------------- +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. + --------------------- 19.9.0 (2019-12-16) --------------------- diff --git a/packages/objectstore/galaxy/project_galaxy_objectstore.py b/packages/objectstore/galaxy/project_galaxy_objectstore.py index 8389a7251b6..cc83b9e9564 100644 --- a/packages/objectstore/galaxy/project_galaxy_objectstore.py +++ b/packages/objectstore/galaxy/project_galaxy_objectstore.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.1.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-objectstore" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/objectstore/requirements.txt b/packages/objectstore/requirements.txt index 623c4906bb2..c3d75e0c839 100644 --- a/packages/objectstore/requirements.txt +++ b/packages/objectstore/requirements.txt @@ -1 +1,2 @@ galaxy-util +pyyaml diff --git a/packages/objectstore/setup.py b/packages/objectstore/setup.py index 444e4337388..a4e59f43496 100644 --- a/packages/objectstore/setup.py +++ b/packages/objectstore/setup.py @@ -55,9 +55,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/objectstore/test-requirements.txt b/packages/objectstore/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/objectstore/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/test/unit/datatypes/dataproviders/__init__.py b/packages/objectstore/tests/__init__.py similarity index 100% rename from test/unit/datatypes/dataproviders/__init__.py rename to packages/objectstore/tests/__init__.py diff --git a/packages/objectstore/tests/objectstore b/packages/objectstore/tests/objectstore new file mode 120000 index 00000000000..d8fee00a393 --- /dev/null +++ b/packages/objectstore/tests/objectstore @@ -0,0 +1 @@ +../../../test/unit/objectstore \ No newline at end of file diff --git a/packages/objectstore/tests/test_objectstore.py b/packages/objectstore/tests/test_objectstore.py deleted file mode 120000 index 0ef5a715802..00000000000 --- a/packages/objectstore/tests/test_objectstore.py +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/test_objectstore.py \ No newline at end of file diff --git a/packages/objectstore/tests/unittest_utils/__init__.py b/packages/objectstore/tests/unittest_utils/__init__.py new file mode 120000 index 00000000000..290bb9ee29b --- /dev/null +++ b/packages/objectstore/tests/unittest_utils/__init__.py @@ -0,0 +1 @@ +../../../../test/unit/unittest_utils/__init__.py \ No newline at end of file diff --git a/packages/objectstore/tests/unittest_utils/objectstore_helpers.py b/packages/objectstore/tests/unittest_utils/objectstore_helpers.py new file mode 120000 index 00000000000..c1eefa086c4 --- /dev/null +++ b/packages/objectstore/tests/unittest_utils/objectstore_helpers.py @@ -0,0 +1 @@ +../../../../test/unit/unittest_utils/objectstore_helpers.py \ No newline at end of file diff --git a/packages/package-dev-requirements.txt b/packages/package-dev-requirements.txt index 034ad598066..bb17505d354 100644 --- a/packages/package-dev-requirements.txt +++ b/packages/package-dev-requirements.txt @@ -1,5 +1,5 @@ # For testing -pytest +-r test-requirements.txt # For dev sphinx diff --git a/packages/package.Makefile b/packages/package.Makefile index 8b5a332977e..a86b62648b8 100644 --- a/packages/package.Makefile +++ b/packages/package.Makefile @@ -12,7 +12,7 @@ VERSION?=$(shell DEV_RELEASE=$(DEV_RELEASE) python $(BUILD_SCRIPTS_DIR)/print_ve PROJECT_NAME?="galaxy-$(shell basename $(CURDIR))" PROJECT_NAME:=$(subst _,-,$(PROJECT_NAME)) BRANCH?=$(shell git rev-parse --abbrev-ref HEAD) -TEST_DIR?=test +TEST_DIR?=tests TESTS?=$(SOURCE_DIR) $(TEST_DIR) .PHONY: clean-pyc clean-build docs clean diff --git a/packages/selenium/HISTORY.rst b/packages/selenium/HISTORY.rst index ec2f0e746e9..72090ac4c88 100644 --- a/packages/selenium/HISTORY.rst +++ b/packages/selenium/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/selenium/galaxy/project_galaxy_selenium.py b/packages/selenium/galaxy/project_galaxy_selenium.py index 73ec5c1cf31..8682a60d8ee 100644 --- a/packages/selenium/galaxy/project_galaxy_selenium.py +++ b/packages/selenium/galaxy/project_galaxy_selenium.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-selenium" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/selenium/setup.py b/packages/selenium/setup.py index a53fe8e6602..fd4dbd42c4b 100644 --- a/packages/selenium/setup.py +++ b/packages/selenium/setup.py @@ -59,9 +59,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/selenium/test-requirements.txt b/packages/selenium/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/selenium/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/test.sh b/packages/test.sh index 2f7a62af826..3e795614c08 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -14,7 +14,6 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" -pip install pytest # ensure ordered by dependency dag PACKAGE_DIRS=( @@ -29,17 +28,19 @@ PACKAGE_DIRS=( web_stack web_framework app - web_apps + webapps ) -# containers has no tests, tool_util not yet working 100%, +# tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(1 1 1 1 1 1 1 1 0 0 0 0) +RUN_TESTS=(1 1 1 1 1 1 1 1 1 1 0 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} run_tests=${RUN_TESTS[$i]} cd "$package_dir" - pip install -e . + pip install -e '.' + pip install -r test-requirements.txt + # Install extras (if needed) if [ "$package_dir" = "util" ]; then pip install -e '.[template,jstree]' diff --git a/packages/test_api/setup.py b/packages/test_api/setup.py index f5b10798a4f..be3583bab49 100644 --- a/packages/test_api/setup.py +++ b/packages/test_api/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/test_api/test-requirements.txt b/packages/test_api/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/test_api/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/test_base/HISTORY.rst b/packages/test_base/HISTORY.rst index 7cb67f838f8..b8a2e6045e6 100644 --- a/packages/test_base/HISTORY.rst +++ b/packages/test_base/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -20.1.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 20.01 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* Initial import from the 20.05 branch of Galaxy. diff --git a/packages/test_base/galaxy/project_galaxy_test_base.py b/packages/test_base/galaxy/project_galaxy_test_base.py index 668485b359f..d2ae57543f9 100644 --- a/packages/test_base/galaxy/project_galaxy_test_base.py +++ b/packages/test_base/galaxy/project_galaxy_test_base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '20.1.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-test-base" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/test_base/setup.py b/packages/test_base/setup.py index d517c2ab9d7..5499f6a9535 100644 --- a/packages/test_base/setup.py +++ b/packages/test_base/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/test_base/test-requirements.txt b/packages/test_base/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/test_base/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/test_driver/HISTORY.rst b/packages/test_driver/HISTORY.rst index 7cb67f838f8..72090ac4c88 100644 --- a/packages/test_driver/HISTORY.rst +++ b/packages/test_driver/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -20.1.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 20.01 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/test_driver/galaxy/project_galaxy_test_driver.py b/packages/test_driver/galaxy/project_galaxy_test_driver.py index 3c665e4adfb..ccf9c1596b8 100644 --- a/packages/test_driver/galaxy/project_galaxy_test_driver.py +++ b/packages/test_driver/galaxy/project_galaxy_test_driver.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '20.1.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-test-driver" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/test_driver/requirements.txt b/packages/test_driver/requirements.txt index e8566b22706..11f4af7564b 100644 --- a/packages/test_driver/requirements.txt +++ b/packages/test_driver/requirements.txt @@ -1,4 +1,12 @@ +galaxy-app galaxy-data +galaxy-test-base +galaxy-tool-util +galaxy-util +galaxy-webapps +galaxy-web-framework nose pytest - +pyyaml +sqlalchemy-utils +six diff --git a/packages/test_driver/setup.py b/packages/test_driver/setup.py index b0d9cce9ded..3d68b544e31 100644 --- a/packages/test_driver/setup.py +++ b/packages/test_driver/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/test_driver/test-requirements.txt b/packages/test_driver/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/test_driver/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/test_selenium/setup.py b/packages/test_selenium/setup.py index dee400c2c05..6a03b956e87 100644 --- a/packages/test_selenium/setup.py +++ b/packages/test_selenium/setup.py @@ -57,9 +57,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/test_selenium/test-requirements.txt b/packages/test_selenium/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/test_selenium/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index d2be3810139..6524c304020 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -6,13 +6,19 @@ History .. to_doc --------------------- -20.5.0.dev0 +20.5.1.dev0 +--------------------- + + + +--------------------- +20.5.0 (2020-07-04) --------------------- * First release from the 20.05 branch of Galaxy. --------------------- -20.1.0.dev0 +20.1.0 (2020-07-04) --------------------- * First release from the 20.01 branch of Galaxy. diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index e1595d959a7..42a4eb677a4 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '20.5.0.dev3' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py index a238e274aa2..c6f0916768a 100644 --- a/packages/tool_util/setup.py +++ b/packages/tool_util/setup.py @@ -76,9 +76,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/tool_util/test-requirements.txt b/packages/tool_util/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/tool_util/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/util/HISTORY.rst b/packages/util/HISTORY.rst index 0a46ed31291..2816300aea2 100644 --- a/packages/util/HISTORY.rst +++ b/packages/util/HISTORY.rst @@ -6,10 +6,16 @@ History .. to_doc --------------------- -20.1.0.dev0 +20.5.1.dev0 --------------------- -* First release from 20.01 branch of Galaxy. + + +--------------------- +20.5.0 (2020-07-03) +--------------------- + +* First release from 20.05 branch of Galaxy. --------------------- 19.9.0 (2019-11-21) diff --git a/packages/util/galaxy/project_galaxy_util.py b/packages/util/galaxy/project_galaxy_util.py index c94f764054a..dd0d57feb07 100644 --- a/packages/util/galaxy/project_galaxy_util.py +++ b/packages/util/galaxy/project_galaxy_util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '20.1.0.dev1' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/util/setup.py b/packages/util/setup.py index 57cda725692..01c3f38502b 100644 --- a/packages/util/setup.py +++ b/packages/util/setup.py @@ -60,9 +60,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/util/test-data b/packages/util/test-data new file mode 120000 index 00000000000..e50575ea576 --- /dev/null +++ b/packages/util/test-data @@ -0,0 +1 @@ +../../test-data \ No newline at end of file diff --git a/packages/util/test-requirements.txt b/packages/util/test-requirements.txt new file mode 100644 index 00000000000..208ec64cdf0 --- /dev/null +++ b/packages/util/test-requirements.txt @@ -0,0 +1,2 @@ +pytest +responses diff --git a/test/unit/web/__init__.py b/packages/util/tests/__init__.py similarity index 100% rename from test/unit/web/__init__.py rename to packages/util/tests/__init__.py diff --git a/packages/util/tests/test_utils.py b/packages/util/tests/test_utils.py deleted file mode 120000 index b770559d38a..00000000000 --- a/packages/util/tests/test_utils.py +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/util/test_utils.py \ No newline at end of file diff --git a/packages/util/tests/util b/packages/util/tests/util new file mode 120000 index 00000000000..d42557d85de --- /dev/null +++ b/packages/util/tests/util @@ -0,0 +1 @@ +../../../test/unit/util \ No newline at end of file diff --git a/packages/web_apps/HISTORY.rst b/packages/web_apps/HISTORY.rst deleted file mode 100644 index ec2f0e746e9..00000000000 --- a/packages/web_apps/HISTORY.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. :changelog: - -History -------- - -.. to_doc - ---------------------- -19.9.0.dev0 ---------------------- - -* Initial import from dev branch of Galaxy during 19.09 development cycle. diff --git a/packages/web_apps/requirements.txt b/packages/web_apps/requirements.txt deleted file mode 100644 index eaf79b2a242..00000000000 --- a/packages/web_apps/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -galaxy-app - diff --git a/packages/web_framework/HISTORY.rst b/packages/web_framework/HISTORY.rst index ec2f0e746e9..72090ac4c88 100644 --- a/packages/web_framework/HISTORY.rst +++ b/packages/web_framework/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/web_framework/galaxy/project_galaxy_web_framework.py b/packages/web_framework/galaxy/project_galaxy_web_framework.py index e6f46b22814..79a5de41962 100644 --- a/packages/web_framework/galaxy/project_galaxy_web_framework.py +++ b/packages/web_framework/galaxy/project_galaxy_web_framework.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-web-framework" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/web_framework/requirements.txt b/packages/web_framework/requirements.txt index 7dcf8f4def7..bde04c26cfe 100644 --- a/packages/web_framework/requirements.txt +++ b/packages/web_framework/requirements.txt @@ -1 +1,11 @@ +babel galaxy-data +galaxy-util +galaxy-web-stack +MarkupSafe +paste +requests +Routes +six +SQLAlchemy +WebOb diff --git a/packages/web_framework/setup.py b/packages/web_framework/setup.py index 8b894882f6b..d2d5add572a 100644 --- a/packages/web_framework/setup.py +++ b/packages/web_framework/setup.py @@ -60,9 +60,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/web_framework/test-requirements.txt b/packages/web_framework/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/web_framework/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/test/unit/web/framework/__init__.py b/packages/web_framework/tests/__init__.py similarity index 100% rename from test/unit/web/framework/__init__.py rename to packages/web_framework/tests/__init__.py diff --git a/packages/web_framework/tests/web_framework b/packages/web_framework/tests/web_framework new file mode 120000 index 00000000000..ee2f25432d2 --- /dev/null +++ b/packages/web_framework/tests/web_framework @@ -0,0 +1 @@ +../../../test/unit/web_framework \ No newline at end of file diff --git a/packages/web_stack/HISTORY.rst b/packages/web_stack/HISTORY.rst index ec2f0e746e9..72090ac4c88 100644 --- a/packages/web_stack/HISTORY.rst +++ b/packages/web_stack/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -19.9.0.dev0 +20.5.1.dev0 --------------------- -* Initial import from dev branch of Galaxy during 19.09 development cycle. + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/web_stack/galaxy/project_galaxy_web_stack.py b/packages/web_stack/galaxy/project_galaxy_web_stack.py index 987fdbbd1fd..a454a258c8d 100644 --- a/packages/web_stack/galaxy/project_galaxy_web_stack.py +++ b/packages/web_stack/galaxy/project_galaxy_web_stack.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' PROJECT_NAME = "galaxy-web-stack" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/web_stack/setup.py b/packages/web_stack/setup.py index e75068a7065..4a14a0c88e8 100644 --- a/packages/web_stack/setup.py +++ b/packages/web_stack/setup.py @@ -56,9 +56,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/web_stack/test-requirements.txt b/packages/web_stack/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/web_stack/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/web_stack/tests/web_stack b/packages/web_stack/tests/web_stack new file mode 120000 index 00000000000..c3dbc746415 --- /dev/null +++ b/packages/web_stack/tests/web_stack @@ -0,0 +1 @@ +../../../test/unit/web_stack \ No newline at end of file diff --git a/packages/webapps/HISTORY.rst b/packages/webapps/HISTORY.rst new file mode 100644 index 00000000000..72090ac4c88 --- /dev/null +++ b/packages/webapps/HISTORY.rst @@ -0,0 +1,18 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +20.5.1.dev0 +--------------------- + + + +--------------------- +20.5.0 (2020-07-04) +--------------------- + +* First release from the 20.05 branch of Galaxy. diff --git a/packages/web_apps/LICENSE b/packages/webapps/LICENSE similarity index 100% rename from packages/web_apps/LICENSE rename to packages/webapps/LICENSE diff --git a/packages/web_apps/MANIFEST.in b/packages/webapps/MANIFEST.in similarity index 100% rename from packages/web_apps/MANIFEST.in rename to packages/webapps/MANIFEST.in diff --git a/packages/web_apps/Makefile b/packages/webapps/Makefile similarity index 100% rename from packages/web_apps/Makefile rename to packages/webapps/Makefile diff --git a/packages/web_apps/README.rst b/packages/webapps/README.rst similarity index 65% rename from packages/web_apps/README.rst rename to packages/webapps/README.rst index 2f28a9fefb6..43024db40ba 100644 --- a/packages/web_apps/README.rst +++ b/packages/webapps/README.rst @@ -1,6 +1,6 @@ -.. image:: https://badge.fury.io/py/galaxy-web-apps.svg - :target: https://pypi.python.org/pypi/galaxy-web-apps/ +.. image:: https://badge.fury.io/py/galaxy-webapps.svg + :target: https://pypi.python.org/pypi/galaxy-webapps/ Overview diff --git a/packages/web_apps/dev-requirements.txt b/packages/webapps/dev-requirements.txt similarity index 100% rename from packages/web_apps/dev-requirements.txt rename to packages/webapps/dev-requirements.txt diff --git a/packages/web_apps/galaxy/__init__.py b/packages/webapps/galaxy/__init__.py similarity index 100% rename from packages/web_apps/galaxy/__init__.py rename to packages/webapps/galaxy/__init__.py diff --git a/packages/web_apps/galaxy/project_galaxy_web_apps.py b/packages/webapps/galaxy/project_galaxy_webapps.py similarity index 85% rename from packages/web_apps/galaxy/project_galaxy_web_apps.py rename to packages/webapps/galaxy/project_galaxy_webapps.py index 3f416bde726..67cb650e0be 100644 --- a/packages/web_apps/galaxy/project_galaxy_web_apps.py +++ b/packages/webapps/galaxy/project_galaxy_webapps.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev0' +__version__ = '20.5.1.dev0' -PROJECT_NAME = "galaxy-web-apps" +PROJECT_NAME = "galaxy-webapps" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" PROJECT_AUTHOR = 'Galaxy Project and Community' diff --git a/packages/web_apps/galaxy/webapps b/packages/webapps/galaxy/webapps similarity index 100% rename from packages/web_apps/galaxy/webapps rename to packages/webapps/galaxy/webapps diff --git a/packages/webapps/requirements.txt b/packages/webapps/requirements.txt new file mode 100644 index 00000000000..8bb9841f7b5 --- /dev/null +++ b/packages/webapps/requirements.txt @@ -0,0 +1,3 @@ +galaxy-app +Cheetah3 +Mako diff --git a/packages/web_apps/scripts b/packages/webapps/scripts similarity index 100% rename from packages/web_apps/scripts rename to packages/webapps/scripts diff --git a/packages/web_apps/setup.cfg b/packages/webapps/setup.cfg similarity index 100% rename from packages/web_apps/setup.cfg rename to packages/webapps/setup.cfg diff --git a/packages/web_apps/setup.py b/packages/webapps/setup.py similarity index 98% rename from packages/web_apps/setup.py rename to packages/webapps/setup.py index d7396f5d88f..b5110d50f66 100644 --- a/packages/web_apps/setup.py +++ b/packages/webapps/setup.py @@ -85,9 +85,7 @@ else: requirements = [] -test_requirements = [ - # TODO: put package test requirements here -] +test_requirements = open("test-requirements.txt").read().split("\n") setup( diff --git a/packages/webapps/test-requirements.txt b/packages/webapps/test-requirements.txt new file mode 100644 index 00000000000..e079f8a6038 --- /dev/null +++ b/packages/webapps/test-requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/packages/web_apps/tool_shed b/packages/webapps/tool_shed similarity index 100% rename from packages/web_apps/tool_shed rename to packages/webapps/tool_shed diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index 227de7ffa3b..6fb9a7f09de 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -12,6 +12,7 @@ import sys galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) sys.path[1:1] = [os.path.join(galaxy_root, "lib"), os.path.join(galaxy_root, "test")] +from galaxy.util import classproperty from galaxy_test.base.api_util import get_master_api_key, get_user_api_key from galaxy_test.driver import driver_util @@ -61,7 +62,7 @@ class SeleniumGalaxyTestDriver(driver_util.GalaxyTestDriver): framework_tool_and_types = True - @driver_util.classproperty + @classproperty def default_web_host(cls): from galaxy_test.selenium.framework import default_web_host_for_selenium_tests return default_web_host_for_selenium_tests() diff --git a/templates/base/base_panels.mako b/templates/base/base_panels.mako index f4a2dfe2252..b00979259fe 100644 --- a/templates/base/base_panels.mako +++ b/templates/base/base_panels.mako @@ -174,13 +174,13 @@
%if self.message_box_visible: -
+
${app.config.message_box_content}
%endif %if self.show_inactivity_warning: -
+
${app.config.inactivity_box_content} Resend verification.
%endif diff --git a/test/functional/test_toolbox.py b/test/functional/test_toolbox.py index 24e45a5c8e7..c527008d145 100644 --- a/test/functional/test_toolbox.py +++ b/test/functional/test_toolbox.py @@ -10,9 +10,9 @@ except ImportError: from galaxy.tool_util.verify.interactor import GalaxyInteractorApi, verify_tool from galaxy.tools import DataManagerTool +from galaxy_test.base.env import setup_keep_outdir, target_url_parts from galaxy_test.base.instrument import register_job_data -from galaxy_test.driver.driver_util import setup_keep_outdir, target_url_parts -from galaxy_test.driver.testcase import FunctionalTestCase +from galaxy_test.driver.testcase import DrivenFunctionalTestCase log = logging.getLogger(__name__) @@ -22,7 +22,7 @@ toolbox = None TOOL_TYPES_NO_TEST = (DataManagerTool, ) -class ToolTestCase(FunctionalTestCase): +class ToolTestCase(DrivenFunctionalTestCase): """Abstract test case that runs tests based on a `galaxy.tools.test.ToolTest`.""" def do_it(self, tool_id=None, tool_version=None, test_index=0, resource_parameters={}): diff --git a/test/functional/tools/metadata_bam.xml b/test/functional/tools/metadata_bam.xml index 2f20be5b531..6cf73b2e9c3 100644 --- a/test/functional/tools/metadata_bam.xml +++ b/test/functional/tools/metadata_bam.xml @@ -1,5 +1,8 @@ - - echo "${ref_names}" > "${output_of_input_metadata}" + + '${output_of_input_metadata}' + ]]> diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 4db887d296f..7945698d06b 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -57,6 +57,7 @@ + diff --git a/test/functional/tools/strict_shell_profile.xml b/test/functional/tools/strict_shell_profile.xml new file mode 100644 index 00000000000..eb03d4517ca --- /dev/null +++ b/test/functional/tools/strict_shell_profile.xml @@ -0,0 +1,23 @@ + + + echo "Hello" > $out_file1 + ; sh -c "exit $exit_code" + ; sh -c "exit 0" + + + + + + + + + + + + + + + + + + diff --git a/test/integration/embedded_pulsar_metadata_job_conf.yml b/test/integration/embedded_pulsar_metadata_job_conf.yml index 76520bf4ed5..451162866a0 100644 --- a/test/integration/embedded_pulsar_metadata_job_conf.yml +++ b/test/integration/embedded_pulsar_metadata_job_conf.yml @@ -13,6 +13,7 @@ execution: runner: pulsar_embed remote_metadata: true default_file_action: copy + metadata_strategy: directory tools: - id: upload1 diff --git a/test/integration/test_configuration_decode.py b/test/integration/test_configuration_decode.py new file mode 100644 index 00000000000..ed662dbca78 --- /dev/null +++ b/test/integration/test_configuration_decode.py @@ -0,0 +1,23 @@ +from galaxy_test.base.populators import ( + LibraryPopulator +) +from galaxy_test.driver import integration_util + + +class ConfigurationDecodeIntegrationTestCase(integration_util.IntegrationTestCase): + + def setUp(self): + super(ConfigurationDecodeIntegrationTestCase, self).setUp() + self.library_populator = LibraryPopulator(self.galaxy_interactor) + + def test_admin_decode_id(self): + new_lib = self.library_populator.new_library('DecodeTestLibrary') + decode_response = self._get("configuration/decode/" + new_lib["id"], admin=True) + response_id = decode_response.json()["decoded_id"] + decoded_library_id = self._app.security.decode_id(new_lib["id"]) + assert decoded_library_id == response_id + # fake valid folder id by prepending F + valid_encoded_folder_id = 'F' + new_lib["id"] + folder_decode_response = self._get("configuration/decode/" + valid_encoded_folder_id, admin=True) + folder_response_id = folder_decode_response.json()["decoded_id"] + assert decoded_library_id == folder_response_id diff --git a/test/integration/test_pulsar_embedded_metadata.py b/test/integration/test_pulsar_embedded_metadata.py index 4ebae4a5037..4ba5a600c7d 100644 --- a/test/integration/test_pulsar_embedded_metadata.py +++ b/test/integration/test_pulsar_embedded_metadata.py @@ -16,8 +16,13 @@ class EmbeddedMetadataPulsarIntegrationInstance(integration_util.IntegrationInst @classmethod def handle_galaxy_config_kwds(cls, config): config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE + config['object_store_store_by'] = 'uuid' + # We set the global metadata_strategy to `extended, but`metadata_strategy is + # being overridden in embedded_pulsar_metadata_job_conf.yml, since extended_metadata does not yet work on pulsar + config['metadata_strategy'] = 'extended' + config['retry_metadata_internally'] = False instance = integration_util.integration_module_instance(EmbeddedMetadataPulsarIntegrationInstance) -test_tools = integration_util.integration_tool_runner(["simple_constructs"]) +test_tools = integration_util.integration_tool_runner(["simple_constructs", "metadata_bam"]) diff --git a/test/unit/auth/__init__.py b/test/unit/auth/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/test_auth.py b/test/unit/auth/test_auth.py similarity index 100% rename from test/unit/test_auth.py rename to test/unit/auth/test_auth.py diff --git a/test/unit/containers/__init__.py b/test/unit/containers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/test_containers.py b/test/unit/containers/test_containers.py similarity index 86% rename from test/unit/test_containers.py rename to test/unit/containers/test_containers.py index 965d30fae00..5bfb271a62f 100644 --- a/test/unit/test_containers.py +++ b/test/unit/containers/test_containers.py @@ -1,4 +1,5 @@ import galaxy.containers +import galaxy.containers.docker_model def test_dummy(): diff --git a/test/unit/data/__init__.py b/test/unit/data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/data/datatypes/__init__.py b/test/unit/data/datatypes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/data/datatypes/converters/__init__.py b/test/unit/data/datatypes/converters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/datatypes/converters/test_interval_to_tabix.py b/test/unit/data/datatypes/converters/test_interval_to_tabix.py similarity index 100% rename from test/unit/datatypes/converters/test_interval_to_tabix.py rename to test/unit/data/datatypes/converters/test_interval_to_tabix.py diff --git a/test/unit/data/datatypes/dataproviders/__init__.py b/test/unit/data/datatypes/dataproviders/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/datatypes/dataproviders/test_base_dataproviders.py b/test/unit/data/datatypes/dataproviders/test_base_dataproviders.py similarity index 90% rename from test/unit/datatypes/dataproviders/test_base_dataproviders.py rename to test/unit/data/datatypes/dataproviders/test_base_dataproviders.py index 9cf8274910e..31cff443c5a 100644 --- a/test/unit/datatypes/dataproviders/test_base_dataproviders.py +++ b/test/unit/data/datatypes/dataproviders/test_base_dataproviders.py @@ -3,12 +3,14 @@ Unit tests for base DataProviders. .. seealso:: galaxy.datatypes.dataproviders.base """ import logging +import os +import tempfile import unittest from six import StringIO from galaxy.datatypes.dataproviders import base, exceptions -from ...unittest_utils import tempfilecache, utility +from galaxy.util import clean_multiline_string log = logging.getLogger(__name__) @@ -34,7 +36,7 @@ class BaseTestCase(unittest.TestCase): def __init__(self, *args): unittest.TestCase.__init__(self, *args) - self.tmpfiles = tempfilecache.TempFileCache(log) + self.tmpfiles = TempFileCache() def setUp(self): log.debug('BEGIN %s %s', ('.' * 40), self._testMethodName) @@ -47,7 +49,7 @@ class BaseTestCase(unittest.TestCase): def format_tmpfile_contents(self, contents=None): contents = contents or self.default_file_contents - contents = utility.clean_multiline_string(contents) + contents = clean_multiline_string(contents) log.debug('file contents:\n%s', contents) return contents @@ -131,7 +133,7 @@ class Test_BaseDataProvider(BaseTestCase): def test_stringio(self): """should work with StringIO """ - contents = utility.clean_multiline_string(""" + contents = clean_multiline_string(""" One Two Three @@ -293,7 +295,7 @@ class Test_MultiSourceDataProvider(BaseTestCase): def contents_and_tmpfile(self, contents=None): # TODO: hmmmm... contents = contents or self.default_file_contents - contents = utility.clean_multiline_string(contents) + contents = clean_multiline_string(contents) return (contents, self.tmpfiles.create_tmpfile(contents)) def test_multiple_sources(self): @@ -319,7 +321,7 @@ class Test_MultiSourceDataProvider(BaseTestCase): Twelve! (<-- http://youtu.be/JZshZp-cxKg) """ ] - contents = [utility.clean_multiline_string(c) for c in contents] + contents = [clean_multiline_string(c) for c in contents] source_list = [open(self.tmpfiles.create_tmpfile(c)) for c in contents] provider = self.provider_class(source_list) @@ -351,7 +353,7 @@ class Test_MultiSourceDataProvider(BaseTestCase): Twelve! (<-- http://youtu.be/JZshZp-cxKg) """ ] - contents = [utility.clean_multiline_string(c) for c in contents] + contents = [clean_multiline_string(c) for c in contents] source_list = [open(self.tmpfiles.create_tmpfile(c)) for c in contents] def no_Fs(string): @@ -372,5 +374,34 @@ class Test_MultiSourceDataProvider(BaseTestCase): self.assertEqual(''.join(data), 'Two\nThree\nNine\nEleven\n') +class TempFileCache(object): + """ + Creates and caches tempfiles with/based-on the given contents. + """ + + def __init__(self): + self._content_dict = {} + + def create_tmpfile(self, contents): + if contents not in self._content_dict: + # create a named tmp and write contents to it, return filename + with tempfile.NamedTemporaryFile(delete=False, mode='w+') as tmpfile: + tmpfile.write(contents) + tmpfile_name = tmpfile.name + log.debug('created tmpfile.name: %s', tmpfile_name) + self._content_dict[contents] = tmpfile_name + + else: + log.debug('(cached): %s', self._content_dict[contents]) + return self._content_dict[contents] + + def clear(self): + for tmpfile in self._content_dict.values(): + if os.path.exists(tmpfile): + log.debug('unlinking tmpfile: %s', tmpfile) + os.unlink(tmpfile) + self._content_dict = {} + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/datatypes/dataproviders/test_line_dataproviders.py b/test/unit/data/datatypes/dataproviders/test_line_dataproviders.py similarity index 100% rename from test/unit/datatypes/dataproviders/test_line_dataproviders.py rename to test/unit/data/datatypes/dataproviders/test_line_dataproviders.py diff --git a/test/unit/datatypes/test_bam.py b/test/unit/data/datatypes/test_bam.py similarity index 100% rename from test/unit/datatypes/test_bam.py rename to test/unit/data/datatypes/test_bam.py diff --git a/test/unit/datatypes/test_bcf.py b/test/unit/data/datatypes/test_bcf.py similarity index 100% rename from test/unit/datatypes/test_bcf.py rename to test/unit/data/datatypes/test_bcf.py diff --git a/test/unit/datatypes/test_cram.py b/test/unit/data/datatypes/test_cram.py similarity index 100% rename from test/unit/datatypes/test_cram.py rename to test/unit/data/datatypes/test_cram.py diff --git a/test/unit/datatypes/test_data.py b/test/unit/data/datatypes/test_data.py similarity index 100% rename from test/unit/datatypes/test_data.py rename to test/unit/data/datatypes/test_data.py diff --git a/test/unit/datatypes/test_datatypes_registry.py b/test/unit/data/datatypes/test_datatypes_registry.py similarity index 100% rename from test/unit/datatypes/test_datatypes_registry.py rename to test/unit/data/datatypes/test_datatypes_registry.py diff --git a/test/unit/datatypes/test_sniff.py b/test/unit/data/datatypes/test_sniff.py similarity index 100% rename from test/unit/datatypes/test_sniff.py rename to test/unit/data/datatypes/test_sniff.py diff --git a/test/unit/datatypes/test_validation.py b/test/unit/data/datatypes/test_validation.py similarity index 100% rename from test/unit/datatypes/test_validation.py rename to test/unit/data/datatypes/test_validation.py diff --git a/test/unit/datatypes/test_vcf.py b/test/unit/data/datatypes/test_vcf.py similarity index 100% rename from test/unit/datatypes/test_vcf.py rename to test/unit/data/datatypes/test_vcf.py diff --git a/test/unit/datatypes/util.py b/test/unit/data/datatypes/util.py similarity index 100% rename from test/unit/datatypes/util.py rename to test/unit/data/datatypes/util.py diff --git a/test/unit/test_id_encode_decode.py b/test/unit/data/security/test_id_encode_decode.py similarity index 100% rename from test/unit/test_id_encode_decode.py rename to test/unit/data/security/test_id_encode_decode.py diff --git a/test/unit/test_security_passwords.py b/test/unit/data/security/test_passwords.py similarity index 100% rename from test/unit/test_security_passwords.py rename to test/unit/data/security/test_passwords.py diff --git a/test/unit/data/security/test_validate_user_input.py b/test/unit/data/security/test_validate_user_input.py new file mode 100644 index 00000000000..63f9b829625 --- /dev/null +++ b/test/unit/data/security/test_validate_user_input.py @@ -0,0 +1,12 @@ +from galaxy.security.validate_user_input import extract_domain + + +def test_extract_full_domain(): + assert extract_domain('jack@foo.com') == 'foo.com' + assert extract_domain('jack@foo.bar.com') == 'foo.bar.com' + + +def test_extract_base_domain(): + # Use case: ignore subdomains to filter out disposable email addresses + assert extract_domain('jack@foo.com', base_only=True) == 'foo.com' + assert extract_domain('jack@foo.bar.com', base_only=True) == 'bar.com' diff --git a/test/unit/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py similarity index 100% rename from test/unit/test_galaxy_mapping.py rename to test/unit/data/test_galaxy_mapping.py diff --git a/test/unit/test_model_copy.py b/test/unit/data/test_model_copy.py similarity index 98% rename from test/unit/test_model_copy.py rename to test/unit/data/test_model_copy.py index 5e52b2c4c11..02ad637b106 100644 --- a/test/unit/test_model_copy.py +++ b/test/unit/data/test_model_copy.py @@ -7,7 +7,7 @@ import galaxy.model import galaxy.model.mapping as mapping from galaxy.model.metadata import MetadataTempFile from galaxy.util import ExecutionTimer -from .test_objectstore import DISK_TEST_CONFIG, TestConfig +from ..unittest_utils.objectstore_helpers import DISK_TEST_CONFIG, TestConfig datatypes_registry = galaxy.datatypes.registry.Registry() diff --git a/test/unit/jobs/test_job_context.py b/test/unit/jobs/test_job_context.py new file mode 100644 index 00000000000..e9a54a04a47 --- /dev/null +++ b/test/unit/jobs/test_job_context.py @@ -0,0 +1,86 @@ +import os +import tempfile + +from galaxy import model +from galaxy.job_execution.output_collect import ( + dataset_collector, + JobContext, +) +from galaxy.model.dataset_collections import builder +from galaxy.tool_util.parser.output_collection_def import FilePatternDatasetCollectionDescription +from ..tools.test_history_imp_exp import _mock_app + + +class PermissionProvider: + + def __init__(self): + self.permissions = [] + + def set_default_hda_permissions(self, primary_data): + pass + + def copy_dataset_permissions(self, init_from, primary_data): + pass + + +class MetadataSourceProvider: + + def get_metadata_source(self, input_name): + return None + + +class Tool: + + def __init__(self, app): + self.app = app + self.sa_session = app.model.context + + +def setup_data(job_working_directory): + for i in range(10): + with open(os.path.join(job_working_directory, 'datasets_{}.txt'.format(i)), 'w') as out: + out.write(str(i)) + + +def test_job_context_discover_outputs_flushes_once(mocker): + app = _mock_app() + sa_session = app.model.context + # mocker is a pytest-mock fixture + + u = model.User(email="collection@example.com", password="password") + h = model.History(name="Test History", user=u) + + tool = Tool(app) + tool_provided_metadata = None + job = model.Job() + job.history = h + sa_session.add(job) + job_working_directory = tempfile.mkdtemp() + setup_data(job_working_directory) + permission_provider = PermissionProvider() + metadata_source_provider = MetadataSourceProvider() + object_store = app.object_store + input_dbkey = '?' + final_job_state = 'ok' + collection_description = FilePatternDatasetCollectionDescription(pattern="__name__") + collection = model.DatasetCollection(collection_type='list', populated=False) + sa_session.add(collection) + job_context = JobContext(tool, tool_provided_metadata, job, job_working_directory, permission_provider, metadata_source_provider, input_dbkey, object_store, final_job_state) + collection_builder = builder.BoundCollectionBuilder(collection) + dataset_collectors = [dataset_collector(collection_description)] + output_name = 'output' + filenames = job_context.find_files(output_name, collection, dataset_collectors) + assert len(filenames) == 10 + spy = mocker.spy(sa_session, 'flush') + job_context.populate_collection_elements( + collection, + collection_builder, + filenames, + name=output_name, + metadata_source_name='', + final_job_state=job_context.final_job_state, + ) + collection_builder.populate() + assert spy.call_count == 1 + assert len(collection.dataset_instances) == 10 + assert collection.dataset_instances[0].dataset.file_size == 1 diff --git a/test/unit/managers/test_markdown_export.py b/test/unit/managers/test_markdown_export.py index 88a06f2dac8..79a154d926a 100644 --- a/test/unit/managers/test_markdown_export.py +++ b/test/unit/managers/test_markdown_export.py @@ -10,6 +10,7 @@ from galaxy.managers.markdown_util import ( ready_galaxy_markdown_for_export, to_basic_markdown, ) +from galaxy.model.orm.now import now from .base import BaseTestCase @@ -33,6 +34,12 @@ class BaseExportTestCase(BaseTestCase): hda.dataset.get_file_name.return_value = t.name return hda + def _new_invocation(self): + invocation = model.WorkflowInvocation() + invocation.id = 1 + invocation.create_time = now() + return invocation + @contextmanager def _expect_get_hda(self, hda, hda_id=1): self.app.hda_manager.get_accessible.return_value = hda @@ -165,6 +172,30 @@ history_dataset_display(history_dataset_id=1) result = self._to_basic(example) assert ' - - - - - -""" - - -DISK_TEST_CONFIG_YAML = """ -type: disk -files_dir: "${temp_directory}/files1" -extra_dirs: - - type: temp - path: "${temp_directory}/tmp1" - - type: job_work - path: "${temp_directory}/job_working_directory1" -""" - - def test_disk_store(): for config_str in [DISK_TEST_CONFIG, DISK_TEST_CONFIG_YAML]: with TestConfig(config_str) as (directory, object_store): @@ -764,53 +741,6 @@ def test_config_parse_azure(): assert len(extra_dirs) == 2 -class TestConfig(object): - def __init__(self, config_str=DISK_TEST_CONFIG, clazz=None, store_by="id"): - self.temp_directory = mkdtemp() - if config_str.startswith("<"): - config_file = "store.xml" - else: - config_file = "store.yaml" - self.write(config_str, config_file) - config = MockConfig(self.temp_directory, config_file, store_by=store_by) - if clazz is None: - self.object_store = objectstore.build_object_store_from_config(config) - elif config_file == "store.xml": - self.object_store = clazz.from_xml(config, XML(config_str)) - else: - self.object_store = clazz(config, yaml.safe_load(StringIO(config_str))) - - def __enter__(self): - return self, self.object_store - - def __exit__(self, type, value, tb): - rmtree(self.temp_directory) - - def write(self, contents, name): - path = os.path.join(self.temp_directory, name) - directory = os.path.dirname(path) - if not os.path.exists(directory): - os.makedirs(directory) - contents_template = Template(contents) - expanded_contents = contents_template.safe_substitute(temp_directory=self.temp_directory) - open(path, "w").write(expanded_contents) - return path - - -class MockConfig(object): - - def __init__(self, temp_directory, config_file, store_by="id"): - self.file_path = temp_directory - self.object_store_config_file = os.path.join(temp_directory, config_file) - self.object_store_check_old_style = False - self.object_store_cache_path = os.path.join(temp_directory, "staging") - self.object_store_store_by = store_by - self.jobs_directory = temp_directory - self.new_file_path = temp_directory - self.umask = 0000 - self.gid = 1000 - - class MockDataset(object): def __init__(self, id): diff --git a/test/unit/test_markdown_validate.py b/test/unit/test_markdown_validate.py index a9bf4671474..2b7a6f2acf3 100644 --- a/test/unit/test_markdown_validate.py +++ b/test/unit/test_markdown_validate.py @@ -86,12 +86,12 @@ job_metrics(THISFAKEID) # assert quotes are fine assert_markdown_valid(""" ```galaxy -job_metrics(output="Moo Cow") +job_metrics(step="Moo Cow") ``` """) assert_markdown_valid(""" ```galaxy -job_metrics(output='Moo Cow') +job_metrics(step='Moo Cow') ``` """) # assert spaces require quotes @@ -121,3 +121,94 @@ job_metrics(output='Moo Cow) job_metrics(output=Moo Cow') ``` """) + + assert_markdown_valid(""" +```galaxy +workflow_display() +``` +""") + + # Test image with a composite path (param needs to be closed, can't be misnamed i.e. pathx) + assert_markdown_valid(""" + +```galaxy +history_dataset_as_image(output="cow", path="foo/bar.png") +``` +""") + assert_markdown_valid(""" + +```galaxy +history_dataset_as_image(output=cow, path="foo/bar.png") +``` +""") + assert_markdown_invalid(""" + +```galaxy +history_dataset_as_image(output="cow", path="foo/bar.png) +``` +""", at_line=3) + assert_markdown_invalid(""" + +```galaxy +history_dataset_as_image(output="cow", pathx="foo/bar.png") +``` +""", at_line=3) + + # Test validation of three arguments + assert_markdown_valid(""" +```galaxy +history_dataset_link(output=moo, path="cow.png", label="my label") +``` +""") + assert_markdown_invalid(""" +```galaxy +history_dataset_link(outputx=moo, path="cow.png", label="my label") +``` +""", at_line=2) + assert_markdown_invalid(""" +```galaxy +history_dataset_link(output=moo, pathx="cow.png", label="my label") +``` +""", at_line=2) + assert_markdown_invalid(""" +```galaxy +history_dataset_link(output=moo, path="cow.png", labelx="my label") +``` +""", at_line=2) + + # Test validation of arguments with different whitespaces + assert_markdown_valid(""" +```galaxy +history_dataset_link(output= moo, path= "cow.png", label= "my label") +``` +""") + assert_markdown_valid(""" +```galaxy +history_dataset_link(output = moo, path = "cow.png", label = "my label") +``` +""") + assert_markdown_valid(""" +```galaxy +history_dataset_link(output = moo, path ="cow.png", label= "my label" ) +``` +""") + assert_markdown_valid(""" +```galaxy +history_dataset_link( output = moo, path ="cow.png", label= "my label" ) +``` +""") + assert_markdown_invalid(""" +```galaxy +history_dataset_link( outputx = moo, path ="cow.png", label= "my label" ) +``` +""", at_line=2) + assert_markdown_invalid(""" +```galaxy +history_dataset_link( output = moo, pathx ="cow.png", label= "my label" ) +``` +""", at_line=2) + assert_markdown_invalid(""" +```galaxy +history_dataset_link( output = moo, path ="cow.png", labelx= "my label" ) +``` +""", at_line=2) diff --git a/test/unit/test_routes.py b/test/unit/test_routes.py index 5c197c89b40..a6645091bd0 100644 --- a/test/unit/test_routes.py +++ b/test/unit/test_routes.py @@ -1,6 +1,6 @@ from galaxy.util.bunch import Bunch from galaxy.web import url_for -from galaxy.web.framework.webapp import WebApplication +from galaxy.webapps.base.webapp import WebApplication from galaxy.webapps.galaxy import buildapp as galaxy_buildapp diff --git a/test/unit/tools/test_cwl_util.py b/test/unit/tool_util/test_cwl_util.py similarity index 100% rename from test/unit/tools/test_cwl_util.py rename to test/unit/tool_util/test_cwl_util.py diff --git a/test/unit/tool_util/test_tool_deps.py b/test/unit/tool_util/test_tool_deps.py index 3c595f31fbe..0da020a4eb0 100644 --- a/test/unit/tool_util/test_tool_deps.py +++ b/test/unit/tool_util/test_tool_deps.py @@ -752,5 +752,7 @@ def __dependency_manager_for_config(app_config, resolution_config=None): class _SimpleDependencyManager(object): - def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + def get_resolver_option(self, resolver, key, explicit_resolver_options=None): + if explicit_resolver_options is None: + explicit_resolver_options = {} return explicit_resolver_options.get(key) diff --git a/test/unit/tools/test_actions.py b/test/unit/tools/test_actions.py index db948079459..d8dfc9f83f7 100644 --- a/test/unit/tools/test_actions.py +++ b/test/unit/tools/test_actions.py @@ -134,12 +134,13 @@ class DefaultToolActionTestCase(unittest.TestCase, tools_support.UsesApp, tools_ if incoming is None: incoming = dict(param1="moo") self._init_tool(contents) - return self.action.execute( + job, out_data, _, _ = self.action.execute( tool=self.tool, trans=self.trans, history=self.history, incoming=incoming, ) + return job, out_data def test_determine_output_format(): diff --git a/test/unit/tools/test_execution.py b/test/unit/tools/test_execution.py index fc547af9375..83185a15510 100644 --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -211,6 +211,9 @@ class MockTrans(object): def get_current_user_roles(self): return [] + def log_event(self, *args, **kwds): + pass + class MockCollectionService(object): diff --git a/test/unit/tools/test_history_imp_exp.py b/test/unit/tools/test_history_imp_exp.py index 63eb9d71dc1..3fc279eff12 100644 --- a/test/unit/tools/test_history_imp_exp.py +++ b/test/unit/tools/test_history_imp_exp.py @@ -9,8 +9,8 @@ from galaxy import model from galaxy.exceptions import MalformedContents from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper, JobImportHistoryArchiveWrapper, unpack_tar_gz_archive from galaxy.tools.imp_exp.export_history import create_archive -from ..test_objectstore import TestConfig from ..unittest_utils.galaxy_mock import MockApp +from ..unittest_utils.objectstore_helpers import TestConfig # good enough for the very specific tests we're writing as of now... diff --git a/test/unit/unittest_utils/objectstore_helpers.py b/test/unit/unittest_utils/objectstore_helpers.py new file mode 100644 index 00000000000..fd17c54fa8a --- /dev/null +++ b/test/unit/unittest_utils/objectstore_helpers.py @@ -0,0 +1,76 @@ +import os +from shutil import rmtree +from string import Template +from tempfile import mkdtemp + +import yaml +from six import StringIO + +from galaxy import objectstore +from galaxy.util import XML + +DISK_TEST_CONFIG = """ + + + + + +""" + + +DISK_TEST_CONFIG_YAML = """ +type: disk +files_dir: "${temp_directory}/files1" +extra_dirs: + - type: temp + path: "${temp_directory}/tmp1" + - type: job_work + path: "${temp_directory}/job_working_directory1" +""" + + +class TestConfig(object): + def __init__(self, config_str=DISK_TEST_CONFIG, clazz=None, store_by="id"): + self.temp_directory = mkdtemp() + if config_str.startswith("<"): + config_file = "store.xml" + else: + config_file = "store.yaml" + self.write(config_str, config_file) + config = MockConfig(self.temp_directory, config_file, store_by=store_by) + if clazz is None: + self.object_store = objectstore.build_object_store_from_config(config) + elif config_file == "store.xml": + self.object_store = clazz.from_xml(config, XML(config_str)) + else: + self.object_store = clazz(config, yaml.safe_load(StringIO(config_str))) + + def __enter__(self): + return self, self.object_store + + def __exit__(self, type, value, tb): + rmtree(self.temp_directory) + + def write(self, contents, name): + path = os.path.join(self.temp_directory, name) + directory = os.path.dirname(path) + if not os.path.exists(directory): + os.makedirs(directory) + contents_template = Template(contents) + expanded_contents = contents_template.safe_substitute(temp_directory=self.temp_directory) + open(path, "w").write(expanded_contents) + return path + + +class MockConfig(object): + + def __init__(self, temp_directory, config_file, store_by="id"): + self.file_path = temp_directory + self.object_store_config_file = os.path.join(temp_directory, config_file) + self.object_store_check_old_style = False + self.object_store_cache_path = os.path.join(temp_directory, "staging") + self.object_store_store_by = store_by + self.jobs_directory = temp_directory + self.new_file_path = temp_directory + self.umask = 0000 + self.gid = 1000 diff --git a/test/unit/unittest_utils/tempfilecache.py b/test/unit/unittest_utils/tempfilecache.py deleted file mode 100644 index a828304a8f9..00000000000 --- a/test/unit/unittest_utils/tempfilecache.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging -import os -import tempfile - -logging.getLogger(__name__) -log = logging - - -class TempFileCache(object): - """ - Creates and caches tempfiles with/based-on the given contents. - """ - - def __init__(self, logger=None): - if logger: - global log - log = logger - super(TempFileCache, self).__init__() - self.clear() - - def clear(self): - self.delete_tmpfiles() - self._content_dict = {} - - def create_tmpfile(self, contents): - if not hasattr(self, '_content_dict'): - self.set_up_tmpfiles() - - if contents not in self._content_dict: - # create a named tmp and write contents to it, return filename - with tempfile.NamedTemporaryFile(delete=False, mode='w+') as tmpfile: - tmpfile.write(contents) - tmpfile_name = tmpfile.name - log.debug('created tmpfile.name: %s', tmpfile_name) - self._content_dict[contents] = tmpfile_name - - else: - log.debug('(cached): %s', self._content_dict[contents]) - return self._content_dict[contents] - - def delete_tmpfiles(self): - if not hasattr(self, '_content_dict') or not self._content_dict: - return - for tmpfile_contents in self._content_dict: - tmpfile = self._content_dict[tmpfile_contents] - if os.path.exists(tmpfile): - log.debug('unlinking tmpfile: %s', tmpfile) - os.unlink(tmpfile) diff --git a/test/unit/unittest_utils/utility.py b/test/unit/unittest_utils/utility.py deleted file mode 100644 index 411ff7367f9..00000000000 --- a/test/unit/unittest_utils/utility.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Unit test utilities. -""" -import textwrap - - -def clean_multiline_string(multiline_string, sep='\n'): - """ - Dedent, split, remove first and last empty lines, rejoin. - """ - multiline_string = textwrap.dedent(multiline_string) - string_list = multiline_string.split(sep) - if not string_list[0]: - string_list = string_list[1:] - if not string_list[-1]: - string_list = string_list[:-1] - # return '\n'.join( docstrings ) - return ''.join((s + '\n') for s in string_list) - - -__all__ = ( - "clean_multiline_string", -) diff --git a/test/unit/util/test_checkers.py b/test/unit/util/test_checkers.py new file mode 100644 index 00000000000..5a01b45dcb9 --- /dev/null +++ b/test/unit/util/test_checkers.py @@ -0,0 +1,19 @@ +import tempfile + +from galaxy.util.checkers import check_html + + +def test_check_html(): + html_text = '

\nLink\n

\n' + assert check_html(html_text, file_path=False) + # Test a non-HTML binary string + assert not check_html(b'No HTML here\nSecond line\n', file_path=False) + with tempfile.NamedTemporaryFile(mode='w') as tmp: + tmp.write(html_text) + tmp.flush() + assert check_html(tmp.name) + # Test a non-UTF8 binary file + with tempfile.NamedTemporaryFile(mode='wb') as tmp: + tmp.write(b'\x1f\x8b') + tmp.flush() + assert not check_html(tmp.name) diff --git a/test/unit/test_compression_util.py b/test/unit/util/test_compression_util.py similarity index 100% rename from test/unit/test_compression_util.py rename to test/unit/util/test_compression_util.py diff --git a/test/unit/tools/test_fill_template.py b/test/unit/util/test_fill_template.py similarity index 100% rename from test/unit/tools/test_fill_template.py rename to test/unit/util/test_fill_template.py diff --git a/test/unit/test_fix_type_error.py b/test/unit/util/test_fix_type_error.py similarity index 100% rename from test/unit/test_fix_type_error.py rename to test/unit/util/test_fix_type_error.py diff --git a/test/unit/test_lazy_process.py b/test/unit/util/test_lazy_process.py similarity index 100% rename from test/unit/test_lazy_process.py rename to test/unit/util/test_lazy_process.py diff --git a/test/unit/test_sockets.py b/test/unit/util/test_sockets.py similarity index 100% rename from test/unit/test_sockets.py rename to test/unit/util/test_sockets.py diff --git a/test/unit/test_sqlite_utils.py b/test/unit/util/test_sqlite_utils.py similarity index 100% rename from test/unit/test_sqlite_utils.py rename to test/unit/util/test_sqlite_utils.py diff --git a/test/unit/test_topsort.py b/test/unit/util/test_topsort.py similarity index 100% rename from test/unit/test_topsort.py rename to test/unit/util/test_topsort.py diff --git a/test/unit/util/test_utils.py b/test/unit/util/test_utils.py index 130c19401d2..6248817f0c7 100644 --- a/test/unit/util/test_utils.py +++ b/test/unit/util/test_utils.py @@ -82,3 +82,12 @@ def test_parse_xml_enoent(): with pytest.raises(IOError) as excinfo: util.parse_xml(path) assert excinfo.value.errno == errno.ENOENT + + +def test_clean_multiline_string(): + x = util.clean_multiline_string(""" + a + b + c +""") + assert x == "a\nb\nc\n" diff --git a/test/unit/visualizations/plugins/test_VisualizationPlugin.py b/test/unit/visualizations/plugins/test_VisualizationPlugin.py index dcf7abfc614..ce3f5b74c0b 100644 --- a/test/unit/visualizations/plugins/test_VisualizationPlugin.py +++ b/test/unit/visualizations/plugins/test_VisualizationPlugin.py @@ -5,12 +5,13 @@ import unittest from six import string_types +from galaxy.util import clean_multiline_string from galaxy.visualization.plugins import ( plugin as vis_plugin, resource_parser, utils as vis_utils ) -from ...unittest_utils import galaxy_mock, utility +from ...unittest_utils import galaxy_mock class VisualizationsPlugin_TestCase(unittest.TestCase): @@ -142,7 +143,7 @@ class VisualizationsPlugin_TestCase(unittest.TestCase): """ # use the python in a template to test for variables that should be there # TODO: gotta be a better way - testing_template = utility.clean_multiline_string("""\ + testing_template = clean_multiline_string("""\ <% found_all = True should_have = [ diff --git a/test/unit/visualizations/plugins/test_VisualizationsRegistry.py b/test/unit/visualizations/plugins/test_VisualizationsRegistry.py index e212d3dd2b0..4baed60b6bb 100644 --- a/test/unit/visualizations/plugins/test_VisualizationsRegistry.py +++ b/test/unit/visualizations/plugins/test_VisualizationsRegistry.py @@ -8,9 +8,10 @@ import unittest from six import text_type from galaxy import model +from galaxy.util import clean_multiline_string from galaxy.visualization.plugins import plugin from galaxy.visualization.plugins.registry import VisualizationsRegistry -from ...unittest_utils import galaxy_mock, utility +from ...unittest_utils import galaxy_mock glx_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir)) template_cache_dir = os.path.join(glx_dir, 'database', 'compiled_templates') @@ -132,7 +133,7 @@ class VisualizationsRegistry_TestCase(unittest.TestCase): def test_interactive_environ_plugin_load(self): """ """ - jupyter_config = utility.clean_multiline_string("""\ + jupyter_config = clean_multiline_string("""\ @@ -212,7 +213,7 @@ class VisualizationsRegistry_TestCase(unittest.TestCase): def test_script_entry(self): """""" - script_entry_config = utility.clean_multiline_string("""\ + script_entry_config = clean_multiline_string("""\ diff --git a/test/unit/web_framework/__init__.py b/test/unit/web_framework/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/web_framework/framework/test_framework_base.py b/test/unit/web_framework/framework/test_framework_base.py new file mode 100644 index 00000000000..bba5aa540d2 --- /dev/null +++ b/test/unit/web_framework/framework/test_framework_base.py @@ -0,0 +1,18 @@ +from galaxy.web.framework.base import WebApplication + + +class TestWebapp(WebApplication): + + def assert_maps(self, url, method="GET", **parts): + map_result = self.mapper.match(url, environ={"REQUEST_METHOD": method}) + for key, expected_value in parts.items(): + actual_value = map_result[key] + if actual_value != expected_value: + message = "Problem mapping route [%s], part %s expected value [%s] but obtained [%s]" + raise AssertionError(message % (url, key, expected_value, actual_value)) + + +def test_add_route(): + test_webapp = TestWebapp() + test_webapp.add_route('/authnz/', controller='authnz', action='index', provider=None) + test_webapp.assert_maps("/authnz/", controller="authnz", action="index") diff --git a/test/unit/web_stack/__init__.py b/test/unit/web_stack/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/web_stack/test_message.py b/test/unit/web_stack/test_message.py new file mode 100644 index 00000000000..56c8d9d7121 --- /dev/null +++ b/test/unit/web_stack/test_message.py @@ -0,0 +1,30 @@ +from galaxy.web_stack.message import ( + JobHandlerMessage, + WorkflowSchedulingMessage, +) + + +def test_validate_job_handler_message(): + message = JobHandlerMessage(task='setup', job_id=42) + message.validate() + + message = JobHandlerMessage(task='setup') + error = None + try: + message.validate() + except AssertionError as e: + error = e + assert error is not None + + +def test_validate_workflow_handler_message(): + message = WorkflowSchedulingMessage(task='setup', workflow_invocation_id=42) + message.validate() + + message = WorkflowSchedulingMessage(task='setup') + error = None + try: + message.validate() + except AssertionError as e: + error = e + assert error is not None diff --git a/test/unit/webapps/__init__.py b/test/unit/webapps/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/web/framework/test_webapp.py b/test/unit/webapps/test_webapp_base.py similarity index 98% rename from test/unit/web/framework/test_webapp.py rename to test/unit/webapps/test_webapp_base.py index 15cd442b517..e45204e0b02 100644 --- a/test/unit/web/framework/test_webapp.py +++ b/test/unit/webapps/test_webapp_base.py @@ -6,8 +6,8 @@ import re import unittest import galaxy.config -from galaxy.web.framework import webapp as Webapp -from ...unittest_utils import galaxy_mock +from galaxy.webapps.base import webapp as Webapp +from ..unittest_utils import galaxy_mock log = logging.getLogger(__name__)