Render useful Markdown components for mapped over steps.

API functionality to fetch and query ImplicitCollectionJobs' jobs.

client/src/api/schema/schema.ts

@@ -5495,6 +5495,11 @@ export interface components {
              * @example 0123456789ABCDEF
              */
             id: string;
+            /**
+             * Implicit Collection Jobs Id
+             * @description Encoded ID for the ICJ object describing the collection of jobs corresponding to this collection
+             */
+            implicit_collection_jobs_id?: string | null;
             /**
              * Job Source ID
              * @description The encoded ID of the Job that produced this dataset collection. Used to track the state of the job.
@@ -16846,6 +16851,7 @@ export interface operations {
             /** @description Limit listing of jobs to those that match the history_id. If none, jobs from any history may be returned. */
             /** @description Limit listing of jobs to those that match the specified workflow ID. If none, jobs from any workflow (or from no workflows) may be returned. */
             /** @description Limit listing of jobs to those that match the specified workflow invocation ID. If none, jobs from any workflow invocation (or from no workflows) may be returned. */
+            /** @description Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned. */
             /** @description Sort results by specified field. */
             /**
              * @description A mix of free text and GitHub-style tags used to filter the index operation.
@@ -16897,6 +16903,7 @@ export interface operations {
                 history_id?: string | null;
                 workflow_id?: string | null;
                 invocation_id?: string | null;
+                implicit_collection_jobs_id?: string | null;
                 order_by?: components["schemas"]["JobIndexSortByEnum"];
                 search?: string | null;
                 limit?: number;

client/src/components/JobMetrics/JobMetrics.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { computed, ref, unref } from "vue";
+import { computed, ref, unref, watch } from "vue";
 
 import { useJobMetricsStore } from "@/stores/jobMetricsStore";
 
@@ -60,7 +60,14 @@ async function getJobMetrics() {
         await jobMetricsStore.fetchJobMetricsForDatasetId(props.datasetId, props.datasetType);
     }
 }
-getJobMetrics();
+
+watch(
+    props,
+    () => {
+        getJobMetrics();
+    },
+    { immediate: true }
+);
 
 const ec2Instances = ref<EC2[]>();
 import("./awsEc2ReferenceData.js").then((data) => (ec2Instances.value = data.ec2Instances));

client/src/components/JobParameters/JobParameters.vue

@@ -108,17 +108,25 @@ export default {
             return parameter ? parameter.value : `Parameter "${this.param}" is not found!`;
         },
     },
+    watch: {
+        jobId: function (newValue) {
+            this.initJob();
+        },
+    },
     created: function () {
-        let url;
-        if (this.jobId) {
-            url = `${getAppRoot()}api/jobs/${this.jobId}/parameters_display`;
-        } else {
-            url = `${getAppRoot()}api/datasets/${this.datasetId}/parameters_display?hda_ldda=${this.datasetType}`;
-        }
-        this.ajaxCall(url);
-        this.isSingleParam = this.param !== undefined && this.param !== "undefined";
+        this.initJob();
     },
     methods: {
+        initJob() {
+            let url;
+            if (this.jobId) {
+                url = `${getAppRoot()}api/jobs/${this.jobId}/parameters_display`;
+            } else {
+                url = `${getAppRoot()}api/datasets/${this.datasetId}/parameters_display?hda_ldda=${this.datasetType}`;
+            }
+            this.ajaxCall(url);
+            this.isSingleParam = this.param !== undefined && this.param !== "undefined";
+        },
         appRoot: function () {
             return getAppRoot();
         },

client/src/components/Markdown/Elements/JobMetrics.vue

@@ -1,18 +1,24 @@
 <script setup lang="ts">
-import { computed, watch } from "vue";
+import { computed, toRef, watch } from "vue";
 
 import { useJobStore } from "@/stores/jobStore";
 
+import { useMappingJobs } from "./handlesMappingJobs";
+
+import JobSelection from "./JobSelection.vue";
 import JobMetrics from "@/components/JobMetrics/JobMetrics.vue";
 import ToolLinkPopover from "@/components/Tool/ToolLinkPopover.vue";
 
 interface JobMetricsProps {
-    jobId: string;
+    jobId?: string;
+    implicitCollectionJobsId?: string;
     title?: string;
     footer?: string;
 }
 
 const props = withDefaults(defineProps<JobMetricsProps>(), {
+    jobId: undefined,
+    implicitCollectionJobsId: undefined,
     title: undefined,
     footer: undefined,
 });
@@ -20,16 +26,32 @@ const props = withDefaults(defineProps<JobMetricsProps>(), {
 const jobStore = useJobStore();
 
 const toolId = computed(() => {
-    return jobStore.getJob(props.jobId)?.tool_id;
+    if (targetJobId.value) {
+        return jobStore.getJob(targetJobId.value)?.tool_id;
+    }
+    return undefined;
 });
 const toolVersion = computed(() => {
-    return jobStore.getJob(props.jobId)?.tool_version;
+    if (targetJobId.value) {
+        return jobStore.getJob(targetJobId.value)?.tool_version;
+    }
+    return undefined;
 });
+const jobIdRef = toRef(props, "jobId");
+const implicitCollectionJobsIdRef = toRef(props, "implicitCollectionJobsId");
+
+const { selectJobOptions, selectedJob, targetJobId } = useMappingJobs(jobIdRef, implicitCollectionJobsIdRef);
+
+async function init() {
+    if (targetJobId.value) {
+        jobStore.fetchJob(targetJobId.value);
+    }
+}
 
 watch(
-    props,
+    targetJobId,
     () => {
-        jobStore.fetchJob(props.jobId);
+        init();
     },
     { immediate: true }
 );
@@ -42,12 +64,19 @@ watch(
             <icon ref="info" icon="info-circle" size="sm" />
             <ToolLinkPopover :target="() => $refs.info" :tool-id="toolId" :tool-version="toolVersion" />
         </b-card-title>
-        <JobMetrics
-            class="job-metrics"
+        <JobSelection
+            v-model="selectedJob"
             :job-id="jobId"
-            :should-show-aws-estimate="false"
-            :should-show-carbon-emissions-estimates="false"
-            :include-title="false" />
+            :implicit-collection-jobs-id="implicitCollectionJobsId"
+            :select-job-options="selectJobOptions">
+            <JobMetrics
+                v-if="targetJobId"
+                class="job-metrics"
+                :job-id="targetJobId"
+                :should-show-aws-estimate="false"
+                :should-show-carbon-emissions-estimates="false"
+                :include-title="false" />
+        </JobSelection>
         <b-card-footer v-if="footer">
             {{ footer }}
         </b-card-footer>

client/src/components/Markdown/Elements/JobParameters.vue

@@ -1,19 +1,25 @@
 <script setup lang="ts">
-import { computed, watch } from "vue";
+import { computed, toRef, watch } from "vue";
 
 import { useJobStore } from "@/stores/jobStore";
 
+import { useMappingJobs } from "./handlesMappingJobs";
+
+import JobSelection from "./JobSelection.vue";
 import JobParameters from "@/components/JobParameters/JobParameters.vue";
 import ToolLinkPopover from "@/components/Tool/ToolLinkPopover.vue";
 
 interface JobParametersProps {
-    jobId: string;
+    jobId?: string;
+    implicitCollectionJobsId?: string;
     param?: string;
     title?: string;
     footer?: string;
 }
 
 const props = withDefaults(defineProps<JobParametersProps>(), {
+    jobId: undefined,
+    implicitCollectionJobsId: undefined,
     param: undefined,
     title: undefined,
     footer: undefined,
@@ -22,16 +28,32 @@ const props = withDefaults(defineProps<JobParametersProps>(), {
 const jobStore = useJobStore();
 
 const toolId = computed(() => {
-    return jobStore.getJob(props.jobId)?.tool_id;
+    if (targetJobId.value) {
+        return jobStore.getJob(targetJobId.value)?.tool_id;
+    }
+    return undefined;
 });
 const toolVersion = computed(() => {
-    return jobStore.getJob(props.jobId)?.tool_version;
+    if (targetJobId.value) {
+        return jobStore.getJob(targetJobId.value)?.tool_version;
+    }
+    return undefined;
 });
+const jobIdRef = toRef(props, "jobId");
+const implicitCollectionJobsIdRef = toRef(props, "implicitCollectionJobsId");
+
+const { selectJobOptions, selectedJob, targetJobId } = useMappingJobs(jobIdRef, implicitCollectionJobsIdRef);
+
+async function init() {
+    if (targetJobId.value) {
+        jobStore.fetchJob(targetJobId.value);
+    }
+}
 
 watch(
-    props,
+    targetJobId,
     () => {
-        jobStore.fetchJob(props.jobId);
+        init();
     },
     { immediate: true }
 );
@@ -44,7 +66,14 @@ watch(
             <icon ref="info" icon="info-circle" size="sm" />
             <ToolLinkPopover :target="() => $refs.info" :tool-id="toolId" :tool-version="toolVersion" />
         </b-card-title>
-        <JobParameters class="job-parameters" :job-id="jobId" :param="param" :include-title="false" />
+        <JobSelection
+            v-model="selectedJob"
+            :job-id="jobId"
+            :implicit-collection-jobs-id="implicitCollectionJobsId"
+            :select-job-options="selectJobOptions">
+            <JobParameters class="job-parameters" :job-id="targetJobId" :param="param" :include-title="false" />
+            {{ targetJobId }}
+        </JobSelection>
         <b-card-footer v-if="footer">
             {{ footer }}
         </b-card-footer>

client/src/components/Markdown/Elements/JobSelection.vue

@@ -0,0 +1,44 @@
+<script setup lang="ts">
+import { SelectOption } from "./handlesMappingJobs";
+
+interface JobSelectionProps {
+    jobId?: string;
+    implicitCollectionJobsId?: string;
+    selectJobOptions: SelectOption[];
+    value?: String | undefined;
+}
+
+const emit = defineEmits<{
+    (e: "input", id: string | undefined): void;
+}>();
+
+function handleInput(value: string | undefined) {
+    emit("input", value);
+}
+defineProps<JobSelectionProps>();
+</script>
+
+<template>
+    <div>
+        <slot v-if="jobId"> </slot>
+        <div v-else>
+            <b-navbar>
+                <b-collapse id="nav-text-collapse" is-nav>
+                    <b-navbar-nav>
+                        <b-nav-text>Select Job</b-nav-text>
+                    </b-navbar-nav>
+                    <b-nav-form>
+                        <b-input-group size="sm">
+                            <b-form-select
+                                :value="value"
+                                class="text-right"
+                                :options="selectJobOptions"
+                                @input="handleInput"></b-form-select>
+                        </b-input-group>
+                    </b-nav-form>
+                </b-collapse>
+            </b-navbar>
+            <slot />
+        </div>
+    </div>
+</template>

client/src/components/Markdown/Elements/handlesMappingJobs.ts

@@ -0,0 +1,51 @@
+import { format, parseISO } from "date-fns";
+import { computed, Ref, ref, watch } from "vue";
+
+import { fetcher } from "@/api/schema";
+
+const jobsFetcher = fetcher.path("/api/jobs").method("get").create();
+
+export interface SelectOption {
+    value: string;
+    text: string;
+}
+
+interface Job {
+    id: string;
+    create_time: string;
+}
+
+export function useMappingJobs(
+    singleJobId: Ref<string | undefined>,
+    implicitCollectionJobsId: Ref<string | undefined>
+) {
+    const selectJobOptions = ref<SelectOption[]>([]);
+    const selectedJob = ref<string | undefined>(undefined);
+    const targetJobId = computed<string | undefined>(() => {
+        if (singleJobId.value) {
+            return singleJobId.value;
+        } else {
+            return selectedJob.value;
+        }
+    });
+    watch(
+        implicitCollectionJobsId,
+        async () => {
+            if (implicitCollectionJobsId.value) {
+                const response = await jobsFetcher({ implicit_collection_jobs_id: implicitCollectionJobsId.value });
+                const jobs: Job[] = response.data as unknown as Job[];
+                selectJobOptions.value = jobs.map((value, index) => {
+                    const isoCreateTime = parseISO(`${value.create_time}Z`);
+                    const prettyTime = format(isoCreateTime, "eeee MMM do H:mm:ss yyyy zz");
+                    return { value: value.id, text: `${index + 1}: ${prettyTime}` };
+                });
+                if (jobs[0]) {
+                    const job: Job = jobs[0];
+                    selectedJob.value = job.id;
+                }
+            }
+        },
+        { immediate: true }
+    );
+    return { selectJobOptions, selectedJob, targetJobId };
+}

client/src/components/Markdown/MarkdownContainer.vue

@@ -151,11 +151,13 @@ function argToBoolean(args, name, booleanDefault) {
             <JobMetrics
                 v-else-if="name == 'job_metrics'"
                 :job-id="args.job_id"
+                :implicit-collection-jobs-id="args.implicit_collection_jobs_id"
                 :title="args.title"
                 :footer="args.footer" />
             <JobParameters
                 v-else-if="name == 'job_parameters'"
                 :job-id="args.job_id"
+                :implicit-collection-jobs-id="args.implicit_collection_jobs_id"
                 :param="args.param || undefined"
                 :title="args.title"
                 :footer="args.footer" />

lib/galaxy/managers/collections_util.py

@@ -144,6 +144,11 @@ def dictify_dataset_collection_instance(
         else:
             element_func = dictify_element_reference
         dict_value["elements"] = [element_func(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements]
+        icj = dataset_collection_instance.implicit_collection_jobs
+        if icj:
+            dict_value["implicit_collection_jobs_id"] = icj.id
+        else:
+            dict_value["implicit_collection_jobs_id"] = None
 
     return dict_value
 

lib/galaxy/managers/jobs.py

@@ -41,6 +41,7 @@ from galaxy.managers.datasets import DatasetManager
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.lddas import LDDAManager
 from galaxy.model import (
+    ImplicitCollectionJobs,
     ImplicitCollectionJobsJobAssociation,
     Job,
     JobParameter,
@@ -105,12 +106,18 @@ class JobManager:
         self.dataset_manager = DatasetManager(app)
 
     def index_query(self, trans, payload: JobIndexQueryPayload) -> sqlalchemy.engine.Result:
+        """The caller is responsible for security checks on the resulting job if
+        history_id, invocation_id, or implicit_collection_jobs_id is set.
+        Otherwise this will only return the user's jobs or all jobs if the requesting
+        user is acting as an admin.
+        """
         is_admin = trans.user_is_admin
         user_details = payload.user_details
         decoded_user_id = payload.user_id
         history_id = payload.history_id
         workflow_id = payload.workflow_id
         invocation_id = payload.invocation_id
+        implicit_collection_jobs_id = payload.implicit_collection_jobs_id
         search = payload.search
         order_by = payload.order_by
 
@@ -200,7 +207,9 @@ class JobManager:
             if user_details:
                 stmt = stmt.outerjoin(Job.user)
         else:
-            stmt = stmt.where(Job.user_id == trans.user.id)
+            if history_id is None and invocation_id is None and implicit_collection_jobs_id is None:
+                stmt = stmt.where(Job.user_id == trans.user.id)
+            # caller better check security
 
         stmt = build_and_apply_filters(stmt, payload.states, lambda s: model.Job.state == s)
         stmt = build_and_apply_filters(stmt, payload.tool_ids, lambda t: model.Job.tool_id == t)
@@ -214,7 +223,15 @@ class JobManager:
         order_by_columns = Job
         if workflow_id or invocation_id:
             stmt, order_by_columns = add_workflow_jobs()
-
+        elif implicit_collection_jobs_id:
+            stmt = (
+                stmt.join(ImplicitCollectionJobsJobAssociation, ImplicitCollectionJobsJobAssociation.job_id == Job.id)
+                .join(
+                    ImplicitCollectionJobs,
+                    ImplicitCollectionJobs.id == ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id,
+                )
+                .where(ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id == implicit_collection_jobs_id)
+            )
         if search:
             stmt = add_search_criteria(stmt)
 

lib/galaxy/managers/markdown_parse.py

@@ -50,10 +50,10 @@ VALID_ARGUMENTS: Dict[str, Union[List[str], DynamicArguments]] = {
     "workflow_display": ["workflow_id", "workflow_checkpoint"],
     "workflow_license": ["workflow_id"],
     "workflow_image": ["workflow_id", "size", "workflow_checkpoint"],
-    "job_metrics": ["step", "job_id"],
-    "job_parameters": ["step", "job_id"],
-    "tool_stderr": ["step", "job_id"],
-    "tool_stdout": ["step", "job_id"],
+    "job_metrics": ["step", "job_id", "implicit_collection_jobs_id"],
+    "job_parameters": ["step", "job_id", "implicit_collection_jobs_id"],
+    "tool_stderr": ["step", "job_id", "implicit_collection_jobs_id"],
+    "tool_stdout": ["step", "job_id", "implicit_collection_jobs_id"],
     "generate_galaxy_version": [],
     "generate_time": [],
     "instance_access_link": [],

lib/galaxy/managers/markdown_util.py

@@ -73,10 +73,10 @@ PATH_LABEL_PATTERN = re.compile(r"path=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX)
 SIZE_PATTERN = re.compile(r"size=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX)
 # STEP_OUTPUT_LABEL_PATTERN = re.compile(r'step_output=([\w_\-]+)/([\w_\-]+)')
 UNENCODED_ID_PATTERN = re.compile(
-    r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([\d]+)"
+    r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|implicit_collection_jobs_id|invocation_id)=([\d]+)"
 )
 ENCODED_ID_PATTERN = re.compile(
-    r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([a-z0-9]+)"
+    r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|implicit_collection_jobs_id|invocation_id)=([a-z0-9]+)"
 )
 INVOCATION_SECTION_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile(r"```\s*galaxy\s*")
 GALAXY_FENCED_BLOCK = re.compile(r"^```\s*galaxy\s*(.*?)^```", re.MULTILINE ^ re.DOTALL)
@@ -930,9 +930,13 @@ history_dataset_collection_display(input={})
         elif step_match:
             target_match = step_match
             name = find_non_empty_group(target_match)
-            ref_object_type = "job"
             invocation_step = invocation.step_invocation_for_label(name)
-            ref_object = invocation_step and invocation_step.job
+            if invocation_step and invocation_step.job:
+                ref_object_type = "job"
+                ref_object = invocation_step.job
+            elif invocation_step and invocation_step.implicit_collection_jobs:
+                ref_object_type = "implicit_collection_jobs"
+                ref_object = invocation_step.implicit_collection_jobs
         else:
             target_match = None
             ref_object = None

lib/galaxy/schema/schema.py

@@ -1057,6 +1057,10 @@ class HDCADetailed(HDCASummary):
     elements_datatypes: Set[str] = Field(
         ..., description="A set containing all the different element datatypes in the collection."
     )
+    implicit_collection_jobs_id: Optional[EncodedDatabaseIdField] = Field(
+        None,
+        description="Encoded ID for the ICJ object describing the collection of jobs corresponding to this collection",
+    )
 
 
 class HistoryBase(Model):
@@ -1411,6 +1415,7 @@ class JobIndexQueryPayload(Model):
     history_id: Optional[DecodedDatabaseIdField] = None
     workflow_id: Optional[DecodedDatabaseIdField] = None
     invocation_id: Optional[DecodedDatabaseIdField] = None
+    implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = None
     order_by: JobIndexSortByEnum = JobIndexSortByEnum.update_time
     search: Optional[str] = None
     limit: int = 500

lib/galaxy/webapps/galaxy/api/jobs.py

@@ -149,6 +149,12 @@ InvocationIdQueryParam: Optional[DecodedDatabaseIdField] = Query(
     description="Limit listing of jobs to those that match the specified workflow invocation ID. If none, jobs from any workflow invocation (or from no workflows) may be returned.",
 )
 
+ImplicitCollectionJobsIdQueryParam: Optional[DecodedDatabaseIdField] = Query(
+    default=None,
+    title="Implicit Collection Jobs ID",
+    description="Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned.",
+)
+
 SortByQueryParam: JobIndexSortByEnum = Query(
     default=JobIndexSortByEnum.update_time,
     title="Sort By",
@@ -215,6 +221,7 @@ class FastAPIJobs:
         history_id: Optional[DecodedDatabaseIdField] = HistoryIdQueryParam,
         workflow_id: Optional[DecodedDatabaseIdField] = WorkflowIdQueryParam,
         invocation_id: Optional[DecodedDatabaseIdField] = InvocationIdQueryParam,
+        implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = ImplicitCollectionJobsIdQueryParam,
         order_by: JobIndexSortByEnum = SortByQueryParam,
         search: Optional[str] = SearchQueryParam,
         limit: int = LimitQueryParam,
@@ -232,6 +239,7 @@ class FastAPIJobs:
             history_id=history_id,
             workflow_id=workflow_id,
             invocation_id=invocation_id,
+            implicit_collection_jobs_id=implicit_collection_jobs_id,
             order_by=order_by,
             search=search,
             limit=limit,

lib/galaxy/webapps/galaxy/services/jobs.py

@@ -11,6 +11,7 @@ from galaxy import (
     model,
 )
 from galaxy.managers import hdas
+from galaxy.managers.base import security_check
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.managers.jobs import (
     JobManager,
@@ -71,13 +72,20 @@ class JobsService(ServiceBase):
             payload.user_details = True
         user_details = payload.user_details
         decoded_user_id = payload.user_id
-
         if not is_admin:
             self._check_nonadmin_access(view, user_details, decoded_user_id, trans.user and trans.user.id)
 
+        check_security_of_jobs = (
+            payload.invocation_id is not None
+            or payload.implicit_collection_jobs_id is not None
+            or payload.history_id is not None
+        )
         jobs = self.job_manager.index_query(trans, payload)
         out = []
         for job in jobs.yield_per(model.YIELD_PER_ROWS):
+            # TODO: optimize if this crucial
+            if check_security_of_jobs and not security_check(trans, job.history, check_accessible=True):
+                raise exceptions.ItemAccessibilityException("Cannot access the request job objects.")
             job_dict = job.to_dict(view, system_details=is_admin)
             if view == JobIndexViewEnum.admin_job_list:
                 job_dict["decoded_job_id"] = job.id

lib/galaxy_test/api/test_jobs.py

@@ -741,6 +741,22 @@ steps:
         search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs)
         self._search(search_payload, expected_search_count=0)
 
+    def test_implicit_collection_jobs(self, history_id):
+        run_response = self._run_map_over_error(history_id)
+        implicit_collection_id = run_response["implicit_collections"][0]["id"]
+        failed_hdca = self.dataset_populator.get_history_collection_details(
+            history_id=history_id,
+            content_id=implicit_collection_id,
+            assert_ok=False,
+        )
+        job_id = run_response["jobs"][0]["id"]
+        icj_id = failed_hdca["implicit_collection_jobs_id"]
+        assert icj_id
+        index = self.__jobs_index(data=dict(implicit_collection_jobs_id=icj_id))
+        assert len(index) == 1
+        assert index[0]["id"] == job_id
+        assert index[0]["state"] == "error", index
+
     @pytest.mark.require_new_history
     def test_search_with_hdca_list_input(self, history_id):
         list_id_a = self.__history_with_ok_collection(collection_type="list", history_id=history_id)