Merge pull request #20288 from jmchilton/fetch_workbooks

Empower Users to More Pragmatically Import Datasets & Collections From Tables

client/src/api/schema/schema.ts

@@ -4626,6 +4626,40 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/api/tools/fetch/workbook": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** Generate a template workbook to use with the activity builder UI */
+        get: operations["tools__fetch_workbook_download"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/tools/fetch/workbook/parse": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** Generate a template workbook to use with the activity builder UI */
+        post: operations["tools__fetch_workbook_parse"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/api/tools/{tool_id}/icon": {
         parameters: {
             query?: never;
@@ -10780,6 +10814,24 @@ export interface components {
              */
             sorting: boolean;
         };
+        /** FillIdentifiers */
+        FillIdentifiers: {
+            /**
+             * Deduplication Index From
+             * @default 1
+             */
+            deduplication_index_from: number;
+            /**
+             * Deduplication Pattern
+             * @default _{#}
+             */
+            deduplication_pattern: string;
+            /**
+             * Fill Inner List Identifiers
+             * @default false
+             */
+            fill_inner_list_identifiers: boolean;
+        };
         /** FillStepDefaultsAction */
         FillStepDefaultsAction: {
             /**
@@ -16885,6 +16937,89 @@ export interface components {
          * @default []
          */
         PageSummaryList: components["schemas"]["PageSummary"][];
+        /** ParseFetchWorkbook */
+        ParseFetchWorkbook: {
+            /**
+             * Workbook Content (Base 64 encoded)
+             * @description The workbook content (the contents of the xlsx file) that have been base64 encoded.
+             */
+            content: string;
+            fill_identifiers?: components["schemas"]["FillIdentifiers"] | null;
+        };
+        /** ParseLogEntry */
+        ParseLogEntry: {
+            /** Message */
+            message: string;
+        };
+        /** ParsedColumn */
+        ParsedColumn: {
+            /** Title */
+            title: string;
+            /**
+             * Type
+             * @enum {string}
+             */
+            type:
+                | "list_identifiers"
+                | "paired_identifier"
+                | "paired_or_unpaired_identifier"
+                | "collection_name"
+                | "name_tag"
+                | "tags"
+                | "group_tags"
+                | "name"
+                | "dbkey"
+                | "hash_sha1"
+                | "hash_md5"
+                | "hash_sha256"
+                | "hash_sha512"
+                | "file_type"
+                | "url"
+                | "url_deferred"
+                | "info"
+                | "ftp_path";
+            /** Type Index */
+            type_index: number;
+        };
+        /** ParsedFetchWorkbookForCollections */
+        ParsedFetchWorkbookForCollections: {
+            /**
+             * Collection Type
+             * @enum {string}
+             */
+            collection_type: "list" | "list:paired" | "list:list" | "list:list:paired" | "list:paired_or_unpaired";
+            /** Columns */
+            columns: components["schemas"]["ParsedColumn"][];
+            /** Parse Log */
+            parse_log: components["schemas"]["ParseLogEntry"][];
+            /** Rows */
+            rows: {
+                [key: string]: string | null;
+            }[];
+            /**
+             * Workbook Type
+             * @default collection
+             * @enum {string}
+             */
+            workbook_type: "datasets" | "collection" | "collections";
+        };
+        /** ParsedFetchWorkbookForDatasets */
+        ParsedFetchWorkbookForDatasets: {
+            /** Columns */
+            columns: components["schemas"]["ParsedColumn"][];
+            /** Parse Log */
+            parse_log: components["schemas"]["ParseLogEntry"][];
+            /** Rows */
+            rows: {
+                [key: string]: string | null;
+            }[];
+            /**
+             * Workbook Type
+             * @default datasets
+             * @enum {string}
+             */
+            workbook_type: "datasets" | "collection" | "collections";
+        };
         /** PastedDataElement */
         PastedDataElement: {
             /** Md5 */
@@ -36422,6 +36557,99 @@ export interface operations {
             };
         };
     };
+    tools__fetch_workbook_download: {
+        parameters: {
+            query?: {
+                /** @description Generate a workbook for simple datasets or a collection. */
+                type?: "datasets" | "collection" | "collections";
+                /** @description Generate workbook for specified collection type (not all collection types are supported) */
+                collection_type?: "list" | "list:paired" | "list:list" | "list:list:paired" | "list:paired_or_unpaired";
+                /** @description Filename of the workbook download to generate */
+                filename?: string | null;
+            };
+            header?: {
+                /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+                "run-as"?: string | null;
+            };
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content?: never;
+            };
+            /** @description Request Error */
+            "4XX": {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["MessageExceptionModel"];
+                };
+            };
+            /** @description Server Error */
+            "5XX": {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["MessageExceptionModel"];
+                };
+            };
+        };
+    };
+    tools__fetch_workbook_parse: {
+        parameters: {
+            query?: never;
+            header?: {
+                /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+                "run-as"?: string | null;
+            };
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["ParseFetchWorkbook"];
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json":
+                        | components["schemas"]["ParsedFetchWorkbookForDatasets"]
+                        | components["schemas"]["ParsedFetchWorkbookForCollections"];
+                };
+            };
+            /** @description Request Error */
+            "4XX": {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["MessageExceptionModel"];
+                };
+            };
+            /** @description Server Error */
+            "5XX": {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["MessageExceptionModel"];
+                };
+            };
+        };
+    };
     get_icon_api_tools__tool_id__icon_get: {
         parameters: {
             query?: never;

client/src/components/Collections/BuildFileSetWizard.vue

@@ -2,16 +2,29 @@
 import { BCardGroup } from "bootstrap-vue";
 import { computed, ref } from "vue";
 
+import { GalaxyApi } from "@/api";
 import { getGalaxyInstance } from "@/app";
 import { attemptCreate, type CollectionCreatorComponent } from "@/components/Collections/common/useCollectionCreator";
+import { useWorkbookDropHandling } from "@/components/Collections/common/useWorkbooks";
 import { rawToTable } from "@/components/Collections/tables";
+import { forBuilder, type ForBuilderResponse } from "@/components/Collections/wizard/fetchWorkbooks";
 import { useWizard } from "@/components/Common/Wizard/useWizard";
 import { useToolRouting } from "@/composables/route";
 import localize from "@/utils/localization";
 
-import type { RemoteFile, RulesCreatingWhat, RulesSourceFrom } from "./wizard/types";
+import type {
+    ParsedFetchWorkbook,
+    ParsedFetchWorkbookForCollectionCollectionType,
+    RuleBuilderMapping,
+    RuleBuilderOptions,
+    RuleElementsType,
+    RulesCreatingWhat,
+    RuleSelectionType,
+    RulesSourceFrom,
+} from "./wizard/types";
 import { useFileSetSources } from "./wizard/useFileSetSources";
 
+import ConfigureFetchWorkbook from "./wizard/ConfigureFetchWorkbook.vue";
 import CreatingWhat from "./wizard/CreatingWhat.vue";
 import PasteData from "./wizard/PasteData.vue";
 import SelectDataset from "./wizard/SelectDataset.vue";
@@ -20,6 +33,8 @@ import SourceFromCollectionApplyRules from "./wizard/SourceFromCollectionApplyRu
 import SourceFromDatasetAsTable from "./wizard/SourceFromDatasetAsTable.vue";
 import SourceFromPastedData from "./wizard/SourceFromPastedData.vue";
 import SourceFromRemoteFiles from "./wizard/SourceFromRemoteFiles.vue";
+import SourceFromWorkbook from "./wizard/SourceFromWorkbook.vue";
+import UploadFetchWorkbook from "./wizard/UploadFetchWorkbook.vue";
 import GenericWizard from "@/components/Common/Wizard/GenericWizard.vue";
 import RuleCollectionBuilder from "@/components/RuleCollectionBuilder.vue";
 
@@ -45,8 +60,14 @@ const sourceInstructions = computed(() => {
     return `${creatingWhatTitle.value} can be created from a set or files, URIs, or existing datasets.`;
 });
 const collectionCreator = ref<CollectionCreatorComponent | undefined>();
+const workbookCompleted = ref<boolean>(false);
+// workbook generation options
+const workbookCollectionType = ref<ParsedFetchWorkbookForCollectionCollectionType>("list");
+const workbookIncludeCollectionName = ref<boolean>(false);
 
 const sourceFrom = ref<RulesSourceFrom>("remote_files");
+// if we upload a workbook - just jump to the end with these properties
+const parsedWorkbookUploadProps = ref<ForBuilderResponse | undefined>(undefined);
 
 const { routeToTool } = useToolRouting();
 
@@ -83,6 +104,18 @@ const wizard = useWizard({
         isValid: () => sourceFrom.value === "dataset_as_table" && tabularDatasetContents.value.length > 0,
         isSkippable: () => sourceFrom.value !== "dataset_as_table",
     },
+    "configure-workbook": {
+        label: "Configure workbook",
+        instructions: "Configure a workbook to fill with URIs and metadata",
+        isValid: () => sourceFrom.value === "workbook" && workbookCollectionType.value !== undefined,
+        isSkippable: () => sourceFrom.value !== "workbook" || creatingWhat.value !== "collections",
+    },
+    "upload-workbook": {
+        label: "Upload workbook",
+        instructions: "Upload a workbook containing with URIs and metadata",
+        isValid: () => sourceFrom.value === "workbook" && workbookCompleted.value,
+        isSkippable: () => sourceFrom.value !== "workbook",
+    },
     "rule-builder": {
         label: "Specify Rules",
         instructions: "Use this form to describe rules for importing",
@@ -101,25 +134,15 @@ const importButtonLabel = computed(() => {
 });
 
 const emit = defineEmits(["dismiss", "created"]);
-type SelectionType = "raw" | "remote_files";
-type ElementsType = RemoteFile[] | string[][];
-
-// it would be nice to have a real type from the rule builder but
-// it is older code. This is really outlining what this component can
-// produce and not what the rule builder can consume which is a wide
-// superset of this.
-interface Entry {
-    dataType: RulesCreatingWhat;
-    ftpUploadSite?: string;
-    elements?: ElementsType | undefined;
-    content?: string;
-    selectionType: SelectionType;
-}
 
 const ruleBuilderModalEntryProps = computed(() => {
-    let elements: ElementsType | undefined = undefined;
-    let selectionType: SelectionType = "raw";
-    if (sourceFrom.value == "remote_files") {
+    let elements: RuleElementsType | undefined = undefined;
+    let initialMappings: RuleBuilderMapping | undefined = undefined;
+    let selectionType: RuleSelectionType = "raw";
+    if (parsedWorkbookUploadProps.value) {
+        elements = parsedWorkbookUploadProps.value.initialElements;
+        initialMappings = parsedWorkbookUploadProps.value.initialMapping;
+    } else if (sourceFrom.value == "remote_files") {
         elements = uris.value;
         selectionType = "remote_files";
     } else if (sourceFrom.value == "pasted_table") {
@@ -127,11 +150,12 @@ const ruleBuilderModalEntryProps = computed(() => {
     } else if (sourceFrom.value == "dataset_as_table") {
         elements = tabularDatasetContents.value;
     }
-    const entry: Entry = {
+    const entry: RuleBuilderOptions = {
         dataType: creatingWhat.value,
         ftpUploadSite: props.ftpUploadSite,
         selectionType: selectionType,
         elements: elements,
+        initialMappings: initialMappings,
     };
     return entry;
 });
@@ -172,6 +196,14 @@ function setSourceForm(newValue: RulesSourceFrom) {
     sourceFrom.value = newValue;
 }
 
+function setWorkbookCollectionType(newValue: ParsedFetchWorkbookForCollectionCollectionType) {
+    workbookCollectionType.value = newValue;
+}
+
+function setWorkbookIncludeCollectionName(newValue: boolean) {
+    workbookIncludeCollectionName.value = newValue;
+}
+
 function onRuleState(newRuleState: boolean) {
     ruleState.value = newRuleState;
 }
@@ -180,10 +212,68 @@ function onRuleCreate() {
     // axios response data for job currently sent, not really used but wanted to document what is available.
     emit("created");
 }
+
+const dropWorkbookTitle = computed(() => {
+    return localize(
+        "If you have a completed data import workbook just drop it here or click this icon to upload your local file and skip ahead to the end of the data import wizard."
+    );
+});
+
+function handleUploadedData(response: ParsedFetchWorkbook) {
+    sourceFrom.value = "workbook";
+    const builderPropsFromUpload = forBuilder(response);
+    creatingWhat.value = builderPropsFromUpload.rulesCreatingWhat;
+    parsedWorkbookUploadProps.value = builderPropsFromUpload;
+    workbookCompleted.value = true;
+    wizard.goTo("rule-builder");
+}
+
+async function handleWorkbook(base64Content: string) {
+    const parseBody = {
+        content: base64Content,
+    };
+    const { data, error } = await GalaxyApi().POST("/api/tools/fetch/workbook/parse", {
+        body: parseBody,
+    });
+    if (data) {
+        handleUploadedData(data);
+    } else {
+        console.log(error);
+        uploadErrorMessage.value = "There was an error processing the file.";
+    }
+}
+
+const {
+    browseFiles,
+    dropZoneClasses,
+    faUpload,
+    FontAwesomeIcon,
+    handleDrop,
+    HiddenWorkbookUploadInput,
+    isDragging,
+    onFileUpload,
+    uploadErrorMessage,
+    uploadRef,
+} = useWorkbookDropHandling(handleWorkbook);
 </script>
 
 <template>
     <GenericWizard :use="wizard" :submit-button-label="importButtonLabel" :title="title" @submit="submit">
+        <template v-slot:header>
+            <h2 data-galaxy-file-drop-target>
+                {{ title }}
+                <FontAwesomeIcon
+                    class="workbook-upload-helper mr-1"
+                    :class="dropZoneClasses"
+                    :title="dropWorkbookTitle"
+                    :icon="faUpload"
+                    @click.prevent="browseFiles"
+                    @drop.prevent="handleDrop"
+                    @dragover.prevent="isDragging = true"
+                    @dragleave.prevent="isDragging = false" />
+                <HiddenWorkbookUploadInput ref="uploadRef" @onFileUpload="onFileUpload" />
+            </h2>
+        </template>
         <div v-if="wizard.isCurrent('select-what')">
             <CreatingWhat :creating-what="creatingWhat" @onChange="setCreatingWhat" />
         </div>
@@ -192,6 +282,10 @@ function onRuleCreate() {
                 <SourceFromRemoteFiles :selected="sourceFrom === 'remote_files'" @select="setSourceForm" />
                 <SourceFromPastedData :selected="sourceFrom === 'pasted_table'" @select="setSourceForm" />
                 <SourceFromDatasetAsTable :selected="sourceFrom === 'dataset_as_table'" @select="setSourceForm" />
+                <SourceFromWorkbook
+                    :creating-what="creatingWhat"
+                    :selected="sourceFrom === 'workbook'"
+                    @select="setSourceForm" />
                 <SourceFromCollectionApplyRules
                     v-if="creatingWhat == 'collections'"
                     :selected="sourceFrom === 'collection'"
@@ -207,6 +301,20 @@ function onRuleCreate() {
         <div v-else-if="wizard.isCurrent('select-dataset')">
             <SelectDataset @onChange="setDatasetContents" />
         </div>
+        <div v-else-if="wizard.isCurrent('configure-workbook')">
+            <ConfigureFetchWorkbook
+                :collection-type="workbookCollectionType"
+                :include-collection-name="workbookIncludeCollectionName"
+                @onCollectionType="setWorkbookCollectionType"
+                @onIncludeCollectionName="setWorkbookIncludeCollectionName" />
+        </div>
+        <div v-else-if="wizard.isCurrent('upload-workbook')">
+            <UploadFetchWorkbook
+                :creating-what="creatingWhat"
+                :collection-type="workbookCollectionType"
+                :include-collection-name="workbookIncludeCollectionName"
+                @workbookContents="handleWorkbook" />
+        </div>
         <div v-else-if="wizard.isCurrent('rule-builder')" style="width: 100%">
             <RuleCollectionBuilder
                 ref="collectionCreator"
@@ -214,9 +322,20 @@ function onRuleCreate() {
                 :import-type="creatingWhat"
                 :elements-type="ruleBuilderModalEntryProps.selectionType"
                 :initial-elements="ruleBuilderElements"
+                :initial-mapping="ruleBuilderModalEntryProps.initialMappings"
                 mode="wizard"
                 @onCreate="onRuleCreate"
                 @validInput="onRuleState" />
         </div>
     </GenericWizard>
 </template>
+
+<style lang="scss" scoped>
+@import "theme/blue.scss";
+@import "@/components/Collections/wizard/workbook-dropzones.scss";
+
+// modeled a bit after upload-helper in the upload component...
+.workbook-upload-helper {
+    color: $border-color;
+}
+</style>

client/src/components/Collections/ListWizard.vue

@@ -17,9 +17,8 @@ import {
 } from "./common/useCollectionCreator";
 import { showHid } from "./common/useCollectionCreator";
 import type { WhichListBuilder } from "./ListWizard/types";
-import { autoPairWithCommonFilters } from "./pairing";
+import { useAutoPairing } from "./usePairing";
 
-import AutoPairing from "./common/AutoPairing.vue";
 import ListCollectionCreator from "./ListCollectionCreator.vue";
 import WhichBuilder from "./ListWizard/WhichBuilder.vue";
 import PairedOrUnpairedListCollectionCreator from "./PairedOrUnpairedListCollectionCreator.vue";
@@ -36,8 +35,8 @@ const isBusy = ref<boolean>(false);
 const whichBuilder = ref<WhichListBuilder>("list");
 const store = useCollectionBuilderItemSelection();
 const { currentHistoryId, createPayload } = useCollectionCreation();
-const currentForwardFilter = ref("");
-const currentReverseFilter = ref("");
+const { countPaired, currentForwardFilter, currentReverseFilter, AutoPairing, autoPair, onFilters } = useAutoPairing();
+
 const creationError = ref<string | null>(null);
 const collectionCreated = ref(false);
 
@@ -45,18 +44,13 @@ type InferrableBuilder = "list" | "list:paired";
 const collectionCreator = ref<CollectionCreatorComponent>();
 const { selectedItems } = storeToRefs(store);
 
-const countPaired = ref(-1);
-const countUnpaired = ref(-1);
 const inferredBuilder = ref<InferrableBuilder>("list");
 const builderInputsValid = ref(false);
 
 async function initialize() {
     isBusy.value = true;
-    const summary = autoPairWithCommonFilters(selectedItems.value, true);
-    currentForwardFilter.value = summary.forwardFilter || "";
-    currentReverseFilter.value = summary.reverseFilter || "";
-    countPaired.value = summary.pairs?.length || 0;
-    countUnpaired.value = summary.unpaired.length;
+    autoPair(selectedItems.value);
+    // updates currentForwardFilter, currentReverseFilter, countPaired
     if (countPaired.value * 2 > selectedItems.value.length * 0.2) {
         whichBuilder.value = "list:paired";
         inferredBuilder.value = "list:paired";
@@ -191,11 +185,6 @@ function setWhichBuilder(newWhichBuilder: WhichListBuilder) {
     whichBuilder.value = newWhichBuilder;
 }
 
-function onFilters(forwardFilter: string, reverseFilter: string) {
-    currentForwardFilter.value = forwardFilter;
-    currentReverseFilter.value = reverseFilter;
-}
-
 function goToAutoPairing() {
     wizard.goTo("auto-pairing");
 }

client/src/components/Collections/PairedOrUnpairedListCollectionCreator.vue

@@ -42,8 +42,6 @@ interface Props {
     suggestedName?: string;
     fromSelection?: boolean;
     extensions?: string[];
-    height?: string;
-    width?: string;
     forwardFilter?: string;
     reverseFilter?: string;
     collectionType: SupportedPairedOrPairedBuilderCollectionTypes;
@@ -93,7 +91,7 @@ const {
 pairingTargetsStore.setShowElementExtension(showElementExtension);
 
 const style = computed(() => {
-    return { width: props.width || "100%", height: props.height || "500px" };
+    return { width: "100%", height: "500px" };
 });
 
 const flatLists = computed(() => props.collectionType.indexOf("paired") == -1);

client/src/components/Collections/auto_pairing_spec.yml

@@ -1,76 +0,0 @@
-- doc: Simple _R1 _R2 fastqs split.
-  inputs:
-  - input_R1.fastq
-  - input_R2.fastq
-  paired:
-    input:
-      forward: input_R1.fastq
-      reverse: input_R2.fastq
-
-- doc: Compressed _R1 _R2 fastqs split.
-  inputs:
-  - input_R1.fastq.gz
-  - input_R2.fastq.gz
-  paired:
-    input:
-      forward: input_R1.fastq.gz
-      reverse: input_R2.fastq.gz
-
-- doc: Joining common prefixes on _R1 _R2.
-  inputs:
-  - input541_R1.fastq
-  - input541_R2.fastq
-  - input521_R1.fastq
-  - input521_R2.fastq
-  paired:
-    input541:
-      forward: input541_R1.fastq
-      reverse: input541_R2.fastq
-    input521:
-      forward: input521_R1.fastq
-      reverse: input521_R2.fastq
-
-- doc: Simple .1.fastq/.2.fastq split.
-  inputs:
-  - input.1.fastq
-  - input.2.fastq
-  paired:
-    input:
-      forward: input.1.fastq
-      reverse: input.2.fastq
-
-- doc: Simple .1.fastq.gz/.2.fastq.gz split.
-  inputs:
-  - input.1.fastq.gz
-  - input.2.fastq.gz
-  paired:
-    input:
-      forward: input.1.fastq.gz
-      reverse: input.2.fastq.gz
-
-- doc: Simple _1/_2 split.
-  inputs:
-  - input_1.fastq
-  - input_2.fastq
-  paired:
-    input:
-      forward: input_1.fastq
-      reverse: input_2.fastq
-
-- doc: Simple _1/_2 split on compressed files (.gz)
-  inputs:
-  - input_1.fastq.gz
-  - input_2.fastq.gz
-  paired:
-    input:
-      forward: input_1.fastq.gz
-      reverse: input_2.fastq.gz
-
-- doc: Simple _1/_2 split on compressed files (.bz2)
-  inputs:
-  - input_1.fastq.bz2
-  - input_2.fastq.bz2
-  paired:
-    input:
-      forward: input_1.fastq.bz2
-      reverse: input_2.fastq.bz2

client/src/components/Collections/auto_pairing_spec.yml

@@ -0,0 +1 @@
+../../../../lib/galaxy/model/dataset_collections/auto_pairing_spec.yml

client/src/components/Collections/common/useWorkbooks.ts

@@ -0,0 +1,113 @@
+import { faUpload } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { computed, ref } from "vue";
+
+import { errorMessageAsString } from "@/utils/simple-error";
+
+import HiddenWorkbookUploadInput from "@/components/Collections/wizard/HiddenWorkbookUploadInput.vue";
+
+export const fileToBase64 = (file: File): Promise<string> =>
+    new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(reader.result as string);
+        reader.onerror = (error) => reject(error);
+        reader.readAsDataURL(file);
+    });
+
+export type WorkbookHandler = (workbookContentBase64: string) => Promise<void>;
+
+export function useWorkbookDropHandling(workbookHandler: WorkbookHandler) {
+    const uploadErrorMessage = ref<string | undefined>(undefined);
+    const isDragging = ref(false);
+    const isProcessingUpload = ref(false);
+
+    async function onFileUpload(event: Event) {
+        const input = event.target as HTMLInputElement;
+        const file = input.files?.[0] ?? null;
+        if (file) {
+            if (file.size > 10 * 1024 * 1024) {
+                // Limit to 10MB
+                uploadErrorMessage.value = "File size exceeds 10MB limit.";
+                return;
+            }
+            const base64Content = await readAsBase64(file);
+            workbookHandler(base64Content);
+        }
+    }
+
+    async function readAsBase64(file: File) {
+        const fileContent = await fileToBase64(file);
+        const base64Content = fileContent.split(",")[1] as string;
+        return base64Content;
+    }
+
+    function checkDrop(event: DragEvent): File | undefined {
+        const file = event.dataTransfer?.files[0];
+        if (!file || !file.name.endsWith(".xlsx")) {
+            uploadErrorMessage.value = "Please drop a valid XLSX file.";
+            return undefined;
+        }
+        return file;
+    }
+
+    const handleDrop = async (event: DragEvent) => {
+        const file = checkDrop(event);
+        if (!file) {
+            return;
+        }
+        if (file.size > 10 * 1024 * 1024) {
+            // Limit to 10MB
+            uploadErrorMessage.value = "File size exceeds 10MB limit.";
+            return;
+        }
+
+        isDragging.value = false;
+        isProcessingUpload.value = true;
+        try {
+            // Read and base64 encode the file
+            const base64Content = await readAsBase64(file);
+            await workbookHandler(base64Content);
+        } catch (error) {
+            console.error("Error uploading file:", error);
+            uploadErrorMessage.value = "Error processing the file: " + errorMessageAsString(error);
+        } finally {
+            isProcessingUpload.value = false;
+        }
+    };
+
+    const uploadRef = ref<InstanceType<typeof HiddenWorkbookUploadInput>>();
+
+    interface HasBrowse {
+        browse: () => void;
+    }
+
+    function browseFiles() {
+        const ref = uploadRef.value;
+        if (ref) {
+            (ref as unknown as HasBrowse).browse();
+        }
+    }
+
+    const dropZoneClasses = computed(() => {
+        const classes = ["dropzone"];
+        if (isDragging.value) {
+            classes.push("highlight");
+        }
+        return classes;
+    });
+
+    return {
+        browseFiles,
+        dropZoneClasses,
+        FontAwesomeIcon,
+        faUpload,
+        readAsBase64,
+        isDragging,
+        isProcessingUpload,
+        handleDrop,
+        HiddenWorkbookUploadInput,
+        onFileUpload,
+        uploadErrorMessage,
+        uploadRef,
+    };
+}

client/src/components/Collections/usePairing.ts

@@ -0,0 +1,35 @@
+import { ref } from "vue";
+
+import { autoPairWithCommonFilters, type HasName } from "./pairing";
+
+import AutoPairing from "./common/AutoPairing.vue";
+
+export function useAutoPairing() {
+    const currentForwardFilter = ref("");
+    const currentReverseFilter = ref("");
+    const countPaired = ref(-1);
+    const countUnpaired = ref(-1);
+
+    function onFilters(forwardFilter: string, reverseFilter: string) {
+        currentForwardFilter.value = forwardFilter;
+        currentReverseFilter.value = reverseFilter;
+    }
+
+    function autoPair(selectedItems: HasName[]) {
+        const summary = autoPairWithCommonFilters(selectedItems, true);
+        currentForwardFilter.value = summary.forwardFilter || "";
+        currentReverseFilter.value = summary.reverseFilter || "";
+        countPaired.value = summary.pairs?.length || 0;
+        countUnpaired.value = summary.unpaired.length;
+    }
+
+    return {
+        AutoPairing,
+        autoPair,
+        countPaired,
+        countUnpaired,
+        currentForwardFilter,
+        currentReverseFilter,
+        onFilters,
+    };
+}

client/src/components/Collections/wizard/CardDownloadWorkbook.vue

@@ -0,0 +1,20 @@
+<script setup lang="ts">
+import { faDownload } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { BCard, BCardTitle, BLink } from "bootstrap-vue";
+
+interface Props {
+    generateWorkbookLink: string;
+}
+
+defineProps<Props>();
+</script>
+
+<template>
+    <BCard>
+        <BCardTitle>
+            <b>Step 1: Download</b>
+        </BCardTitle>
+        <BLink :href="generateWorkbookLink"><FontAwesomeIcon size="xl" :icon="faDownload" /> Download workbook.</BLink>
+    </BCard>
+</template>

client/src/components/Collections/wizard/CardEditWorkbook.vue

@@ -0,0 +1,15 @@
+<script setup lang="ts">
+import { faEdit } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { BCard, BCardTitle } from "bootstrap-vue";
+</script>
+
+<template>
+    <BCard>
+        <BCardTitle>
+            <b>Step 2: Fill Out</b>
+        </BCardTitle>
+        <FontAwesomeIcon size="xl" :icon="faEdit" />Fill out the workbook with your favorite spreadsheet application and
+        save it to your local disk.
+    </BCard>
+</template>

client/src/components/Collections/wizard/CardUploadWorkbook.vue

@@ -0,0 +1,58 @@
+<script setup lang="ts">
+import { faUpload } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { BAlert, BCard, BCardTitle, BLink } from "bootstrap-vue";
+
+import { useWorkbookDropHandling } from "@/components/Collections/common/useWorkbooks";
+
+async function handleWorkbook(base64Content: string) {
+    emit("workbookContents", base64Content);
+}
+
+const {
+    browseFiles,
+    dropZoneClasses,
+    handleDrop,
+    HiddenWorkbookUploadInput,
+    isDragging,
+    onFileUpload,
+    uploadErrorMessage,
+    uploadRef,
+} = useWorkbookDropHandling(handleWorkbook);
+
+const emit = defineEmits(["workbookContents"]);
+</script>
+<template>
+    <BCard
+        data-galaxy-file-drop-target
+        :class="dropZoneClasses"
+        @drop.prevent="handleDrop"
+        @dragover.prevent="isDragging = true"
+        @dragleave.prevent="isDragging = false">
+        <BCardTitle>
+            <b>Step 3: Upload</b>
+        </BCardTitle>
+        <div v-if="uploadErrorMessage">
+            <BAlert variant="danger" show>
+                {{ uploadErrorMessage }}
+            </BAlert>
+        </div>
+        <div>
+            <BLink href="#" @click.prevent="browseFiles">
+                <FontAwesomeIcon size="xl" :icon="faUpload" />
+                Drop completed workbook here or click to upload.
+            </BLink>
+            <HiddenWorkbookUploadInput ref="uploadRef" @onFileUpload="onFileUpload" />
+        </div>
+    </BCard>
+</template>
+
+<style scoped>
+@import "theme/blue.scss";
+@import "@/components/Collections/wizard/workbook-dropzones.scss";
+
+.dropzone {
+    padding: 7px !important;
+    width: 100%;
+}
+</style>

client/src/components/Collections/wizard/ConfigureFetchWorkbook.vue

@@ -0,0 +1,50 @@
+<script setup lang="ts">
+import { BFormCheckbox } from "bootstrap-vue";
+import { ref, watch } from "vue";
+
+import type { ParsedFetchWorkbookForCollectionCollectionType } from "./types";
+
+import WhichWorkbookCollectionType from "./WhichWorkbookCollectionType.vue";
+
+interface Props {
+    collectionType: ParsedFetchWorkbookForCollectionCollectionType;
+    includeCollectionName: boolean;
+}
+
+const props = defineProps<Props>();
+
+const currentCollectionType = ref<ParsedFetchWorkbookForCollectionCollectionType>(props.collectionType);
+const currentIncludeCollectionName = ref<boolean>(props.includeCollectionName);
+
+const emit = defineEmits(["onCollectionType", "onIncludeCollectionName"]);
+
+function setCollectionType(type: ParsedFetchWorkbookForCollectionCollectionType) {
+    currentCollectionType.value = type;
+    emit("onCollectionType", currentCollectionType.value);
+}
+
+watch(currentIncludeCollectionName, (newValue) => {
+    emit("onIncludeCollectionName", newValue);
+});
+</script>
+
+<template>
+    <div class="configure-workbook">
+        <div class="which-question">What type of collection do you want to create?</div>
+        <WhichWorkbookCollectionType :value="collectionType" @onChange="setCollectionType" />
+        <BFormCheckbox v-model="currentIncludeCollectionName" class="multiple-collections" size="lg"
+            >Include a collection name column so multiple collections can be created using the workbook?</BFormCheckbox
+        >
+    </div>
+</template>
+
+<style scoped lang="scss">
+.which-question {
+    font-size: 1.2rem;
+}
+
+.multiple-collections {
+    margin-top: 1rem;
+    font-size: 1.2rem;
+}
+</style>

client/src/components/Collections/wizard/HiddenWorkbookUploadInput.vue

@@ -0,0 +1,24 @@
+<script setup lang="ts">
+import { ref } from "vue";
+
+const emit = defineEmits(["onFileUpload"]);
+const fileInputRef = ref<HTMLInputElement>();
+
+function browse() {
+    fileInputRef.value?.click();
+}
+
+function onFileUpload(event: Event) {
+    emit("onFileUpload", event);
+}
+
+defineExpose({
+    browse,
+});
+</script>
+
+<template>
+    <label style="display: none">
+        <input ref="fileInputRef" type="file" accept=".xlsx" @change="onFileUpload" />
+    </label>
+</template>

client/src/components/Collections/wizard/SourceFromWorkbook.vue

@@ -0,0 +1,41 @@
+<script setup lang="ts">
+import { BCard, BCardTitle } from "bootstrap-vue";
+import { computed } from "vue";
+
+import { borderVariant } from "@/components/Common/Wizard/utils";
+
+import type { RulesCreatingWhat } from "./types";
+
+interface Props {
+    selected: boolean;
+    creatingWhat: RulesCreatingWhat;
+}
+
+const props = defineProps<Props>();
+
+const emit = defineEmits(["select"]);
+
+const whatText = computed(() => {
+    if (props.creatingWhat == "datasets") {
+        return "dataset and metadata";
+    } else {
+        return "collection element metadata and structure";
+    }
+});
+</script>
+
+<template>
+    <BCard
+        data-import-source-from="workbook"
+        class="wizard-selection-card"
+        :border-variant="borderVariant(selected)"
+        @click="emit('select', 'workbook')">
+        <BCardTitle>
+            <b>External Workbook</b>
+        </BCardTitle>
+        <div>
+            This option lets you fill in URLs/URIs in an external workbook using tools like Excel or Google Sheets and
+            specify {{ whatText }} in the workbook also.
+        </div>
+    </BCard>
+</template>

client/src/components/Collections/wizard/UploadFetchWorkbook.vue

@@ -0,0 +1,51 @@
+<script setup lang="ts">
+import { BCardGroup } from "bootstrap-vue";
+import { computed } from "vue";
+
+import type { ParsedFetchWorkbookForCollectionCollectionType, RulesCreatingWhat } from "./types";
+
+import CardDownloadWorkbook from "./CardDownloadWorkbook.vue";
+import CardEditWorkbook from "./CardEditWorkbook.vue";
+import CardUploadWorkbook from "./CardUploadWorkbook.vue";
+
+async function handleWorkbook(base64Content: string) {
+    emit("workbookContents", base64Content);
+}
+
+interface Props {
+    creatingWhat: RulesCreatingWhat;
+    collectionType: ParsedFetchWorkbookForCollectionCollectionType;
+    includeCollectionName: boolean;
+}
+
+const props = defineProps<Props>();
+
+const emit = defineEmits(["workbookContents"]);
+
+const generateWorkbookHref = computed(() => {
+    let type;
+    if (props.creatingWhat === "datasets") {
+        type = "dataset";
+    } else if (props.includeCollectionName) {
+        type = "collections";
+    } else {
+        type = "collection";
+    }
+    // const type = props.creatingWhat === "collections" ? "collection" : "dataset";
+    let url = `/api/tools/fetch/workbook?type=${type}`;
+    if (type === "collection" || type === "collections") {
+        url = `${url}&collection_type=${props.collectionType}`;
+    }
+    return url;
+});
+</script>
+
+<template>
+    <div class="generate-workbook">
+        <BCardGroup deck>
+            <CardDownloadWorkbook :generate-workbook-link="generateWorkbookHref" />
+            <CardEditWorkbook />
+            <CardUploadWorkbook @workbookContents="handleWorkbook" />
+        </BCardGroup>
+    </div>
+</template>

client/src/components/Collections/wizard/WhichWorkbookCollectionType.vue

@@ -0,0 +1,92 @@
+<script setup lang="ts">
+import { BCard, BCardGroup, BCardTitle } from "bootstrap-vue";
+
+import { borderVariant } from "@/components/Common/Wizard/utils";
+
+import type { ParsedFetchWorkbookForCollectionCollectionType } from "./types";
+
+interface Props {
+    value: ParsedFetchWorkbookForCollectionCollectionType;
+}
+
+const emit = defineEmits(["onChange"]);
+
+defineProps<Props>();
+</script>
+
+<template>
+    <div>
+        <BCardGroup deck>
+            <BCard
+                data-which-builder="list"
+                class="wizard-selection-card"
+                :border-variant="borderVariant(value === 'list')"
+                @click="emit('onChange', 'list')">
+                <BCardTitle>
+                    <b>Flat List</b>
+                </BCardTitle>
+                <div>
+                    This options creates a simple flat list of files. If your data isn't nested and does not contained
+                    paired datasets, this is the the option to choose.
+                </div>
+            </BCard>
+            <BCard
+                data-which-builder="list:paired"
+                class="wizard-selection-card"
+                :border-variant="borderVariant(value === 'list:paired')"
+                @click="emit('onChange', 'list:paired')">
+                <BCardTitle>
+                    <b>List of Paired Datasets</b>
+                </BCardTitle>
+                <div>
+                    This options creates a uniform list of paired datasets, use this option if all of your data should
+                    be paired off.
+                </div>
+            </BCard>
+        </BCardGroup>
+        <span>
+            <BCardGroup deck class="mt-3">
+                <BCard
+                    data-which-builder="list:paired_or_unpaired"
+                    class="wizard-selection-card"
+                    :border-variant="borderVariant(value === 'list:paired_or_unpaired')"
+                    @click="emit('onChange', 'list:paired_or_unpaired')">
+                    <BCardTitle>
+                        <b>Mixed List of Paired and Unpaired Datasets</b>
+                    </BCardTitle>
+                    <div>
+                        For studies that have a mix of paired and unpaired data, this may be the right choice. Existing
+                        tools and workflows may have to be updated to allow this modality though.
+                    </div>
+                </BCard>
+                <BCard
+                    data-which-builder="list:list"
+                    class="wizard-selection-card"
+                    :border-variant="borderVariant(value === 'list:list')"
+                    @click="emit('onChange', 'list:list')">
+                    <BCardTitle>
+                        <b>Nested List of Datasets</b>
+                    </BCardTitle>
+                    <div>
+                        A nested list of datasets. This type of collection is a list where each element is in turn a
+                        list of datasets. The outer list identifier might group samples together by condition, sample
+                        type, etc..
+                    </div>
+                </BCard>
+                <BCard
+                    data-which-builder="list:list:paired"
+                    class="wizard-selection-card"
+                    :border-variant="borderVariant(value === 'list:list:paired')"
+                    @click="emit('onChange', 'list:list:paired')">
+                    <BCardTitle>
+                        <b>Nested List of Dataset Pairs</b>
+                    </BCardTitle>
+                    <div>
+                        A nested list of datasets. This type of collection is a list where each element is in turn a
+                        list of pairs of datasets.
+                    </div>
+                </BCard>
+            </BCardGroup>
+        </span>
+    </div>
+</template>

client/src/components/Collections/wizard/fetchWorkbooks.test.ts

@@ -0,0 +1,33 @@
+import { forBuilder } from "./fetchWorkbooks";
+import type { ParsedFetchWorkbook } from "./types";
+
+describe("forBuilder", () => {
+    it("should return the correct ForBuilderResponse for a valid ParsedFetchWorkbook", () => {
+        const parsedWorkbook: ParsedFetchWorkbook = {
+            rows: [
+                { list_identifiers: "Row1", url: "http://example.com/1", dbkey: "db1" },
+                { list_identifiers: "Row2", url: "http://example.com/2", dbkey: "db2" },
+            ],
+            columns: [
+                { type: "list_identifiers", title: "Name", type_index: 0 },
+                { type: "url", title: "URI", type_index: 0 },
+                { type: "dbkey", title: "Genome", type_index: 0 },
+            ],
+            workbook_type: "datasets",
+            parse_log: [],
+        };
+
+        const result = forBuilder(parsedWorkbook);
+
+        expect(result.initialElements).toEqual([
+            ["Row1", "http://example.com/1", "db1"],
+            ["Row2", "http://example.com/2", "db2"],
+        ]);
+        expect(result.rulesCreatingWhat).toBe("datasets");
+        expect(result.initialMapping).toEqual([
+            { type: "list_identifiers", columns: [0] },
+            { type: "url", columns: [1] },
+            { type: "dbkey", columns: [2] },
+        ]);
+    });
+});

client/src/components/Collections/wizard/fetchWorkbooks.ts

@@ -0,0 +1,79 @@
+// utilities for populating the rule builder from parsed "fetch workbook"s.
+import type {
+    ColumnMappingType,
+    ParsedFetchWorkbook,
+    ParsedFetchWorkbookColumn,
+    RawRowData,
+    RuleBuilderMapping,
+    RulesCreatingWhat,
+} from "./types";
+
+export function hasData(parsedWorkbook: ParsedFetchWorkbook): boolean {
+    return parsedWorkbook.rows.length > 0;
+}
+
+export interface ForBuilderResponse {
+    initialElements: string[][];
+    rulesCreatingWhat: RulesCreatingWhat;
+    initialMapping: RuleBuilderMapping;
+}
+
+export function forBuilder(parsedWorkbook: ParsedFetchWorkbook): ForBuilderResponse {
+    const initialElements: RawRowData = [];
+    const rulesCreatingWhat = creatingWhat(parsedWorkbook);
+    for (const row of parsedWorkbook.rows) {
+        const rowAsString: string[] = [];
+        for (const column of parsedWorkbook.columns) {
+            const rowKey: string = columnToRowKey(column);
+            const cellValue = row[rowKey];
+            if (cellValue === undefined) {
+                throw Error("Error processing server response.");
+            }
+            rowAsString.push(cellValue || "");
+        }
+        initialElements.push(rowAsString);
+    }
+    const initialMapping = buildInitialMapping(parsedWorkbook);
+    return {
+        initialElements,
+        initialMapping,
+        rulesCreatingWhat,
+    };
+}
+
+function columnToRowKey(column: ParsedFetchWorkbookColumn): string {
+    if (column.type_index == 0) {
+        return column.type;
+    } else {
+        return `${column.type}_${column.type_index}`;
+    }
+}
+
+function creatingWhat(parsedWorkbook: ParsedFetchWorkbook): RulesCreatingWhat {
+    if (parsedWorkbook.workbook_type == "datasets") {
+        return "datasets";
+    } else {
+        return "collections";
+    }
+}
+
+function buildInitialMapping(parsedWorkbook: ParsedFetchWorkbook): RuleBuilderMapping {
+    const columnMappings: RuleBuilderMapping = [];
+    for (let index = 0; index < parsedWorkbook.columns.length; index++) {
+        const column = parsedWorkbook.columns[index] as ParsedFetchWorkbookColumn;
+        const type: ColumnMappingType = column.type;
+        if (column.type_index > 0) {
+            for (const columnMapping of columnMappings) {
+                if (columnMapping.type == type) {
+                    columnMapping.columns.push(index);
+                }
+            }
+        } else {
+            columnMappings.push({
+                type: type,
+                columns: [index],
+            });
+        }
+    }
+    return columnMappings;
+}

client/src/components/Collections/wizard/types.ts

@@ -1,8 +1,42 @@
 import type { components } from "@/api/schema";
+import type { MAPPING_TARGETS } from "@/components/RuleBuilder/rule-definitions";
 
 export type RulesCreatingWhat = "datasets" | "collections";
-export type RulesSourceFrom = "remote_files" | "pasted_table" | "dataset_as_table" | "collection";
+export type RulesSourceFrom = "remote_files" | "pasted_table" | "dataset_as_table" | "collection" | "workbook";
 
 export type ListUriResponse = components["schemas"]["ListUriResponse"];
 export type RemoteFile = components["schemas"]["RemoteFile"];
 export type RemoteDirectory = components["schemas"]["RemoteDirectory"];
+export type ParsedFetchWorkbookForCollections = components["schemas"]["ParsedFetchWorkbookForCollections"];
+export type ParsedFetchWorkbookForDatasets = components["schemas"]["ParsedFetchWorkbookForDatasets"];
+export type ParsedFetchWorkbook = ParsedFetchWorkbookForCollections | ParsedFetchWorkbookForDatasets;
+export type ParsedFetchWorkbookColumn = components["schemas"]["ParsedColumn"];
+export type ParsedFetchWorkbookForCollectionCollectionType =
+    components["schemas"]["ParsedFetchWorkbookForCollections"]["collection_type"];
+
+export type RawRowData = string[][];
+
+// types and helpers around initializing the rule builder with data
+export type RuleSelectionType = "raw" | "remote_files";
+export type RuleElementsType = RemoteFile[] | string[][];
+export type ColumnMappingType = keyof typeof MAPPING_TARGETS;
+export type ParsedFetchWorkbookColumnType = ParsedFetchWorkbookColumn["type"];
+
+export interface RuleBuilderSingleMapping {
+    type: ColumnMappingType;
+    columns: number[];
+}
+export type RuleBuilderMapping = RuleBuilderSingleMapping[];
+
+// it would be nice to have a real type from the rule builder but
+// it is older code. This is really outlining what this component can
+// produce and not what the rule builder can consume which is a wide
+// superset of this.
+export interface RuleBuilderOptions {
+    dataType: RulesCreatingWhat;
+    ftpUploadSite?: string;
+    elements?: RuleElementsType | undefined;
+    content?: string;
+    selectionType: RuleSelectionType;
+    initialMappings?: RuleBuilderMapping;
+}

client/src/components/Collections/wizard/workbook-dropzones.scss

@@ -0,0 +1,9 @@
+@import "theme/blue.scss";
+
+.dropzone.highlight {
+    border-width: 2px;
+    border-color: $border-color;
+    border-style: dashed;
+    border-radius: $border-radius-large;
+    -moz-border-radius: $border-radius-large;
+}

client/src/components/Help/HelpText.vue

@@ -23,13 +23,4 @@ const helpTarget = ref();
     </span>
 </template>
 
-<style scoped>
-/* Give visual indication of mouseover info */
-.help-text {
-    text-decoration-line: underline;
-    text-decoration-style: dashed;
-}
-.title-help-text {
-    text-decoration-thickness: 1px;
-}
-</style>
+<style scoped src="@/components/Help/help-text.scss" />

client/src/components/Help/help-text.scss

@@ -0,0 +1,8 @@
+/* Give visual indication of mouseover info */
+.help-text {
+    text-decoration-line: underline;
+    text-decoration-style: dashed;
+}
+.title-help-text {
+    text-decoration-thickness: 1px;
+}

client/src/components/RuleBuilder/ColumnSelector.vue

@@ -1,7 +1,7 @@
 <template>
     <div v-if="!multiple || !ordered" class="rule-column-selector">
         <label class="d-flex justify-content-end align-items-center">
-            <span v-b-tooltip.hover class="mr-auto" :title="help">{{ label }}</span>
+            <span v-b-tooltip.hover class="mr-auto help-text" :title="help">{{ label }}</span>
             <div v-b-tooltip.hover class="mr-1" :title="title">
                 <Select2 :value="target" :multiple="multiple" @input="handleInput">
                     <option v-for="(col, index) in colHeaders" :key="col" :value="index">{{ col }}</option>
@@ -11,7 +11,7 @@
         </label>
     </div>
     <div v-else class="rule-column-selector">
-        <span>{{ label }}</span>
+        <span class="help-text" :title="help">{{ label }}</span>
         <slot></slot>
         <ol>
             <li v-for="(targetEl, index) in target" :key="targetEl" :index="index" class="rule-column-selector-target">
@@ -136,3 +136,5 @@ export default {
     },
 };
 </script>
+
+<style scoped src="@/components/Help/help-text.scss" />

client/src/components/RuleBuilder/rule-definitions.js

@@ -2,6 +2,8 @@ import pyre from "pyre-to-regexp";
 import _ from "underscore";
 import _l from "utils/localization";
 
+import MAPPING_TARGETS from "./rule_targets.yml";
+
 const NEW_COLUMN = "new";
 
 const multiColumnsToString = function (targetColumns, colHeaders) {
@@ -796,96 +798,6 @@ const RULES = {
     },
 };
 
-const MAPPING_TARGETS = {
-    list_identifiers: {
-        multiple: true,
-        label: _l("List Identifier(s)"),
-        columnHeader: _l("List Identifier"),
-        help: _l(
-            "This should be a short description of the replicate, sample name, condition, etc... that describes each level of the list structure."
-        ),
-        importType: "collections",
-    },
-    paired_identifier: {
-        label: _l("Paired-end Indicator"),
-        columnHeader: _l("Paired Indicator"),
-        help: _l(
-            "This should be set to '1', 'R1', 'forward', 'f', or 'F' to indicate forward reads, and '2', 'r', 'reverse', 'R2', 'R', or 'R2' to indicate reverse reads."
-        ),
-        importType: "collections",
-    },
-    collection_name: {
-        label: _l("Collection Name"),
-        help: _l(
-            "If this is set, all rows with the same collection name will be joined into a collection and it is possible to create multiple collections at once."
-        ),
-        modes: ["raw", "ftp", "datasets", "library_datasets"],
-        importType: "collections",
-    },
-    name_tag: {
-        label: _l("Name Tag"),
-        help: _l("Add a name tag or hash tag based on the specified column value for imported datasets."),
-        importType: "datasets",
-        modes: ["raw", "ftp"],
-    },
-    tags: {
-        multiple: true,
-        label: _l("General Purpose Tag(s)"),
-        help: _l(
-            "Add a general purpose tag based on the specified column value, use : to separate key-value pairs if desired. These tags are not propagated to derived datasets the way name and group tags are."
-        ),
-        modes: ["raw", "ftp", "datasets", "library_datasets", "collection_contents"],
-    },
-    group_tags: {
-        multiple: true,
-        label: _l("Group Tag(s)"),
-        help: _l(
-            "Add a group tag based on the specified column value, use : to separate key-value pairs. These tags are propagated to derived datasets and may be useful for factorial experiments."
-        ),
-        modes: ["raw", "ftp", "datasets", "library_datasets", "collection_contents"],
-    },
-    name: {
-        label: _l("Name"),
-        importType: "datasets",
-    },
-    dbkey: {
-        label: _l("Genome"),
-        modes: ["raw", "ftp"],
-    },
-    file_type: {
-        label: _l("Type"),
-        modes: ["raw", "ftp"],
-        help: _l("This should be the Galaxy file type corresponding to this file."),
-    },
-    url: {
-        label: _l("URL"),
-        modes: ["raw"],
-        help: _l("This should be a URL (or Galaxy-aware URI) the file can be downloaded from."),
-    },
-    url_deferred: {
-        label: _l("Deferred URL"),
-        modes: ["raw"],
-        help: _l(
-            "This should be a URL (or Galaxy-aware URI) th efile can be downloaded from - the file will not be downloaded until it used by a tool."
-        ),
-    },
-    info: {
-        label: _l("Info"),
-        help: _l(
-            "Unstructured text associated with the dataset that shows up in the history panel, this is optional and can be whatever you would like."
-        ),
-        modes: ["raw", "ftp"],
-    },
-    ftp_path: {
-        label: _l("FTP Path"),
-        modes: ["raw", "ftp"],
-        help: _l(
-            "This should be the path to the target file to include relative to your FTP directory on the Galaxy server"
-        ),
-        requiresFtp: true,
-    },
-};
-
 const columnDisplay = function (columns, colHeaders) {
     let columnNames;
     if (typeof columns == "object") {

client/src/components/RuleBuilder/rule_targets.yml

@@ -0,0 +1 @@
+../../../../lib/galaxy/model/dataset_collections/rule_targets.yml

client/src/components/RuleCollectionBuilder.vue

@@ -311,11 +311,25 @@
                                 </button>
                                 <div class="dropdown-menu" role="menu">
                                     <a
-                                        v-for="target in unmappedTargets"
+                                        v-for="target in basicUnmappedTargets"
                                         :key="target"
                                         :index="target"
                                         class="dropdown-item"
                                         href="javascript:void(0)"
+                                        :title="mappingTargets()[target].help"
+                                        :class="'rule-add-mapping-' + target.replace(/_/g, '-')"
+                                        @click="addIdentifier(target)"
+                                        >{{ mappingTargets()[target].label }}</a
+                                    >
+                                    <li><hr class="dropdown-divider" /></li>
+                                    <li><h6 class="dropdown-header">Advanced</h6></li>
+                                    <a
+                                        v-for="target in advancedUnmappedTargets"
+                                        :key="target"
+                                        :index="target"
+                                        class="dropdown-item"
+                                        href="javascript:void(0)"
+                                        :title="mappingTargets()[target].help"
                                         :class="'rule-add-mapping-' + target.replace(/_/g, '-')"
                                         @click="addIdentifier(target)"
                                         >{{ mappingTargets()[target].label }}</a
@@ -676,6 +690,11 @@ export default {
             required: false,
             type: Object,
         },
+        initialMapping: {
+            // only respected if elementsType is raw currently - other element types have their own default behaviors that make sense (e.g. assigning ftp paths to a URI implicitly)
+            required: false,
+            type: Array,
+        },
         defaultHideSourceItems: {
             type: Boolean,
             required: false,
@@ -722,6 +741,8 @@ export default {
                 mapping = [{ type: "url", columns: [0] }];
             } else if (this.elementsType == "datasets") {
                 mapping = [{ type: "list_identifiers", columns: [1] }];
+            } else if (this.initialMapping) {
+                mapping = this.initialMapping;
             } else {
                 mapping = [];
             }
@@ -957,6 +978,18 @@ export default {
             }
             return targets;
         },
+        basicUnmappedTargets() {
+            const unmappedTargets = this.unmappedTargets;
+            return unmappedTargets.filter((target) => {
+                return !MAPPING_TARGETS[target].advanced;
+            });
+        },
+        advancedUnmappedTargets() {
+            const unmappedTargets = this.unmappedTargets;
+            return unmappedTargets.filter((target) => {
+                return MAPPING_TARGETS[target].advanced;
+            });
+        },
         colHeaders() {
             const { data, columns } = this.hotData;
             return RuleDefs.colHeadersFor(data, columns);
@@ -1058,6 +1091,13 @@ export default {
                     collectionType = "paired";
                 }
             }
+            if (this.mappingAsDict.paired_or_unpaired_identifier) {
+                if (collectionType) {
+                    collectionType += ":paired_or_unpaired";
+                } else {
+                    collectionType = "paired_or_unpaired";
+                }
+            }
             return collectionType;
         },
         validName() {
@@ -1252,6 +1292,11 @@ export default {
                     console.log("Error in RuleCollectionBuilder, unable to load genomes", err);
                 });
         }
+        // is this comparable to watch immediate in newer Vue code?, I just need that event to
+        // to flair if it is initially okay also.
+        if (this.validInput) {
+            this.$emit("validInput", true);
+        }
     },
     mounted() {
         // something bizarre is up with the rendering of hands-on-table, needs a click to render.
@@ -1557,6 +1602,9 @@ export default {
             if (this.mappingAsDict.paired_identifier) {
                 identifierColumns.push(this.mappingAsDict.paired_identifier.columns[0]);
             }
+            if (this.mappingAsDict.paired_or_unpaired_identifier) {
+                identifierColumns.push(this.mappingAsDict.paired_or_unpaired_identifier.columns[0]);
+            }
             return identifierColumns;
         },
         buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) {
@@ -1611,15 +1659,24 @@ export default {
                         let identifier = String(rowData[identifierColumns[identifierColumnIndex]]);
                         if (identifierColumnIndex + 1 == numIdentifierColumns) {
                             // At correct final position in nested structure for this dataset.
-                            if (collectionTypeAtDepth === "paired") {
+                            if (["paired", "paired_or_unpaired"].indexOf(collectionTypeAtDepth) > -1) {
                                 if (["f", "1", "r1", "forward"].indexOf(identifier.toLowerCase()) > -1) {
                                     identifier = "forward";
                                 } else if (["r", "2", "r2", "reverse"].indexOf(identifier.toLowerCase()) > -1) {
                                     identifier = "reverse";
+                                } else if (
+                                    collectionTypeAtDepth == "paired_or_unpaired" &&
+                                    ["unpaired", "u"].indexOf(identifier.toLowerCase()) > -1
+                                ) {
+                                    // assert collectionTypeAtDepth == paired_or_unpaired
+                                    identifier = "unpaired";
                                 } else {
                                     this.state = "error";
-                                    this.errorMessage =
-                                        "Unknown indicator of paired status encountered - only values of F, R, 1, 2, R1, R2, forward, or reverse are allowed.";
+                                    const allowedIndicators = ["F", "R", "1", "2", "R1", "R2", "forward", "reverse"];
+                                    if (collectionTypeAtDepth == "paired_or_unpaired") {
+                                        allowedIndicators.push("unpaired", "u");
+                                    }
+                                    this.errorMessage = `Unknown indicator (${identifier}) of paired status encountered - only values of (${allowedIndicators}) are allowed.`;
                                     return;
                                 }
                             }
@@ -1658,6 +1715,29 @@ export default {
                     }
                 }
 
+                // Recursively descend elements to handle "paired_or_unpaired" collections
+                const updateUnpairedIdentifiers = (elements) => {
+                    for (const value of Object.values(elements)) {
+                        if (typeof value !== "object" || value === null) {
+                            continue;
+                        }
+                        if (value.src === "new_collection" && value.collection_type === "paired_or_unpaired") {
+                            const subElements = value.elements;
+                            if (subElements["forward"] && !subElements["reverse"]) {
+                                subElements["unpaired"] = subElements["forward"];
+                                delete subElements["forward"];
+                            }
+                        }
+                        if (value.elements) {
+                            updateUnpairedIdentifiers(value.elements);
+                        }
+                    }
+                };
+
+                if (collectionType.endsWith("paired_or_unpaired")) {
+                    updateUnpairedIdentifiers(elements);
+                }
+
                 elementsByName[collectionName] = elements;
             }
 
@@ -1819,6 +1899,24 @@ export default {
                 const info = data[dataIndex][infoColumn];
                 res["info"] = info;
             }
+            const hashTypes = [
+                { key: "hash_md5", function: "MD5" },
+                { key: "hash_sha1", function: "SHA1" },
+                { key: "hash_sha256", function: "SHA256" },
+                { key: "hash_sha515", function: "SHA512" },
+            ];
+
+            hashTypes.forEach(({ key, function: hashFunction }) => {
+                if (mappingAsDict[key]) {
+                    const hashColumn = mappingAsDict[key].columns[0];
+                    const hash = data[dataIndex][hashColumn];
+                    if (res.hashes === undefined) {
+                        res["hashes"] = [];
+                    }
+                    res["hashes"].push({ hash_function: hashFunction, hash_value: hash });
+                }
+            });
+
             const tags = [];
             if (mappingAsDict.tags) {
                 const tagColumns = mappingAsDict.tags.columns;

lib/galaxy/dependencies/pinned-requirements.txt

@@ -118,6 +118,7 @@ numpy==2.0.2 ; python_full_version < '3.10'
 numpy==2.2.6 ; python_full_version == '3.10.*'
 numpy==2.3.0 ; python_full_version >= '3.11'
 oauthlib==3.2.2
+openpyxl==3.1.5
 orjson==3.10.18
 oyaml==1.0
 packaging==24.2 ; python_full_version < '3.10'

lib/galaxy/managers/collections.py

@@ -748,14 +748,19 @@ class DatasetCollectionManager:
 
                 if i + 1 == len(identifier_columns):
                     # At correct final position in nested structure for this dataset.
-                    if collection_type_at_depth.collection_type == "paired":
+                    if collection_type_at_depth.collection_type in ["paired", "paired_or_unpaired"]:
                         if identifier.lower() in ["f", "1", "r1", "forward"]:
                             identifier = "forward"
                         elif identifier.lower() in ["r", "2", "r2", "reverse"]:
                             identifier = "reverse"
+                        elif identifier.lower() in ["u", "unpaired"]:
+                            identifier = "unpaired"
                         else:
-                            raise Exception(
-                                "Unknown indicator of paired status encountered - only values of F, R, 1, 2, R1, R2, forward, or reverse are allowed."
+                            allow_identifiers = ["F", "R", "1", "2", "R1", "R2", "forward", "reverse"]
+                            if collection_type_at_depth == "paired_or_unpaired":
+                                allow_identifiers.extend(["unpaired", "u"])
+                            raise RequestParameterInvalidException(
+                                f"Unknown indicator of paired status encountered ({identifier}) - only values from ({allow_identifiers}) are allowed."
                             )
 
                     tags = []
@@ -773,7 +778,6 @@ class DatasetCollectionManager:
 
                     effective_dataset = handle_dataset(sources[data_index]["dataset"], tags)
                     elements_at_depth[identifier] = effective_dataset
-                    # log.info("Handling dataset [%s] with sources [%s], need to add tags [%s]" % (effective_dataset, sources, tags))
                 else:
                     collection_type_at_depth = collection_type_at_depth.child_collection_type_description()
                     found = False
@@ -792,6 +796,21 @@ class DatasetCollectionManager:
                         # Subsequent loop fills elements of newly created collection
                         elements_at_depth = sub_collection["elements"]
 
+        # Recursively descend elements to handle "paired_or_unpaired" collections
+        def update_unpaired_identifiers(elements):
+            for value in elements.values():
+                if not isinstance(value, dict):
+                    continue
+                if value.get("src") == "new_collection" and value.get("collection_type") == "paired_or_unpaired":
+                    sub_elements = value["elements"]
+                    if "forward" in sub_elements and "reverse" not in sub_elements:
+                        sub_elements["unpaired"] = sub_elements.pop("forward")
+                if "elements" in value:
+                    update_unpaired_identifiers(value["elements"])
+
+        if collection_type_description.collection_type.endswith("paired_or_unpaired"):
+            update_unpaired_identifiers(elements)
+
         return elements
 
     def __init_rule_data(self, elements, collection_type_description, parent_identifiers=None, parent_indices=None):

lib/galaxy/model/dataset_collections/auto_identifiers.py

@@ -0,0 +1,71 @@
+"""Code around assigning implicit list identifiers for collections."""
+
+import os.path
+from typing import (
+    List,
+    Optional,
+    Set,
+    Tuple,
+)
+from urllib.parse import urlparse
+
+from pydantic import BaseModel
+
+
+class FillIdentifiers(BaseModel):
+    fill_inner_list_identifiers: bool = False
+    deduplication_pattern: str = "_{#}"
+    deduplication_index_from: int = 1
+
+
+def filename_to_element_identifier(filename_or_uri: str):
+    basename = guess_filename_from_url(filename_or_uri)
+    for zip_extension in [".zip", ".tar.gz", ".tar", ".tgz", ".gz", "bz2", ".bz"]:
+        if basename.endswith(zip_extension):
+            basename = basename[: -len(zip_extension)]
+    if "." in basename:
+        basename = basename.split(".")[0]
+    return basename
+
+
+def fill_in_identifiers(
+    uris_to_identifiers: List[Tuple[str, Optional[str]]], config: Optional[FillIdentifiers]
+) -> List[Optional[str]]:
+    if config is None:
+        config = FillIdentifiers()
+
+    new_identifiers: List[Optional[str]] = []
+    seen_identifiers: Set[Optional[str]] = set()
+    for uri, identifier in uris_to_identifiers:
+        if identifier is None and config.fill_inner_list_identifiers:
+            basename = filename_to_element_identifier(uri)
+            identifier = basename
+            assert identifier
+            if identifier in seen_identifiers:
+                # Handle deduplication
+                index = config.deduplication_index_from
+                suffix = config.deduplication_pattern.replace("{#}", str(index))
+                new_identifier = f"{basename}{suffix}"
+                while new_identifier in seen_identifiers:
+                    index += 1
+                    new_identifier = f"{basename}{config.deduplication_pattern.format(index=index)}"
+                identifier = new_identifier
+            seen_identifiers.add(identifier)
+            new_identifiers.append(identifier)
+        else:
+            seen_identifiers.add(identifier)
+            new_identifiers.append(identifier)
+
+    return new_identifiers
+
+
+def guess_filename_from_url(url: str) -> str:
+    """
+    Guess the filename for a given URL.
+
+    :param url: The URL to extract the filename from.
+    :return: The guessed filename.
+    """
+    parsed_url = urlparse(url)
+    filename = os.path.basename(parsed_url.path)
+    return filename or ""

lib/galaxy/model/dataset_collections/auto_pairing.py

@@ -0,0 +1,153 @@
+import re
+from dataclasses import dataclass
+from typing import (
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Protocol,
+    Tuple,
+    TypeVar,
+)
+
+from .auto_identifiers import filename_to_element_identifier
+
+
+class HasName(Protocol):
+    name: str
+
+
+T = TypeVar("T", bound=HasName)
+
+# matches pairing.ts in the client
+COMMON_FILTERS: Dict[str, Tuple[str, str]] = {
+    "illumina": ("_1", "_2"),
+    "Rs": ("_R1", "_R2"),
+    "dot12s": (".1.fastq", ".2.fastq"),
+}
+
+
+def paired_element_list_identifier(forward: str, reverse: str) -> Optional[str]:
+    for forward_filter, reverse_filter in COMMON_FILTERS.values():
+        if forward_filter in forward and reverse_filter in reverse:
+            forward_base = filename_to_element_identifier(re.sub(f"{forward_filter}", "", forward))
+            reverse_base = filename_to_element_identifier(re.sub(f"{reverse_filter}", "", reverse))
+            common_base = longest_prefix(forward_base, reverse_base)
+            return common_base
+
+    forward_base = filename_to_element_identifier(forward)
+    reverse_base = filename_to_element_identifier(reverse)
+    common_base = longest_prefix(forward_base, reverse_base)
+    return common_base
+
+
+def longest_prefix(str1: str, str2: str) -> str:
+    prefix = ""
+    min_len = min(len(str1), len(str2))
+    for i in range(min_len):
+        if str1[i] == str2[i]:
+            prefix += str1[i]
+        else:
+            break
+    return prefix
+
+
+@dataclass
+class Pair(Generic[T]):
+    name: str
+    forward: T
+    reverse: T
+
+
+@dataclass
+class PartialPair(Generic[T]):
+    name: str
+    forward: Optional[T]
+    reverse: Optional[T]
+
+    def to_pair(self) -> Pair[T]:
+        assert self.forward
+        assert self.reverse
+        return Pair(name=self.name, forward=self.forward, reverse=self.reverse)
+
+
+@dataclass
+class AutoPairResponse(Generic[T]):
+    paired: List[Pair[T]]
+    unpaired: List[T]
+
+
+def auto_pair(elements: List[T]) -> AutoPairResponse[T]:
+    filter_type = guess_initial_filter_type(elements)
+    if filter_type:
+        forward_filter, reverse_filter = COMMON_FILTERS[filter_type]
+        forward_elements, reverse_elements = split_elements_by_filter(elements, forward_filter, reverse_filter)
+        partial_pairs: Dict[str, PartialPair[T]] = {}
+        for forward_element in forward_elements:
+            forward_base = filename_to_element_identifier(re.sub(f"{forward_filter}", "", forward_element.name))
+
+            if forward_base not in partial_pairs:
+                partial_pairs[forward_base] = PartialPair(name=forward_base, forward=forward_element, reverse=None)
+        for reverse_element in reverse_elements:
+            reverse_base = filename_to_element_identifier(re.sub(f"{reverse_filter}", "", reverse_element.name))
+            if reverse_base not in partial_pairs:
+                partial_pairs[reverse_base] = PartialPair(name=reverse_base, forward=None, reverse=reverse_element)
+            else:
+                partial_pairs[reverse_base].reverse = reverse_element
+
+        unpaired: List[T] = elements.copy()
+        for forward_element in forward_elements:
+            unpaired.remove(forward_element)
+        for reverse_element in reverse_elements:
+            unpaired.remove(reverse_element)
+
+        full_pairs: List[Pair[T]] = []
+        for partial_pair in partial_pairs.values():
+            if partial_pair.forward is None:
+                assert partial_pair.reverse
+                unpaired.append(partial_pair.reverse)
+            elif partial_pair.reverse is None:
+                assert partial_pair.forward
+                unpaired.append(partial_pair.forward)
+            else:
+                full_pairs.append(partial_pair.to_pair())
+        return AutoPairResponse(paired=full_pairs, unpaired=unpaired)
+    else:
+        return AutoPairResponse(paired=[], unpaired=elements)
+
+
+def guess_initial_filter_type(elements: List[T]) -> Optional[str]:
+    illumina = 0
+    dot12s = 0
+    Rs = 0
+
+    # Iterate through elements and count occurrences of filter patterns
+    for element in elements:
+        if ".1.fastq" in element.name or ".2.fastq" in element.name:
+            dot12s += 1
+        elif "_R1" in element.name or "_R2" in element.name:
+            Rs += 1
+        elif "_1" in element.name or "_2" in element.name:
+            illumina += 1
+
+    # Determine the most likely filter type
+    if illumina == 0 and dot12s == 0 and Rs == 0:
+        return None
+    elif illumina > dot12s and illumina > Rs:
+        return "illumina"
+    elif dot12s > illumina and dot12s > Rs:
+        return "dot12s"
+    elif Rs > illumina and Rs > dot12s:
+        return "Rs"
+    else:
+        return "illumina"
+
+
+def split_elements_by_filter(elements: List[T], forward_filter: str, reverse_filter: str) -> Tuple[List[T], List[T]]:
+    filters = [re.compile(forward_filter), re.compile(reverse_filter)]
+    split: Tuple[List[T], List[T]] = ([], [])
+    for element in elements:
+        for i, filter in enumerate(filters):
+            if element.name and filter.search(element.name):
+                split[i].append(element)
+    return split

lib/galaxy/model/dataset_collections/auto_pairing_spec.yml

@@ -0,0 +1,76 @@
+- doc: Simple _R1 _R2 fastqs split.
+  inputs:
+  - input_R1.fastq
+  - input_R2.fastq
+  paired:
+    input:
+      forward: input_R1.fastq
+      reverse: input_R2.fastq
+
+- doc: Compressed _R1 _R2 fastqs split.
+  inputs:
+  - input_R1.fastq.gz
+  - input_R2.fastq.gz
+  paired:
+    input:
+      forward: input_R1.fastq.gz
+      reverse: input_R2.fastq.gz
+
+- doc: Joining common prefixes on _R1 _R2.
+  inputs:
+  - input541_R1.fastq
+  - input541_R2.fastq
+  - input521_R1.fastq
+  - input521_R2.fastq
+  paired:
+    input541:
+      forward: input541_R1.fastq
+      reverse: input541_R2.fastq
+    input521:
+      forward: input521_R1.fastq
+      reverse: input521_R2.fastq
+
+- doc: Simple .1.fastq/.2.fastq split.
+  inputs:
+  - input.1.fastq
+  - input.2.fastq
+  paired:
+    input:
+      forward: input.1.fastq
+      reverse: input.2.fastq
+
+- doc: Simple .1.fastq.gz/.2.fastq.gz split.
+  inputs:
+  - input.1.fastq.gz
+  - input.2.fastq.gz
+  paired:
+    input:
+      forward: input.1.fastq.gz
+      reverse: input.2.fastq.gz
+
+- doc: Simple _1/_2 split.
+  inputs:
+  - input_1.fastq
+  - input_2.fastq
+  paired:
+    input:
+      forward: input_1.fastq
+      reverse: input_2.fastq
+
+- doc: Simple _1/_2 split on compressed files (.gz)
+  inputs:
+  - input_1.fastq.gz
+  - input_2.fastq.gz
+  paired:
+    input:
+      forward: input_1.fastq.gz
+      reverse: input_2.fastq.gz
+
+- doc: Simple _1/_2 split on compressed files (.bz2)
+  inputs:
+  - input_1.fastq.bz2
+  - input_2.fastq.bz2
+  paired:
+    input:
+      forward: input_1.fastq.bz2
+      reverse: input_2.fastq.bz2

lib/galaxy/model/dataset_collections/rule_target_column_specification.yml

@@ -0,0 +1,136 @@
+# The idea that we reserve some column names for reserved purposes in Galaxy
+# is going to span fetch workbooks and sample sheet workbooks - and we need
+# to validate it both on the backend and the frontend. So having a set of tests
+# that validate what are reserved column names that we can run across both languages
+# utilities for doing this is valuable.
+
+- doc: "Name (with uppercase) maps directly to the name target type"
+  column_header: "Name"
+  maps_to: "name"
+- doc: "name maps directly to the name target type"
+  column_header: "name"
+  maps_to: "name"
+- doc: "NAME (all uppercase) maps directly to the name target type"
+  column_header: "NAME"
+  maps_to: "name"
+- doc: "'List Name' maps to the collection_name target type"
+  column_header: "List Name"
+  maps_to: "collection_name"
+- doc: "'list name' maps to the collection_name target type"
+  column_header: "list name"
+  maps_to: "collection_name"
+- doc: "'listname' maps to the collection_name target type"
+  column_header: "listname"
+  maps_to: "collection_name"
+
+- doc: "'collection name' maps to the collection_name target type"
+  column_header: "collection name"
+  maps_to: "collection_name"
+
+- doc: "uri maps to the url target type"
+  column_header: "uri"
+  maps_to: "url"
+- doc: "url maps to the url target type"
+  column_header: "url"
+  maps_to: "url"
+
+- doc: "'url deferred' maps to the url_deferred target type"
+  column_header: "url deferred"
+  maps_to: "url_deferred"
+
+- doc: "'deferred url' maps to the url_deferred target type"
+  column_header: "deferred url"
+  maps_to: "url_deferred"
+
+- doc: "genome maps to the dbkey target type"
+  column_header: "genome"
+  maps_to: "dbkey"
+- doc: "dbkey maps to the dbkey target type"
+  column_header: "dbkey"
+  maps_to: "dbkey"
+
+- doc: "filetype maps to the file_type target type"
+  column_header: "file type"
+  maps_to: "file_type"
+- doc: "extension maps to the file_type target type"
+  column_header: "extension"
+  maps_to: "file_type"
+
+- doc: "info maps to the info target type"
+  column_header: "info"
+  maps_to: "info"
+
+- doc: "tag maps to the tags target type"
+  column_header: "tag"
+  maps_to: "tags"
+
+- doc: "grouptag maps to the group_tags target type"
+  column_header: "grouptag"
+  maps_to: "group_tags"
+
+- doc: "nametag maps to the name_tag target type"
+  column_header: "nametag"
+  maps_to: "name_tag"
+
+- doc: "'List Identifier' maps to the list_identifiers target type"
+  column_header: "List Identifier"
+  maps_to: "list_identifiers"
+
+- doc: "'paired identifier' maps to the paired_identifier target type"
+  column_header: "Paired Identifier"
+  maps_to: "paired_identifier"
+- doc: "'paired indicator' maps to the paired_identifier target type"
+  column_header: "Paired Identifier"
+  maps_to: "paired_identifier"
+
+- doc: "'MD5Sum' maps to hash_md5" 
+  column_header: "MD5Sum"
+  maps_to: "hash_md5"
+- doc: "'MD5' maps to hash_md5" 
+  column_header: "MD5"
+  maps_to: "hash_md5"
+- doc: "'Hash MD5' maps to hash_md5" 
+  column_header: "Hash MD5"
+  maps_to: "hash_md5"
+- doc: "'MD5 Hash' maps to hash_md5" 
+  column_header: "MD5 Hash"
+  maps_to: "hash_md5"
+
+- doc: "'SHA-1 Sum' maps to hash_sha1" 
+  column_header: "SHA-1 Sum"
+  maps_to: "hash_sha1"
+- doc: "'SHA1' maps to hash_sha1" 
+  column_header: "SHA1"
+  maps_to: "hash_sha1"
+- doc: "'Hash SHA-1' maps to hash_sha1" 
+  column_header: "Hash SHA-1"
+  maps_to: "hash_sha1"
+- doc: "'SHA-1 Hash' maps to hash_sha1" 
+  column_header: "SHA-1 Hash"
+  maps_to: "hash_sha1"
+
+- doc: "'SHA-256 Sum' maps to hash_sha256" 
+  column_header: "SHA-256 Sum"
+  maps_to: "hash_sha256"
+- doc: "'SHA256' maps to hash_sha256" 
+  column_header: "SHA256"
+  maps_to: "hash_sha256"
+- doc: "'Hash SHA-256' maps to hash_sha256" 
+  column_header: "Hash SHA-256"
+  maps_to: "hash_sha256"
+- doc: "'SHA-256 Hash' maps to hash_sha256" 
+  column_header: "SHA-256 Hash"
+  maps_to: "hash_sha256"
+
+- doc: "'SHA-512 Sum' maps to hash_sha512" 
+  column_header: "SHA-512 Sum"
+  maps_to: "hash_sha512"
+- doc: "'SHA512' maps to hash_sha512" 
+  column_header: "SHA512"
+  maps_to: "hash_sha512"
+- doc: "'Hash SHA-512' maps to hash_sha512" 
+  column_header: "Hash SHA-512"
+  maps_to: "hash_sha512"
+- doc: "'SHA-512 Hash' maps to hash_sha512" 
+  column_header: "SHA-512 Hash"
+  maps_to: "hash_sha512"

lib/galaxy/model/dataset_collections/rule_target_columns.py

@@ -0,0 +1,162 @@
+import re
+from dataclasses import dataclass
+from typing import (
+    Dict,
+    List,
+    Optional,
+)
+
+from pydantic import BaseModel
+
+from galaxy.model.dataset_collections.rule_target_models import (
+    ColumnTarget,
+    RuleBuilderMappingTargetKey,
+    target_model_by_type,
+)
+
+COLUMN_TITLE_PREFIXES: Dict[str, RuleBuilderMappingTargetKey] = {
+    "name": "name",
+    "listname": "collection_name",
+    "collectionname": "collection_name",
+    "uri": "url",
+    "url": "url",
+    "urldeferred": "url_deferred",
+    "deferredurl": "url_deferred",
+    "genome": "dbkey",
+    "dbkey": "dbkey",
+    "genomebuild": "dbkey",
+    "filetype": "file_type",
+    "extension": "file_type",
+    "fileextension": "file_type",
+    "info": "info",
+    "tag": "tags",
+    "grouptag": "group_tags",
+    "nametag": "name_tag",
+    "listidentifier": "list_identifiers",
+    "pairedidentifier": "paired_identifier",
+    "hashmd5sum": "hash_md5",
+    "hashmd5": "hash_md5",
+    "md5sum": "hash_md5",
+    "md5": "hash_md5",
+    "sha1hash": "hash_sha1",
+    "hashsha1sum": "hash_sha1",
+    "hashsha1": "hash_sha1",
+    "sha1sum": "hash_sha1",
+    "sha1": "hash_sha1",
+    "sha256hash": "hash_sha256",
+    "hashsha256sum": "hash_sha256",
+    "hashsha256": "hash_sha256",
+    "sha256sum": "hash_sha256",
+    "sha256": "hash_sha256",
+    "sha512hash": "hash_sha512",
+    "hashsha512sum": "hash_sha512",
+    "hashsha512": "hash_sha512",
+    "sha512sum": "hash_sha512",
+    "sha512": "hash_sha512",
+}
+
+
+def column_title_to_target_type(column_title: str) -> Optional[RuleBuilderMappingTargetKey]:
+    normalized_title = re.sub(r"[\s\(\)\-\_]|optional", "", column_title.lower())
+    if normalized_title not in COLUMN_TITLE_PREFIXES:
+        for key in COLUMN_TITLE_PREFIXES.keys():
+            if normalized_title.startswith(key):
+                normalized_title = key
+                break
+            elif normalized_title.endswith(key):
+                normalized_title = key
+                break
+
+    if normalized_title not in COLUMN_TITLE_PREFIXES:
+        return None
+
+    column_type: RuleBuilderMappingTargetKey = COLUMN_TITLE_PREFIXES[normalized_title]
+    return column_type
+
+
+@dataclass
+class HeaderColumn:
+    type: RuleBuilderMappingTargetKey
+    title: str  # user facing
+    # e.g. for paired data will have two columns of URIs, record types maybe have any number
+    # and after dataset hash may have multiples of those also
+    type_index: int
+
+    @property
+    def width(self):
+        if self.type in ["url", "url_deferred", "ftp_path"]:
+            return 50
+        else:
+            return 20
+
+    @property
+    def name(self):
+        if self.type_index == 0:
+            return self.type
+        else:
+            return f"{self.type}_{self.type_index}"
+
+    @property
+    def help(self) -> str:
+        column_target = _column_header_to_column_target(self)
+        return column_target.help if column_target.help else ""
+
+    @property
+    def parsed_column(self) -> "ParsedColumn":
+        """Return a ParsedColumn representation of this header."""
+        return ParsedColumn(
+            type=self.type,
+            type_index=self.type_index,
+            title=self.title,
+        )
+
+
+def _column_header_to_column_target(column_header: HeaderColumn) -> ColumnTarget:
+    return target_model_by_type(column_header.type)
+
+
+class ParsedColumn(BaseModel):
+    type: RuleBuilderMappingTargetKey
+    type_index: int
+    title: str
+
+    @property
+    def name(self):
+        if self.type_index == 0:
+            return self.type
+        else:
+            return f"{self.type}_{self.type_index}"
+
+
+def column_titles_to_headers(column_titles: List[str]) -> List[HeaderColumn]:
+    headers: List[HeaderColumn] = []
+    headers_of_type_seen: Dict[RuleBuilderMappingTargetKey, int] = {}
+
+    for column_title in column_titles:
+        column_type_: Optional[RuleBuilderMappingTargetKey] = column_title_to_target_type(column_title)
+        if not column_type_:
+            # make a note in parse log that this column was skipped
+            continue
+        column_type: RuleBuilderMappingTargetKey = column_type_
+        if column_type in headers_of_type_seen:
+            type_index = headers_of_type_seen[column_type]
+            headers_of_type_seen[column_type] += 1
+        else:
+            type_index = 0
+            headers_of_type_seen[column_type] = 1
+        header_column = HeaderColumn(
+            type=column_type,
+            type_index=type_index,
+            title=column_title,
+        )
+        if header_column.type == "paired_identifier" and implied_paired_or_unpaired_column_header(header_column):
+            header_column.type = "paired_or_unpaired_identifier"
+        headers.append(header_column)
+
+    return headers
+
+
+def implied_paired_or_unpaired_column_header(column_header: HeaderColumn) -> bool:
+    # URI 2 - Reverse (Optional), Optional URI 2, Optional Paired Identifier all
+    # imply paired_or_unpaired lists using this logic.
+    return "optional" in column_header.title.lower()

lib/galaxy/model/dataset_collections/rule_target_models.py

@@ -0,0 +1,93 @@
+from typing import (
+    Dict,
+    List,
+    Literal,
+    Optional,
+)
+
+import yaml
+from pydantic import (
+    BaseModel,
+    RootModel,
+)
+
+from galaxy.util.resources import resource_string
+
+RuleBuilderImportType = Literal["datasets", "collections"]
+RuleBuilderModes = Literal[
+    "raw",
+    "ftp",
+    "datasets",
+    "library_datasets",
+    "collection_contents",
+]
+
+
+class ColumnTarget(BaseModel):
+    label: str
+    help: Optional[str]
+    modes: Optional[List[RuleBuilderModes]] = None
+    importType: Optional[RuleBuilderImportType] = None
+    multiple: Optional[bool] = False
+    columnHeader: Optional[str] = None
+    advanced: Optional[bool] = False
+    requiresFtp: Optional[bool] = False
+    example_column_names: Optional[List[str]] = None
+
+    @property
+    def example_column_names_as_str(self) -> Optional[str]:
+        if self.example_column_names:
+            return '"' + '", "'.join(self.example_column_names) + '"'
+        return ""
+
+
+RuleBuilderMappingTargetKey = Literal[
+    "list_identifiers",
+    "paired_identifier",
+    "paired_or_unpaired_identifier",
+    "collection_name",
+    "name_tag",
+    "tags",
+    "group_tags",
+    "name",
+    "dbkey",
+    "hash_sha1",
+    "hash_md5",
+    "hash_sha256",
+    "hash_sha512",
+    "file_type",
+    "url",
+    "url_deferred",
+    "info",
+    "ftp_path",
+]
+
+
+ColumnTargetsConfig = Dict[RuleBuilderMappingTargetKey, ColumnTarget]
+ColumnTargetsConfigRootModel = RootModel[ColumnTargetsConfig]
+
+
+def target_models() -> ColumnTargetsConfig:
+    column_targets_str = resource_string(__name__, "rule_targets.yml")
+    column_targets_raw = yaml.safe_load(column_targets_str)
+    return ColumnTargetsConfigRootModel.model_validate(column_targets_raw).root
+
+
+def target_model_by_type(type: RuleBuilderMappingTargetKey) -> ColumnTarget:
+    return target_models()[type]
+
+
+# Any URI-based workbook should allow specifying these metadata column types.
+COMMON_COLUMN_TARGETS = [
+    target_model_by_type("name"),
+    target_model_by_type("dbkey"),
+    target_model_by_type("info"),
+    target_model_by_type("file_type"),
+    target_model_by_type("tags"),
+    target_model_by_type("group_tags"),
+    target_model_by_type("name_tag"),
+    target_model_by_type("hash_md5"),
+    target_model_by_type("hash_sha1"),
+    target_model_by_type("hash_sha256"),
+    target_model_by_type("hash_sha512"),
+]

lib/galaxy/model/dataset_collections/rule_targets.yml

@@ -0,0 +1,175 @@
+# This file defines the column targets for the rule builder.
+list_identifiers:
+  multiple: true
+  label: "List Identifier(s)"
+  columnHeader: "List Identifier"
+  help: |
+    This should be a short description of the replicate, sample name, 
+    condition, etc... that describes each level of the list structure.
+  importType: "collections"
+paired_identifier:
+  label: "Paired-end Indicator"
+  columnHeader: "Paired Indicator"
+  help: |
+    This should be set to '1', 'R1', 'forward', 'f', or 'F' to indicate 
+    forward reads, and '2', 'r', 'reverse', 'R2', 'R', or 'R2' to indicate 
+    reverse reads.
+  importType: "collections"
+paired_or_unpaired_identifier:
+  label: "Optional Paired-end Indicator"
+  columnHeader: "Optional Paired Indicator"
+  help: |
+    This should be set to '1', 'R1', 'forward', 'f', or 'F' to indicate 
+    forward reads, and '2', 'r', 'reverse', 'R2', 'R', or 'R2' to indicate 
+    reverse reads. Unmatched '1' for 'forward' elements will be 'unpaired' 
+    in the resulting list, alternatively this column can be set to 'u' or 
+    'unpaired' to force the element to be unpaired.
+  importType: "collections"
+  advanced: true
+collection_name:
+  label: "Collection Name"
+  help: |
+    If this is set, all rows with the same collection name will be joined 
+    into a collection and it is possible to create multiple collections 
+    at once.
+  modes:
+    - "raw"
+    - "ftp"
+    - "datasets"
+    - "library_datasets"
+  importType: "collections"
+name_tag:
+  label: "Name Tag"
+  help: |
+    Add a name tag or hash tag based on the specified column value for 
+    imported datasets.
+  importType: "datasets"
+  modes:
+    - "raw"
+    - "ftp"
+  example_column_names: ["Name Tag", "nametag"]  
+tags:
+  multiple: true
+  label: "General Purpose Tag(s)"
+  help: |
+    Add a general purpose tag based on the specified column value, use : 
+    to separate key-value pairs if desired. These tags are not propagated 
+    to derived datasets the way name and group tags are.
+  modes:
+    - "raw"
+    - "ftp"
+    - "datasets"
+    - "library_datasets"
+    - "collection_contents"
+  example_column_names: ["Tag"]  
+group_tags:
+  multiple: true
+  label: "Group Tag(s)"
+  help: |
+    Add a group tag based on the specified column value, use : to separate 
+    key-value pairs. These tags are propagated to derived datasets and may 
+    be useful for factorial experiments.
+  modes:
+    - "raw"
+    - "ftp"
+    - "datasets"
+    - "library_datasets"
+    - "collection_contents"
+  example_column_names: ["Group Tag", "grouptag"]
+name:
+  label: "Name"
+  importType: "datasets"
+  help: |
+    This is just the name of the dataset as it appears in the Galaxy history
+    panel.
+  example_column_names: ["Name"]
+dbkey:
+  label: "Genome"
+  help: |
+    This should be the genome build (e.g. hg19, mm9, etc...) that the data 
+    is associated with. If this is not set, there will be no genome associated
+    with the dataset.
+  modes:
+    - "raw"
+    - "ftp"
+  example_column_names: ["DBKey", "DB Key", "Genome", "Genome Build"]
+hash_sha1:
+  label: "Hash (SHA1)"
+  help: |
+    This is the SHA1 hash of the URI, this is used to verify that the file 
+    has not been corrupted during transfer.
+  modes:
+    - "raw"
+    - "ftp"
+  advanced: true
+  example_column_names: ["SHA1", "sha-1 hash", "SHA1-Sum"]
+hash_md5:
+  label: "Hash (MD5)"
+  help: |
+    This is the MD5 hash of the URI, this is used to verify that the file 
+    has not been corrupted during transfer.
+  modes:
+    - "raw"
+    - "ftp"
+  advanced: true
+  example_column_names: ["MD5", "MD5 hash", "MD5-Sum"]
+hash_sha256:
+  label: "Hash (SHA256)"
+  help: |
+    This is the SHA256 hash of the URI, this is used to verify that the file 
+    has not been corrupted during transfer.
+  modes:
+    - "raw"
+    - "ftp"
+  advanced: true
+  example_column_names: ["SHA256", "sha-256 hash", "SHA256-Sum"]
+hash_sha512:
+  label: "Hash (SHA512)"
+  help: |
+    This is the SHA512 hash of the URI, this is used to verify that the file 
+    has not been corrupted during transfer.
+  modes:
+    - "raw"
+    - "ftp"
+  advanced: true
+  example_column_names: ["SHA512", "sha-512 hash", "SHA512-Sum"]
+file_type:
+  label: "Type"
+  modes:
+    - "raw"
+    - "ftp"
+  help: |
+    This should be the Galaxy file type corresponding to this file.
+  example_column_names: ["File Type", "file extension", "Extension"]
+url:
+  label: "URL"
+  modes:
+    - "raw"
+  help: |
+    This should be a URL (or Galaxy-aware URI) the file can be downloaded 
+    from.
+url_deferred:
+  label: "Deferred URL"
+  modes:
+    - "raw"
+  help: |
+    This should be a URL (or Galaxy-aware URI) the file can be downloaded 
+    from - the file will not be downloaded until it used by a tool.
+info:
+  label: "Info"
+  help: |
+    Unstructured text associated with the dataset that shows up in the 
+    history panel, this is optional and can be whatever you would like.
+  modes:
+    - "raw"
+    - "ftp"
+  example_column_names: [Info]
+ftp_path:
+  label: "FTP Path"
+  modes:
+    - "raw"
+    - "ftp"
+  help: |
+    This should be the path to the target file to include relative to your 
+    FTP directory on the Galaxy server.
+  requiresFtp: true

lib/galaxy/model/dataset_collections/workbook_util.py

@@ -0,0 +1,206 @@
+"""Shared code for working with workbook data.
+
+Module was developed for sample sheet work but extracted for reuse with fetch
+workbooks.
+"""
+
+import base64
+from dataclasses import dataclass
+from io import BytesIO
+from textwrap import wrap
+from typing import List
+
+from openpyxl import (
+    load_workbook,
+    Workbook,
+)
+from openpyxl.styles import Font
+from openpyxl.worksheet.datavalidation import DataValidation
+from openpyxl.worksheet.worksheet import Worksheet
+
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.model.dataset_collections.rule_target_models import (
+    ColumnTarget,
+)
+
+Base64StringT = str
+
+
+def freeze_header_row(worksheet: Worksheet) -> None:
+    worksheet.freeze_panes = "A2"
+
+
+def make_headers_bold(worksheet: Worksheet, headers: list) -> None:
+    for index in range(len(headers)):
+        make_bold(worksheet, 1, index)
+
+
+def make_bold(worksheet: Worksheet, row: int, column: int):
+    """Make the target cell bold in the specified worksheet."""
+    bold_font = Font(bold=True)
+    worksheet[f"{index_to_excel_column(column)}{row}"].font = bold_font
+
+
+def index_to_excel_column(index: int) -> str:
+    """Converts a numeric index (0-based) into an Excel column label."""
+    if index < 0:
+        raise ValueError("Index must be 0 or greater")
+
+    column_label = ""
+    while index >= 0:
+        column_label = chr(index % 26 + 65) + column_label
+        index = index // 26 - 1  # Move to the next "digit" in base-26, adjusting for zero-based indexing
+
+    return column_label
+
+
+def uri_data_validation(column: str) -> DataValidation:
+    # Add data validation for "URI" column
+    # We cannot assume http/https since drs, gxfiles, etc... are all fine
+    uri_validation = DataValidation(type="custom", formula1=f'=ISNUMBER(FIND("://", {column}2))', allow_blank=True)
+    uri_validation.error = "Invalid URI"
+    uri_validation.errorTitle = "Error"
+    uri_validation.showErrorMessage = True
+    return uri_validation
+
+
+def add_column_validation(column: str, data_validation: DataValidation, worksheet: Worksheet):
+    data_validation.add(f"{column}2:{column}1048576")
+    worksheet.add_data_validation(data_validation)
+
+
+def set_column_width(worksheet: Worksheet, column_index: int, width: int):
+    worksheet.column_dimensions[index_to_excel_column(column_index)].width = width
+
+
+def load_workbook_from_base64(content: str) -> Workbook:
+    decoded_content = base64.b64decode(content)
+    file_like = BytesIO(decoded_content)
+    try:
+        workbook = load_workbook(file_like, data_only=True)
+    except Exception:
+        raise RequestParameterInvalidException(
+            "The provided content is not a valid Excel file. Please check the content and try again."
+        )
+    return workbook
+
+
+def workbook_to_bytes(workbook: "Workbook") -> BytesIO:
+    output = BytesIO()
+    workbook.save(output)
+    output.seek(0)
+    return output
+
+
+@dataclass
+class HasHelp:
+    title: str
+    help: str
+
+
+@dataclass
+class HelpConfiguration:
+    instructions: List[str]
+    columns: List[HasHelp]
+    text_width: int
+    column_width: int
+    help_row_start: int = 3
+
+
+@dataclass
+class ExtraColumnsHelpConfiguration:
+    instructions: List[str]
+    text_width: int
+    column_targets: List[ColumnTarget]
+
+
+def wrap_instructions(instruction: str, help_width: int) -> List[str]:
+    return wrap(instruction, width=help_width)
+
+
+def add_extra_column_help_as_new_sheet(workbook: Workbook, extra_columns_help: ExtraColumnsHelpConfiguration) -> None:
+    worksheet = workbook.create_sheet(title="Supplying Extra Metadata")
+    worksheet.cell(row=1, column=1, value="Metadata")
+    worksheet.cell(row=1, column=2, value="Description")
+    worksheet.cell(row=1, column=3, value="Example Column Names")
+    make_bold(worksheet, 1, 0)
+    make_bold(worksheet, 1, 1)
+    make_bold(worksheet, 1, 2)
+
+    set_column_width(worksheet, 0, 40)
+    set_column_width(worksheet, 1, 140)
+    set_column_width(worksheet, 2, 40)
+
+    current_row = 2
+    for column_target in extra_columns_help.column_targets:
+        worksheet.cell(row=current_row, column=1, value=column_target.label)
+        worksheet.cell(row=current_row, column=2, value=column_target.help)
+        worksheet.cell(row=current_row, column=3, value=column_target.columnHeader or "")
+        current_row += 1
+
+    help_label_index = 6
+    current_row = 2
+    worksheet.cell(row=current_row, column=help_label_index, value="Instructions")
+    make_bold(worksheet, current_row, help_label_index - 1)
+    current_row += 1
+    write_instructions_to_sheet(
+        worksheet, extra_columns_help.instructions, current_row, help_label_index, extra_columns_help.text_width
+    )
+
+
+def add_instructions_to_sheet(worksheet: Worksheet, help_configuration: HelpConfiguration):
+    columns = help_configuration.columns
+    num_columns = len(columns)
+    help_label_index = num_columns + 2
+    # why is the width not +1
+    set_column_width(worksheet, help_label_index, help_configuration.column_width)
+
+    help_start_row = help_configuration.help_row_start
+    current_row = help_start_row
+
+    worksheet.cell(row=current_row, column=help_label_index, value="Instructions")
+    make_bold(worksheet, current_row, help_label_index - 1)
+
+    current_row += 1
+
+    current_row = write_instructions_to_sheet(
+        worksheet, help_configuration.instructions, current_row, help_label_index, help_configuration.text_width
+    )
+
+    current_row += 2
+    worksheet.cell(row=current_row, column=help_label_index, value="Columns")
+    make_bold(worksheet, current_row, help_label_index - 1)
+    current_row += 1
+
+    for column in columns:
+        worksheet.cell(row=current_row, column=help_label_index, value=column.title)
+        worksheet.cell(row=current_row, column=help_label_index + 1, value=column.help)
+        current_row += 1
+
+
+def write_instructions_to_sheet(
+    worksheet: Worksheet, instructions: List[str], start_row: int, help_label_index: int, help_width: int
+) -> int:
+    current_row = start_row
+
+    for instruction_index, instruction in enumerate(instructions):
+        worksheet.cell(row=current_row, column=help_label_index, value=f"> {instruction_index + 1}.")
+        instruction_lines = wrap_instructions(instruction, help_width)
+        for line in instruction_lines:
+            worksheet.cell(row=current_row, column=help_label_index + 1, value=line)
+            current_row += 1
+
+    return current_row
+
+
+def read_column_header_titles(worksheet: Worksheet) -> List[str]:
+    """Read the first row of the worksheet and return a list of these column titles."""
+    index = 1
+    titles: List[str] = []
+    while True:
+        value = worksheet.cell(1, index).value
+        if not value:
+            break
+        titles.append(value)
+        index += 1
+    return titles

lib/galaxy/tools/fetch/workbooks.py

@@ -0,0 +1,467 @@
+from dataclasses import dataclass
+from typing import (
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from openpyxl import Workbook
+from openpyxl.worksheet.worksheet import Worksheet
+from pydantic import (
+    BaseModel,
+    Field,
+)
+from typing_extensions import Literal
+
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.model.dataset_collections.auto_identifiers import (
+    fill_in_identifiers,
+    FillIdentifiers,
+)
+from galaxy.model.dataset_collections.auto_pairing import paired_element_list_identifier
+from galaxy.model.dataset_collections.rule_target_columns import (
+    column_titles_to_headers,
+    HeaderColumn,
+    implied_paired_or_unpaired_column_header,
+    ParsedColumn,
+)
+from galaxy.model.dataset_collections.rule_target_models import (
+    COMMON_COLUMN_TARGETS,
+)
+from galaxy.model.dataset_collections.workbook_util import (
+    add_column_validation,
+    add_extra_column_help_as_new_sheet,
+    add_instructions_to_sheet,
+    Base64StringT,
+    ExtraColumnsHelpConfiguration,
+    freeze_header_row,
+    HasHelp,
+    HelpConfiguration,
+    load_workbook_from_base64,
+    make_headers_bold,
+    read_column_header_titles,
+    set_column_width,
+    uri_data_validation,
+)
+
+FetchWorkbookType = Literal["datasets", "collection", "collections"]
+FetchWorkbookCollectionType = Literal["list", "list:paired", "list:list", "list:list:paired", "list:paired_or_unpaired"]
+
+DEFAULT_FETCH_WORKBOOK_TYPE: FetchWorkbookType = "datasets"
+DEFAULT_FETCH_WORKBOOK_COLLECTION_TYPE: FetchWorkbookCollectionType = "list"
+DEFAULT_WORKBOOK_TITLE: str = "Galaxy Data Import"
+
+INSTRUCTION_USE_THIS = "Use this spreadsheet to describe your datasets. For each dataset (i.e. each file), ensure all the labeled columns are specified and correct."
+INSTRUCTION_EXTRA_COLUMNS = "Additional columns can be added if we you wish to specify additional metadata."  # mention second tab when implemented
+INSTRUCTION_ONCE_COMPLETE_DATASETS = (
+    "Once data entry is complete, drop this file back into Galaxy to finish creating datasets for your inputs."
+)
+INSTRUCTION_ONCE_COMPLETE_COLLECTION = (
+    "Once data entry is complete, drop this file back into Galaxy to finish creating a collection for your inputs."
+)
+INSTRUCTION_ONCE_COMPLETE_COLLECTIONS = (
+    "Once data entry is complete, drop this file back into Galaxy to finish creating collections for your inputs."
+)
+INSTRUCTIONS_BY_TYPE: Dict[FetchWorkbookType, List[str]] = {
+    "datasets": [
+        INSTRUCTION_USE_THIS,
+        INSTRUCTION_EXTRA_COLUMNS,
+        INSTRUCTION_ONCE_COMPLETE_DATASETS,
+    ],
+    "collection": [
+        INSTRUCTION_USE_THIS,
+        INSTRUCTION_EXTRA_COLUMNS,
+        INSTRUCTION_ONCE_COMPLETE_COLLECTION,
+    ],
+    "collections": [
+        INSTRUCTION_USE_THIS,
+        INSTRUCTION_EXTRA_COLUMNS,
+        INSTRUCTION_ONCE_COMPLETE_COLLECTIONS,
+    ],
+}
+
+EXCEPTION_TOO_MANY_URI_COLUMNS = "Too many URI/URL columns in the supplied workbook."
+EXCEPTION_NO_URIS_FOUND = "Failed to find any URI or URL like column in the supplied workbook."
+EXTRA_COLUMN_INSTRUCTIONS = [
+    "Extra metadata for the uploaded datasets can be specified by just adding columns with special headers to the first sheet of this workbook.",
+    "The columns can appear in order on the first sheet of the workbook and the names are fairly flexible - extra white space and casing do not matter.",
+    "The list of supported column metadata types appear in this help sheet to the left along with example column names Galaxy would support.",
+]
+
+
+@dataclass
+class GenerateFetchWorkbookRequest:
+    type: FetchWorkbookType = DEFAULT_FETCH_WORKBOOK_TYPE
+    collection_type: FetchWorkbookCollectionType = DEFAULT_FETCH_WORKBOOK_COLLECTION_TYPE
+    title: str = DEFAULT_WORKBOOK_TITLE
+
+
+WorkbookContentField: Base64StringT = Field(
+    ...,
+    max_length=1024 * 1024 * 10,  # 10MB
+    title="Workbook Content (Base 64 encoded)",
+    description="The workbook content (the contents of the xlsx file) that have been base64 encoded.",
+)
+
+
+class ParseFetchWorkbook(BaseModel):
+    content: Base64StringT = WorkbookContentField
+    fill_identifiers: Optional[FillIdentifiers] = None
+
+
+def header_column_to_parsed_column(header_column: HeaderColumn) -> "ParsedColumn":
+    return header_column.parsed_column
+
+
+def generate(request: GenerateFetchWorkbookRequest) -> Workbook:
+    workbook = Workbook()
+    worksheet = workbook.active
+    worksheet.title = request.title
+    header_columns = _request_to_columns(request)
+
+    header_titles = [h.title for h in header_columns]
+    worksheet.append(header_titles)
+    make_headers_bold(worksheet, header_titles)
+    freeze_header_row(worksheet)
+    for i, column in enumerate(header_columns):
+        set_column_width(worksheet, i, column.width)
+
+    uri_data_validator = uri_data_validation("A")
+    add_column_validation("A", uri_data_validator, worksheet)
+
+    columns_for_help = [HasHelp(c.title, c.help) for c in header_columns]
+    help_configuration = HelpConfiguration(
+        instructions=INSTRUCTIONS_BY_TYPE[request.type],
+        columns=columns_for_help,
+        text_width=50,
+        column_width=50,
+    )
+    add_instructions_to_sheet(
+        worksheet,
+        help_configuration,
+    )
+    extra_column_configuration = ExtraColumnsHelpConfiguration(
+        EXTRA_COLUMN_INSTRUCTIONS, text_width=50, column_targets=COMMON_COLUMN_TARGETS
+    )
+    add_extra_column_help_as_new_sheet(workbook, extra_column_configuration)
+    return workbook
+
+
+ParsedRow = Dict[str, Optional[str]]
+ParsedRows = List[ParsedRow]
+
+
+class ParseLogEntry(BaseModel):
+    message: str
+
+
+class SplitUpPairedDataLogEntry(ParseLogEntry):
+    message: str
+    old_forward_column: ParsedColumn
+    old_reverse_column: ParsedColumn
+    new_paired_status_column: int
+
+
+class InferredCollectionTypeLogEntry(ParseLogEntry):
+    message: str
+    from_columns: List[ParsedColumn]
+
+
+ParseLog = List[ParseLogEntry]
+
+
+class BaseParsedFetchWorkbook(BaseModel):
+    rows: ParsedRows
+    columns: List[ParsedColumn]
+    workbook_type: FetchWorkbookType
+    parse_log: ParseLog
+
+
+class ParsedFetchWorkbookForDatasets(BaseParsedFetchWorkbook):
+    workbook_type: FetchWorkbookType = "datasets"
+
+
+class ParsedFetchWorkbookForCollections(BaseParsedFetchWorkbook):
+    workbook_type: FetchWorkbookType = "collection"
+    collection_type: FetchWorkbookCollectionType
+
+
+ParsedFetchWorkbook = Union[ParsedFetchWorkbookForDatasets, ParsedFetchWorkbookForCollections]
+
+
+def parse(payload: ParseFetchWorkbook) -> ParsedFetchWorkbook:
+    parse_log: ParseLog = []
+    workbook = load_workbook_from_base64(payload.content)
+    column_headers = _read_column_headers(workbook.active)
+    _validate_parsed_column_headers(column_headers)
+    raw_rows = _load_row_data(workbook, payload)
+    # the rule builder does require splitting the paired data in this way but it might
+    # be worth it to do it with an "initial rule" instead to demo how you'd do it
+    # with actual rule builder rules? Not sure.
+    rows, column_headers, split_data_log_entry = _split_paired_data_if_needed(raw_rows, column_headers)
+    if split_data_log_entry:
+        parse_log.append(split_data_log_entry)
+    columns = [ParsedColumn(title=c.title, type=c.type, type_index=c.type_index) for c in column_headers]
+    if _is_fetch_workbook_for_collections(column_headers):
+        collection_type, log_entry = _infer_fetch_workbook_collection_type(column_headers)
+        parse_log.append(log_entry)
+        assert collection_type in ["list", "list:paired", "list:list", "list:list:paired", "list:paired_or_unpaired"]
+        rows = _fill_in_identifier_column_if_needed(rows, column_headers, payload.fill_identifiers)
+        return ParsedFetchWorkbookForCollections(
+            collection_type=collection_type, rows=rows, columns=columns, parse_log=parse_log
+        )
+    else:
+        return ParsedFetchWorkbookForDatasets(rows=rows, columns=columns, parse_log=parse_log)
+
+
+def _validate_parsed_column_headers(column_headers: List[HeaderColumn]) -> None:
+    uri_like_columns = _uri_like_columns(column_headers)
+    if len(uri_like_columns) > 2:
+        raise RequestParameterInvalidException(
+            f"{EXCEPTION_TOO_MANY_URI_COLUMNS}. Relevant headers are {[c.title for c in uri_like_columns]}"
+        )
+    if len(uri_like_columns) == 0:
+        raise RequestParameterInvalidException(EXCEPTION_NO_URIS_FOUND)
+
+
+def _request_to_columns(request: GenerateFetchWorkbookRequest) -> List[HeaderColumn]:
+    if request.type == "datasets":
+        return [
+            HeaderColumn("url", "URI", 0),
+            HeaderColumn("name", "Name", 0),
+        ]
+    else:
+        if request.collection_type == "list":
+            header_columns = [
+                HeaderColumn("url", "URI", 0),
+                HeaderColumn("list_identifiers", "List Identifier", 0),
+            ]
+        elif request.collection_type == "list:paired":
+            header_columns = [
+                HeaderColumn("url", "URI 1 (Forward)", 0),
+                HeaderColumn("url", "URI 2 (Reverse)", 1),
+                HeaderColumn("list_identifiers", "List Identifier", 0),
+            ]
+        elif request.collection_type == "list:list":
+            header_columns = [
+                HeaderColumn("url", "URI", 0),
+                HeaderColumn("list_identifiers", "Outer List Identifier", 0),
+                HeaderColumn("list_identifiers", "Inner List Identifier", 1),
+            ]
+        elif request.collection_type == "list:list:paired":
+            header_columns = [
+                HeaderColumn("url", "URI 1 (Forward)", 0),
+                HeaderColumn("url", "URI 2 (Reverse)", 1),
+                HeaderColumn("list_identifiers", "Outer List Identifier", 0),
+                HeaderColumn("list_identifiers", "Inner List Identifier", 1),
+            ]
+        elif request.collection_type == "list:paired_or_unpaired":
+            header_columns = [
+                HeaderColumn("url", "URI 1 (Forward)", 0),
+                HeaderColumn("url", "URI 2 (Optional/Reverse)", 1),
+                HeaderColumn("list_identifiers", "List Identifier", 0),
+            ]
+        else:
+            raise NotImplementedError()
+        if request.type == "collections":
+            header_columns.append(
+                HeaderColumn(
+                    type="collection_name",
+                    title="Collection Name",
+                    type_index=0,
+                )
+            )
+        return header_columns
+
+
+def _load_row_data(workbook: Workbook, payload: ParseFetchWorkbook) -> ParsedRows:
+    sheet = workbook.active  # Get the first sheet
+
+    rows: ParsedRows = []
+
+    column_headers = _read_column_headers(sheet)
+    columns_to_read = len(column_headers)
+    index_of_first_uri = _index_of_fist_uri_column(column_headers)
+
+    for row_index, row in enumerate(sheet.iter_rows(max_col=columns_to_read, values_only=True)):
+        if row_index == 0:  # skip column headers
+            continue
+        if not row[index_of_first_uri]:
+            break
+        parsed_row: ParsedRow = {}
+        for value, column in zip(row, column_headers):
+            parsed_row[column.name] = value
+        rows.append(parsed_row)
+    return rows
+
+
+def _split_paired_data_if_needed(
+    rows: ParsedRows, column_headers: List[HeaderColumn]
+) -> Tuple[ParsedRows, List[HeaderColumn], Optional[SplitUpPairedDataLogEntry]]:
+    split_rows: ParsedRows = []
+    uri_like_columns = _uri_like_columns(column_headers)
+    if len(_uri_like_columns(column_headers)) != 2:
+        return rows, column_headers, None
+
+    hash_columns_to_split: List[Tuple[HeaderColumn, HeaderColumn]] = []
+    for column_type in ["hash_sha1", "hash_md5", "hash_sha256", "hash_sha512"]:
+        hash_columns = [c for c in column_headers if c.type == column_type]
+        if len(hash_columns) == 2:
+            hash_columns_to_split.append((hash_columns[0], hash_columns[1]))
+
+    url_column_0 = uri_like_columns[0]
+    url_column_1 = uri_like_columns[1]
+
+    # Split the data up
+    split_rows = []
+    for row in rows:
+        row_to_copy = row.copy()
+
+        url_0 = row_to_copy.pop(url_column_0.name)
+        url_1 = row_to_copy.pop(url_column_1.name)
+
+        # before we break up the pairs, do any implicit identifier filling in
+        list_identifier_columns = [c for c in column_headers if c.type == "list_identifiers"]
+        if len(list_identifier_columns) > 0:
+            inner_list_identifier_column = list_identifier_columns[-1]
+            list_identifier = row_to_copy[inner_list_identifier_column.name]
+            if not list_identifier and url_0 and url_1:
+                list_identifier = paired_element_list_identifier(url_0, url_1)
+                row_to_copy[inner_list_identifier_column.name] = list_identifier
+
+        row_0 = row_to_copy.copy()
+        row_1 = row_to_copy.copy()
+        row_0[url_column_0.name] = url_0
+        row_1[url_column_0.name] = url_1
+        row_0["paired_identifier"] = "1"
+        row_1["paired_identifier"] = "2"
+
+        for hash_column_0, hash_column_1 in hash_columns_to_split:
+            # remove all the hashes from copied row before we...
+            row_0.pop(hash_column_0.name, None)
+            row_1.pop(hash_column_0.name, None)
+            row_0.pop(hash_column_1.name, None)
+            row_1.pop(hash_column_1.name, None)
+
+            # ... merge both hash columns in the original into one column in the duplicates
+            row_0[hash_column_0.name] = row.pop(hash_column_0.name)
+            row_1[hash_column_0.name] = row.pop(hash_column_1.name)
+
+        split_rows.append(row_0)
+        split_rows.append(row_1)
+
+    # Adjust the columns accordingly
+    column_headers.remove(url_column_1)
+    for _, hash_column_1 in hash_columns_to_split:
+        column_headers.remove(hash_column_1)
+
+    column_headers.append(
+        HeaderColumn(
+            type="paired_identifier",
+            title="Paired Identifier",
+            type_index=0,
+        )
+    )
+    split_log_entry = SplitUpPairedDataLogEntry(
+        message="Merged paired data URIs columns into single column and added paired identifier column.",
+        old_forward_column=header_column_to_parsed_column(url_column_0),
+        old_reverse_column=header_column_to_parsed_column(url_column_1),
+        new_paired_status_column=len(column_headers) - 1,
+    )
+    return split_rows, column_headers, split_log_entry
+
+
+def _fill_in_identifier_column_if_needed(
+    rows: ParsedRows, columns: List[HeaderColumn], config: Optional[FillIdentifiers]
+) -> ParsedRows:
+    list_identifiers_columns = [c for c in columns if c.type == "list_identifiers"]
+    uri_columns = _uri_like_columns(columns)
+    if len(uri_columns) == 0 or len(list_identifiers_columns) == 0:
+        # this doesn't look like a list creation, don't need identifiers to be filled in
+        return rows
+    if config is None or not config.fill_inner_list_identifiers:
+        # no filling in identifiers requested, just return the rows as is
+        return rows
+
+    uri_column = uri_columns[0]
+    inner_list_identifier_column = list_identifiers_columns[-1]
+    uris_to_identifiers: List[Tuple[str, Optional[str]]] = []
+    for row in rows:
+        uri = row.get(uri_column.name)
+        if not uri:
+            continue
+        list_identifier = row.get(inner_list_identifier_column.name)
+        uris_to_identifiers.append((uri, list_identifier))
+    new_identifiers = fill_in_identifiers(uris_to_identifiers, config)
+    for row, new_identifier in zip(rows, new_identifiers):
+        row[inner_list_identifier_column.name] = new_identifier
+    return rows
+
+
+def _read_column_headers(worksheet: Worksheet) -> List[HeaderColumn]:
+    column_titles = read_column_header_titles(worksheet)
+    return column_titles_to_headers(column_titles)
+
+
+def _uri_like_columns(column_headers: List[HeaderColumn]) -> List[HeaderColumn]:
+    return [c for c in column_headers if c.type == "url" or c.type == "url_deferred"]
+
+
+def _index_of_fist_uri_column(column_headers: List[HeaderColumn]) -> int:
+    for index, column_header in enumerate(column_headers):
+        if column_header.type == "url" or column_header.type == "url_deferred":
+            return index
+    raise RequestParameterInvalidException(
+        f"{EXCEPTION_NO_URIS_FOUND}. Relevant headers are {[c.title for c in column_headers]}"
+    )
+
+
+def _infer_fetch_workbook_collection_type(
+    column_headers: List[HeaderColumn],
+) -> Tuple[str, InferredCollectionTypeLogEntry]:
+    paired_identifier_columns = [c for c in column_headers if c.type == "paired_identifier"]
+    paired_or_unpaired_identifier_columns = [c for c in column_headers if c.type == "paired_or_unpaired_identifier"]
+    any_paired = len(paired_identifier_columns) > 0
+    uri_columns = _uri_like_columns(column_headers)
+    num_uris = len(uri_columns)
+
+    inference_on_columns: List[ParsedColumn] = []
+
+    list_type: str = ""
+    for column_header in column_headers:
+        if column_header.type == "list_identifiers":
+            inference_on_columns.append(header_column_to_parsed_column(column_header))
+            if list_type:
+                list_type = f"list:{list_type}"
+            else:
+                list_type = "list"
+    if any_paired or num_uris == 2:
+        if num_uris == 2:
+            inference_on_columns.append(header_column_to_parsed_column(uri_columns[0]))
+            inference_on_columns.append(header_column_to_parsed_column(uri_columns[1]))
+            if implied_paired_or_unpaired_column_header(uri_columns[1]):
+                collection_type = f"{list_type}:paired_or_unpaired"
+            else:
+                collection_type = f"{list_type}:paired"
+        else:
+            paired_identifier_column = paired_identifier_columns[0]
+            inference_on_columns.append(header_column_to_parsed_column(paired_identifier_column))
+            if implied_paired_or_unpaired_column_header(paired_identifier_column):
+                collection_type = f"{list_type}:paired_or_unpaired"
+            else:
+                collection_type = f"{list_type}:paired"
+    elif len(paired_or_unpaired_identifier_columns) > 0:
+        inference_on_columns.append(header_column_to_parsed_column(paired_or_unpaired_identifier_columns[0]))
+        collection_type = f"{list_type}:paired_or_unpaired"
+    else:
+        collection_type = list_type
+    return collection_type, InferredCollectionTypeLogEntry(
+        message="Inferred collection type from column headers.",
+        from_columns=inference_on_columns,
+    )
+
+
+def _is_fetch_workbook_for_collections(column_headers: List[HeaderColumn]) -> bool:
+    return _infer_fetch_workbook_collection_type(column_headers)[0] != ""

lib/galaxy/util/rules_dsl.py

@@ -596,6 +596,8 @@ class RuleSet:
             identifier_columns.extend(mapping_as_dict["list_identifiers"]["columns"])
         if "paired_identifier" in mapping_as_dict:
             identifier_columns.append(mapping_as_dict["paired_identifier"]["columns"][0])
+        if "paired_or_unpaired_identifier" in mapping_as_dict:
+            identifier_columns.append(mapping_as_dict["paired_or_unpaired_identifier"]["columns"][0])
 
         return identifier_columns
 
@@ -609,6 +611,11 @@ class RuleSet:
                 collection_type += ":paired"
             else:
                 collection_type = "paired"
+        if "paired_or_unpaired_identifier" in mapping_as_dict:
+            if collection_type:
+                collection_type += ":paired_or_unpaired"
+            else:
+                collection_type = "paired_or_unpaired"
         return collection_type
 
     @property

lib/galaxy/webapps/galaxy/api/common.py

@@ -1,5 +1,6 @@
 """This module contains utility functions shared across the api package."""
 
+from io import BytesIO
 from typing import (
     Any,
     List,
@@ -13,6 +14,7 @@ from fastapi import (
     Query,
     Request,
 )
+from starlette.responses import StreamingResponse
 from typing_extensions import Annotated
 
 from galaxy.schema import (
@@ -294,3 +296,12 @@ def query_parameter_as_list(query):
         return elements
 
     return parse_elements
+
+
+def serve_workbook(content: BytesIO, filename: Optional[str]) -> StreamingResponse:
+    filename = filename or "galaxy_sample_sheet_workbook.xlsx"
+    return StreamingResponse(
+        content,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers={"Content-Disposition": f"attachment; filename={filename}"},
+    )

lib/galaxy/webapps/galaxy/api/tools.py

@@ -16,12 +16,14 @@ from typing import (
 from fastapi import (
     Body,
     Depends,
+    Query,
     Request,
     Response,
     UploadFile,
 )
 from fastapi.responses import FileResponse
 from starlette.datastructures import UploadFile as StarletteUploadFile
+from starlette.responses import StreamingResponse
 
 from galaxy import (
     exceptions,
@@ -33,6 +35,7 @@ from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.context import ProvidesHistoryContext
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.histories import HistoryManager
+from galaxy.model.dataset_collections.workbook_util import workbook_to_bytes
 from galaxy.schema.fetch_data import (
     FetchDataFormPayload,
     FetchDataPayload,
@@ -40,6 +43,15 @@ from galaxy.schema.fetch_data import (
 from galaxy.tool_util.verify import ToolTestDescriptionDict
 from galaxy.tool_util_models import UserToolSource
 from galaxy.tools.evaluation import global_tool_errors
+from galaxy.tools.fetch.workbooks import (
+    FetchWorkbookCollectionType,
+    FetchWorkbookType,
+    generate,
+    GenerateFetchWorkbookRequest,
+    parse,
+    ParsedFetchWorkbook,
+    ParseFetchWorkbook,
+)
 from galaxy.util.hash_util import (
     HashFunctionNameEnum,
     memory_bound_hexdigest,
@@ -53,6 +65,7 @@ from galaxy.web import (
 )
 from galaxy.webapps.base.controller import UsesVisualizationMixin
 from galaxy.webapps.base.webapp import GalaxyWebTransaction
+from galaxy.webapps.galaxy.api.common import serve_workbook
 from galaxy.webapps.galaxy.services.tools import ToolsService
 from . import (
     APIContentTypeRoute,
@@ -86,6 +99,22 @@ class PNGIconResponse(FileResponse):
     media_type = "image/png"
 
 
+FetchWorkbookTypeQueryParam: FetchWorkbookType = Query(
+    default="datasets",
+    title="Workbook Type",
+    description="Generate a workbook for simple datasets or a collection.",
+)
+FetchWorkbookCollectionTypeQueryParam: FetchWorkbookCollectionType = Query(
+    default="list",
+    title="Collection Type",
+    description="Generate workbook for specified collection type (not all collection types are supported)",
+)
+FetchWorkbookFilenameQueryParam: Optional[str] = Query(
+    None,
+    description="Filename of the workbook download to generate",
+)
+
+
 router = Router(tags=["tools"])
 
 FetchDataForm = as_form(FetchDataFormPayload)
@@ -123,6 +152,39 @@ class FetchTools:
     ):
         return self.service.create_fetch(trans, payload, files)
 
+    @router.get(
+        "/api/tools/fetch/workbook",
+        summary="Generate a template workbook to use with the activity builder UI",
+        response_class=StreamingResponse,
+        operation_id="tools__fetch_workbook_download",
+    )
+    def fetch_workbook(
+        self,
+        trans: ProvidesHistoryContext = DependsOnTrans,
+        type: FetchWorkbookType = FetchWorkbookTypeQueryParam,
+        collection_type: FetchWorkbookCollectionType = FetchWorkbookCollectionTypeQueryParam,
+        filename: Optional[str] = FetchWorkbookFilenameQueryParam,
+    ):
+        generate_request = GenerateFetchWorkbookRequest(
+            type=type,
+            collection_type=collection_type,
+        )
+        workbook = generate(generate_request)
+        contents = workbook_to_bytes(workbook)
+        return serve_workbook(contents, filename)
+
+    @router.post(
+        "/api/tools/fetch/workbook/parse",
+        summary="Generate a template workbook to use with the activity builder UI",
+        operation_id="tools__fetch_workbook_parse",
+    )
+    def parse_workbook(
+        self,
+        trans: ProvidesHistoryContext = DependsOnTrans,
+        payload: ParseFetchWorkbook = Body(...),
+    ) -> ParsedFetchWorkbook:
+        return parse(payload)
+
     @router.get(
         "/api/tools/{tool_id:path}/icon",
         summary="Get the icon image associated with a tool",

lib/galaxy_test/api/test_tools.py

@@ -23,6 +23,8 @@ from galaxy.util import galaxy_root_path
 from galaxy.util.unittest_utils import skip_if_github_down
 from galaxy_test.base import rules_test_data
 from galaxy_test.base.api_asserts import (
+    assert_error_code_is,
+    assert_file_looks_like_xlsx,
     assert_has_keys,
     assert_status_code_is,
 )
@@ -412,6 +414,10 @@ class TestToolsApi(ApiTestCase, TestsTools):
             )
             assert sanitized_address.strip() == cool_name_without_quote
 
+    def test_fetch_workbook_generate(self):
+        workbook_path = self.dataset_populator.download_fetch_workbook()
+        assert_file_looks_like_xlsx(workbook_path)
+
     @skip_without_tool("composite_output")
     def test_test_data_filepath_security(self):
         test_data_response = self._get("tools/composite_output/test_data_path?filename=../CONTRIBUTORS.md", admin=True)
@@ -876,6 +882,55 @@ class TestToolsApi(ApiTestCase, TestsTools):
             output_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=output_hid, wait=False)
             example["check"](output_hdca, self.dataset_populator)
 
+    def test_apply_rules_with_error_in_mapping(self):
+        # this would produce a list:paired but the child identifiers are incorrectly spelled
+        example_with_mapping_error = {
+            "rules": {
+                "rules": [
+                    {
+                        "type": "add_column_metadata",
+                        "value": "identifier0",
+                    },
+                    {
+                        "type": "add_column_metadata",
+                        "value": "identifier1",
+                    },
+                ],
+                "mapping": [
+                    {
+                        "type": "list_identifiers",
+                        "columns": [0],
+                    },
+                    {
+                        "type": "paired_identifier",
+                        "columns": [1],
+                    },
+                ],
+            },
+            "test_data": {
+                "type": "list:list",
+                "elements": [
+                    {
+                        "identifier": "sample1",
+                        "elements": [
+                            {"identifier": "floorward", "class": "File", "contents": "TestData123forward"},
+                            {"identifier": "reverb", "class": "File", "contents": "TestData123reverse"},
+                        ],
+                    },
+                ],
+            },
+        }
+        with self.dataset_populator.test_history(require_new=False) as history_id:
+            inputs = stage_rules_example(self.galaxy_interactor, history_id, example_with_mapping_error)
+            hdca = inputs["input"]
+            inputs = {"input": {"src": "hdca", "id": hdca["id"]}, "rules": example_with_mapping_error["rules"]}
+
+            self.dataset_populator.wait_for_history(history_id)
+            response = self._run("__APPLY_RULES__", history_id, inputs, assert_ok=False)
+            assert_status_code_is(response, 400)
+            assert_error_code_is(response, 400008)
+            assert "Unknown indicator of paired status encountered (floorward)" in response.json()["err_msg"]
+
     def test_apply_rules_flatten_paired_unpaired(self):
         self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_PAIRED_OR_UNPAIRED)
 
@@ -897,6 +952,9 @@ class TestToolsApi(ApiTestCase, TestsTools):
     def test_apply_rules_6(self):
         self._apply_rules_and_check(rules_test_data.EXAMPLE_6)
 
+    def test_apply_rules_create_paired_or_unpaired_list(self):
+        self._apply_rules_and_check(rules_test_data.EXAMPLE_CREATE_PAIRED_OR_UNPAIRED_COLLECTION)
+
     def test_apply_rules_flatten_with_indices(self):
         self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_USING_INDICES)
 

lib/galaxy_test/base/api_asserts.py

@@ -113,3 +113,11 @@ def _as_dict(response: Union[Response, dict]) -> Dict[str, Any]:
 
 
 assert_has_key = assert_has_keys
+
+
+def assert_file_looks_like_xlsx(xlsx_file: str):
+    # Check the file header
+    with open(xlsx_file, "rb") as file:
+        header = file.read(4)
+        # The ZIP file signature is 0x50 0x4B 0x03 0x04
+        return header == b"\x50\x4b\x03\x04"

lib/galaxy_test/base/populators.py

@@ -420,8 +420,11 @@ class BasePopulator(metaclass=ABCMeta):
         """Perform a _get and store the result in a tempfile."""
         get_response = self._get(route, **kwd)
         get_response.raise_for_status()
+        return self._get_response_to_tempfile(get_response)
+
+    def _get_response_to_tempfile(self, response, suffix=None) -> str:
         temp_file = tempfile.NamedTemporaryFile("wb", delete=False, suffix=suffix)
-        temp_file.write(get_response.content)
+        temp_file.write(response.content)
         temp_file.flush()
         return temp_file.name
 
@@ -1108,6 +1111,12 @@ class BaseDatasetPopulator(BasePopulator):
         response.raise_for_status()
         return response.json()
 
+    def download_fetch_workbook(self) -> str:
+        url = "tools/fetch/workbook"
+        download_response = self._get(url)
+        api_asserts.assert_status_code_is_ok(download_response)
+        return self._get_response_to_tempfile(download_response)
+
     def run_tool_payload(self, tool_id: Optional[str], inputs: dict, history_id: str, **kwds) -> dict:
         # Remove files_%d|file_data parameters from inputs dict and attach
         # as __files dictionary.

lib/galaxy_test/base/rules_test_data.py

@@ -348,6 +348,100 @@ EXAMPLE_FLATTEN_USING_INDICES = {
 }
 
 
+def check_example_create_paired_or_unpaired_with_unmatched_forward_becoming_unpaired(hdca, dataset_populator):
+    assert hdca["collection_type"] == "list:paired_or_unpaired"
+    assert hdca["element_count"] == 4
+    sample1_el = hdca["elements"][0]
+    assert "object" in sample1_el, hdca
+    assert "element_identifier" in sample1_el
+    assert sample1_el["element_identifier"] == "sample1", hdca
+    child_collection_level = sample1_el["object"]
+    assert child_collection_level["collection_type"] == "paired_or_unpaired"
+    assert child_collection_level["elements"][0]["element_identifier"] == "forward", hdca
+
+    sample2_el = hdca["elements"][1]
+    assert "object" in sample2_el, hdca
+    assert "element_identifier" in sample2_el
+    assert sample2_el["element_identifier"] == "sample2", hdca
+    child_collection_level = sample2_el["object"]
+    assert child_collection_level["collection_type"] == "paired_or_unpaired"
+    assert child_collection_level["elements"][0]["element_identifier"] == "unpaired", hdca
+
+    sample3_el = hdca["elements"][2]
+    assert "object" in sample3_el, hdca
+    assert "element_identifier" in sample3_el
+    assert sample3_el["element_identifier"] == "sample3", hdca
+    child_collection_level = sample3_el["object"]
+    assert child_collection_level["collection_type"] == "paired_or_unpaired"
+    assert child_collection_level["elements"][0]["element_identifier"] == "unpaired", hdca
+
+    sample4_el = hdca["elements"][2]
+    assert "object" in sample4_el, hdca
+    assert "element_identifier" in sample4_el
+    assert sample4_el["element_identifier"] == "sample3", hdca
+    child_collection_level = sample4_el["object"]
+    assert child_collection_level["collection_type"] == "paired_or_unpaired"
+    assert child_collection_level["elements"][0]["element_identifier"] == "unpaired", hdca
+
+
+EXAMPLE_CREATE_PAIRED_OR_UNPAIRED_COLLECTION = {
+    "rules": {
+        "rules": [
+            {
+                "type": "add_column_metadata",
+                "value": "identifier0",
+            },
+            {
+                "type": "add_column_metadata",
+                "value": "identifier1",
+            },
+        ],
+        "mapping": [
+            {
+                "type": "list_identifiers",
+                "columns": [0],
+            },
+            {
+                "type": "paired_or_unpaired_identifier",
+                "columns": [1],
+            },
+        ],
+    },
+    "test_data": {
+        "type": "list:list",
+        "elements": [
+            {
+                "identifier": "sample1",
+                "elements": [
+                    {"identifier": "forward", "class": "File", "contents": "TestData123forward"},
+                    {"identifier": "reverse", "class": "File", "contents": "TestData123reverse"},
+                ],
+            },
+            {
+                "identifier": "sample2",
+                "elements": [
+                    {"identifier": "unpaired", "class": "File", "contents": "TestData123unpaired"},
+                ],
+            },
+            {
+                "identifier": "sample3",
+                "elements": [
+                    {"identifier": "u", "class": "File", "contents": "TestData123unpaired-2"},
+                ],
+            },
+            {
+                "identifier": "sample4",
+                "elements": [
+                    {"identifier": "forward", "class": "File", "contents": "TestData123unpaired-3"},
+                ],
+            },
+        ],
+    },
+    "check": check_example_create_paired_or_unpaired_with_unmatched_forward_becoming_unpaired,
+    "output_hid": 12,
+}
+
+
 def check_example_flatten_paired_or_unpaired(hdca, dataset_populator):
     assert hdca["collection_type"] == "list"
     assert hdca["element_count"] == 3

lib/galaxy_test/selenium/test_tool_form.py

@@ -392,6 +392,12 @@ class TestLoggedInToolForm(SeleniumTestCase):
         self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_PAIRED_OR_UNPAIRED)
         self.screenshot("tool_apply_rules_example_flatten_paired_unpaired_final")
 
+    @selenium_test
+    @managed_history
+    def test_run_apply_rules_create_paired_or_unpaired_list(self):
+        self._apply_rules_and_check(rules_test_data.EXAMPLE_CREATE_PAIRED_OR_UNPAIRED_COLLECTION)
+        self.screenshot("tool_apply_rules_example_flatten_paired_unpaired_final")
+
     @selenium_test
     def test_run_apply_rules_flatten_with_indices(self):
         self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_USING_INDICES)

packages/data/MANIFEST.in

@@ -3,6 +3,7 @@ include galaxy/datatypes/set_metadata_tool.xml
 include galaxy/datatypes/converters/*.xml
 graft galaxy/datatypes/display_applications/configs/
 graft galaxy/datatypes/display_applications/xsd/
+include galaxy/model/dataset_collections/rule_targets.yml
 include galaxy/model/migrations/alembic.ini
 recursive-include galaxy/model/migrations/alembic/ *.py
 include galaxy/model/migrations/README.md

packages/data/setup.cfg

@@ -49,6 +49,7 @@ install_requires =
     msal
     mrcfile
     numpy
+    openpyxl>=3.1.5
     parsley
     pycryptodome
     pydantic[email]>=2.7.4

pyproject.toml

@@ -62,6 +62,7 @@ dependencies = [
     "nodeenv",
     "numpy>=1.26.0",  # Python 3.12 support
     "numpy>=2.1.0 ; python_version>='3.10'",  # Python 3.13 support
+    "openpyxl>=3.1.5",  # Minimum version ever tested with
     "packaging",
     "paramiko!=2.9.0,!=2.9.1",  # https://github.com/paramiko/paramiko/issues/1961
     "Parsley",

test/unit/app/tools/test_fetch_workbooks.py

@@ -0,0 +1,314 @@
+import base64
+import os
+
+import pytest
+
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.model.dataset_collections.auto_identifiers import FillIdentifiers
+from galaxy.model.dataset_collections.rule_target_columns import column_titles_to_headers
+from galaxy.model.dataset_collections.rule_target_models import target_models
+from galaxy.model.dataset_collections.workbook_util import read_column_header_titles
+from galaxy.tools.fetch.workbooks import (
+    _infer_fetch_workbook_collection_type,
+    _validate_parsed_column_headers,
+    DEFAULT_WORKBOOK_TITLE,
+    EXCEPTION_NO_URIS_FOUND,
+    EXCEPTION_TOO_MANY_URI_COLUMNS,
+    generate,
+    GenerateFetchWorkbookRequest,
+    parse,
+    ParseFetchWorkbook,
+)
+from galaxy.util.resources import resource_path
+
+WRITE_TEST_WORKBOOKS = False
+
+
+def test_fetch_datasets_workbook():
+    request = GenerateFetchWorkbookRequest()
+    workbook = generate(request)
+    assert workbook
+    worksheet = workbook.active
+    assert worksheet.title == DEFAULT_WORKBOOK_TITLE
+    assert worksheet.cell(1, 1).value == "URI"
+
+    header_titles = read_column_header_titles(worksheet)
+    assert header_titles == ["URI", "Name"]
+
+    if WRITE_TEST_WORKBOOKS:
+        path = "~/fetch_workbook.xlsx"
+        expanded_path = os.path.expanduser(path)
+        workbook.save(expanded_path)
+
+
+def test_fetch_list_workbook():
+    request = GenerateFetchWorkbookRequest(
+        type="collection",
+        collection_type="list",
+    )
+    workbook = generate(request)
+    assert workbook
+    worksheet = workbook.active
+
+    header_titles = read_column_header_titles(worksheet)
+    assert header_titles == ["URI", "List Identifier"]
+
+
+def test_fetch_multiple_lists_workbook():
+    request = GenerateFetchWorkbookRequest(
+        type="collections",
+        collection_type="list",
+    )
+    workbook = generate(request)
+    assert workbook
+    worksheet = workbook.active
+
+    header_titles = read_column_header_titles(worksheet)
+    assert header_titles == ["URI", "List Identifier", "Collection Name"]
+
+
+def test_fetch_list_paired_workbook():
+    request = GenerateFetchWorkbookRequest(
+        type="collection",
+        collection_type="list:paired",
+    )
+    workbook = generate(request)
+    assert workbook
+    worksheet = workbook.active
+
+    header_titles = read_column_header_titles(worksheet)
+    assert header_titles == ["URI 1 (Forward)", "URI 2 (Reverse)", "List Identifier"]
+
+    if WRITE_TEST_WORKBOOKS:
+        path = "~/fetch_workbook_paired.xlsx"
+        expanded_path = os.path.expanduser(path)
+        workbook.save(expanded_path)
+
+
+def test_fetch_list_paired_or_unpaired_workbook():
+    request = GenerateFetchWorkbookRequest(
+        type="collection",
+        collection_type="list:paired_or_unpaired",
+    )
+    workbook = generate(request)
+    assert workbook
+    worksheet = workbook.active
+
+    header_titles = read_column_header_titles(worksheet)
+    assert header_titles == ["URI 1 (Forward)", "URI 2 (Optional/Reverse)", "List Identifier"]
+
+
+def test_parse_datasets():
+    content = unittest_file_to_base64("fetch_workbook.xlsx")
+    parse_request = ParseFetchWorkbook(
+        content=content,
+    )
+    parsed = parse(parse_request)
+    assert len(parsed.rows) == 1
+    row0 = parsed.rows[0]
+    assert row0["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed"
+    assert row0["name"] == "4.bed"
+
+    assert len(parsed.columns) == 2
+
+
+def test_parse_paired_list():
+    # workbook has URI 1 and URI 2 columns - make sure they are broken out and have a paired_indicator column
+    # for the rule builder.
+    content = unittest_file_to_base64("fetch_workbook_paired.xlsx")
+    parse_request = ParseFetchWorkbook(
+        content=content,
+    )
+    parsed = parse(parse_request)
+    assert len(parsed.rows) == 2
+    row0 = parsed.rows[0]
+    assert row0["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed"
+    assert row0["list_identifiers"] == "sample1"
+    assert row0["paired_identifier"] == "1"
+
+    row1 = parsed.rows[1]
+    assert row1["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed"
+    assert row1["list_identifiers"] == "sample1"
+    assert row1["paired_identifier"] == "2"
+
+    assert len(parsed.columns) == 3
+
+
+def test_parsed_paired_with_hashes():
+    content = unittest_file_to_base64("fetch_workbook_paired_with_hashes.xlsx")
+    parse_request = ParseFetchWorkbook(
+        content=content,
+    )
+    parsed = parse(parse_request)
+    assert len(parsed.rows) == 2
+
+    row0 = parsed.rows[0]
+    assert row0["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed"
+    assert row0["list_identifiers"] == "sample1"
+    assert row0["paired_identifier"] == "1"
+    assert row0["list_identifiers"] == "sample1"
+    assert row0["hash_md5"] == "37b59762b59fff860460522d271bc111"
+    assert (
+        row0["hash_sha512"]
+        == "b83327251deae7e8c865948573d325a6657eaef10b274ba98d8ba6835f073f787c5621a4a9afd6db894f87bbd4299770f062369a3a39dea8b0263860559ff939"
+    )
+
+    row1 = parsed.rows[1]
+    assert row1["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed"
+    assert row1["list_identifiers"] == "sample1"
+    assert row1["paired_identifier"] == "2"
+    assert row1["hash_md5"] == "29e9dd693af0a946e67cd1861b987d13"
+    assert (
+        row1["hash_sha512"]
+        == "9be59f9dc30bc2b35016dc45f5726a249baef2cdee2eb840665d3cdf4d0a0539c50eea1aa96bfd84f3a619782fc9321e5817e005a8a30c318d45c6759c9601bd"
+    )
+
+
+def test_parsed_list_with_auto_identifiers():
+    content = unittest_file_to_base64("fetch_workbook_list_without_ids_example.xlsx")
+    parse_request = ParseFetchWorkbook(
+        content=content, fill_identifiers=FillIdentifiers(fill_inner_list_identifiers=True)
+    )
+    parsed = parse(parse_request)
+    assert len(parsed.rows) == 2
+    row0 = parsed.rows[0]
+    assert row0["url"] == "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"
+    assert row0["list_identifiers"] == "DRR000770"
+    row1 = parsed.rows[1]
+    assert row1["url"] == "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz"
+    assert row1["list_identifiers"] == "DRR000771"
+
+
+def test_parsed_list_of_pairs_with_auto_identifiers():
+    content = unittest_file_to_base64("fetch_workbook_list_pairs_without_ids_example.xlsx")
+    parse_request = ParseFetchWorkbook(
+        content=content, fill_identifiers=FillIdentifiers(fill_inner_list_identifiers=True)
+    )
+    parsed = parse(parse_request)
+    assert len(parsed.rows) == 4
+    row0 = parsed.rows[0]
+    assert row0["url"] == "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR039/DRR039919/DRR039919_1.fastq.gz"
+    assert row0["list_identifiers"] == "DRR039919"
+
+
+def test_read_column_headers_from_titles():
+    # datasets...
+    column_headers = column_titles_to_headers(["URI", "Name", "Genome"])
+    assert len(column_headers) == 3
+    assert column_headers[0].type == "url"
+    assert column_headers[0].title == "URI"
+    assert column_headers[1].type == "name"
+    assert column_headers[1].title == "Name"
+    assert column_headers[2].type == "dbkey"
+    assert column_headers[2].title == "Genome"
+
+    # simple list...
+    column_headers = column_titles_to_headers(["URI", "List Identifier"])
+    assert len(column_headers) == 2
+    assert column_headers[0].type == "url"
+    assert column_headers[0].title == "URI"
+    assert column_headers[1].type == "list_identifiers"
+    assert column_headers[1].title == "List Identifier"
+
+    # paired list with list two URIs per row....
+    column_headers = column_titles_to_headers(["URI 1 (Forward)", "URI 2 (Reverse)", "List Identifier"])
+    assert len(column_headers) == 3
+    assert column_headers[0].type == "url"
+    assert column_headers[0].title == "URI 1 (Forward)"
+    assert column_headers[0].type_index == 0
+
+    assert column_headers[1].type == "url"
+    assert column_headers[1].title == "URI 2 (Reverse)"
+    assert column_headers[1].type_index == 1
+
+    assert column_headers[2].type == "list_identifiers"
+    assert column_headers[2].title == "List Identifier"
+
+    # paired list with paired identifier as a row...
+    column_headers = column_titles_to_headers(["URI", "List Identifier", "Paired Identifier"])
+    assert len(column_headers) == 3
+    assert column_headers[0].type == "url"
+    assert column_headers[0].title == "URI"
+    assert column_headers[0].type_index == 0
+
+    assert column_headers[2].type == "paired_identifier"
+    assert column_headers[2].title == "Paired Identifier"
+    assert column_headers[2].type_index == 0
+
+    # nested list
+    column_headers = column_titles_to_headers(["URI", "Outer List Identifier", "Inner List Identifier"])
+
+    assert len(column_headers) == 3
+    assert column_headers[0].type == "url"
+    assert column_headers[0].title == "URI"
+    assert column_headers[0].type_index == 0
+
+    assert column_headers[1].type == "list_identifiers"
+    assert column_headers[1].title == "Outer List Identifier"
+    assert column_headers[1].type_index == 0
+
+    assert column_headers[2].type == "list_identifiers"
+    assert column_headers[2].title == "Inner List Identifier"
+    assert column_headers[2].type_index == 1
+
+
+def test_infer_fetch_workbook_collection_type():
+    column_headers = column_titles_to_headers(["URI", "List Identifier", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list"
+
+    column_headers = column_titles_to_headers(["URI", "List Identifier 1", "List Identifier 2", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:list"
+
+    column_headers = column_titles_to_headers(["URI", "List Identifier", "Paired Identifier", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:paired"
+
+    # probably more usable - two URI style list:paired
+    column_headers = column_titles_to_headers(["URI 1", "URI 2", "List Identifier", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:paired"
+
+    column_headers = column_titles_to_headers(
+        ["URI", "List Identifier 1", "List Identifier 2", "Paired Identifier", "Genome"]
+    )
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:list:paired"
+
+    column_headers = column_titles_to_headers(["URI 1", "URI 2", "List Identifier 1", "List Identifier 2", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:list:paired"
+
+    # paired/unpaired sheets
+    column_headers = column_titles_to_headers(["URI 1", "URI 2 (Optional)", "List Identifier", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:paired_or_unpaired"
+
+    # paired/unpaired sheets
+    column_headers = column_titles_to_headers(["URI", "List Identifier", "Paired Identifier (Optional)", "Genome"])
+    collection_type = _infer_fetch_workbook_collection_type(column_headers)[0]
+    assert collection_type == "list:paired_or_unpaired"
+
+
+def test_column_target_model_parsing():
+    target_models()
+
+
+def test_validate_parsed_column_headers():
+    headers = column_titles_to_headers(["URI 1", "URI 2", "URI 3"])
+    with pytest.raises(RequestParameterInvalidException) as exception_info:
+        _validate_parsed_column_headers(headers)
+    assert EXCEPTION_TOO_MANY_URI_COLUMNS in str(exception_info.value)
+
+    headers = column_titles_to_headers(["Name", "Paired Indicator"])
+    with pytest.raises(RequestParameterInvalidException) as exception_info:
+        _validate_parsed_column_headers(headers)
+    assert EXCEPTION_NO_URIS_FOUND in str(exception_info.value)
+
+
+def unittest_file_to_base64(filename: str) -> str:
+    path = resource_path("galaxy.app_unittest_utils", filename)
+    example_as_bytes = path.read_bytes()
+    content_base64 = base64.b64encode(example_as_bytes).decode("utf-8")
+    return content_base64

test/unit/data/dataset_collections/test_auto_identifiers.py

@@ -0,0 +1,44 @@
+from galaxy.model.dataset_collections.auto_identifiers import (
+    fill_in_identifiers,
+    FillIdentifiers,
+)
+
+
+def test_fill_in_identifiers_simple():
+    uris = {
+        "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz": None,
+        "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz": None,
+    }
+    new_map = run_fill_on_dict(uris)
+    assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"] == "DRR000770"
+    assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz"] == "DRR000771"
+
+
+def test_fill_in_identifiers_retains_existing_maps():
+    uris = {
+        "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz": "abc",
+        "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz": None,
+    }
+    new_map = run_fill_on_dict(uris)
+    assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"] == "abc"
+    assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz"] == "DRR000771"
+
+
+def test_fill_in_identifiers_avoids_duplicates():
+    uris = {
+        "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz": None,
+        "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000770.fastq.gz": None,
+    }
+    new_map = run_fill_on_dict(uris)
+    assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"] == "DRR000770"
+    assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000770.fastq.gz"] == "DRR000770_1"
+
+
+def run_fill_on_dict(uris):
+    config = FillIdentifiers(fill_inner_list_identifiers=True)
+    as_tuples = list(uris.items())
+    new_ids = fill_in_identifiers(as_tuples, config)
+    new_uris = {}
+    for uri, identifier in zip(uris.keys(), new_ids):
+        new_uris[uri] = identifier
+    return new_uris

test/unit/data/dataset_collections/test_auto_pairing.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+from typing import (
+    Dict,
+    List,
+)
+
+import yaml
+from pydantic import (
+    BaseModel,
+    RootModel,
+)
+
+from galaxy.model.dataset_collections.auto_pairing import auto_pair
+from galaxy.util.resources import resource_string
+
+
+@dataclass
+class MockDataset:
+    name: str
+
+
+class PairedItem(BaseModel):
+    forward: str
+    reverse: str
+
+
+class AutoPairingSpecItem(BaseModel):
+    doc: str
+    inputs: List[str]
+    paired: Dict[str, PairedItem]
+
+
+AutoPairingSpecification = RootModel[List[AutoPairingSpecItem]]
+
+
+def test_auto_pairing_specification():
+    auto_pairing_specification = resource_string("galaxy.model.dataset_collections", "auto_pairing_spec.yml")
+    auto_pairing_specification_dicts = yaml.safe_load(auto_pairing_specification)
+    tests = AutoPairingSpecification.model_validate(auto_pairing_specification_dicts)
+    for test in tests.root:
+        inputs = [MockDataset(name) for name in test.inputs]
+        summary = auto_pair(inputs)
+        pairs = summary.paired
+        for name, expected_pair in test.paired.items():
+            pair = next((p for p in pairs if p.name == name), None)
+            assert pair is not None, f"Pair {name} not found"
+            assert pair.forward.name == expected_pair.forward, f"Forward mismatch for pair {name}"
+            assert pair.reverse.name == expected_pair.reverse, f"Reverse mismatch for pair {name}"

test/unit/data/dataset_collections/test_column_to_target_mapping.py

@@ -0,0 +1,21 @@
+"""Run rule_target_column_specification.yml defined tests using pytest."""
+
+import yaml
+
+from galaxy.model.dataset_collections.rule_target_columns import column_title_to_target_type
+from galaxy.util.resources import resource_string
+
+
+def test_column_to_target_mapping():
+    rule_target_tests_str = resource_string("galaxy.model.dataset_collections", "rule_target_column_specification.yml")
+    rule_target_tests = yaml.safe_load(rule_target_tests_str)
+    for rule_target_test in rule_target_tests:
+        # - doc: "NAME (all uppercase) maps directly to the name target type"
+        #   column_header: "NAME"
+        #   maps_to: "name"
+        doc = rule_target_test.get("doc", "")
+        column_header = rule_target_test["column_header"]
+        maps_to = rule_target_test["maps_to"]
+        assert (
+            column_title_to_target_type(column_header) == maps_to
+        ), f"Failed for column header '{column_header}' with doc '{doc}'"

test/unit/data/dataset_collections/test_workbook_util.py

@@ -0,0 +1,24 @@
+import base64
+
+import pytest
+
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.model.dataset_collections.workbook_util import (
+    index_to_excel_column,
+    load_workbook_from_base64,
+)
+
+
+def test_index_to_excel_column():
+    assert index_to_excel_column(0) == "A"
+    assert index_to_excel_column(25) == "Z"
+    assert index_to_excel_column(26) == "AA"
+    assert index_to_excel_column(700) == "ZY"
+    assert index_to_excel_column(701) == "ZZ"
+    assert index_to_excel_column(702) == "AAA"
+
+
+def test_loading_invalid_workbook():
+    base64Tabular = base64.b64encode(b"1\t2\t3\n").decode("utf-8")
+    with pytest.raises(RequestParameterInvalidException):
+        load_workbook_from_base64(base64Tabular)