Merge pull request #21106 from guerler/remove_tabular_chunked_mako

Remove tabular chunked, binary and large file mako
This commit is contained in:
Aysam Guerler
2025-10-29 12:35:33 +03:00
committed by GitHub
17 changed files with 196 additions and 254 deletions
+1 -15
View File
@@ -1,23 +1,9 @@
/**
* The list of horrible globals we expose on window.bundleEntries.
*
* Everything that is exposed on this global variable is something that the python templates
* require for their hardcoded initializations. These objects are going to have to continue
* to exist until such time as we replace the overall application with a Vue component which
* will handle initializations for components individually.
*
* The list of globals we expose on window.bundleEntries.
*/
import { replaceChildrenWithComponent } from "utils/mountVueComponent";
import TabularChunkedView from "components/Visualizations/Tabular/TabularChunkedView.vue";
// legacy/grid_base.mako
export { default as LegacyGridView } from "legacy/grid/grid-view";
// webapps/reports/run_stats.mako
export { create_chart, create_histogram } from "reports/run_stats";
// webapps/galaxy/dataset/{ display | tabular_chunked }.mako
export const createTabularDatasetChunkedView = (options) => {
return replaceChildrenWithComponent(options.parent_elt, TabularChunkedView, { options });
};
@@ -0,0 +1,105 @@
<script setup lang="ts">
import { BAlert } from "bootstrap-vue";
import { storeToRefs } from "pinia";
import { computed, ref, watch } from "vue";
import { useDatasetStore } from "@/stores/datasetStore";
import { useUserStore } from "@/stores/userStore";
import { withPrefix } from "@/utils/redirect";
import { errorMessageAsString } from "@/utils/simple-error";
import { bytesToString } from "@/utils/utils";
import Alert from "@/components/Alert.vue";
import TabularChunkedView from "@/components/Dataset/Tabular/TabularChunkedView.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
import CenterFrame from "@/entry/analysis/modules/CenterFrame.vue";
interface Props {
datasetId: string;
isBinary: boolean;
}
const { getDataset, isLoadingDataset } = useDatasetStore();
const emit = defineEmits(["load"]);
const props = defineProps<Props>();
const contentTruncated = ref<number | null>(null);
const contentChunked = ref<boolean>(false);
const errorMessage = ref<string>("");
const sanitizedJobImported = ref<boolean>(false);
const sanitizedToolId = ref<string | null>(null);
const { isAdmin } = storeToRefs(useUserStore());
const dataset = computed(() => getDataset(props.datasetId));
const datasetUrl = computed(() => `/datasets/${props.datasetId}/display`);
const downloadUrl = computed(() => withPrefix(`${datasetUrl.value}?to_ext=${dataset.value?.file_ext}`));
const isLoading = computed(() => isLoadingDataset(props.datasetId));
const previewUrl = computed(() => `${datasetUrl.value}?preview=True`);
const sanitizedMessage = computed(() => {
const plainText = "Contents are shown as plain text.";
if (sanitizedJobImported.value) {
return `Dataset has been imported. ${plainText}`;
} else if (sanitizedToolId.value) {
return `Dataset created by a tool that is not known to create safe HTML. ${plainText}`;
}
return undefined;
});
watch(
() => props.datasetId,
async () => {
try {
const { headers } = await fetch(withPrefix(previewUrl.value), { method: "HEAD" });
contentChunked.value = !!headers.get("x-content-chunked");
contentTruncated.value = headers.get("x-content-truncated")
? Number(headers.get("x-content-truncated"))
: null;
sanitizedJobImported.value = !!headers.get("x-sanitized-job-imported");
sanitizedToolId.value = headers.get("x-sanitized-tool-id");
errorMessage.value = "";
} catch (e) {
errorMessage.value = errorMessageAsString(e);
console.error(e);
}
},
{ immediate: true },
);
</script>
<template>
<BAlert v-if="errorMessage" variant="danger" show>
{{ errorMessage }}
</BAlert>
<LoadingSpan v-else-if="isLoading || !dataset" message="Loading dataset content" />
<div v-else class="dataset-display h-100">
<Alert v-if="sanitizedMessage" :dismissible="true" variant="warning" data-description="sanitization warning">
{{ sanitizedMessage }}
<span v-if="isAdmin && sanitizedToolId">
<br />
<router-link data-description="allowlist link" to="/admin/sanitize_allow">Review Allowlist</router-link>
if outputs of {{ sanitizedToolId }} are trusted and should be shown as HTML.
</span>
</Alert>
<div v-if="dataset.deleted" id="deleted-data-message" class="errormessagelarge">
You are viewing a deleted dataset.
</div>
<TabularChunkedView v-if="contentChunked" :options="dataset" />
<div v-else class="h-100">
<div v-if="isBinary">
This is a binary (or unknown to Galaxy) dataset of size {{ bytesToString(dataset.file_size) }}. Preview
is not implemented for this filetype. Displaying as ASCII text.
</div>
<div v-if="contentTruncated" class="warningmessagelarge">
<div>
This dataset is large and only the first {{ bytesToString(contentTruncated) }} is shown below.
</div>
<a :href="downloadUrl">Download</a>
</div>
<CenterFrame :src="previewUrl" @load="emit('load')" />
</div>
</div>
</template>
+17 -29
View File
@@ -14,13 +14,13 @@ import { bytesToString } from "@/utils/utils";
import DatasetError from "../DatasetInformation/DatasetError.vue";
import LoadingSpan from "../LoadingSpan.vue";
import DatasetAsImage from "./DatasetAsImage/DatasetAsImage.vue";
import DatasetDisplay from "./DatasetDisplay.vue";
import DatasetState from "./DatasetState.vue";
import Heading from "@/components/Common/Heading.vue";
import DatasetAttributes from "@/components/DatasetInformation/DatasetAttributes.vue";
import DatasetDetails from "@/components/DatasetInformation/DatasetDetails.vue";
import VisualizationsList from "@/components/Visualizations/Index.vue";
import VisualizationFrame from "@/components/Visualizations/VisualizationFrame.vue";
import CenterFrame from "@/entry/analysis/modules/CenterFrame.vue";
const datasetStore = useDatasetStore();
const datatypeStore = useDatatypeStore();
@@ -61,6 +61,13 @@ const downloadUrl = computed(() => withPrefix(`/datasets/${props.datasetId}/disp
const preferredVisualization = computed(
() => dataset.value && datatypeStore.getPreferredVisualization(dataset.value.file_ext),
);
const isBinaryDataset = computed(() => {
if (!dataset.value?.file_ext || !datatypesMapperStore.datatypesMapper) {
return false;
}
return datatypesMapperStore.datatypesMapper.isSubTypeOfAny(dataset.value.file_ext, ["galaxy.datatypes.binary"]);
});
const isImageDataset = computed(() => {
if (!dataset.value?.file_ext || !datatypesMapperStore.datatypesMapper) {
return false;
@@ -93,7 +100,7 @@ watch(
<template>
<LoadingSpan v-if="isLoading || !dataset" message="Loading dataset details" />
<div v-else class="dataset-view d-flex flex-column h-100">
<div v-else class="dataset-view d-flex flex-column">
<header v-if="!displayOnly" :key="`dataset-header-${dataset.id}`" class="dataset-header flex-shrink-0">
<div class="d-flex">
<Heading
@@ -181,18 +188,13 @@ watch(
<FontAwesomeIcon :icon="faBug" class="mr-1" /> Error
</BNavItem>
</BNav>
<div v-if="tab === 'preview'" class="h-100">
<div v-if="tab === 'preview'" class="tab-content-panel">
<VisualizationFrame
v-if="preferredVisualization"
:dataset-id="datasetId"
:visualization="preferredVisualization"
@load="iframeLoading = false" />
<CenterFrame
v-else-if="isPdfDataset"
:src="`/datasets/${datasetId}/display/?preview=True`"
:is-preview="true"
@load="iframeLoading = false" />
<div v-else-if="isAutoDownloadType" class="auto-download-message p-4">
<div v-else-if="isAutoDownloadType && !isPdfDataset" class="auto-download-message p-4">
<div class="alert alert-info">
<h4>Download Required</h4>
<p>This file type ({{ dataset.file_ext }}) will download automatically when accessed directly.</p>
@@ -203,23 +205,14 @@ watch(
</div>
</div>
<DatasetAsImage
v-else-if="isImageDataset"
v-else-if="isImageDataset && !isPdfDataset"
:history-dataset-id="datasetId"
:allow-size-toggle="true"
class="p-3" />
<CenterFrame
v-else
:src="`/datasets/${datasetId}/display/?preview=True`"
:is-preview="true"
@load="iframeLoading = false" />
<DatasetDisplay v-else :dataset-id="datasetId" :is-binary="isBinaryDataset" @load="iframeLoading = false" />
</div>
<div v-else-if="tab === 'raw'" class="h-100">
<CenterFrame
v-if="isPdfDataset"
:src="`/datasets/${datasetId}/display/?preview=True`"
:is-preview="true"
@load="iframeLoading = false" />
<div v-else-if="isAutoDownloadType" class="auto-download-message p-4">
<div v-else-if="tab === 'raw'" class="tab-content-panel">
<div v-if="isAutoDownloadType && !isPdfDataset" class="auto-download-message p-4">
<div class="alert alert-info">
<h4>Download Required</h4>
<p>This file type ({{ dataset.file_ext }}) will download automatically when accessed directly.</p>
@@ -229,11 +222,7 @@ watch(
</a>
</div>
</div>
<CenterFrame
v-else
:src="`/datasets/${datasetId}/display/?preview=True`"
:is-preview="true"
@load="iframeLoading = false" />
<DatasetDisplay v-else :dataset-id="datasetId" :is-binary="isBinaryDataset" @load="iframeLoading = false" />
</div>
<div v-else-if="tab === 'visualize'" class="tab-content-panel">
<VisualizationsList :dataset-id="datasetId" />
@@ -318,8 +307,7 @@ watch(
.tab-content-panel {
display: flex;
flex-direction: column;
overflow: hidden;
overflow-y: auto;
overflow: auto;
height: 100%;
}
@@ -4,6 +4,7 @@ import axios from "axios";
import { parse } from "csv-parse/sync";
import { computed, onMounted, reactive, ref, watch } from "vue";
import type { HDADetailed } from "@/api";
import { getAppRoot } from "@/onload/loadConfig";
interface TabularChunk {
@@ -12,17 +13,14 @@ interface TabularChunk {
data_line_offset: number;
}
interface TabularDataset extends HDADetailed {
metadata_columns?: number;
metadata_column_types?: string[];
metadata_column_names?: string[];
}
interface TabularChunkedViewProps {
options: {
dataset_config: {
id: string;
file_ext: string;
first_data_chunk: TabularChunk;
metadata_columns: number;
metadata_column_types: string[];
metadata_column_names: string[];
};
};
options: TabularDataset;
}
const props = defineProps<TabularChunkedViewProps>();
@@ -37,10 +35,10 @@ const tabularData = reactive<{ rows: string[][] }>({
});
const columns = computed(() => {
const columns = Array(props.options.dataset_config.metadata_columns);
const columns = Array(props.options.metadata_columns);
// for each column_name, inject header
if (props.options.dataset_config.metadata_column_names?.length > 0) {
props.options.dataset_config.metadata_column_names.forEach((column_name, index) => {
if (props.options.metadata_column_names && props.options.metadata_column_names?.length > 0) {
props.options.metadata_column_names.forEach((column_name, index) => {
columns[index] = column_name;
});
}
@@ -48,9 +46,9 @@ const columns = computed(() => {
});
const columnStyle = computed(() => {
const columnStyle = Array(props.options.dataset_config.metadata_columns);
if (props.options.dataset_config.metadata_column_types?.length > 0) {
props.options.dataset_config.metadata_column_types.forEach((column_type, index) => {
const columnStyle = Array(props.options.metadata_columns);
if (props.options.metadata_column_types && props.options.metadata_column_types?.length > 0) {
props.options.metadata_column_types.forEach((column_type, index) => {
columnStyle[index] = column_type === "str" || column_type === "list" ? "string-align" : "number-align";
});
}
@@ -58,11 +56,11 @@ const columnStyle = computed(() => {
});
const delimiter = computed(() => {
return props.options.dataset_config.file_ext === "csv" ? "," : "\t";
return props.options.file_ext === "csv" ? "," : "\t";
});
const chunkUrl = computed(() => {
return `${getAppRoot()}dataset/display?dataset_id=${props.options.dataset_config.id}`;
return `${getAppRoot()}dataset/display?dataset_id=${props.options.id}`;
});
// Loading more data on user scroll to (near) bottom.
@@ -163,11 +161,8 @@ function nextChunk() {
}
onMounted(() => {
// Render first chunk if available.
if (props.options.dataset_config.first_data_chunk) {
processChunk(props.options.dataset_config.first_data_chunk);
loading.value = false;
}
// Fetch and render first chunk
nextChunk();
});
</script>
@@ -91,7 +91,7 @@ onMounted(async () => {
</script>
<template>
<div aria-labelledby="dataset-attributes-heading">
<div class="dataset-attributes" aria-labelledby="dataset-attributes-heading">
<Heading id="dataset-attributes-heading" h1 separator inline size="md">
{{ localize("Edit Dataset Attributes") }}
</Heading>
@@ -230,3 +230,10 @@ onMounted(async () => {
</div>
</div>
</template>
<style>
.dataset-attributes {
overflow-x: hidden;
overflow-y: auto;
}
</style>
@@ -155,7 +155,8 @@ onUnmounted(() => {
display: flex;
flex-direction: column;
gap: 1rem;
overflow-x: hidden;
overflow-y: auto;
.dataset-peek {
word-break: break-all;
}
@@ -95,7 +95,7 @@ onMounted(() => {
</script>
<template>
<div class="position-relative h-100">
<div class="position-relative h-100 overflow-hidden">
<div v-if="isLoading" class="iframe-loading bg-light">
<LoadingSpan message="Loading preview" />
</div>
@@ -1,11 +1,8 @@
<script setup lang="ts">
import { storeToRefs } from "pinia";
import { computed, ref, watch } from "vue";
import { computed, ref } from "vue";
import { useUserStore } from "@/stores/userStore";
import { withPrefix } from "@/utils/redirect";
import Alert from "@/components/Alert.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const emit = defineEmits(["load"]);
@@ -13,53 +10,15 @@ const props = withDefaults(
defineProps<{
id?: string;
src?: string;
isPreview?: boolean;
}>(),
{
id: "frame",
src: "",
isPreview: false,
},
);
const { isAdmin } = storeToRefs(useUserStore());
const srcWithRoot = computed(() => withPrefix(props.src));
const sanitizedImport = ref(false);
const sanitizedToolId = ref<String | false>(false);
const isLoading = ref(true);
watch(
() => srcWithRoot.value,
async () => {
sanitizedImport.value = false;
sanitizedToolId.value = false;
if (props.isPreview) {
try {
const response = await fetch(srcWithRoot.value, { method: "HEAD" });
const isImported = response.headers.get("x-sanitized-job-imported");
const toolId = response.headers.get("x-sanitized-tool-id");
if (isImported !== null) {
sanitizedImport.value = true;
} else if (toolId !== null) {
sanitizedToolId.value = toolId;
}
} catch (e) {
// I guess that's fine and the center panel will show something
console.error(e);
}
}
},
{ immediate: true },
);
const plainText = "Contents are shown as plain text.";
const sanitizedMessage = computed(() => {
if (sanitizedImport.value) {
return `Dataset has been imported. ${plainText}`;
} else if (sanitizedToolId.value) {
return `Dataset created by a tool that is not known to create safe HTML. ${plainText}`;
}
return undefined;
});
function onLoad(ev: Event) {
isLoading.value = false;
@@ -70,20 +29,12 @@ function onLoad(ev: Event) {
emit("load");
}
} catch (err) {
console.warn("CenterFrame - onLoad location access forbidden.", ev, location);
console.warn("[CenterFrame] onLoad location access forbidden.", ev, location);
}
}
</script>
<template>
<div class="h-100">
<Alert v-if="sanitizedMessage" :dismissible="true" variant="warning" data-description="sanitization warning">
{{ sanitizedMessage }}
<span v-if="isAdmin && sanitizedToolId">
<br />
<router-link data-description="allowlist link" to="/admin/sanitize_allow">Review Allowlist</router-link>
if outputs of {{ sanitizedToolId }} are trusted and should be shown as HTML.
</span>
</Alert>
<div class="h-100 overflow-hidden">
<LoadingSpan v-if="isLoading">Loading ...</LoadingSpan>
<iframe
:id="id"
@@ -165,6 +165,11 @@ dataset_view:
details_tab: ".nav-item[title='View detailed information about this dataset'] > a.nav-link"
visualize_tab: ".nav-item[title='Explore available visualizations for this dataset'] > a.nav-link"
dataset_display:
selectors:
container: .dataset-display
content: body
history_panel:
menu:
labels:
+2 -20
View File
@@ -819,26 +819,8 @@ class BamNative(CompressedArchive, _BamOrSam):
elif to_ext or not preview:
return super().display_data(trans, dataset, preview, filename, to_ext, **kwd)
else:
column_names = dataset.metadata.column_names
if not column_names:
column_names = []
column_types = dataset.metadata.column_types
if not column_types:
column_types = []
column_number = dataset.metadata.columns
if column_number is None:
column_number = 1
return (
trans.fill_template(
"/dataset/tabular_chunked.mako",
dataset=dataset,
chunk=self.get_chunk(trans, dataset, 0),
column_number=column_number,
column_names=column_names,
column_types=column_types,
),
headers,
)
headers["x-content-chunked"] = "true"
return self.get_chunk(trans, dataset, 0), headers
def validate(self, dataset: DatasetProtocol, **kwd) -> DatatypeValidation:
if not BamNative.is_bam(dataset.get_file_name()):
+7 -21
View File
@@ -509,17 +509,10 @@ class Data(metaclass=DataMeta):
def _serve_binary_file_contents_as_text(self, trans, data, headers, file_size, max_peek_size):
headers["content-type"] = "text/html"
if file_size > max_peek_size:
headers["x-content-truncated"] = max_peek_size
with open(data.get_file_name(), "rb") as fh:
return (
trans.fill_template_mako(
"/dataset/binary_file.mako",
data=data,
file_contents=fh.read(max_peek_size),
file_size=util.nice_size(file_size),
truncated=file_size > max_peek_size,
),
headers,
)
return unicodify(fh.read(max_peek_size)), headers
def _serve_file_contents(self, trans, data, headers, preview, file_size, max_peek_size):
from galaxy.datatypes import images
@@ -531,14 +524,8 @@ class Data(metaclass=DataMeta):
with compression_utils.get_fileobj(data.get_file_name(), "rb") as fh:
# preview large text file
headers["content-type"] = "text/html"
return (
trans.fill_template_mako(
"/dataset/large_file.mako",
truncated_data=fh.read(max_peek_size),
data=data,
),
headers,
)
headers["x-content-truncated"] = max_peek_size
return unicodify(fh.read(max_peek_size)), headers
def display_data(
self,
@@ -618,9 +605,8 @@ class Data(metaclass=DataMeta):
else: # displaying
trans.log_event(f"Display dataset id: {str(dataset.id)}")
max_peek_size = _get_max_peek_size(dataset)
if (
_is_binary_file(dataset) and preview and hasattr(trans, "fill_template_mako")
): # preview file which format is unknown (to Galaxy), we still try to display this as text
if _is_binary_file(dataset) and preview:
# preview file which format is unknown (to Galaxy), we still try to display this as text
return self._serve_binary_file_contents_as_text(trans, dataset, headers, file_size, max_peek_size)
else: # text/html, or image, or display was called without preview flag
return self._serve_file_contents(trans, dataset, headers, preview, file_size, max_peek_size)
+2 -4
View File
@@ -337,10 +337,8 @@ class Sequence(data.Text):
mime = "text/plain"
self._clean_and_set_mime_type(trans, mime, headers)
return chunk[:-1], headers
return (
trans.fill_template_mako("/dataset/large_file.mako", truncated_data=chunk[:-1], data=dataset),
headers,
)
headers["x-content-truncated"] = max_peek_size
return util.unicodify(chunk[:-1]), headers
else:
return super().display_data(trans, dataset, preview, filename, to_ext, **kwd)
+4 -30
View File
@@ -196,38 +196,12 @@ class TabularData(Text):
return open(dataset.get_file_name(), mode="rb"), headers
else:
headers["content-type"] = "text/html"
headers["x-content-truncated"] = max_peek_size
with compression_utils.get_fileobj(dataset.get_file_name(), "rb") as fh:
return (
trans.fill_template_mako(
"/dataset/large_file.mako",
truncated_data=fh.read(max_peek_size),
data=dataset,
),
headers,
)
return util.unicodify(fh.read(max_peek_size)), headers
else:
column_names = "null"
if dataset.metadata.column_names:
column_names = dataset.metadata.column_names
elif hasattr(dataset.datatype, "column_names"):
column_names = dataset.datatype.column_names
column_types = dataset.metadata.column_types
if not column_types:
column_types = []
column_number = dataset.metadata.columns
if column_number is None:
column_number = "null"
return (
trans.fill_template(
"/dataset/tabular_chunked.mako",
dataset=dataset,
chunk=self.get_chunk(trans, dataset, 0),
column_number=column_number,
column_names=column_names,
column_types=column_types,
),
headers,
)
headers["x-content-chunked"] = "true"
return self.get_chunk(trans, dataset, 0), headers
def display_as_markdown(self, dataset_instance: DatasetProtocol) -> str:
with open(dataset_instance.get_file_name()) as f:
@@ -10,9 +10,29 @@ TEST_ANNOTATION = "my cool annotation"
TEST_INFO = "my cool info"
class TestHistoryPanel(SeleniumTestCase):
class TestDataset(SeleniumTestCase):
ensure_registered = True
@selenium_test
@selenium_only("Not yet migrated to support Playwright backend")
@managed_history
def test_history_dataset_display_text(self):
original_name = "1.txt"
history_entry = self.perform_single_upload(self.get_filename(original_name))
hid = history_entry.hid
self.wait_for_history()
self.history_panel_wait_for_hid_ok(hid)
self.display_dataset(hid=hid)
dataset_display = self.components.dataset_display.container
dataset_display.wait_for_visible()
self.switch_to_frame()
text = self.components.dataset_display.content.wait_for_text()
assert "chr1 4225 19670" in text
self.driver.switch_to.default_content()
@selenium_test
@managed_history
def test_history_dataset_rename(self):
@@ -1,17 +0,0 @@
<%inherit file="/base.mako"/>
<%namespace file="/dataset/display.mako" import="render_deleted_data_message" />
${ render_deleted_data_message( data ) }
<div class="warningmessagelarge">
This is a binary (or unknown to Galaxy) dataset of size ${ file_size }. Preview is not implemented for this filetype. Displaying
%if truncated:
first 100KB
%endif
as ASCII text<br/>
<a href="${h.url_for( controller='dataset', action='display', dataset_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">Download</a>
</div>
<pre>
${ util.unicodify( file_contents ) | h }
</pre>
@@ -1,14 +0,0 @@
<%inherit file="/base.mako"/>
<%namespace file="/dataset/display.mako" import="render_deleted_data_message" />
${ render_deleted_data_message( data ) }
<div class="warningmessagelarge">
This dataset is large and only the first megabyte is shown below.<br />
<a href="${h.url_for( controller='dataset', action='display', dataset_id=trans.security.encode_id( data.id ), filename='' )}">Show all</a> |
<a href="${h.url_for( controller='dataset', action='display', dataset_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">Save</a>
</div>
<pre>
${ util.unicodify( truncated_data ) | h }
</pre>
@@ -1,25 +0,0 @@
<%inherit file="/base.mako"/>
<%namespace file="/dataset/display.mako" import="render_deleted_data_message" />
<%def name="title()">Dataset Display</%def>
<%def name="javascript_app()">
<!-- tabular_chunked.mako javascript_app() -->
${parent.javascript_app()}
<script type="text/javascript">
config.addInitialization(function(galaxy) {
var dataset = ${ h.dumps( trans.security.encode_dict_ids( dataset.to_dict() ) )};
var firstChunk = ${chunk};
var datasetConfig = Object.assign(dataset, { first_data_chunk: firstChunk })
window.bundleEntries.createTabularDatasetChunkedView({
dataset_config : datasetConfig,
parent_elt : document.body
});
});
</script>
</%def>
${ render_deleted_data_message( dataset ) }