opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-11-03 12:53:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adds state-saving for installed viz (plugin_build_hash) like we do for

Browse Source 
built.  Additional code cleanup.

Rebase me -- bug where we should write a modified hash when changes are
outstanding?
This commit is contained in:
Dannon Baker
2025-04-23 07:55:52 -04:00
parent 874040811e
commit b47d4f00a1
1 changed files with 121 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/gulpfile.js
+121 -92
View File
@@ -12,7 +12,7 @@ const xml2js = require("xml2js");
* un-built visualizations in the repository; for performance and
* simplicity just add them one at a time until we upgrade older viz's.
*/
const STATIC_PLUGIN_BUILD_IDS = [
const PLUGIN_BUILD_IDS = [
"annotate_image",
"chiraviz",
"drawrna",
@@ -46,8 +46,6 @@ const INSTALL_PLUGIN_BUILD_IDS = [
"vitessce",
"vizarr",
]; // todo: derive from XML
const DIST_PLUGIN_BUILD_IDS = ["new_user"];
const PLUGIN_BUILD_IDS = Array.prototype.concat(DIST_PLUGIN_BUILD_IDS, STATIC_PLUGIN_BUILD_IDS);
const args = process.argv.slice(2);
const limitIndex = args.indexOf("--limit");
@@ -65,9 +63,6 @@ const failOnError =
const PATHS = {
nodeModules: "./node_modules",
stagedLibraries: {
// This is a stepping stone towards having all this staged
// automatically. Eventually, this dictionary and staging step will
// not be necessary.
backbone: ["backbone.js", "backbone.js"],
jquery: ["dist/jquery.js", "jquery/jquery.js"],
"jquery-migrate": ["dist/jquery-migrate.js", "jquery/jquery.migrate.js"],
@@ -95,8 +90,8 @@ PATHS.pluginBuildModules = [
function stageLibs(callback) {
Object.keys(PATHS.stagedLibraries).forEach((lib) => {
var p1 = path.resolve(path.join(PATHS.nodeModules, lib, PATHS.stagedLibraries[lib][0]));
var p2 = path.resolve(path.join("src", "libs", PATHS.stagedLibraries[lib][1]));
const p1 = path.resolve(path.join(PATHS.nodeModules, lib, PATHS.stagedLibraries[lib][0]));
const p2 = path.resolve(path.join("src", "libs", PATHS.stagedLibraries[lib][1]));
if (fs.existsSync(p1)) {
del.sync(p2);
fs.createReadStream(p1).pipe(fs.createWriteStream(p2));
@@ -121,48 +116,37 @@ function stagePlugins() {
function buildPlugins(callback, forceRebuild) {
/*
* Walk pluginBuildModules glob and attempt to build modules.
* */
*/
const packageJsons = globSync(PATHS.pluginBuildModules, {});
packageJsons.forEach((file) => {
let skipBuild = false;
const pluginDir = path.dirname(file);
const pluginName = pluginDir.split(path.sep).pop();
const hashFilePath = path.join(
pluginDir,
DIST_PLUGIN_BUILD_IDS.indexOf(pluginName) > -1 ? "dist" : "static",
"plugin_build_hash.txt"
);
const hashFilePath = path.join(pluginDir, "static", "plugin_build_hash.txt");
if (forceRebuild) {
skipBuild = false;
} else {
// Try reading existing plugin_build_hash.txt
if (fs.existsSync(hashFilePath)) {
const hashFileContent = fs.readFileSync(hashFilePath, "utf8").trim();
const isHash = /^[0-9a-f]{7,40}$/.test(hashFileContent); // Check for a 7 to 40 character hexadecimal string
const oldHash = fs.readFileSync(hashFilePath, "utf8").trim();
const isHash = /^[0-9a-f]{7,40}$/.test(oldHash);
if (!isHash) {
console.log(`Hash file for ${pluginName} exists but does not have a valid git hash.`);
skipBuild = false;
} else {
skipBuild =
child_process.spawnSync("git", ["diff", "--quiet", hashFileContent, "--", pluginDir], {
stdio: "inherit",
shell: true,
}).status === 0;
}
skipBuild =
child_process.spawnSync("git", ["diff", "--quiet", `$(cat ${hashFilePath})`, "--", pluginDir], {
// Check if there are changes since the stored hash
const diffResult = child_process.spawnSync("git", ["diff", "--quiet", oldHash, "--", pluginDir], {
stdio: "inherit",
shell: true,
}).status === 0;
if (!skipBuild) {
// Hash exists and is outdated, triggering a rebuild.
// Stage current hash to .orig for debugging and to
// force a plugin rebuild in the event of a failure
// (i.e. -- we're committed to a new build of this plugin).
fs.renameSync(hashFilePath, `${hashFilePath}.orig`);
});
skipBuild = diffResult.status === 0;
if (!skipBuild) {
// Hash exists and is outdated, rename to .orig
fs.renameSync(hashFilePath, `${hashFilePath}.orig`);
}
}
} else {
console.log(`No build hashfile detected for ${pluginName}, generating now.`);
@@ -170,11 +154,10 @@ function buildPlugins(callback, forceRebuild) {
}
if (skipBuild) {
console.log(`No changes detected for ${pluginName}`);
console.log(`No changes detected for ${pluginName}, skipping build.`);
} else {
console.log(`Installing Dependencies for ${pluginName}`);
// Else we call yarn install and yarn build
// Yarn install
child_process.spawnSync(
"yarn",
["install", "--production=false", "--network-timeout=300000", "--check-files"],
@@ -184,32 +167,34 @@ function buildPlugins(callback, forceRebuild) {
shell: true,
}
);
console.log(`Building ${pluginName}`);
const opts = {
cwd: pluginDir,
stdio: "inherit",
shell: true,
};
// if node version is >16, set NODE_OPTIONS to use legacy openssl provider
if (process.versions.node.split(".")[0] > "16") {
// Node >16 fix
if (Number(process.versions.node.split(".")[0]) > 16) {
opts.env = {
...process.env,
PARCEL_WORKER_BACKEND: "process",
NODE_OPTIONS: "--openssl-legacy-provider",
};
}
if (child_process.spawnSync("yarn", ["build"], opts).status === 0) {
const buildResult = child_process.spawnSync("yarn", ["build"], opts);
if (buildResult.status === 0) {
console.log(`Successfully built, saving build state to ${hashFilePath}`);
child_process.exec(`(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hashFilePath}`);
// Save new hash
child_process.execSync(`git rev-parse HEAD > ${hashFilePath}`, {
stdio: "inherit",
shell: true,
});
} else {
console.error(
`Error building ${pluginName}, not saving build state. Please report this issue to the Galaxy Team.`
);
console.error(`Error building ${pluginName}, not saving build state.`);
if (failOnError) {
// Fail on error.
console.error(
"Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set, see error(s) above."
);
console.error("Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set.");
process.exit(1);
}
}
@@ -218,61 +203,101 @@ function buildPlugins(callback, forceRebuild) {
return callback();
}
async function installPlugins(callback) {
// iterate through install_plugin_build_ids, identify xml files and install dependencies
for (const plugin_name of INSTALL_PLUGIN_BUILD_IDS.filter(applyPluginFilter)) {
const pluginDir = path.join(PATHS.pluginBaseDir, `visualizations/${plugin_name}`);
const xmlPath = path.join(pluginDir, `config/${plugin_name}.xml`);
// Check if the file exists
if (fs.existsSync(xmlPath)) {
await installDependenciesFromXML(xmlPath, pluginDir);
} else {
/**
* Same logic as buildPlugins, but for 'installed' plugins that rely on an XML
* specifying npm dependencies (the INSTALL_PLUGIN_BUILD_IDS).
*/
async function installPlugins(callback, forceReinstall) {
for (const pluginName of INSTALL_PLUGIN_BUILD_IDS.filter(applyPluginFilter)) {
const pluginDir = path.join(PATHS.pluginBaseDir, `visualizations/${pluginName}`);
const xmlPath = path.join(pluginDir, `config/${pluginName}.xml`);
if (!fs.existsSync(xmlPath)) {
console.error(`XML file not found: ${xmlPath}`);
continue;
}
const hashFilePath = path.join(pluginDir, "static", "plugin_build_hash.txt");
let skipInstall = false;
if (!forceReinstall && fs.existsSync(hashFilePath)) {
const oldHash = fs.readFileSync(hashFilePath, "utf8").trim();
const isHash = /^[0-9a-f]{7,40}$/.test(oldHash);
if (!isHash) {
console.log(`Hash file for ${pluginName} exists but is not a valid git hash.`);
} else {
const diffResult = child_process.spawnSync("git", ["diff", "--quiet", oldHash, "--", pluginDir], {
stdio: "inherit",
shell: true,
});
skipInstall = diffResult.status === 0;
if (!skipInstall) {
fs.renameSync(hashFilePath, `${hashFilePath}.orig`);
}
}
} else if (!fs.existsSync(hashFilePath)) {
console.log(`No install hashfile detected for ${pluginName}, generating now.`);
}
if (skipInstall) {
console.log(`No changes detected for ${pluginName}, skipping re-installation.`);
} else {
console.log(`Installing npm dependencies for installed plugin ${pluginName}...`);
try {
await installDependenciesFromXML(xmlPath, pluginDir);
console.log(`Finished installing for ${pluginName}, saving new hash.`);
// Save the current commit hash
child_process.execSync(`git rev-parse HEAD > ${hashFilePath}`, {
stdio: "inherit",
shell: true,
});
} catch (err) {
console.error(`Error installing for ${pluginName}:`, err);
if (failOnError) {
console.error(
"Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set, see error(s) above."
);
process.exit(1);
}
}
}
}
return callback();
}
// Function to parse the XML and install dependencies
// Function to parse XML and install specified npm packages
async function installDependenciesFromXML(xmlPath, pluginDir) {
try {
const pluginXML = fs.readFileSync(xmlPath);
const parsedXML = await xml2js.parseStringPromise(pluginXML);
const requirements = parsedXML.visualization.requirements[0].requirement;
const pluginXML = fs.readFileSync(xmlPath);
const parsedXML = await xml2js.parseStringPromise(pluginXML);
const requirements = parsedXML?.visualization?.requirements?.[0]?.requirement || [];
const installPromises = requirements.map(async (dep) => {
const { type: reqType, package: pkgName, version } = dep.$;
if (reqType === "npm" && pkgName && version) {
try {
const installResult = child_process.spawnSync(
"npm",
["install", "--silent", "--no-save", "--prefix .", `${pkgName}@${version}`],
{
cwd: pluginDir,
stdio: "inherit",
shell: true,
}
);
if (installResult.status === 0) {
await fs.copy(
path.join(pluginDir, "node_modules", pkgName, "static"),
path.join(pluginDir, "static")
);
console.log(`Installed package ${pkgName}@${version} in ${pluginDir}`);
} else {
console.error(`Error installing package ${pkgName}@${version} in ${pluginDir}`);
}
} catch (err) {
console.error(`Error handling package ${pkgName}@${version} in ${pluginDir}:`, err);
// For each <requirement type="npm" package="xxx" version="yyy" />
for (const dep of requirements) {
const { type: reqType, package: pkgName, version } = dep.$;
if (reqType === "npm" && pkgName && version) {
console.log(` → Installing ${pkgName}@${version} in ${pluginDir}`);
const installResult = child_process.spawnSync(
"npm",
["install", "--silent", "--no-save", `--prefix=${pluginDir}`, `${pkgName}@${version}`],
{
cwd: pluginDir,
stdio: "inherit",
shell: true,
}
}
});
);
await Promise.all(installPromises);
} catch (err) {
console.error(`Error processing XML file ${xmlPath}:`, err);
if (installResult.status === 0) {
// Installed successfully, copy over '/static' assets
const potentialStaticPath = path.join(pluginDir, "node_modules", pkgName, "static");
if (fs.existsSync(potentialStaticPath)) {
await fs.copy(potentialStaticPath, path.join(pluginDir, "static"));
}
} else {
throw new Error(`npm install of ${pkgName}@${version} failed in ${pluginDir}`);
}
}
}
}
@@ -280,13 +305,17 @@ function forceBuildPlugins(callback) {
return buildPlugins(callback, true);
}
function forceInstallPlugins(callback) {
return installPlugins(callback, true);
}
function cleanPlugins() {
return del(["../static/plugins/{visualizations,welcome_page}/*"], { force: true });
}
const client = parallel(stageLibs, icons);
const plugins = series(buildPlugins, installPlugins, cleanPlugins, stagePlugins);
const pluginsRebuild = series(forceBuildPlugins, installPlugins, cleanPlugins, stagePlugins);
const pluginsRebuild = series(forceBuildPlugins, forceInstallPlugins, cleanPlugins, stagePlugins);
function watchPlugins() {
const BUILD_PLUGIN_WATCH_GLOB = [