diff --git a/client/gulpfile.js b/client/gulpfile.js index e63d755d376..e93908e3bdb 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -12,7 +12,7 @@ const xml2js = require("xml2js"); * un-built visualizations in the repository; for performance and * simplicity just add them one at a time until we upgrade older viz's. */ -const STATIC_PLUGIN_BUILD_IDS = [ +const PLUGIN_BUILD_IDS = [ "annotate_image", "chiraviz", "drawrna", @@ -46,8 +46,6 @@ const INSTALL_PLUGIN_BUILD_IDS = [ "vitessce", "vizarr", ]; // todo: derive from XML -const DIST_PLUGIN_BUILD_IDS = ["new_user"]; -const PLUGIN_BUILD_IDS = Array.prototype.concat(DIST_PLUGIN_BUILD_IDS, STATIC_PLUGIN_BUILD_IDS); const args = process.argv.slice(2); const limitIndex = args.indexOf("--limit"); @@ -65,9 +63,6 @@ const failOnError = const PATHS = { nodeModules: "./node_modules", stagedLibraries: { - // This is a stepping stone towards having all this staged - // automatically. Eventually, this dictionary and staging step will - // not be necessary. backbone: ["backbone.js", "backbone.js"], jquery: ["dist/jquery.js", "jquery/jquery.js"], "jquery-migrate": ["dist/jquery-migrate.js", "jquery/jquery.migrate.js"], @@ -95,8 +90,8 @@ PATHS.pluginBuildModules = [ function stageLibs(callback) { Object.keys(PATHS.stagedLibraries).forEach((lib) => { - var p1 = path.resolve(path.join(PATHS.nodeModules, lib, PATHS.stagedLibraries[lib][0])); - var p2 = path.resolve(path.join("src", "libs", PATHS.stagedLibraries[lib][1])); + const p1 = path.resolve(path.join(PATHS.nodeModules, lib, PATHS.stagedLibraries[lib][0])); + const p2 = path.resolve(path.join("src", "libs", PATHS.stagedLibraries[lib][1])); if (fs.existsSync(p1)) { del.sync(p2); fs.createReadStream(p1).pipe(fs.createWriteStream(p2)); @@ -121,48 +116,37 @@ function stagePlugins() { function buildPlugins(callback, forceRebuild) { /* * Walk pluginBuildModules glob and attempt to build modules. - * */ + */ const packageJsons = globSync(PATHS.pluginBuildModules, {}); packageJsons.forEach((file) => { let skipBuild = false; const pluginDir = path.dirname(file); const pluginName = pluginDir.split(path.sep).pop(); - const hashFilePath = path.join( - pluginDir, - DIST_PLUGIN_BUILD_IDS.indexOf(pluginName) > -1 ? "dist" : "static", - "plugin_build_hash.txt" - ); + const hashFilePath = path.join(pluginDir, "static", "plugin_build_hash.txt"); if (forceRebuild) { skipBuild = false; } else { + // Try reading existing plugin_build_hash.txt if (fs.existsSync(hashFilePath)) { - const hashFileContent = fs.readFileSync(hashFilePath, "utf8").trim(); - const isHash = /^[0-9a-f]{7,40}$/.test(hashFileContent); // Check for a 7 to 40 character hexadecimal string + const oldHash = fs.readFileSync(hashFilePath, "utf8").trim(); + const isHash = /^[0-9a-f]{7,40}$/.test(oldHash); if (!isHash) { console.log(`Hash file for ${pluginName} exists but does not have a valid git hash.`); skipBuild = false; } else { - skipBuild = - child_process.spawnSync("git", ["diff", "--quiet", hashFileContent, "--", pluginDir], { - stdio: "inherit", - shell: true, - }).status === 0; - } - - skipBuild = - child_process.spawnSync("git", ["diff", "--quiet", `$(cat ${hashFilePath})`, "--", pluginDir], { + // Check if there are changes since the stored hash + const diffResult = child_process.spawnSync("git", ["diff", "--quiet", oldHash, "--", pluginDir], { stdio: "inherit", shell: true, - }).status === 0; - if (!skipBuild) { - // Hash exists and is outdated, triggering a rebuild. - // Stage current hash to .orig for debugging and to - // force a plugin rebuild in the event of a failure - // (i.e. -- we're committed to a new build of this plugin). - fs.renameSync(hashFilePath, `${hashFilePath}.orig`); + }); + skipBuild = diffResult.status === 0; + if (!skipBuild) { + // Hash exists and is outdated, rename to .orig + fs.renameSync(hashFilePath, `${hashFilePath}.orig`); + } } } else { console.log(`No build hashfile detected for ${pluginName}, generating now.`); @@ -170,11 +154,10 @@ function buildPlugins(callback, forceRebuild) { } if (skipBuild) { - console.log(`No changes detected for ${pluginName}`); + console.log(`No changes detected for ${pluginName}, skipping build.`); } else { console.log(`Installing Dependencies for ${pluginName}`); - - // Else we call yarn install and yarn build + // Yarn install child_process.spawnSync( "yarn", ["install", "--production=false", "--network-timeout=300000", "--check-files"], @@ -184,32 +167,34 @@ function buildPlugins(callback, forceRebuild) { shell: true, } ); + console.log(`Building ${pluginName}`); const opts = { cwd: pluginDir, stdio: "inherit", shell: true, }; - // if node version is >16, set NODE_OPTIONS to use legacy openssl provider - if (process.versions.node.split(".")[0] > "16") { + // Node >16 fix + if (Number(process.versions.node.split(".")[0]) > 16) { opts.env = { ...process.env, PARCEL_WORKER_BACKEND: "process", NODE_OPTIONS: "--openssl-legacy-provider", }; } - if (child_process.spawnSync("yarn", ["build"], opts).status === 0) { + + const buildResult = child_process.spawnSync("yarn", ["build"], opts); + if (buildResult.status === 0) { console.log(`Successfully built, saving build state to ${hashFilePath}`); - child_process.exec(`(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hashFilePath}`); + // Save new hash + child_process.execSync(`git rev-parse HEAD > ${hashFilePath}`, { + stdio: "inherit", + shell: true, + }); } else { - console.error( - `Error building ${pluginName}, not saving build state. Please report this issue to the Galaxy Team.` - ); + console.error(`Error building ${pluginName}, not saving build state.`); if (failOnError) { - // Fail on error. - console.error( - "Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set, see error(s) above." - ); + console.error("Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set."); process.exit(1); } } @@ -218,61 +203,101 @@ function buildPlugins(callback, forceRebuild) { return callback(); } -async function installPlugins(callback) { - // iterate through install_plugin_build_ids, identify xml files and install dependencies - for (const plugin_name of INSTALL_PLUGIN_BUILD_IDS.filter(applyPluginFilter)) { - const pluginDir = path.join(PATHS.pluginBaseDir, `visualizations/${plugin_name}`); - const xmlPath = path.join(pluginDir, `config/${plugin_name}.xml`); - // Check if the file exists - if (fs.existsSync(xmlPath)) { - await installDependenciesFromXML(xmlPath, pluginDir); - } else { +/** + * Same logic as buildPlugins, but for 'installed' plugins that rely on an XML + * specifying npm dependencies (the INSTALL_PLUGIN_BUILD_IDS). + */ +async function installPlugins(callback, forceReinstall) { + for (const pluginName of INSTALL_PLUGIN_BUILD_IDS.filter(applyPluginFilter)) { + const pluginDir = path.join(PATHS.pluginBaseDir, `visualizations/${pluginName}`); + const xmlPath = path.join(pluginDir, `config/${pluginName}.xml`); + + if (!fs.existsSync(xmlPath)) { console.error(`XML file not found: ${xmlPath}`); + continue; + } + + const hashFilePath = path.join(pluginDir, "static", "plugin_build_hash.txt"); + + let skipInstall = false; + if (!forceReinstall && fs.existsSync(hashFilePath)) { + const oldHash = fs.readFileSync(hashFilePath, "utf8").trim(); + const isHash = /^[0-9a-f]{7,40}$/.test(oldHash); + + if (!isHash) { + console.log(`Hash file for ${pluginName} exists but is not a valid git hash.`); + } else { + const diffResult = child_process.spawnSync("git", ["diff", "--quiet", oldHash, "--", pluginDir], { + stdio: "inherit", + shell: true, + }); + skipInstall = diffResult.status === 0; + if (!skipInstall) { + fs.renameSync(hashFilePath, `${hashFilePath}.orig`); + } + } + } else if (!fs.existsSync(hashFilePath)) { + console.log(`No install hashfile detected for ${pluginName}, generating now.`); + } + + if (skipInstall) { + console.log(`No changes detected for ${pluginName}, skipping re-installation.`); + } else { + console.log(`Installing npm dependencies for installed plugin ${pluginName}...`); + try { + await installDependenciesFromXML(xmlPath, pluginDir); + console.log(`Finished installing for ${pluginName}, saving new hash.`); + + // Save the current commit hash + child_process.execSync(`git rev-parse HEAD > ${hashFilePath}`, { + stdio: "inherit", + shell: true, + }); + } catch (err) { + console.error(`Error installing for ${pluginName}:`, err); + if (failOnError) { + console.error( + "Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set, see error(s) above." + ); + process.exit(1); + } + } } } return callback(); } -// Function to parse the XML and install dependencies +// Function to parse XML and install specified npm packages async function installDependenciesFromXML(xmlPath, pluginDir) { - try { - const pluginXML = fs.readFileSync(xmlPath); - const parsedXML = await xml2js.parseStringPromise(pluginXML); - const requirements = parsedXML.visualization.requirements[0].requirement; + const pluginXML = fs.readFileSync(xmlPath); + const parsedXML = await xml2js.parseStringPromise(pluginXML); + const requirements = parsedXML?.visualization?.requirements?.[0]?.requirement || []; - const installPromises = requirements.map(async (dep) => { - const { type: reqType, package: pkgName, version } = dep.$; - - if (reqType === "npm" && pkgName && version) { - try { - const installResult = child_process.spawnSync( - "npm", - ["install", "--silent", "--no-save", "--prefix .", `${pkgName}@${version}`], - { - cwd: pluginDir, - stdio: "inherit", - shell: true, - } - ); - - if (installResult.status === 0) { - await fs.copy( - path.join(pluginDir, "node_modules", pkgName, "static"), - path.join(pluginDir, "static") - ); - console.log(`Installed package ${pkgName}@${version} in ${pluginDir}`); - } else { - console.error(`Error installing package ${pkgName}@${version} in ${pluginDir}`); - } - } catch (err) { - console.error(`Error handling package ${pkgName}@${version} in ${pluginDir}:`, err); + // For each + for (const dep of requirements) { + const { type: reqType, package: pkgName, version } = dep.$; + if (reqType === "npm" && pkgName && version) { + console.log(` → Installing ${pkgName}@${version} in ${pluginDir}`); + const installResult = child_process.spawnSync( + "npm", + ["install", "--silent", "--no-save", `--prefix=${pluginDir}`, `${pkgName}@${version}`], + { + cwd: pluginDir, + stdio: "inherit", + shell: true, } - } - }); + ); - await Promise.all(installPromises); - } catch (err) { - console.error(`Error processing XML file ${xmlPath}:`, err); + if (installResult.status === 0) { + // Installed successfully, copy over '/static' assets + const potentialStaticPath = path.join(pluginDir, "node_modules", pkgName, "static"); + if (fs.existsSync(potentialStaticPath)) { + await fs.copy(potentialStaticPath, path.join(pluginDir, "static")); + } + } else { + throw new Error(`npm install of ${pkgName}@${version} failed in ${pluginDir}`); + } + } } } @@ -280,13 +305,17 @@ function forceBuildPlugins(callback) { return buildPlugins(callback, true); } +function forceInstallPlugins(callback) { + return installPlugins(callback, true); +} + function cleanPlugins() { return del(["../static/plugins/{visualizations,welcome_page}/*"], { force: true }); } const client = parallel(stageLibs, icons); const plugins = series(buildPlugins, installPlugins, cleanPlugins, stagePlugins); -const pluginsRebuild = series(forceBuildPlugins, installPlugins, cleanPlugins, stagePlugins); +const pluginsRebuild = series(forceBuildPlugins, forceInstallPlugins, cleanPlugins, stagePlugins); function watchPlugins() { const BUILD_PLUGIN_WATCH_GLOB = [