From f5253b40270643a777aa091ffc740d88108abaec Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Mon, 20 Apr 2020 18:17:48 +0200 Subject: [PATCH 1/7] skip visualization build, if not changes were registered --- client/gulpfile.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/client/gulpfile.js b/client/gulpfile.js index af0d5c6369c..c560cdae6ff 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -73,14 +73,28 @@ function buildPlugins(callback){ /* * Walk plugin build glob and attempt to build anything with a package.json * */ + + const client_hash_path = '../static/client_build_hash.txt'; paths.plugin_build_dirs.map( build_dir => { glob(build_dir, {}, (er, files) => { files.map( file => { + let skip_build = false; const f = path.join(process.cwd(), file).slice(0, -12); - console.log("Installing Dependencies for", f); - spawn('yarn', ['install', '--production=false', '--network-timeout=300000', '--check-files'], { cwd: f, stdio: 'inherit', shell: true }); - console.log("Building ", f); - spawn('yarn', ['build'], { cwd: f, stdio: 'inherit', shell: true }); + + if (fs.existsSync(client_hash_path)) { + skip_build = spawn('git', ['diff', '--quiet', '"$(cat ../static/client_build_hash.txt)"', '--', f], { + stdio: 'pipe', + shell: true + }).status === 0; + } + if(skip_build){ + console.log("No changes detected for", f) + } else { + console.log("Installing Dependencies for", f); + spawn('yarn', ['install', '--production=false', '--network-timeout=300000', '--check-files'], { cwd: f, stdio: 'inherit', shell: true }); + console.log("Building ", f); + spawn('yarn', ['build'], { cwd: f, stdio: 'inherit', shell: true }); + } }); }); }); From b2d89658026edb862d63ae6b2290c2876597bd61 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Tue, 21 Apr 2020 18:16:04 +0200 Subject: [PATCH 2/7] make individual plugin build hash for every viz --- .gitignore | 2 ++ client/gulpfile.js | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e4d13b558e7..804aba6a4ba 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,5 @@ packages/*/*.egg-info config/plugins/visualizations/**/static/script.js # TODO: Really need to follow up on this and make it standard. config/plugins/visualizations/**/static/main.css + +config/plugins/visualizations/**/static/*_plugin_build_hash.txt diff --git a/client/gulpfile.js b/client/gulpfile.js index c560cdae6ff..da32a8a6d9d 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -74,26 +74,33 @@ function buildPlugins(callback){ * Walk plugin build glob and attempt to build anything with a package.json * */ - const client_hash_path = '../static/client_build_hash.txt'; - paths.plugin_build_dirs.map( build_dir => { + const static_viz_path = '../static/plugins/visualizations'; + paths.plugin_build_dirs.map(build_dir => { glob(build_dir, {}, (er, files) => { files.map( file => { let skip_build = false; const f = path.join(process.cwd(), file).slice(0, -12); - if (fs.existsSync(client_hash_path)) { - skip_build = spawn('git', ['diff', '--quiet', '"$(cat ../static/client_build_hash.txt)"', '--', f], { + const plugin_name = path.dirname(file).split(path.sep).pop() + const hash_file_name = plugin_name + '_plugin_build_hash.txt' + const hash_file_path = path.join(static_viz_path, plugin_name, 'static', hash_file_name) + + if (fs.existsSync(hash_file_path)) { + skip_build = spawn('git', ['diff', '--quiet', '"$(cat ' + hash_file_path + ')"', '--', f], { stdio: 'pipe', shell: true }).status === 0; } + if(skip_build){ - console.log("No changes detected for", f) + console.log("No changes detected for", plugin_name) } else { - console.log("Installing Dependencies for", f); + console.log("Installing Dependencies for", plugin_name); spawn('yarn', ['install', '--production=false', '--network-timeout=300000', '--check-files'], { cwd: f, stdio: 'inherit', shell: true }); - console.log("Building ", f); + console.log("Building ", plugin_name); spawn('yarn', ['build'], { cwd: f, stdio: 'inherit', shell: true }); + // hash_file_name is copied to static during stagePlugins() + spawn('bash', ['-c', '"(git rev-parse HEAD 2>/dev/null || echo ``) > ' + f + '/static/' + hash_file_name + '"'], {shell: true}); } }); }); @@ -106,7 +113,7 @@ function cleanPlugins() { } client = parallel(fonts, stageLibs); -plugins = series(cleanPlugins, buildPlugins, stagePlugins); +plugins = series(buildPlugins, cleanPlugins, stagePlugins); module.exports.client = client; module.exports.plugins = plugins; From 52e60271094fd4b9afd42ac6b71e85e7c1383173 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 21 Apr 2020 16:53:00 -0400 Subject: [PATCH 3/7] Standardize formatting --- client/gulpfile.js | 58 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/client/gulpfile.js b/client/gulpfile.js index da32a8a6d9d..8564ecafd23 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -9,7 +9,7 @@ const paths = { node_modules: "./node_modules", plugin_dirs: [ "../config/plugins/{visualizations,interactive_environments}/*/static/**/*", - "../config/plugins/{visualizations,interactive_environments}/*/*/static/**/*" + "../config/plugins/{visualizations,interactive_environments}/*/*/static/**/*", ], /* * We'll want a flexible glob down the road, but for now there are no @@ -20,9 +20,7 @@ const paths = { // "../config/plugins/{visualizations,interactive_environments}/*/package.json", // "../config/plugins/{visualizations,interactive_environments}/*/*/package.json" //], - plugin_build_dirs: [ - "../config/plugins/visualizations/{annotate_image,hyphyvision,openlayers,editor}/package.json", - ], + plugin_build_dirs: ["../config/plugins/visualizations/{annotate_image,hyphyvision,openlayers,editor}/package.json"], lib_locs: { // This is a stepping stone towards having all this staged // automatically. Eventually, this dictionary and staging step will @@ -37,13 +35,13 @@ const paths = { "jquery-mousewheel": ["jquery.mousewheel.js", "jquery/jquery.mousewheel.js"], "raven-js": ["dist/raven.js", "raven.js"], requirejs: ["require.js", "require.js"], - underscore: ["underscore.js", "underscore.js"] + underscore: ["underscore.js", "underscore.js"], }, - libs: ["galaxy/scripts/libs/**/*.js"] + libs: ["galaxy/scripts/libs/**/*.js"], }; function stageLibs(callback) { - Object.keys(paths.lib_locs).forEach(lib => { + Object.keys(paths.lib_locs).forEach((lib) => { var p1 = path.resolve(path.join(paths.node_modules, lib, paths.lib_locs[lib][0])); var p2 = path.resolve(path.join("galaxy", "scripts", "libs", paths.lib_locs[lib][1])); if (fs.existsSync(p1)) { @@ -69,38 +67,50 @@ function stagePlugins() { return src(paths.plugin_dirs).pipe(dest("../static/plugins/")); } -function buildPlugins(callback){ +function buildPlugins(callback) { /* - * Walk plugin build glob and attempt to build anything with a package.json + * Walk plugin build glob and attempt to build anything with a package.json * */ - const static_viz_path = '../static/plugins/visualizations'; - paths.plugin_build_dirs.map(build_dir => { + const static_viz_path = "../static/plugins/visualizations"; + paths.plugin_build_dirs.map((build_dir) => { glob(build_dir, {}, (er, files) => { - files.map( file => { + files.map((file) => { let skip_build = false; const f = path.join(process.cwd(), file).slice(0, -12); - const plugin_name = path.dirname(file).split(path.sep).pop() - const hash_file_name = plugin_name + '_plugin_build_hash.txt' - const hash_file_path = path.join(static_viz_path, plugin_name, 'static', hash_file_name) + const plugin_name = path.dirname(file).split(path.sep).pop(); + const hash_file_name = plugin_name + "_plugin_build_hash.txt"; + const hash_file_path = path.join(static_viz_path, plugin_name, "static", hash_file_name); if (fs.existsSync(hash_file_path)) { - skip_build = spawn('git', ['diff', '--quiet', '"$(cat ' + hash_file_path + ')"', '--', f], { - stdio: 'pipe', - shell: true - }).status === 0; + skip_build = + spawn("git", ["diff", "--quiet", '"$(cat ' + hash_file_path + ')"', "--", f], { + stdio: "pipe", + shell: true, + }).status === 0; } - if(skip_build){ - console.log("No changes detected for", plugin_name) + if (skip_build) { + console.log("No changes detected for", plugin_name); } else { console.log("Installing Dependencies for", plugin_name); - spawn('yarn', ['install', '--production=false', '--network-timeout=300000', '--check-files'], { cwd: f, stdio: 'inherit', shell: true }); + spawn("yarn", ["install", "--production=false", "--network-timeout=300000", "--check-files"], { + cwd: f, + stdio: "inherit", + shell: true, + }); console.log("Building ", plugin_name); - spawn('yarn', ['build'], { cwd: f, stdio: 'inherit', shell: true }); + spawn("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }); // hash_file_name is copied to static during stagePlugins() - spawn('bash', ['-c', '"(git rev-parse HEAD 2>/dev/null || echo ``) > ' + f + '/static/' + hash_file_name + '"'], {shell: true}); + spawn( + "bash", + [ + "-c", + '"(git rev-parse HEAD 2>/dev/null || echo ``) > ' + f + "/static/" + hash_file_name + '"', + ], + { shell: true } + ); } }); }); From b772abebdf51ba5f92218b7c554f915703b5e902 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 22 Apr 2020 17:06:44 -0400 Subject: [PATCH 4/7] Simplify hash file logic; minor refactoring. --- .gitignore | 3 +-- client/gulpfile.js | 20 ++++++-------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 804aba6a4ba..e66973d8067 100644 --- a/.gitignore +++ b/.gitignore @@ -178,5 +178,4 @@ packages/*/*.egg-info config/plugins/visualizations/**/static/script.js # TODO: Really need to follow up on this and make it standard. config/plugins/visualizations/**/static/main.css - -config/plugins/visualizations/**/static/*_plugin_build_hash.txt +config/plugins/visualizations/**/static/plugin_build_hash.txt diff --git a/client/gulpfile.js b/client/gulpfile.js index 8564ecafd23..5e026534090 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -72,7 +72,6 @@ function buildPlugins(callback) { * Walk plugin build glob and attempt to build anything with a package.json * */ - const static_viz_path = "../static/plugins/visualizations"; paths.plugin_build_dirs.map((build_dir) => { glob(build_dir, {}, (er, files) => { files.map((file) => { @@ -80,13 +79,12 @@ function buildPlugins(callback) { const f = path.join(process.cwd(), file).slice(0, -12); const plugin_name = path.dirname(file).split(path.sep).pop(); - const hash_file_name = plugin_name + "_plugin_build_hash.txt"; - const hash_file_path = path.join(static_viz_path, plugin_name, "static", hash_file_name); + const hash_file_path = path.join(f, "static", "plugin_build_hash.txt"); if (fs.existsSync(hash_file_path)) { skip_build = - spawn("git", ["diff", "--quiet", '"$(cat ' + hash_file_path + ')"', "--", f], { - stdio: "pipe", + spawn("git", ["diff", "--quiet", `$(cat ${hash_file_path})`, "--", f], { + stdio: "inherit", shell: true, }).status === 0; } @@ -102,15 +100,9 @@ function buildPlugins(callback) { }); console.log("Building ", plugin_name); spawn("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }); - // hash_file_name is copied to static during stagePlugins() - spawn( - "bash", - [ - "-c", - '"(git rev-parse HEAD 2>/dev/null || echo ``) > ' + f + "/static/" + hash_file_name + '"', - ], - { shell: true } - ); + spawn("bash", ["-c", `"(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hash_file_path} "`], { + shell: true, + }); } }); }); From 64cf1d0aff8f4199fcc71748affdcc58541a6d24 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 22 Apr 2020 17:07:23 -0400 Subject: [PATCH 5/7] Add log statement explaining generation of the hashfile --- client/gulpfile.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/gulpfile.js b/client/gulpfile.js index 5e026534090..5a2cc0e5e13 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -87,6 +87,8 @@ function buildPlugins(callback) { stdio: "inherit", shell: true, }).status === 0; + } else { + console.log(`No build hashfile detected for ${plugin_name}, generating now.`); } if (skip_build) { From aafb00ff1d236d1f089d4cb890975eaeb9f3b89f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 22 Apr 2020 17:09:13 -0400 Subject: [PATCH 6/7] Update doc, swap to string literals for log statements --- client/gulpfile.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/gulpfile.js b/client/gulpfile.js index 5a2cc0e5e13..d270fb30d57 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -69,7 +69,7 @@ function stagePlugins() { function buildPlugins(callback) { /* - * Walk plugin build glob and attempt to build anything with a package.json + * Walk plugin_build_dirs glob and attempt to build modules. * */ paths.plugin_build_dirs.map((build_dir) => { @@ -77,7 +77,6 @@ function buildPlugins(callback) { files.map((file) => { let skip_build = false; const f = path.join(process.cwd(), file).slice(0, -12); - const plugin_name = path.dirname(file).split(path.sep).pop(); const hash_file_path = path.join(f, "static", "plugin_build_hash.txt"); @@ -92,15 +91,15 @@ function buildPlugins(callback) { } if (skip_build) { - console.log("No changes detected for", plugin_name); + console.log(`No changes detected for ${plugin_name}`); } else { - console.log("Installing Dependencies for", plugin_name); + console.log(`Installing Dependencies for ${plugin_name}`); spawn("yarn", ["install", "--production=false", "--network-timeout=300000", "--check-files"], { cwd: f, stdio: "inherit", shell: true, }); - console.log("Building ", plugin_name); + console.log(`Building ${plugin_name}`); spawn("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }); spawn("bash", ["-c", `"(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hash_file_path} "`], { shell: true, From 0dad76dd7c13faaec83a3b3af50b51a409566454 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 22 Apr 2020 17:16:33 -0400 Subject: [PATCH 7/7] Be explicit that we're using spawnSync here; swap to exec for git command to avoid explicitly using bash --- client/gulpfile.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/client/gulpfile.js b/client/gulpfile.js index d270fb30d57..0c67ec7a220 100644 --- a/client/gulpfile.js +++ b/client/gulpfile.js @@ -2,7 +2,7 @@ const path = require("path"); const fs = require("fs"); const del = require("del"); const { src, dest, series, parallel } = require("gulp"); -const spawn = require("child_process").spawnSync; +const child_process = require("child_process"); const glob = require("glob"); const paths = { @@ -71,7 +71,6 @@ function buildPlugins(callback) { /* * Walk plugin_build_dirs glob and attempt to build modules. * */ - paths.plugin_build_dirs.map((build_dir) => { glob(build_dir, {}, (er, files) => { files.map((file) => { @@ -82,7 +81,7 @@ function buildPlugins(callback) { if (fs.existsSync(hash_file_path)) { skip_build = - spawn("git", ["diff", "--quiet", `$(cat ${hash_file_path})`, "--", f], { + child_process.spawnSync("git", ["diff", "--quiet", `$(cat ${hash_file_path})`, "--", f], { stdio: "inherit", shell: true, }).status === 0; @@ -94,16 +93,18 @@ function buildPlugins(callback) { console.log(`No changes detected for ${plugin_name}`); } else { console.log(`Installing Dependencies for ${plugin_name}`); - spawn("yarn", ["install", "--production=false", "--network-timeout=300000", "--check-files"], { - cwd: f, - stdio: "inherit", - shell: true, - }); + child_process.spawnSync( + "yarn", + ["install", "--production=false", "--network-timeout=300000", "--check-files"], + { + cwd: f, + stdio: "inherit", + shell: true, + } + ); console.log(`Building ${plugin_name}`); - spawn("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }); - spawn("bash", ["-c", `"(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hash_file_path} "`], { - shell: true, - }); + child_process.spawnSync("yarn", ["build"], { cwd: f, stdio: "inherit", shell: true }); + child_process.exec(`"(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hash_file_path} "`); } }); });