From 03bd478aaa42dbbb21a4baf56a53e6abc65432ae Mon Sep 17 00:00:00 2001 From: Kamran Ahmed Date: Wed, 26 Apr 2023 13:47:19 +0100 Subject: [PATCH] Add data aggregation command --- .gitignore | 3 +- bin/page-data-agg.cjs | 129 ++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + pnpm-lock.yaml | 11 +++- 4 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 bin/page-data-agg.cjs diff --git a/.gitignore b/.gitignore index 1cb2f0fcc..c78473237 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ pnpm-debug.log* /test-results/ /playwright-report/ /playwright/.cache/ -tests-examples \ No newline at end of file +tests-examples +*.csv \ No newline at end of file diff --git a/bin/page-data-agg.cjs b/bin/page-data-agg.cjs new file mode 100644 index 000000000..e449e64c1 --- /dev/null +++ b/bin/page-data-agg.cjs @@ -0,0 +1,129 @@ +const csv = require('csv-parser'); +const fs = require('fs'); +const path = require('path'); + +const csvFilePath = path.join(__dirname, '../data.csv'); + +const results = {}; +const pageSummary = {}; + +fs.createReadStream(csvFilePath) + .pipe( + csv({ + separator: ',', + mapHeaders: ({ header, index }) => + header.toLowerCase().replace(/ /g, '_'), + mapValues: ({ header, index, value }) => { + if (header === 'page') { + return ( + value + .replace(/"/g, '') + .replace(/'/g, '') + .replace(/`/g, '') + .replace(/\?r=/g, '#r#') + .replace(/\?.+?$/g, '') + .replace(/#r#/g, '?r=') + .replace(/\/$/g, '') || '/' + ); + } + + if (header !== 'month_of_year') { + return parseInt(value, 10); + } + + return value; + }, + }) + ) + .on('data', (data) => { + const { page, month_of_year, unique_pageviews, users } = data; + const pageData = results[page] || {}; + const existingPageMonthData = pageData[month_of_year] || {}; + + const existingViews = existingPageMonthData.views || 0; + const existingUsers = existingPageMonthData.users || 0; + + const newViews = existingViews + unique_pageviews; + const newUsers = existingUsers + users; + + pageData[month_of_year] = { + views: newViews, + users: newUsers, + }; + + results[page] = pageData; + + pageSummary[page] = pageSummary[page] || { views: 0, users: 0 }; + pageSummary[page].views += unique_pageviews; + pageSummary[page].users += users; + }) + .on('end', () => { + const csvHeader = [ + 'Page', + 'Jan 2022', + 'Feb 2022', + 'Mar 2022', + 'Apr 2022', + 'May 2022', + 'Jun 2022', + 'Jul 2022', + 'Aug 2022', + 'Sep 2022', + 'Oct 2022', + 'Nov 2022', + 'Dec 2022', + 'Jan 2023', + 'Feb 2023', + 'Mar 2023', + 'Apr 2023', + 'May 2023', + 'Jun 2023', + 'Jul 2023', + 'Aug 2023', + 'Sep 2023', + 'Oct 2023', + 'Nov 2023', + 'Dec 2023', + ]; + + const csvRows = Object.keys(pageSummary) + .filter(pageUrl => pageSummary[pageUrl].views > 10) + .filter(pageUrl => !['/upcoming', '/pdfs', '/signup', '/login', '/@'].includes(pageUrl)) + .sort((pageA, pageB) => { + const aViews = pageSummary[pageA].views; + const bViews = pageSummary[pageB].views; + + return bViews - aViews; + }) + .map((pageUrl) => { + const rawPageResult = results[pageUrl]; + const pageResultCsvRow = []; + + csvHeader.forEach((csvHeaderItem) => { + if (csvHeaderItem === 'Page') { + pageResultCsvRow.push(pageUrl); + return; + } + + const csvHeaderItemAlt = csvHeaderItem + .replace(/ /g, '_') + .toLowerCase(); + + const result = rawPageResult[csvHeaderItem || csvHeaderItemAlt] || {}; + const views = result.views || 0; + const users = result.users || 0; + + pageResultCsvRow.push(users); + }); + + return pageResultCsvRow; + }); + + const finalCsvRows = [csvHeader, ...csvRows]; + const csvRowStrings = finalCsvRows.map((row) => { + return row.join(','); + }); + + const csvString = csvRowStrings.join('\n'); + fs.writeFileSync(path.join(__dirname, '../data-agg.csv'), csvString); + }); diff --git a/package.json b/package.json index 177c0b9c2..6d6bd7089 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@playwright/test": "^1.32.3", "@tailwindcss/typography": "^0.5.9", "@types/js-cookie": "^3.0.3", + "csv-parser": "^3.0.0", "gh-pages": "^5.0.0", "js-yaml": "^4.1.0", "markdown-it": "^13.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae151ecc8..393f6ae3e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,7 @@ specifiers: '@types/js-cookie': ^3.0.3 astro: ^2.2.3 astro-compress: ^1.1.35 + csv-parser: ^3.0.0 gh-pages: ^5.0.0 jose: ^4.13.2 js-cookie: ^3.0.1 @@ -50,6 +51,7 @@ devDependencies: '@playwright/test': 1.32.3 '@tailwindcss/typography': 0.5.9_tailwindcss@3.3.1 '@types/js-cookie': 3.0.3 + csv-parser: 3.0.0 gh-pages: 5.0.0 js-yaml: 4.1.0 markdown-it: 13.0.1 @@ -1787,6 +1789,14 @@ packages: css-tree: 2.2.1 dev: false + /csv-parser/3.0.0: + resolution: {integrity: sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==} + engines: {node: '>= 10'} + hasBin: true + dependencies: + minimist: 1.2.8 + dev: true + /debug/4.3.4: resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -3605,7 +3615,6 @@ packages: /minimist/1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: false /minipass-collect/1.0.2: resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==}