diff --git a/package-lock.json b/package-lock.json index 323a78bed2f0..068f551e7501 100644 --- a/package-lock.json +++ b/package-lock.json @@ -209,6 +209,7 @@ "copy-webpack-plugin": "^10.1.0", "css-loader": "^6.7.2", "csv-parse": "^5.5.5", + "csv-writer": "^1.6.0", "diff-so-fancy": "^1.3.0", "dotenv": "^16.0.3", "electron": "^29.4.6", @@ -21224,6 +21225,12 @@ "dev": true, "license": "MIT" }, + "node_modules/csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==", + "dev": true + }, "node_modules/dag-map": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/dag-map/-/dag-map-1.0.2.tgz", diff --git a/package.json b/package.json index f4ac5dbe8c3b..2fcce0665a29 100644 --- a/package.json +++ b/package.json @@ -266,6 +266,7 @@ "copy-webpack-plugin": "^10.1.0", "css-loader": "^6.7.2", "csv-parse": "^5.5.5", + "csv-writer": "^1.6.0", "diff-so-fancy": "^1.3.0", "dotenv": "^16.0.3", "electron": "^29.4.6", diff --git a/scripts/aggregateGitHubDataFromUpwork.ts b/scripts/aggregateGitHubDataFromUpwork.ts new file mode 100644 index 000000000000..f47b2b43e5cc --- /dev/null +++ b/scripts/aggregateGitHubDataFromUpwork.ts @@ -0,0 +1,175 @@ +/** + * This script is used for categorizing upwork costs into cost buckets for accounting purposes. + * + * To run this script from the root of E/App: + * + * ts-node ./scripts/aggregateGitHubDataFromUpwork.js + * + * The input file must be a CSV with a single column containing just the GitHub issue number. The CSV must have a single header row. + */ +import {getOctokitOptions, GitHub} from '@actions/github/lib/utils'; +import {paginateRest} from '@octokit/plugin-paginate-rest'; +import {throttling} from '@octokit/plugin-throttling'; +import {createObjectCsvWriter} from 'csv-writer'; +import fs from 'fs'; + +type OctokitOptions = {method: string; url: string; request: {retryCount: number}}; +type IssueType = 'bug' | 'feature' | 'other'; + +if (process.argv.length < 3) { + throw new Error('Error: must provide filepath for CSV data'); +} + +if (process.argv.length < 4) { + throw new Error('Error: must provide GitHub token'); +} + +if (process.argv.length < 5) { + throw new Error('Error: must provide output file path'); +} + +// Get filepath for csv +const inputFilepath = process.argv.at(2); +if (!inputFilepath) { + throw new Error('Error: must provide filepath for CSV data'); +} + +// Get GitHub token +const token = (process.argv.at(3) ?? '').trim(); +if (!token) { + throw new Error('Error: must provide GitHub token'); +} + +const Octokit = GitHub.plugin(throttling, paginateRest); +const octokit = new Octokit( + getOctokitOptions(token, { + throttle: { + onRateLimit: (retryAfter: number, options: OctokitOptions) => { + console.warn(`Request quota exhausted for request ${options.method} ${options.url}`); + + // Retry once after hitting a rate limit error, then give up + if (options.request.retryCount <= 1) { + console.log(`Retrying after ${retryAfter} seconds!`); + return true; + } + }, + onAbuseLimit: (retryAfter: number, options: OctokitOptions) => { + // does not retry, only logs a warning + console.warn(`Abuse detected for request ${options.method} ${options.url}`); + }, + }, + }), +); + +// Get output filepath +const outputFilepath = process.argv.at(4); +if (!outputFilepath) { + throw new Error('Error: must provide output file path'); +} + +// Get data from csv +const issues = fs + .readFileSync(inputFilepath) + .toString() + .split('\n') + .reduce((acc, issue) => { + if (!issue) { + return acc; + } + const issueNum = Number(issue.trim()); + if (!issueNum) { + return acc; + } + acc.push(issueNum); + return acc; + }, [] as number[]); + +const csvWriter = createObjectCsvWriter({ + path: outputFilepath, + header: [ + {id: 'number', title: 'number'}, + {id: 'title', title: 'title'}, + {id: 'labels', title: 'labels'}, + {id: 'type', title: 'type'}, + {id: 'capSWProjects', title: 'capSWProjects'}, + ], +}); + +function getIssueTypeFromLabels(labels: string[]): IssueType { + if (labels.includes('NewFeature')) { + return 'feature'; + } + if (labels.includes('Bug')) { + return 'bug'; + } + return 'other'; +} + +/** + * Returns a comma-delimited string with all projects associated with the given issue. + */ +async function getProjectsForIssue(issueNumber: number): Promise { + const response = await octokit.graphql( + ` + { + repository(owner: "Expensify", name: "App") { + issue(number: ${issueNumber}) { + projectsV2(last: 30) { + nodes { + title + } + } + } + } + } + `, + ); + return (response as {repository: {issue: {projectsV2: {nodes: Array<{title: string}>}}}}).repository.issue.projectsV2.nodes.map((node) => node.title).join(','); +} + +async function getGitHubData() { + const gitHubData = []; + // Note: we fetch issues in a loop rather than in parallel to help address rate limiting issues with a PAT + for (const issueNumber of issues) { + console.info(`Fetching ${issueNumber}`); + const result = await octokit.rest.issues + .get({ + owner: 'Expensify', + repo: 'App', + // eslint-disable-next-line @typescript-eslint/naming-convention + issue_number: issueNumber, + }) + .catch(() => { + console.warn(`Error getting issue ${issueNumber}`); + }); + if (result) { + const issue = result.data; + const labels = issue.labels.reduce((acc, label) => { + if (typeof label === 'string') { + acc.push(label); + } else if (label.name) { + acc.push(label.name); + } + return acc; + }, [] as string[]); + const type = getIssueTypeFromLabels(labels); + let capSWProjects = ''; + if (type === 'feature') { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + capSWProjects = await getProjectsForIssue(issueNumber); + } + gitHubData.push({ + number: issue.number, + title: issue.title, + labels, + type, + capSWProjects, + }); + } + } + return gitHubData; +} + +getGitHubData() + .then((gitHubData) => csvWriter.writeRecords(gitHubData)) + .then(() => console.info(`Done ✅ Wrote file to ${outputFilepath}`));