From c28ac4b0784792921d66d6ae2075b97e9660c7d2 Mon Sep 17 00:00:00 2001 From: Kamran Ahmed Date: Mon, 27 Mar 2023 04:11:34 +0100 Subject: [PATCH] Add content writing command --- bin/readme.md | 12 ++++- bin/roadmap-content.cjs | 108 ++++++++++++++++++++++++++++++---------- package.json | 1 + pnpm-lock.yaml | 66 ++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 27 deletions(-) diff --git a/bin/readme.md b/bin/readme.md index 5b38e692d..73ea128c8 100644 --- a/bin/readme.md +++ b/bin/readme.md @@ -9,12 +9,22 @@ Generates a list of all the resources links in any roadmap file. Compresses all the JSON files in the `public/jsons` folder +## `update-sponsors.cjs` + +Updates the sponsor ads on each roadmap page with the latest sponsor information in the Excel sheet. + ## `roadmap-content.cjs` +Currently, for any new roadmaps that we add, we do create the interactive roadmap but we end up leaving the content empty in the roadmap till we get time to fill it up manually. + +This script populates all the content files with some minimal content from OpenAI so that the users visiting the website have something to read in the interactive roadmap till we get time to fill it up manually. + +## `roadmap-dirs.cjs` + This command is used to create the content folders and files for the interactivity of the roadmap. You can use the below command to generate the roadmap skeletons inside a roadmap directory: ```bash -npm run roadmap-content [frontend|backend|devops|...] +npm run roadmap-dirs [frontend|backend|devops|...] ``` For the content skeleton to be generated, we should have proper grouping, and the group names in the project files. You can follow the steps listed below in order to add the meta information to the roadmap. diff --git a/bin/roadmap-content.cjs b/bin/roadmap-content.cjs index 03ae72171..03436bc48 100644 --- a/bin/roadmap-content.cjs +++ b/bin/roadmap-content.cjs @@ -20,6 +20,12 @@ if (!allowedRoadmapIds.includes(roadmapId)) { } const ROADMAP_CONTENT_DIR = path.join(ALL_ROADMAPS_DIR, roadmapId, 'content'); +const { Configuration, OpenAIApi } = require('openai'); +const configuration = new Configuration({ + apiKey: OPEN_AI_API_KEY, +}); + +const openai = new OpenAIApi(configuration); function getFilesInFolder(folderPath, fileList = {}) { const files = fs.readdirSync(folderPath); @@ -44,38 +50,88 @@ function getFilesInFolder(folderPath, fileList = {}) { return fileList; } -const topicUrlToPathMapping = getFilesInFolder(ROADMAP_CONTENT_DIR); +function writeTopicContent(topicTitle) { + console.log(`Genearting '${topicTitle}'...`); -const roadmapJson = require(path.join(ROADMAP_JSON_DIR, `${roadmapId}.json`)); -const groups = roadmapJson?.mockup?.controls?.control?.filter( - (control) => control.typeID === '__group__' && !control.properties?.controlName?.startsWith('ext_link') -); + const instruction = `Write a short paragraph explaining '${topicTitle}' in ${roadmapId}. Avoid any text similar to "In ${roadmapId}, ${topicTitle} refers to" and write it as a direct explanation of the topic.`; -groups.forEach((group) => { - const topicId = group?.properties?.controlName; - const topicTitle = group?.children?.controls?.control?.find((control) => control?.typeID === 'Label')?.properties - ?.text; - const currTopicUrl = topicId.replace(/^\d+-/g, '/').replace(/:/g, '/'); - const contentFilePath = topicUrlToPathMapping[currTopicUrl]; + return new Promise((resolve, reject) => { + openai + .createChatCompletion({ + model: 'gpt-4', + messages: [ + { + role: 'user', + content: instruction, + }, + ], + }) + .then((response) => { + const article = response.data.choices[0].message.content; - const currentFileContent = fs.readFileSync(contentFilePath, 'utf8'); - const isFileEmpty = currentFileContent.replace(/^#.+/, ``).trim() == ''; + resolve(article); + }) + .catch((err) => { + reject(err); + }); + }); +} - if (!isFileEmpty) { - console.log(`${topicId} not empty. Ignoring...`); - return; - } +async function run() { + const topicUrlToPathMapping = getFilesInFolder(ROADMAP_CONTENT_DIR); - let newFileContent = `# ${topicTitle}`; + const roadmapJson = require(path.join(ROADMAP_JSON_DIR, `${roadmapId}.json`)); + const groups = roadmapJson?.mockup?.controls?.control?.filter( + (control) => control.typeID === '__group__' && !control.properties?.controlName?.startsWith('ext_link') + ); if (!OPEN_AI_API_KEY) { - console.log(`OPEN_AI_API_KEY not set. Only adding title to ${topicId}..`); - fs.writeFileSync(contentFilePath, newFileContent, 'utf8'); - return; + console.log('----------------------------------------'); + console.log('OPEN_AI_API_KEY not found. Skipping openai api calls...'); + console.log('----------------------------------------'); } - // console.log(currentFileContent); - // console.log(currTopicUrl); - // console.log(topicTitle); - // console.log(topicUrlToPathMapping[currTopicUrl]); -}); + for (let group of groups) { + const topicId = group?.properties?.controlName; + const topicTitle = group?.children?.controls?.control?.find((control) => control?.typeID === 'Label')?.properties + ?.text; + const currTopicUrl = topicId.replace(/^\d+-/g, '/').replace(/:/g, '/'); + const contentFilePath = topicUrlToPathMapping[currTopicUrl]; + + const currentFileContent = fs.readFileSync(contentFilePath, 'utf8'); + const isFileEmpty = currentFileContent.replace(/^#.+/, ``).trim() == ''; + + if (!isFileEmpty) { + console.log(`Ignoring ${topicId}. Not empty.`); + continue; + } + + let newFileContent = `# ${topicTitle}`; + + if (!OPEN_AI_API_KEY) { + console.log(`Writing ${topicId}..`); + fs.writeFileSync(contentFilePath, newFileContent, 'utf8'); + continue; + } + + const topicContent = await writeTopicContent(topicTitle); + newFileContent += `\n\n${topicContent}`; + + console.log(`Writing ${topicId}..`); + fs.writeFileSync(contentFilePath, newFileContent, 'utf8'); + + // console.log(currentFileContent); + // console.log(currTopicUrl); + // console.log(topicTitle); + // console.log(topicUrlToPathMapping[currTopicUrl]); + } +} + +run() + .then(() => { + console.log('Done'); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/package.json b/package.json index dea093918..3ad71e7b7 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "gh-pages": "^5.0.0", "js-yaml": "^4.1.0", "markdown-it": "^13.0.1", + "openai": "^3.2.1", "prettier": "^2.8.7", "prettier-plugin-astro": "^0.8.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dc6572cf6..dd276377a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,7 @@ specifiers: markdown-it: ^13.0.1 node-html-parser: ^6.1.5 npm-check-updates: ^16.8.0 + openai: ^3.2.1 prettier: ^2.8.7 prettier-plugin-astro: ^0.8.0 rehype-external-links: ^2.0.1 @@ -35,6 +36,7 @@ devDependencies: gh-pages: 5.0.0 js-yaml: 4.1.0 markdown-it: 13.0.1 + openai: 3.2.1 prettier: 2.8.7 prettier-plugin-astro: 0.8.0 @@ -1185,6 +1187,10 @@ packages: resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==} dev: true + /asynckit/0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true + /autoprefixer/10.4.13_postcss@8.4.21: resolution: {integrity: sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg==} engines: {node: ^10 || ^12 || >=14} @@ -1201,6 +1207,14 @@ packages: postcss-value-parser: 4.2.0 dev: false + /axios/0.26.1: + resolution: {integrity: sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==} + dependencies: + follow-redirects: 1.15.2 + transitivePeerDependencies: + - debug + dev: true + /bail/2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} dev: false @@ -1563,6 +1577,13 @@ packages: color-string: 1.9.1 dev: false + /combined-stream/1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + dependencies: + delayed-stream: 1.0.0 + dev: true + /comma-separated-tokens/2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} dev: false @@ -1743,6 +1764,11 @@ packages: /defined/1.0.1: resolution: {integrity: sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==} + /delayed-stream/1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dev: true + /delegates/1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} dev: false @@ -2075,11 +2101,30 @@ packages: pkg-dir: 4.2.0 dev: false + /follow-redirects/1.15.2: + resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: true + /form-data-encoder/2.1.4: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} dev: false + /form-data/4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: true + /fp-and-or/0.1.3: resolution: {integrity: sha512-wJaE62fLaB3jCYvY2ZHjZvmKK2iiLiiehX38rz5QZxtdN8fVPJDeZUiVvJrHStdTc+23LHlyZuSEKgFc0pxi2g==} engines: {node: '>=10'} @@ -3384,6 +3429,18 @@ packages: braces: 3.0.2 picomatch: 2.3.1 + /mime-db/1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + dev: true + + /mime-types/2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: true + /mime/3.0.0: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} @@ -3804,6 +3861,15 @@ packages: is-docker: 2.2.1 is-wsl: 2.2.0 + /openai/3.2.1: + resolution: {integrity: sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==} + dependencies: + axios: 0.26.1 + form-data: 4.0.0 + transitivePeerDependencies: + - debug + dev: true + /ora/6.1.2: resolution: {integrity: sha512-EJQ3NiP5Xo94wJXIzAyOtSb0QEIAUu7m8t6UZ9krbz0vAJqr92JpcK/lEXg91q6B9pEGqrykkd2EQplnifDSBw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}