Add content writing command

bin/readme.md

@@ -9,12 +9,22 @@ Generates a list of all the resources links in any roadmap file.
 
 Compresses all the JSON files in the `public/jsons` folder
 
+## `update-sponsors.cjs`
+
+Updates the sponsor ads on each roadmap page with the latest sponsor information in the Excel sheet.
+
 ## `roadmap-content.cjs`
 
+Currently, for any new roadmaps that we add, we do create the interactive roadmap but we end up leaving the content empty in the roadmap till we get time to fill it up manually.
+
+This script populates all the content files with some minimal content from OpenAI so that the users visiting the website have something to read in the interactive roadmap till we get time to fill it up manually.
+
+## `roadmap-dirs.cjs`
+
 This command is used to create the content folders and files for the interactivity of the roadmap. You can use the below command to generate the roadmap skeletons inside a roadmap directory:
 
 ```bash
-npm run roadmap-content [frontend|backend|devops|...]
+npm run roadmap-dirs [frontend|backend|devops|...]
 ```
 
 For the content skeleton to be generated, we should have proper grouping, and the group names in the project files. You can follow the steps listed below in order to add the meta information to the roadmap.

bin/roadmap-content.cjs

@@ -20,6 +20,12 @@ if (!allowedRoadmapIds.includes(roadmapId)) {
 }
 
 const ROADMAP_CONTENT_DIR = path.join(ALL_ROADMAPS_DIR, roadmapId, 'content');
+const { Configuration, OpenAIApi } = require('openai');
+const configuration = new Configuration({
+  apiKey: OPEN_AI_API_KEY,
+});
+
+const openai = new OpenAIApi(configuration);
 
 function getFilesInFolder(folderPath, fileList = {}) {
   const files = fs.readdirSync(folderPath);
@@ -44,38 +50,88 @@ function getFilesInFolder(folderPath, fileList = {}) {
   return fileList;
 }
 
-const topicUrlToPathMapping = getFilesInFolder(ROADMAP_CONTENT_DIR);
+function writeTopicContent(topicTitle) {
+  console.log(`Genearting '${topicTitle}'...`);
 
-const roadmapJson = require(path.join(ROADMAP_JSON_DIR, `${roadmapId}.json`));
-const groups = roadmapJson?.mockup?.controls?.control?.filter(
-  (control) => control.typeID === '__group__' && !control.properties?.controlName?.startsWith('ext_link')
-);
+  const instruction = `Write a short paragraph explaining '${topicTitle}' in ${roadmapId}. Avoid any text similar to "In ${roadmapId}, ${topicTitle} refers to" and write it as a direct explanation of the topic.`;
 
-groups.forEach((group) => {
-  const topicId = group?.properties?.controlName;
-  const topicTitle = group?.children?.controls?.control?.find((control) => control?.typeID === 'Label')?.properties
-    ?.text;
-  const currTopicUrl = topicId.replace(/^\d+-/g, '/').replace(/:/g, '/');
-  const contentFilePath = topicUrlToPathMapping[currTopicUrl];
+  return new Promise((resolve, reject) => {
+    openai
+      .createChatCompletion({
+        model: 'gpt-4',
+        messages: [
+          {
+            role: 'user',
+            content: instruction,
+          },
+        ],
+      })
+      .then((response) => {
+        const article = response.data.choices[0].message.content;
 
-  const currentFileContent = fs.readFileSync(contentFilePath, 'utf8');
-  const isFileEmpty = currentFileContent.replace(/^#.+/, ``).trim() == '';
+        resolve(article);
+      })
+      .catch((err) => {
+        reject(err);
+      });
+  });
+}
 
-  if (!isFileEmpty) {
-    console.log(`${topicId} not empty. Ignoring...`);
-    return;
-  }
+async function run() {
+  const topicUrlToPathMapping = getFilesInFolder(ROADMAP_CONTENT_DIR);
 
-  let newFileContent = `# ${topicTitle}`;
+  const roadmapJson = require(path.join(ROADMAP_JSON_DIR, `${roadmapId}.json`));
+  const groups = roadmapJson?.mockup?.controls?.control?.filter(
+    (control) => control.typeID === '__group__' && !control.properties?.controlName?.startsWith('ext_link')
+  );
 
   if (!OPEN_AI_API_KEY) {
-    console.log(`OPEN_AI_API_KEY not set. Only adding title to ${topicId}..`);
-    fs.writeFileSync(contentFilePath, newFileContent, 'utf8');
-    return;
+    console.log('----------------------------------------');
+    console.log('OPEN_AI_API_KEY not found. Skipping openai api calls...');
+    console.log('----------------------------------------');
   }
 
-  // console.log(currentFileContent);
-  // console.log(currTopicUrl);
-  // console.log(topicTitle);
-  // console.log(topicUrlToPathMapping[currTopicUrl]);
-});
+  for (let group of groups) {
+    const topicId = group?.properties?.controlName;
+    const topicTitle = group?.children?.controls?.control?.find((control) => control?.typeID === 'Label')?.properties
+      ?.text;
+    const currTopicUrl = topicId.replace(/^\d+-/g, '/').replace(/:/g, '/');
+    const contentFilePath = topicUrlToPathMapping[currTopicUrl];
+
+    const currentFileContent = fs.readFileSync(contentFilePath, 'utf8');
+    const isFileEmpty = currentFileContent.replace(/^#.+/, ``).trim() == '';
+
+    if (!isFileEmpty) {
+      console.log(`Ignoring ${topicId}. Not empty.`);
+      continue;
+    }
+
+    let newFileContent = `# ${topicTitle}`;
+
+    if (!OPEN_AI_API_KEY) {
+      console.log(`Writing ${topicId}..`);
+      fs.writeFileSync(contentFilePath, newFileContent, 'utf8');
+      continue;
+    }
+
+    const topicContent = await writeTopicContent(topicTitle);
+    newFileContent += `\n\n${topicContent}`;
+
+    console.log(`Writing ${topicId}..`);
+    fs.writeFileSync(contentFilePath, newFileContent, 'utf8');
+
+    // console.log(currentFileContent);
+    // console.log(currTopicUrl);
+    // console.log(topicTitle);
+    // console.log(topicUrlToPathMapping[currTopicUrl]);
+  }
+}
+
+run()
+  .then(() => {
+    console.log('Done');
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

package.json

@@ -35,6 +35,7 @@
     "gh-pages": "^5.0.0",
     "js-yaml": "^4.1.0",
     "markdown-it": "^13.0.1",
+    "openai": "^3.2.1",
     "prettier": "^2.8.7",
     "prettier-plugin-astro": "^0.8.0"
   }

pnpm-lock.yaml

@@ -12,6 +12,7 @@ specifiers:
   markdown-it: ^13.0.1
   node-html-parser: ^6.1.5
   npm-check-updates: ^16.8.0
+  openai: ^3.2.1
   prettier: ^2.8.7
   prettier-plugin-astro: ^0.8.0
   rehype-external-links: ^2.0.1
@@ -35,6 +36,7 @@ devDependencies:
   gh-pages: 5.0.0
   js-yaml: 4.1.0
   markdown-it: 13.0.1
+  openai: 3.2.1
   prettier: 2.8.7
   prettier-plugin-astro: 0.8.0
 
@@ -1185,6 +1187,10 @@ packages:
     resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==}
     dev: true
 
+  /asynckit/0.4.0:
+    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+    dev: true
+
   /autoprefixer/10.4.13_postcss@8.4.21:
     resolution: {integrity: sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg==}
     engines: {node: ^10 || ^12 || >=14}
@@ -1201,6 +1207,14 @@ packages:
       postcss-value-parser: 4.2.0
     dev: false
 
+  /axios/0.26.1:
+    resolution: {integrity: sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==}
+    dependencies:
+      follow-redirects: 1.15.2
+    transitivePeerDependencies:
+      - debug
+    dev: true
+
   /bail/2.0.2:
     resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==}
     dev: false
@@ -1563,6 +1577,13 @@ packages:
       color-string: 1.9.1
     dev: false
 
+  /combined-stream/1.0.8:
+    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      delayed-stream: 1.0.0
+    dev: true
+
   /comma-separated-tokens/2.0.3:
     resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==}
     dev: false
@@ -1743,6 +1764,11 @@ packages:
   /defined/1.0.1:
     resolution: {integrity: sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==}
 
+  /delayed-stream/1.0.0:
+    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+    engines: {node: '>=0.4.0'}
+    dev: true
+
   /delegates/1.0.0:
     resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
     dev: false
@@ -2075,11 +2101,30 @@ packages:
       pkg-dir: 4.2.0
     dev: false
 
+  /follow-redirects/1.15.2:
+    resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
+    engines: {node: '>=4.0'}
+    peerDependencies:
+      debug: '*'
+    peerDependenciesMeta:
+      debug:
+        optional: true
+    dev: true
+
   /form-data-encoder/2.1.4:
     resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==}
     engines: {node: '>= 14.17'}
     dev: false
 
+  /form-data/4.0.0:
+    resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
+    engines: {node: '>= 6'}
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      mime-types: 2.1.35
+    dev: true
+
   /fp-and-or/0.1.3:
     resolution: {integrity: sha512-wJaE62fLaB3jCYvY2ZHjZvmKK2iiLiiehX38rz5QZxtdN8fVPJDeZUiVvJrHStdTc+23LHlyZuSEKgFc0pxi2g==}
     engines: {node: '>=10'}
@@ -3384,6 +3429,18 @@ packages:
       braces: 3.0.2
       picomatch: 2.3.1
 
+  /mime-db/1.52.0:
+    resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
+    engines: {node: '>= 0.6'}
+    dev: true
+
+  /mime-types/2.1.35:
+    resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
+    engines: {node: '>= 0.6'}
+    dependencies:
+      mime-db: 1.52.0
+    dev: true
+
   /mime/3.0.0:
     resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}
     engines: {node: '>=10.0.0'}
@@ -3804,6 +3861,15 @@ packages:
       is-docker: 2.2.1
       is-wsl: 2.2.0
 
+  /openai/3.2.1:
+    resolution: {integrity: sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==}
+    dependencies:
+      axios: 0.26.1
+      form-data: 4.0.0
+    transitivePeerDependencies:
+      - debug
+    dev: true
+
   /ora/6.1.2:
     resolution: {integrity: sha512-EJQ3NiP5Xo94wJXIzAyOtSb0QEIAUu7m8t6UZ9krbz0vAJqr92JpcK/lEXg91q6B9pEGqrykkd2EQplnifDSBw==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}