feat: migrate content to database

package.json

@@ -31,6 +31,7 @@
     "migrate:editor-roadmaps": "tsx ./scripts/migrate-editor-roadmap.ts",
     "sync:content-to-repo": "tsx ./scripts/sync-content-to-repo.ts",
     "sync:repo-to-database": "tsx ./scripts/sync-repo-to-database.ts",
+    "migrate:content-repo-to-database": "tsx ./scripts/migrate-content-repo-to-database.ts",
     "test:e2e": "playwright test"
   },
   "dependencies": {

scripts/migrate-content-repo-to-database.ts

@@ -0,0 +1,256 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import type { OfficialRoadmapDocument } from '../src/queries/official-roadmap';
+import { parse } from 'node-html-parser';
+import { markdownToHtml } from '../src/lib/markdown';
+import { htmlToMarkdown } from '../src/lib/html';
+import matter from 'gray-matter';
+import type { RoadmapFrontmatter } from '../src/lib/roadmap';
+import {
+  allowedOfficialRoadmapTopicResourceType,
+  type AllowedOfficialRoadmapTopicResourceType,
+  type SyncToDatabaseTopicContent,
+} from '../src/queries/official-roadmap-topic';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const args = process.argv.slice(2);
+const secret = args
+  .find((arg) => arg.startsWith('--secret='))
+  ?.replace('--secret=', '');
+if (!secret) {
+  throw new Error('Secret is required');
+}
+
+let roadmapJsonCache: Map<string, OfficialRoadmapDocument> = new Map();
+export async function fetchRoadmapJson(
+  roadmapId: string,
+): Promise<OfficialRoadmapDocument> {
+  if (roadmapJsonCache.has(roadmapId)) {
+    return roadmapJsonCache.get(roadmapId)!;
+  }
+
+  const response = await fetch(
+    `https://roadmap.sh/api/v1-official-roadmap/${roadmapId}`,
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch roadmap json: ${response.statusText} for ${roadmapId}`,
+    );
+  }
+
+  const data = await response.json();
+  if (data.error) {
+    throw new Error(
+      `Failed to fetch roadmap json: ${data.error} for ${roadmapId}`,
+    );
+  }
+
+  roadmapJsonCache.set(roadmapId, data);
+  return data;
+}
+
+export async function syncContentToDatabase(
+  topics: SyncToDatabaseTopicContent[],
+) {
+  const response = await fetch(
+    // `https://roadmap.sh/api/v1-sync-official-roadmap-topics`,
+    `http://localhost:8080/v1-sync-official-roadmap-topics`,
+    {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        topics,
+        secret,
+      }),
+    },
+  );
+
+  if (!response.ok) {
+    const error = await response.json();
+    throw new Error(
+      `Failed to sync content to database: ${response.statusText} ${JSON.stringify(error, null, 2)}`,
+    );
+  }
+
+  return response.json();
+}
+
+// Directory containing the roadmaps
+const ROADMAP_CONTENT_DIR = path.join(__dirname, '../src/data/roadmaps');
+const allRoadmaps = await fs.readdir(ROADMAP_CONTENT_DIR);
+
+const editorRoadmapIds = new Set<string>();
+for (const roadmapId of allRoadmaps) {
+  const roadmapFrontmatterDir = path.join(
+    ROADMAP_CONTENT_DIR,
+    roadmapId,
+    `${roadmapId}.md`,
+  );
+  const roadmapFrontmatterRaw = await fs.readFile(
+    roadmapFrontmatterDir,
+    'utf-8',
+  );
+  const { data } = matter(roadmapFrontmatterRaw);
+
+  const roadmapFrontmatter = data as RoadmapFrontmatter;
+  if (roadmapFrontmatter.renderer === 'editor') {
+    editorRoadmapIds.add(roadmapId);
+  }
+}
+
+for (const roadmapId of editorRoadmapIds) {
+  try {
+    const roadmap = await fetchRoadmapJson(roadmapId);
+
+    const files = await fs.readdir(
+      path.join(ROADMAP_CONTENT_DIR, roadmapId, 'content'),
+    );
+
+    console.log(`🚀 Starting ${files.length} files for ${roadmapId}`);
+    const topics: SyncToDatabaseTopicContent[] = [];
+
+    for (const file of files) {
+      const isContentFile = file.endsWith('.md');
+      if (!isContentFile) {
+        console.log(`🚨 Skipping ${file} because it is not a content file`);
+        continue;
+      }
+
+      const nodeSlug = file.replace('.md', '');
+      if (!nodeSlug) {
+        console.error(`🚨 Node id is required: ${file}`);
+        continue;
+      }
+
+      const nodeId = nodeSlug.split('@')?.[1];
+      if (!nodeId) {
+        console.error(`🚨 Node id is required: ${file}`);
+        continue;
+      }
+
+      const node = roadmap.nodes.find((node) => node.id === nodeId);
+      if (!node) {
+        console.error(`🚨 Node not found: ${file}`);
+        continue;
+      }
+
+      const filePath = path.join(
+        ROADMAP_CONTENT_DIR,
+        roadmapId,
+        'content',
+        `${nodeSlug}.md`,
+      );
+
+      const fileExists = await fs
+        .stat(filePath)
+        .then(() => true)
+        .catch(() => false);
+      if (!fileExists) {
+        console.log(`🚨 File not found: ${filePath}`);
+        continue;
+      }
+
+      const content = await fs.readFile(filePath, 'utf8');
+      const html = markdownToHtml(content, false);
+      const rootHtml = parse(html);
+
+      let ulWithLinks: HTMLElement | undefined;
+      rootHtml.querySelectorAll('ul').forEach((ul) => {
+        const listWithJustLinks = Array.from(ul.querySelectorAll('li')).filter(
+          (li) => {
+            const link = li.querySelector('a');
+            return link && link.textContent?.trim() === li.textContent?.trim();
+          },
+        );
+
+        if (listWithJustLinks.length > 0) {
+          // @ts-expect-error - TODO: fix this
+          ulWithLinks = ul;
+        }
+      });
+
+      const listLinks: SyncToDatabaseTopicContent['resources'] =
+        ulWithLinks !== undefined
+          ? Array.from(ulWithLinks.querySelectorAll('li > a'))
+              .map((link) => {
+                const typePattern = /@([a-z.]+)@/;
+                let linkText = link.textContent || '';
+                const linkHref = link.getAttribute('href') || '';
+                let linkType = linkText.match(typePattern)?.[1] || 'article';
+                linkType = allowedOfficialRoadmapTopicResourceType.includes(
+                  linkType as any,
+                )
+                  ? linkType
+                  : 'article';
+
+                linkText = linkText.replace(typePattern, '');
+
+                if (!linkText || !linkHref) {
+                  return null;
+                }
+
+                return {
+                  title: linkText,
+                  url: linkHref,
+                  type: linkType as AllowedOfficialRoadmapTopicResourceType,
+                };
+              })
+              .filter((link) => link !== null)
+              .sort((a, b) => {
+                const order = [
+                  'official',
+                  'opensource',
+                  'article',
+                  'video',
+                  'feed',
+                ];
+                return order.indexOf(a!.type) - order.indexOf(b!.type);
+              })
+          : [];
+
+      const title = rootHtml.querySelector('h1');
+      ulWithLinks?.remove();
+      title?.remove();
+
+      const allParagraphs = rootHtml.querySelectorAll('p');
+      if (listLinks.length > 0 && allParagraphs.length > 0) {
+        // to remove the view more see more from the description
+        const lastParagraph = allParagraphs[allParagraphs.length - 1];
+        lastParagraph?.remove();
+      }
+
+      const htmlStringWithoutLinks = rootHtml.toString();
+      const description = htmlToMarkdown(htmlStringWithoutLinks);
+
+      const updatedDescription =
+        `# ${title?.textContent}\n\n${description}`.trim();
+
+      const label = node?.data?.label as string;
+      if (!label) {
+        console.error(`🚨 Label is required: ${file}`);
+        continue;
+      }
+
+      topics.push({
+        roadmapSlug: roadmapId,
+        nodeId,
+        description: updatedDescription,
+        resources: listLinks,
+      });
+    }
+
+    await syncContentToDatabase(topics);
+    console.log(
+      `✅ Synced ${topics.length} topics to database for ${roadmapId}`,
+    );
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
+  }
+}

scripts/sync-content-to-repo.ts

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { slugify } from '../src/lib/slugger';
 import type { OfficialRoadmapDocument } from '../src/queries/official-roadmap';
+import type { OfficialRoadmapTopicContentDocument } from '../src/queries/official-roadmap-topic';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -19,36 +20,6 @@ if (!roadmapSlug || roadmapSlug === '__default__') {
 }
 
 console.log(`🚀 Starting ${roadmapSlug}`);
-export const allowedOfficialRoadmapTopicResourceType = [
-  'roadmap',
-  'official',
-  'opensource',
-  'article',
-  'course',
-  'podcast',
-  'video',
-  'book',
-  'feed',
-] as const;
-export type AllowedOfficialRoadmapTopicResourceType =
-  (typeof allowedOfficialRoadmapTopicResourceType)[number];
-
-export type OfficialRoadmapTopicResource = {
-  _id?: string;
-  type: AllowedOfficialRoadmapTopicResourceType;
-  title: string;
-  url: string;
-};
-
-export interface OfficialRoadmapTopicContentDocument {
-  _id?: string;
-  roadmapSlug: string;
-  nodeId: string;
-  description: string;
-  resources: OfficialRoadmapTopicResource[];
-  createdAt: Date;
-  updatedAt: Date;
-}
 
 export async function roadmapTopics(
   roadmapId: string,

scripts/sync-repo-to-database.ts

@@ -5,37 +5,11 @@ import type { OfficialRoadmapDocument } from '../src/queries/official-roadmap';
 import { parse } from 'node-html-parser';
 import { markdownToHtml } from '../src/lib/markdown';
 import { htmlToMarkdown } from '../src/lib/html';
-
+import {
-export const allowedOfficialRoadmapTopicResourceType = [
+  allowedOfficialRoadmapTopicResourceType,
-  'roadmap',
+  type AllowedOfficialRoadmapTopicResourceType,
-  'official',
+  type SyncToDatabaseTopicContent,
-  'opensource',
+} from '../src/queries/official-roadmap-topic';
-  'article',
-  'course',
-  'podcast',
-  'video',
-  'book',
-  'feed',
-] as const;
-export type AllowedOfficialRoadmapTopicResourceType =
-  (typeof allowedOfficialRoadmapTopicResourceType)[number];
-
-export type OfficialRoadmapTopicResource = {
-  _id?: string;
-  type: AllowedOfficialRoadmapTopicResourceType;
-  title: string;
-  url: string;
-};
-
-export interface OfficialRoadmapTopicContentDocument {
-  _id?: string;
-  roadmapSlug: string;
-  nodeId: string;
-  description: string;
-  resources: OfficialRoadmapTopicResource[];
-  createdAt: Date;
-  updatedAt: Date;
-}
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -82,10 +56,7 @@ export async function fetchRoadmapJson(
 }
 
 export async function syncContentToDatabase(
-  topics: Omit<
+  topics: SyncToDatabaseTopicContent[],
-    OfficialRoadmapTopicContentDocument,
-    'createdAt' | 'updatedAt' | '_id'
-  >[],
 ) {
   const response = await fetch(
     `https://roadmap.sh/api/v1-sync-official-roadmap-topics`,
@@ -125,10 +96,7 @@ console.log(`🚀 Starting ${files.length} files`);
 const ROADMAP_CONTENT_DIR = path.join(__dirname, '../src/data/roadmaps');
 
 try {
-  const topics: Omit<
+  const topics: SyncToDatabaseTopicContent[] = [];
-    OfficialRoadmapTopicContentDocument,
-    'createdAt' | 'updatedAt' | '_id'
-  >[] = [];
 
   for (const file of files) {
     const isContentFile = file.endsWith('.md') && file.includes('content/');
@@ -198,7 +166,7 @@ try {
       }
     });
 
-    const listLinks: Omit<OfficialRoadmapTopicResource, '_id'>[] =
+    const listLinks: SyncToDatabaseTopicContent['resources'] =
       ulWithLinks !== undefined
         ? Array.from(ulWithLinks.querySelectorAll('li > a'))
             .map((link) => {

src/pages/[roadmapId]/[...topicId].astro

@@ -55,26 +55,32 @@ if (isTopic) {
     `${topicPath}.md`,
   );
 
+  const nodeId = topicPath.split('@')?.[1];
+  if (!nodeId) {
+    Astro.response.status = 404;
+    Astro.response.statusText = 'Not found';
+    return Astro.rewrite('/404');
+  }
+
   const topic = await getOfficialRoadmapTopic({
     roadmapSlug: roadmapId,
-    nodeId: topicPath,
+    nodeId,
   });
 
-  // Check if file exists
+  if (!topic) {
-  if (!fs.existsSync(contentPath) || !topic) {
     Astro.response.status = 404;
     Astro.response.statusText = 'Not found';
 
     return Astro.rewrite('/404');
   }
 
+  const md = MarkdownIt();
+  htmlContent = await md.renderAsync(prepareOfficialRoadmapTopicContent(topic));
+
   const fileWithoutBasePath = contentPath.replace(
     /.+?\/src\/data/,
     '/src/data',
   );
-
-  const md = MarkdownIt();
-  htmlContent = await md.renderAsync(prepareOfficialRoadmapTopicContent(topic));
   gitHubUrl = `https://github.com/kamranahmedse/developer-roadmap/tree/master${fileWithoutBasePath}`;
 } else {
   guide = await getOfficialGuide(topicId, roadmapId);

src/queries/official-roadmap-topic.ts

@@ -14,7 +14,7 @@ export const allowedOfficialRoadmapTopicResourceType = [
 export type AllowedOfficialRoadmapTopicResourceType =
   (typeof allowedOfficialRoadmapTopicResourceType)[number];
 
-type OfficialRoadmapTopicResource = {
+export type OfficialRoadmapTopicResource = {
   _id: string;
   type: AllowedOfficialRoadmapTopicResourceType;
   title: string;
@@ -36,6 +36,13 @@ type GetOfficialRoadmapTopicOptions = {
   nodeId: string;
 };
 
+export type SyncToDatabaseTopicContent = Omit<
+  OfficialRoadmapTopicContentDocument,
+  'createdAt' | 'updatedAt' | '_id' | 'resources'
+> & {
+  resources: Omit<OfficialRoadmapTopicResource, '_id'>[];
+};
+
 export async function getOfficialRoadmapTopic(
   options: GetOfficialRoadmapTopicOptions,
 ) {