info/developer-roadmap
1
0
Fork 0
mirror of https://github.com/kamranahmedse/developer-roadmap.git synced 2025-08-30 12:40:03 +02:00
Code Issues Releases Wiki Activity

fix: sync repo to db

Browse Source
This commit is contained in:
Arik Chakma
2025-08-20 14:38:44 +06:00
committed by Kamran Ahmed
parent d70582411e
commit 885e95399e
1 changed files with 150 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
scripts/sync-repo-to-database.ts
View File

@@ -6,9 +6,11 @@ import type { OfficialRoadmapDocument } from '../src/queries/official-roadmap';
import { parse } from 'node-html-parser'; import { parse } from 'node-html-parser';
import { markdownToHtml } from '../src/lib/markdown'; import { markdownToHtml } from '../src/lib/markdown';
import { htmlToMarkdown } from '../src/lib/html'; import { htmlToMarkdown } from '../src/lib/html';
import type { import {
OfficialRoadmapTopicContentDocument, allowedOfficialRoadmapTopicResourceType,
OfficialRoadmapTopicResource, type AllowedOfficialRoadmapTopicResourceType,
type OfficialRoadmapTopicContentDocument,
type OfficialRoadmapTopicResource,
} from './sync-content-to-repo'; } from './sync-content-to-repo';
const __filename = fileURLToPath(import.meta.url); const __filename = fileURLToPath(import.meta.url);
@@ -46,151 +48,171 @@ export async function fetchRoadmapJson(
return data; return data;
} }
export const allowedOfficialRoadmapTopicResourceType = [ export async function syncContentToDatabase(
'official', topics: Omit<
'opensource', OfficialRoadmapTopicContentDocument,
'article', 'createdAt' | 'updatedAt' | '_id'
'course', >[],
'podcast', ) {
'video', const response = await fetch(
'book', `https://roadmap.sh/api/v1-sync-official-roadmap-topics`,
'feed', {
] as const; method: 'POST',
export type AllowedOfficialRoadmapTopicResourceType = body: JSON.stringify({
(typeof allowedOfficialRoadmapTopicResourceType)[number]; topics,
secret,
}),
},
);
if (!response.ok) {
throw new Error(
`Failed to sync content to database: ${response.statusText}`,
);
}
return response.json();
}
const files = allFiles.split(' '); const files = allFiles.split(' ');
console.log(`🚀 Starting ${files.length} files`); console.log(`🚀 Starting ${files.length} files`);
const ROADMAP_CONTENT_DIR = path.join(__dirname, '../src/data/roadmaps'); const ROADMAP_CONTENT_DIR = path.join(__dirname, '../src/data/roadmaps');
const topics: Omit< try {
OfficialRoadmapTopicContentDocument, const topics: Omit<
'createdAt' | 'updatedAt' | '_id' OfficialRoadmapTopicContentDocument,
>[] = []; 'createdAt' | 'updatedAt' | '_id'
>[] = [];
for (const file of files) { for (const file of files) {
const isContentFile = file.endsWith('.md') && file.includes('content/'); const isContentFile = file.endsWith('.md') && file.includes('content/');
if (!isContentFile) { if (!isContentFile) {
console.log(`🚨 Skipping ${file} because it is not a content file`); console.log(`🚨 Skipping ${file} because it is not a content file`);
continue; continue;
} }
const pathParts = file.replace('src/data/roadmaps/', '').split('/'); const pathParts = file.replace('src/data/roadmaps/', '').split('/');
const roadmapSlug = pathParts?.[0]; const roadmapSlug = pathParts?.[0];
if (!roadmapSlug) { if (!roadmapSlug) {
console.error(`🚨 Roadmap slug is required: ${file}`); console.error(`🚨 Roadmap slug is required: ${file}`);
continue; continue;
} }
const nodeSlug = pathParts?.[2]?.replace('.md', '');
if (!nodeSlug) {
console.error(`🚨 Node id is required: ${file}`);
continue;
}
const nodeId = nodeSlug.split('@')?.[1]; const nodeSlug = pathParts?.[2]?.replace('.md', '');
if (!nodeId) { if (!nodeSlug) {
console.error(`🚨 Node id is required: ${file}`); console.error(`🚨 Node id is required: ${file}`);
continue; continue;
} }
const roadmap = await fetchRoadmapJson(roadmapSlug); const nodeId = nodeSlug.split('@')?.[1];
const node = roadmap.nodes.find((node) => node.id === nodeId); if (!nodeId) {
if (!node) { console.error(`🚨 Node id is required: ${file}`);
console.error(`🚨 Node not found: ${file}`); continue;
continue; }
}
const filePath = path.join( const roadmap = await fetchRoadmapJson(roadmapSlug);
ROADMAP_CONTENT_DIR, const node = roadmap.nodes.find((node) => node.id === nodeId);
roadmapSlug, if (!node) {
'content', console.error(`🚨 Node not found: ${file}`);
`${nodeSlug}.md`, continue;
); }
const content = await fs.readFile(filePath, 'utf8'); const filePath = path.join(
const html = markdownToHtml(content, false); ROADMAP_CONTENT_DIR,
const rootHtml = parse(html); roadmapSlug,
'content',
let ulWithLinks: HTMLElement | undefined; `${nodeSlug}.md`,
rootHtml.querySelectorAll('ul').forEach((ul) => {
const listWithJustLinks = Array.from(ul.querySelectorAll('li')).filter(
(li) => {
const link = li.querySelector('a');
return link && link.textContent?.trim() === li.textContent?.trim();
},
); );
if (listWithJustLinks.length > 0) { const content = await fs.readFile(filePath, 'utf8');
// @ts-expect-error - TODO: fix this const html = markdownToHtml(content, false);
ulWithLinks = ul; const rootHtml = parse(html);
let ulWithLinks: HTMLElement | undefined;
rootHtml.querySelectorAll('ul').forEach((ul) => {
const listWithJustLinks = Array.from(ul.querySelectorAll('li')).filter(
(li) => {
const link = li.querySelector('a');
return link && link.textContent?.trim() === li.textContent?.trim();
},
);
if (listWithJustLinks.length > 0) {
// @ts-expect-error - TODO: fix this
ulWithLinks = ul;
}
});
const listLinks: Omit<OfficialRoadmapTopicResource, '_id'>[] =
ulWithLinks !== undefined
? Array.from(ulWithLinks.querySelectorAll('li > a'))
.map((link) => {
const typePattern = /@([a-z.]+)@/;
let linkText = link.textContent || '';
const linkHref = link.getAttribute('href') || '';
let linkType = linkText.match(typePattern)?.[1] || 'article';
linkType = allowedOfficialRoadmapTopicResourceType.includes(
linkType as any,
)
? linkType
: 'article';
linkText = linkText.replace(typePattern, '');
return {
title: linkText,
url: linkHref,
type: linkType as AllowedOfficialRoadmapTopicResourceType,
};
})
.sort((a, b) => {
const order = [
'official',
'opensource',
'article',
'video',
'feed',
];
return order.indexOf(a.type) - order.indexOf(b.type);
})
: [];
const title = rootHtml.querySelector('h1');
ulWithLinks?.remove();
title?.remove();
const allParagraphs = rootHtml.querySelectorAll('p');
if (listLinks.length > 0 && allParagraphs.length > 0) {
// to remove the view more see more from the description
const lastParagraph = allParagraphs[allParagraphs.length - 1];
lastParagraph?.remove();
} }
});
const listLinks: Omit<OfficialRoadmapTopicResource, '_id'>[] = const htmlStringWithoutLinks = rootHtml.toString();
ulWithLinks !== undefined const description = htmlToMarkdown(htmlStringWithoutLinks);
? Array.from(ulWithLinks.querySelectorAll('li > a'))
.map((link) => {
const typePattern = /@([a-z.]+)@/;
let linkText = link.textContent || '';
const linkHref = link.getAttribute('href') || '';
let linkType = linkText.match(typePattern)?.[1] || 'article';
linkType = allowedOfficialRoadmapTopicResourceType.includes(
linkType as any,
)
? linkType
: 'article';
linkText = linkText.replace(typePattern, ''); const updatedDescription = `# ${title?.textContent}
return {
title: linkText,
url: linkHref,
type: linkType as AllowedOfficialRoadmapTopicResourceType,
};
})
.sort((a, b) => {
const order = [
'official',
'opensource',
'article',
'video',
'feed',
];
return order.indexOf(a.type) - order.indexOf(b.type);
})
: [];
const title = rootHtml.querySelector('h1');
ulWithLinks?.remove();
title?.remove();
if (listLinks.length > 0) {
const lastParagraph = rootHtml.querySelector('p:last-child');
console.log(lastParagraph?.textContent);
lastParagraph?.remove();
}
const htmlStringWithoutLinks = rootHtml.toString();
const description = htmlToMarkdown(htmlStringWithoutLinks);
const updatedDescription = `# ${title?.textContent}
${description}`.trim(); ${description}`.trim();
const label = node?.data?.label as string; const label = node?.data?.label as string;
if (!label) { if (!label) {
console.error(`🚨 Label is required: ${file}`); console.error(`🚨 Label is required: ${file}`);
continue; continue;
}
topics.push({
roadmapSlug,
nodeId,
title: label,
description: updatedDescription,
resources: listLinks,
});
} }
topics.push({ await syncContentToDatabase(topics);
roadmapSlug, } catch (error) {
nodeId, console.error(error);
title: label, process.exit(1);
description: updatedDescription,
resources: listLinks,
});
} }
console.log(JSON.stringify(topics, null, 2));