Use Wikipedia REST API for article content

index.js

@@ -7,7 +7,10 @@ import { initEmbedder } from './services/embedder.js';
 const app = express();
 const PORT = process.env.PORT || 3000;
 
-app.use( cors() );
+app.use( cors( {
+	origin: true,
+	credentials: true
+} ) );
 app.use( express.json() );
 
 // Routes

services/chunker.js

@@ -27,7 +27,8 @@ export function chunkArticle( html, sections = [] ) {
 		}
 	} );
 
-	const container = doc.querySelector( '.mw-parser-output' );
+	// Parsoid REST API puts mw-parser-output class on body, Action API wraps in a div
+	const container = doc.querySelector( '.mw-parser-output' ) || doc.body;
 	if ( !container ) {
 		return chunks;
 	}

services/wikipedia.js

@@ -12,6 +12,18 @@ async function fetchJSON( url ) {
 	return response.json();
 }
 
+async function fetchHTML( url ) {
+	const response = await fetch( url, {
+		headers: { 'User-Agent': USER_AGENT }
+	} );
+
+	if ( !response.ok ) {
+		throw new Error( `HTTP ${ response.status }: ${ response.statusText }` );
+	}
+
+	return response.text();
+}
+
 /**
  * Fetch article metadata including revision ID using the Action API
  */
@@ -52,46 +64,26 @@ export async function getArticleMetadata( title ) {
 }
 
 /**
- * Fetch parsed HTML for an article using the Action API with mobile format and Parsoid
- * Returns cleaner HTML optimized for mobile viewing (no infoboxes, cleaner structure)
+ * Fetch parsed HTML for an article using the REST API (Parsoid)
+ * Returns HTML with proper section structure for chunking
  */
 export async function getArticleHtml( title ) {
-	const params = new URLSearchParams( {
-		action: 'parse',
-		format: 'json',
-		page: title,
-		prop: 'text|sections|displaytitle',
-		redirects: '1',
-		disableeditsection: '1',
-		disabletoc: '1',
-		mobileformat: '1',
-		useparsoid: '1'
-	} );
-
-	const url = `https://en.wikipedia.org/w/api.php?${ params }`;
+	const encodedTitle = encodeURIComponent( title.replace( / /g, '_' ) );
+	const url = `https://en.wikipedia.org/api/rest_v1/page/html/${ encodedTitle }`;
 
 	try {
-		const data = await fetchJSON( url );
-
-		if ( data.error ) {
-			if ( data.error.code === 'missingtitle' ) {
-				return null;
-			}
-			throw new Error( data.error.info );
-		}
+		let html = await fetchHTML( url );
 
-		let html = data.parse.text[ '*' ];
+		// Remove the <base> tag to prevent it from affecting URL resolution in the client
+		html = html.replace( /<base[^>]*>/gi, '' );
 
 		// Convert relative links to absolute Wikipedia URLs
-		html = html.replace( /href="\/wiki\//gi, 'href="https://en.wikipedia.org/wiki/' );
-		html = html.replace( /href="\/w\//gi, 'href="https://en.wikipedia.org/w/' );
 		html = html.replace( /href="\.\/([^"]+)"/gi, 'href="https://en.wikipedia.org/wiki/$1"' );
 
-		// Convert protocol-relative URLs to https (only at start of URL, not ://)
+		// Convert protocol-relative URLs to https
 		html = html.replace( /src="\/\//gi, 'src="https://' );
-		// Handle srcset with multiple entries - only replace // at start of URLs, not :// in the middle
+		html = html.replace( /href="\/\//gi, 'href="https://' );
 		html = html.replace( /srcset="([^"]*)"/gi, ( _, srcset ) => {
-			// Replace space followed by // (new URL entry) or start of string //
 			const fixed = srcset
 				.replace( /^\/\//g, 'https://' )
 				.replace( /(\s)\/\//g, '$1https://' );
@@ -101,14 +93,15 @@ export async function getArticleHtml( title ) {
 		// Add lazy loading to images
 		html = html.replace( /<img /gi, '<img loading="lazy" ' );
 
-		// Wrap in a container div with class for styling
-		html = `<div class="mw-parser-output">${ html }</div>`;
+		// Extract title from the HTML
+		const titleMatch = html.match( /<title>([^<]+)<\/title>/ );
+		const extractedTitle = titleMatch ? titleMatch[ 1 ] : title;
 
 		return {
 			html,
-			title: data.parse.title,
-			displayTitle: data.parse.displaytitle,
-			sections: data.parse.sections
+			title: extractedTitle,
+			displayTitle: extractedTitle,
+			sections: [] // REST API doesn't return sections separately, but they're in the HTML
 		};
 	} catch ( error ) {
 		if ( error.message.includes( '404' ) ) {