feat(node): autocomplete node docs with best effort fallback as before

reoslves #122

package.json

@@ -35,6 +35,7 @@
 		"@ltd/j-toml": "^1.38.0",
 		"@vercel/postgres": "^0.9.0",
 		"algoliasearch": "^4.19.1",
+		"cheerio": "^1.0.0-rc.12",
 		"discord-api-types": "^0.37.83",
 		"dotenv": "^16.3.1",
 		"he": "^1.2.0",
@@ -45,7 +46,7 @@
 		"readdirp": "^3.6.0",
 		"reflect-metadata": "^0.2.2",
 		"turndown": "^7.1.2",
-		"undici": "^5.28.3"
+		"undici": "^6.19.3"
 	},
 	"devDependencies": {
 		"@commitlint/cli": "^17.7.1",

src/functions/autocomplete/nodeAutoComplete.ts

@@ -0,0 +1,86 @@
+import process from 'node:process';
+import { stringify } from 'node:querystring';
+import { InteractionResponseType } from 'discord-api-types/v10';
+import type { Response } from 'polka';
+import { fetch } from 'undici';
+import { API_BASE_ORAMA, AUTOCOMPLETE_MAX_ITEMS } from '../../util/constants.js';
+import { prepareHeader } from '../../util/respond.js';
+import { truncate } from '../../util/truncate.js';
+
+type OramaDocument = {
+	id: string;
+	pageSectionTitle: string;
+	pageTitle: string;
+	path: string;
+	siteSection: string;
+};
+
+type OramaHit = {
+	document: OramaDocument;
+	id: string;
+	score: number;
+};
+
+type OramaResult = {
+	count: number;
+	elapsed: { formatted: string; raw: number };
+	facets: { siteSection: { count: number; values: { docs: number } } };
+	hits: OramaHit[];
+};
+
+function autoCompleteMap(elements: OramaDocument[]) {
+	return elements.map((element) => {
+		const cleanSectionTitle = element.pageSectionTitle.replaceAll('`', '');
+		const name = truncate(`${element.pageTitle} > ${cleanSectionTitle}`, 90, '');
+		if (element.path.length > 100) {
+			return {
+				name: truncate(`[path too long] ${element.pageTitle} > ${cleanSectionTitle}`, 100, ''),
+				value: element.pageTitle,
+			};
+		}
+
+		return {
+			name,
+			// we cannot use the full url with the node api base appended here, since discord only allows string values of length 100
+			// some of `crypto` results are longer, if prefixed
+			value: element.path,
+		};
+	});
+}
+
+export async function nodeAutoComplete(res: Response, query: string): Promise<Response> {
+	const full = `${API_BASE_ORAMA}/indexes/${process.env.ORAMA_CONTAINER}/search?api-key=${process.env.ORAMA_KEY}`;
+
+	const result = (await fetch(full, {
+		method: 'post',
+		body: stringify({
+			version: '1.3.2',
+			id: process.env.ORAMA_ID,
+			// eslint-disable-next-line id-length
+			q: JSON.stringify({
+				term: query,
+				mode: 'fulltext',
+				limit: 25,
+				threshold: 0,
+				boost: { pageSectionTitle: 4, pageSectionContent: 2.5, pageTitle: 1.5 },
+				facets: { siteSection: {} },
+				returning: ['path', 'pageSectionTitle', 'pageTitle', 'path', 'siteSection'],
+			}),
+		}),
+		headers: {
+			'Content-Type': 'application/x-www-form-urlencoded',
+		},
+	}).then(async (res) => res.json())) as OramaResult;
+
+	prepareHeader(res);
+	res.write(
+		JSON.stringify({
+			data: {
+				choices: autoCompleteMap(result.hits?.slice(0, AUTOCOMPLETE_MAX_ITEMS - 1).map((hit) => hit.document) ?? []),
+			},
+			type: InteractionResponseType.ApplicationCommandAutocompleteResult,
+		}),
+	);
+
+	return res;
+}

src/functions/node.ts

@@ -2,14 +2,18 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 
+import { URL } from 'node:url';
 import { bold, hideLinkEmbed, hyperlink, inlineCode, italic, underscore, userMention } from '@discordjs/builders';
+import * as cheerio from 'cheerio';
 import type { Response } from 'polka';
 import TurndownService from 'turndown';
 import { fetch } from 'undici';
 import type { NodeDocs } from '../types/NodeDocs.js';
 import { API_BASE_NODE, EMOJI_ID_NODE } from '../util/constants.js';
 import { logger } from '../util/logger.js';
 import { prepareErrorResponse, prepareResponse } from '../util/respond.js';
+import { truncate } from '../util/truncate.js';
+import { urlOption } from '../util/url.js';
 
 const td = new TurndownService({ codeBlockStyle: 'fenced' });
 
@@ -66,27 +70,68 @@ function docsUrl(version: string, source: string, anchorTextRaw: string) {
 	return `${API_BASE_NODE}/docs/${version}/api/${parsePageFromSource(source)}.html#${formatAnchorText(anchorTextRaw)}`;
 }
 
-const cache: Map<string, NodeDocs> = new Map();
+const jsonCache: Map<string, NodeDocs> = new Map();
+const docsCache: Map<string, string> = new Map();
+
+export async function nodeAutoCompleteResolve(res: Response, query: string, ephemeral?: boolean) {
+	const url = urlOption(`${API_BASE_NODE}/${query}`);
+
+	if (!url || !query.startsWith('docs')) {
+		return nodeSearch(res, query, undefined, ephemeral);
+	}
+
+	const key = `${url.origin}${url.pathname}`;
+	let html = docsCache.get(key);
+
+	if (!html) {
+		const data = await fetch(url.toString()).then(async (response) => response.text());
+		docsCache.set(key, data);
+		html = data;
+	}
+
+	const $ = cheerio.load(html);
+
+	const possible = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'];
+
+	const headingBaseSelectorParts = possible.map((prefix) => `${prefix}:has(${url.hash})`);
+	const heaidngSelector = headingBaseSelectorParts.join(', ');
+	const headingCodeSelector = headingBaseSelectorParts.map((part) => `${part} > code`).join(', ');
+	const paragraphSelector = headingBaseSelectorParts.join(', ');
+
+	const heading = $(heaidngSelector).text().replaceAll('#', '');
+	const headingCode = $(headingCodeSelector).text();
+	const paragraph = $(paragraphSelector).nextUntil('h4', 'p');
+
+	const text = paragraph.text();
+	const fullSentence = text.split('. ')?.[0];
+	const partSentence = text.split('.')?.[0];
+
+	prepareResponse(
+		res,
+		[
+			`<:node:${EMOJI_ID_NODE}> ${hyperlink(inlineCode(headingCode.length ? headingCode : heading), url.toString())}`,
+			`${fullSentence ?? partSentence ?? `${truncate(text, 20, '')}..`}.`,
+		].join('\n'),
+		ephemeral ?? false,
+	);
+
+	return res;
+}
 
 export async function nodeSearch(
 	res: Response,
 	query: string,
-	version = 'latest-v18.x',
+	version = 'latest-v20.x',
 	ephemeral?: boolean,
 ): Promise<Response> {
 	const trimmedQuery = query.trim();
 	try {
 		const url = `${API_BASE_NODE}/dist/${version}/docs/api/all.json`;
-		let allNodeData = cache.get(url);
+		let allNodeData = jsonCache.get(url);
 
 		if (!allNodeData) {
-			// Get the data for this version
 			const data = (await fetch(url).then(async (response) => response.json())) as NodeDocs;
-
-			// Set it to the map for caching
-			cache.set(url, data);
-
-			// Set the local parameter for further processing
+			jsonCache.set(url, data);
 			allNodeData = data;
 		}
 

src/handling/handleApplicationCommand.ts

@@ -8,7 +8,7 @@ import { algoliaResponse } from '../functions/algoliaResponse.js';
 import { resolveOptionsToDocsAutoComplete } from '../functions/autocomplete/docsAutoComplete.js';
 import { djsDocs } from '../functions/docs.js';
 import { mdnSearch } from '../functions/mdn.js';
-import { nodeSearch } from '../functions/node.js';
+import { nodeAutoCompleteResolve } from '../functions/node.js';
 import type { Tag } from '../functions/tag.js';
 import { showTag, reloadTags } from '../functions/tag.js';
 import { testTag } from '../functions/testtag.js';
@@ -117,7 +117,7 @@ export async function handleApplicationCommand(
 
 			case 'node': {
 				const castArgs = args as ArgumentsOf<typeof NodeCommand>;
-				await nodeSearch(res, castArgs.query, castArgs.version, castArgs.hide);
+				await nodeAutoCompleteResolve(res, castArgs.query, castArgs.hide);
 				break;
 			}
 

src/handling/handleApplicationCommandAutocomplete.ts

@@ -5,14 +5,16 @@ import type { Response } from 'polka';
 import { algoliaAutoComplete } from '../functions/autocomplete/algoliaAutoComplete.js';
 import { djsAutoComplete } from '../functions/autocomplete/docsAutoComplete.js';
 import { mdnAutoComplete } from '../functions/autocomplete/mdnAutoComplete.js';
+import { nodeAutoComplete } from '../functions/autocomplete/nodeAutoComplete.js';
 import { tagAutoComplete } from '../functions/autocomplete/tagAutoComplete.js';
 import type { Tag } from '../functions/tag.js';
 import type { DTypesCommand } from '../interactions/discordtypes.js';
 import type { GuideCommand } from '../interactions/guide.js';
+import type { NodeCommand } from '../interactions/node.js';
 import type { MDNIndexEntry } from '../types/mdn.js';
 import { transformInteraction } from '../util/interactionOptions.js';
 
-type CommandAutoCompleteName = 'discorddocs' | 'docs' | 'dtypes' | 'guide' | 'mdn' | 'tag';
+type CommandAutoCompleteName = 'discorddocs' | 'docs' | 'dtypes' | 'guide' | 'mdn' | 'node' | 'tag';
 
 export async function handleApplicationCommandAutocomplete(
 	res: Response,
@@ -23,6 +25,12 @@ export async function handleApplicationCommandAutocomplete(
 	const data = message.data;
 	const name = data.name as CommandAutoCompleteName;
 	switch (name) {
+		case 'node': {
+			const args = transformInteraction<typeof NodeCommand>(data.options);
+			await nodeAutoComplete(res, args.query);
+			break;
+		}
+
 		case 'docs': {
 			await djsAutoComplete(res, data.options);
 			break;

src/index.ts

@@ -122,7 +122,6 @@ process.on('uncaughtException', (err, origin) => {
 });
 
 process.on('unhandledRejection', (reason, promise) => {
-	// eslint-disable-next-line no-console
 	logger.error('Unhandled Rejection at:', promise, 'reason:', reason);
 });
 

src/interactions/node.ts

@@ -7,28 +7,9 @@ export const NodeCommand = {
 		{
 			type: ApplicationCommandOptionType.String,
 			name: 'query',
-			description: 'Class, method or event to search for',
+			description: 'Phrase to search for',
 			required: true,
-		},
-		{
-			type: ApplicationCommandOptionType.String,
-			name: 'version',
-			description: 'Node.js version to search documentation for',
-			required: false,
-			choices: [
-				{
-					name: 'v16',
-					value: 'latest-v16.x',
-				},
-				{
-					name: 'v18 (default)',
-					value: 'latest-v18.x',
-				},
-				{
-					name: 'v20 (current)',
-					value: 'latest-v20.x',
-				},
-			],
+			autocomplete: true,
 		},
 		{
 			type: ApplicationCommandOptionType.Boolean,

src/util/constants.ts

@@ -28,6 +28,7 @@ export const API_BASE_MDN = 'https://developer.mozilla.org' as const;
 export const API_BASE_NODE = 'https://nodejs.org' as const;
 export const API_BASE_ALGOLIA = 'algolia.net' as const;
 export const API_BASE_DISCORD = 'https://discord.com/api/v9' as const;
+export const API_BASE_ORAMA = 'https://cloud.orama.run/v1' as const;
 export const AUTOCOMPLETE_MAX_ITEMS = 25;
 export const MAX_MESSAGE_LENGTH = 4_000;
 export const REMOTE_TAG_URL = 'https://raw.githubusercontent.com/discordjs/discord-utils-bot/main/tags' as const;