Merge remote-tracking branch 'upstream/main' into Wolvinny

package.json

@@ -35,6 +35,7 @@
 		"@ltd/j-toml": "^1.38.0",
 		"@vercel/postgres": "^0.9.0",
 		"algoliasearch": "^4.19.1",
+		"cheerio": "^1.0.0-rc.12",
 		"discord-api-types": "^0.37.83",
 		"dotenv": "^16.3.1",
 		"he": "^1.2.0",
@@ -45,7 +46,7 @@
 		"readdirp": "^3.6.0",
 		"reflect-metadata": "^0.2.2",
 		"turndown": "^7.1.2",
-		"undici": "^5.28.3"
+		"undici": "^6.19.3"
 	},
 	"devDependencies": {
 		"@commitlint/cli": "^17.7.1",

src/functions/autocomplete/docsAutoComplete.ts

@@ -6,19 +6,12 @@ import type {
 } from 'discord-api-types/v10';
 import { ApplicationCommandOptionType, InteractionResponseType } from 'discord-api-types/v10';
 import type { Response } from 'polka';
-import { fetch } from 'undici';
 import { AUTOCOMPLETE_MAX_ITEMS } from '../../util/constants.js';
 import { getDjsVersions } from '../../util/djsdocs.js';
 import { logger } from '../../util/logger.js';
-import { truncate } from '../../util/truncate.js';
+import { queryDocs } from '../docs.js';
 
-const BASE_SEARCH = `https://search.discordjs.dev/`;
-
-function searchURL(pack: string, version: string) {
-	return `${BASE_SEARCH}indexes/${pack}-${version.replaceAll('.', '-')}/search`;
-}
-
-function parseDocsPath(path: string) {
+export function parseDocsPath(path: string) {
 	// /0   /1       /2       /3   /4
 	// /docs/packages/builders/main/EmbedBuilder:Class
 	// /docs/packages/builders/main/EmbedImageData:Interface#proxyURL
@@ -64,55 +57,17 @@ export async function djsAutoComplete(
 	}
 
 	const version = versionOptionData?.value ?? versions.versions.get(option.name)?.at(1) ?? 'main';
-
-	const searchRes = await fetch(searchURL(option.name, version), {
-		method: 'post',
-		body: JSON.stringify({
-			limit: 100,
-			// eslint-disable-next-line id-length
-			q: queryOptionData.value,
-		}),
-		headers: {
-			Authorization: `Bearer ${process.env.DJS_DOCS_BEARER!}`,
-			'Content-Type': 'application/json',
-		},
-	});
-
-	const docsResult = (await searchRes.json()) as any;
-	docsResult.hits.sort((one: any, other: any) => {
-		const oneScore = one.kind === 'Class' ? 1 : 0;
-		const otherScore = other.kind === 'Class' ? 1 : 0;
-
-		return otherScore - oneScore;
-	});
-
+	const docsResult = await queryDocs(queryOptionData.value, option.name, version);
 	const choices = [];
 
 	for (const hit of docsResult.hits) {
 		if (choices.length >= AUTOCOMPLETE_MAX_ITEMS) {
 			break;
 		}
 
-		const parsed = parseDocsPath(hit.path);
-
-		let name = '';
-		const isMember = ['Property', 'Method', 'Event', 'PropertySignature', 'EnumMember'].includes(hit.kind);
-		if (isMember) {
-			name += `${parsed.item}#${hit.name}${hit.kind === 'Method' ? '()' : ''}`;
-		} else {
-			name += hit.name;
-		}
-
-		const itemKind = isMember ? 'Class' : hit.kind;
-		const parts = [parsed.package, parsed.item.toLocaleLowerCase(), parsed.kind];
-
-		if (isMember) {
-			parts.push(hit.name);
-		}
-
 		choices.push({
-			name: truncate(`${name}${hit.summary ? ` - ${hit.summary}` : ''}`, 100, ' '),
-			value: parts.join('|'),
+			name: hit.autoCompleteName,
+			value: hit.autoCompleteValue,
 		});
 	}
 

src/functions/autocomplete/nodeAutoComplete.ts

@@ -0,0 +1,86 @@
+import process from 'node:process';
+import { stringify } from 'node:querystring';
+import { InteractionResponseType } from 'discord-api-types/v10';
+import type { Response } from 'polka';
+import { fetch } from 'undici';
+import { API_BASE_ORAMA, AUTOCOMPLETE_MAX_ITEMS } from '../../util/constants.js';
+import { prepareHeader } from '../../util/respond.js';
+import { truncate } from '../../util/truncate.js';
+
+type OramaDocument = {
+	id: string;
+	pageSectionTitle: string;
+	pageTitle: string;
+	path: string;
+	siteSection: string;
+};
+
+type OramaHit = {
+	document: OramaDocument;
+	id: string;
+	score: number;
+};
+
+type OramaResult = {
+	count: number;
+	elapsed: { formatted: string; raw: number };
+	facets: { siteSection: { count: number; values: { docs: number } } };
+	hits: OramaHit[];
+};
+
+function autoCompleteMap(elements: OramaDocument[]) {
+	return elements.map((element) => {
+		const cleanSectionTitle = element.pageSectionTitle.replaceAll('`', '');
+		const name = truncate(`${element.pageTitle} > ${cleanSectionTitle}`, 90, '');
+		if (element.path.length > 100) {
+			return {
+				name: truncate(`[path too long] ${element.pageTitle} > ${cleanSectionTitle}`, 100, ''),
+				value: element.pageTitle,
+			};
+		}
+
+		return {
+			name,
+			// we cannot use the full url with the node api base appended here, since discord only allows string values of length 100
+			// some of `crypto` results are longer, if prefixed
+			value: element.path,
+		};
+	});
+}
+
+export async function nodeAutoComplete(res: Response, query: string): Promise<Response> {
+	const full = `${API_BASE_ORAMA}/indexes/${process.env.ORAMA_CONTAINER}/search?api-key=${process.env.ORAMA_KEY}`;
+
+	const result = (await fetch(full, {
+		method: 'post',
+		body: stringify({
+			version: '1.3.2',
+			id: process.env.ORAMA_ID,
+			// eslint-disable-next-line id-length
+			q: JSON.stringify({
+				term: query,
+				mode: 'fulltext',
+				limit: 25,
+				threshold: 0,
+				boost: { pageSectionTitle: 4, pageSectionContent: 2.5, pageTitle: 1.5 },
+				facets: { siteSection: {} },
+				returning: ['path', 'pageSectionTitle', 'pageTitle', 'path', 'siteSection'],
+			}),
+		}),
+		headers: {
+			'Content-Type': 'application/x-www-form-urlencoded',
+		},
+	}).then(async (res) => res.json())) as OramaResult;
+
+	prepareHeader(res);
+	res.write(
+		JSON.stringify({
+			data: {
+				choices: autoCompleteMap(result.hits?.slice(0, AUTOCOMPLETE_MAX_ITEMS - 1).map((hit) => hit.document) ?? []),
+			},
+			type: InteractionResponseType.ApplicationCommandAutocompleteResult,
+		}),
+	);
+
+	return res;
+}

src/functions/docs.ts

@@ -26,6 +26,9 @@ import {
 import { logger } from '../util/logger.js';
 import { prepareErrorResponse, prepareResponse } from '../util/respond.js';
 import { truncate } from '../util/truncate.js';
+import { parseDocsPath } from './autocomplete/docsAutoComplete.js';
+
+const BASE_SEARCH = 'https://search.discordjs.dev/';
 
 /**
  * Vercel blob store format
@@ -42,6 +45,59 @@ type CacheEntry = {
 
 const docsCache = new Map<string, CacheEntry>();
 
+function searchURL(pack: string, version: string) {
+	return `${BASE_SEARCH}indexes/${pack}-${version.replaceAll('.', '-')}/search`;
+}
+
+function sanitizeText(name: string) {
+	return name.replaceAll('*', '');
+}
+
+export async function queryDocs(query: string, pack: string, version: string) {
+	const searchRes = await fetch(searchURL(pack, version), {
+		method: 'post',
+		body: JSON.stringify({
+			limit: 100,
+			// eslint-disable-next-line id-length
+			q: query,
+		}),
+		headers: {
+			Authorization: `Bearer ${process.env.DJS_DOCS_BEARER!}`,
+			'Content-Type': 'application/json',
+		},
+	});
+
+	const docsResult = (await searchRes.json()) as any;
+
+	return {
+		...docsResult,
+		hits: docsResult.hits.map((hit: any) => {
+			const parsed = parseDocsPath(hit.path);
+
+			let name = '';
+			const isMember = ['Property', 'Method', 'Event', 'PropertySignature', 'EnumMember'].includes(hit.kind);
+			if (isMember) {
+				name += `${parsed.item}#${hit.name}${hit.kind === 'Method' ? '()' : ''}`;
+			} else {
+				name += hit.name;
+			}
+
+			const parts = [parsed.package, parsed.item.toLocaleLowerCase(), parsed.kind];
+
+			if (isMember) {
+				parts.push(hit.name);
+			}
+
+			return {
+				...hit,
+				autoCompleteName: truncate(`${name}${hit.summary ? ` - ${sanitizeText(hit.summary)}` : ''}`, 100, ' '),
+				autoCompleteValue: parts.join('|'),
+				isMember,
+			};
+		}),
+	};
+}
+
 export async function fetchDocItem(
 	_package: string,
 	branch: string,
@@ -210,10 +266,29 @@ function formatItem(_item: any, _package: string, version: string, member?: stri
 	return lines.join('\n');
 }
 
-export async function djsDocs(res: Response, branch: string, query: string, ephemeral = false) {
-	const [_package, itemName, itemKind, member] = query.split('|');
+async function resolveDjsDocsQuery(query: string, source: string, branch: string) {
+	if (query.includes('|')) {
+		return query;
+	} else {
+		const searchResult = await queryDocs(query, source, branch);
+		const bestHit = searchResult.hits[0];
+		if (bestHit) {
+			return bestHit.autoCompleteValue;
+		}
+
+		return null;
+	}
+}
 
+export async function djsDocs(res: Response, branch: string, _query: string, source: string, ephemeral = false) {
 	try {
+		const query = await resolveDjsDocsQuery(_query, source, branch);
+		if (!query) {
+			prepareErrorResponse(res, 'Cannot find any hits for the provided query - consider using auto complete.');
+			return res.end();
+		}
+
+		const [_package, itemName, itemKind, member] = query.split('|');
 		const item = await fetchDocItem(_package, branch, itemName, itemKind.toLowerCase());
 		if (!item) {
 			prepareErrorResponse(res, `Could not fetch doc entry for query ${inlineCode(query)}.`);

src/functions/node.ts

@@ -2,14 +2,18 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 
+import { URL } from 'node:url';
 import { bold, hideLinkEmbed, hyperlink, inlineCode, italic, underscore, userMention } from '@discordjs/builders';
+import * as cheerio from 'cheerio';
 import type { Response } from 'polka';
 import TurndownService from 'turndown';
 import { fetch } from 'undici';
 import type { NodeDocs } from '../types/NodeDocs.js';
 import { API_BASE_NODE, EMOJI_ID_NODE } from '../util/constants.js';
 import { logger } from '../util/logger.js';
 import { prepareErrorResponse, prepareResponse } from '../util/respond.js';
+import { truncate } from '../util/truncate.js';
+import { urlOption } from '../util/url.js';
 
 const td = new TurndownService({ codeBlockStyle: 'fenced' });
 
@@ -66,27 +70,68 @@ function docsUrl(version: string, source: string, anchorTextRaw: string) {
 	return `${API_BASE_NODE}/docs/${version}/api/${parsePageFromSource(source)}.html#${formatAnchorText(anchorTextRaw)}`;
 }
 
-const cache: Map<string, NodeDocs> = new Map();
+const jsonCache: Map<string, NodeDocs> = new Map();
+const docsCache: Map<string, string> = new Map();
+
+export async function nodeAutoCompleteResolve(res: Response, query: string, ephemeral?: boolean) {
+	const url = urlOption(`${API_BASE_NODE}/${query}`);
+
+	if (!url || !query.startsWith('docs')) {
+		return nodeSearch(res, query, undefined, ephemeral);
+	}
+
+	const key = `${url.origin}${url.pathname}`;
+	let html = docsCache.get(key);
+
+	if (!html) {
+		const data = await fetch(url.toString()).then(async (response) => response.text());
+		docsCache.set(key, data);
+		html = data;
+	}
+
+	const $ = cheerio.load(html);
+
+	const possible = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'];
+
+	const headingBaseSelectorParts = possible.map((prefix) => `${prefix}:has(${url.hash})`);
+	const heaidngSelector = headingBaseSelectorParts.join(', ');
+	const headingCodeSelector = headingBaseSelectorParts.map((part) => `${part} > code`).join(', ');
+	const paragraphSelector = headingBaseSelectorParts.join(', ');
+
+	const heading = $(heaidngSelector).text().replaceAll('#', '');
+	const headingCode = $(headingCodeSelector).text();
+	const paragraph = $(paragraphSelector).nextUntil('h4', 'p');
+
+	const text = paragraph.text();
+	const fullSentence = text.split('. ')?.[0];
+	const partSentence = text.split('.')?.[0];
+
+	prepareResponse(
+		res,
+		[
+			`<:node:${EMOJI_ID_NODE}> ${hyperlink(inlineCode(headingCode.length ? headingCode : heading), url.toString())}`,
+			`${fullSentence ?? partSentence ?? `${truncate(text, 20, '')}..`}.`,
+		].join('\n'),
+		ephemeral ?? false,
+	);
+
+	return res;
+}
 
 export async function nodeSearch(
 	res: Response,
 	query: string,
-	version = 'latest-v18.x',
+	version = 'latest-v20.x',
 	ephemeral?: boolean,
 ): Promise<Response> {
 	const trimmedQuery = query.trim();
 	try {
 		const url = `${API_BASE_NODE}/dist/${version}/docs/api/all.json`;
-		let allNodeData = cache.get(url);
+		let allNodeData = jsonCache.get(url);
 
 		if (!allNodeData) {
-			// Get the data for this version
 			const data = (await fetch(url).then(async (response) => response.json())) as NodeDocs;
-
-			// Set it to the map for caching
-			cache.set(url, data);
-
-			// Set the local parameter for further processing
+			jsonCache.set(url, data);
 			allNodeData = data;
 		}