diff --git a/.env.example b/.env.example index e49d276..049c3d5 100644 --- a/.env.example +++ b/.env.example @@ -1,22 +1,27 @@ -SPOTIFY_AUTH_URL= +LOG_LEVEL=debug + +APP_URL=http://localhost:3000 +DATABASE_PATH=./sqlite/db.sqlite + +SPOTIFY_API_URL=https://api.spotify.com/v1/search +SPOTIFY_AUTH_URL=https://accounts.spotify.com/api/token SPOTIFY_CLIENT_ID= SPOTIFY_CLIENT_SECRET= SPOTIFY_CLIENT_VERSION= -SPOTIFY_API_URL=https://api.spotify.com/v1/search +TIDAL_BASE_URL=https://tidal.com/browse +TIDAL_API_URL=https://openapi.tidal.com/v2/searchresults +TIDAL_AUTH_URL=https://auth.tidal.com/v1/oauth2/token +TIDAL_CLIENT_ID= +TIDAL_CLIENT_SECRET= -YOUTUBE_MUSIC_URL=https://music.youtube.com/search -YOUTUBE_COOKIES= +YOUTUBE_API_URL=https://content-youtube.googleapis.com/youtube/v3/search +YOUTUBE_MUSIC_BASE_URL=https://music.youtube.com +YOUTUBE_API_KEY= DEEZER_API_URL=https://api.deezer.com/search - APPLE_MUSIC_API_URL=https://music.apple.com/us - SOUNDCLOUD_BASE_URL=https://soundcloud.com -TIDAL_BASE_URL=https://listen.tidal.com - -DATABASE_PATH=./sqlite/db.sqlite -APP_URL=http://bit:3000 URL_SHORTENER_API_URL=http://localhost:4000/api/links -URL_SHORTENER_API_KEY= +URL_SHORTENER_API_KEY=secure_12345 diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..c6ebb3c --- /dev/null +++ b/.env.test @@ -0,0 +1,27 @@ +LOG_LEVEL=fatal + +APP_URL=http://localhost:3000 +DATABASE_PATH=./sqlite/db.sqlite + +SPOTIFY_API_URL=https://api.spotify.com/v1/search +SPOTIFY_AUTH_URL=https://accounts.spotify.com/api/token +SPOTIFY_CLIENT_ID=spotify_client_id +SPOTIFY_CLIENT_SECRET=spotify_client_secret +SPOTIFY_CLIENT_VERSION=spotify_client_version + +TIDAL_BASE_URL=https://tidal.com/browse +TIDAL_API_URL=https://openapi.tidal.com/v2/searchresults +TIDAL_AUTH_URL=https://auth.tidal.com/v1/oauth2/token +TIDAL_CLIENT_ID=tidal_client_id +TIDAL_CLIENT_SECRET=tidal_client_secret + +YOUTUBE_API_URL=https://content-youtube.googleapis.com/youtube/v3/search +YOUTUBE_MUSIC_BASE_URL=https://music.youtube.com +YOUTUBE_API_KEY=youtube_api_key + +DEEZER_API_URL=https://api.deezer.com/search +APPLE_MUSIC_API_URL=https://music.apple.com/ca +SOUNDCLOUD_BASE_URL=https://soundcloud.com + +URL_SHORTENER_API_URL=http://localhost:4000/api/links +URL_SHORTENER_API_KEY=url_shortener_api_key diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e77fbc5..48b02c0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,22 +4,18 @@ on: pull_request: branches: [master] -env: - SPOTIFY_AUTH_URL: https://accounts.spotify.com/api/token - SPOTIFY_CLIENT_ID: ${{ secrets.SPOTIFY_CLIENT_ID }} - SPOTIFY_CLIENT_SECRET: ${{ secrets.YOUTUBE_COOKIES }} - SPOTIFY_CLIENT_VERSION: ${{ secrets.SPOTIFY_CLIENT_VERSION }} - SPOTIFY_API_URL: https://api.spotify.com/v1/search - YOUTUBE_MUSIC_URL: https://music.youtube.com/search - YOUTUBE_COOKIES: ${{ secrets.YOUTUBE_COOKIES }} - DEEZER_API_URL: https://api.deezer.com/search - APPLE_MUSIC_API_URL: https://music.apple.com/ca - SOUNDCLOUD_BASE_URL: https://soundcloud.com - TIDAL_BASE_URL: https://listen.tidal.com - jobs: tests: runs-on: ubuntu-latest + services: + bit: + image: sjdonado/bit + env_file: .env.test + ports: + - 4000:4000 + options: >- + --volume sqlite_data:/app/sqlite + steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v1 @@ -27,3 +23,7 @@ jobs: run: bun install - name: Run integration tests run: bun run test + env_file: .env.test + +volumes: + sqlite_data: diff --git a/Dockerfile b/Dockerfile index c10839a..f7121cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,7 @@ EXPOSE 3000/tcp WORKDIR /usr/src/app -RUN apk update && apk add --no-cache chromium nodejs python3 - -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true -ENV CHROME_PATH=/usr/bin/chromium -ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium +RUN apk update && apk add --no-cache nodejs python3 COPY package.json bun.lockb . RUN bun install --frozen-lockfile diff --git a/README.md b/README.md index 373b018..f5c421f 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Adapters represent the streaming services supported by the Web App and the Rayca ![Uptime Badge](https://uptime.sjdonado.com/api/badge/2/uptime/24?labelPrefix=Web%20Page%20&labelSuffix=h) ![Uptime Badge](https://uptime.sjdonado.com/api/badge/2/ping/24?labelPrefix=Web%20Page%20)
- + image
## Raycast Extension diff --git a/bun.lockb b/bun.lockb index d8f4c75..6c5fe79 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/docker-compose.yml b/docker-compose.yml index 45e334a..907034d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: volumes: - ./sqlite:/usr/src/app/sqlite ports: - - 3001:3000 + - 3000:3000 depends_on: - bit bit: @@ -14,7 +14,7 @@ services: environment: APP_URL: http://localhost:4001 ADMIN_NAME: 'Admin' - ADMIN_API_KEY: 'E7gWaEu8JOIGKR/jTmOOgbmBCup2h48jmux2YvIzpxk=' + ADMIN_API_KEY: 'secure_12345' volumes: - sqlite_data:/app/sqlite ports: diff --git a/package.json b/package.json index d7bc9b8..c9f9c15 100644 --- a/package.json +++ b/package.json @@ -18,13 +18,11 @@ "cache-manager": "^5.7.3", "cache-manager-bun-sqlite3": "^0.2.0", "cheerio": "^1.0.0", - "elysia": "^1.1.24", - "elysia-compress": "^1.2.1", + "elysia": "^1.1.26", "howler": "^2.2.4", "notyf": "^3.10.0", "pino": "^8.20.0", "pino-pretty": "^11.0.0", - "puppeteer": "^22.7.0", "string-similarity": "^4.0.4" }, "devDependencies": { diff --git a/src/adapters/apple-music.ts b/src/adapters/apple-music.ts index 56397a2..c3f5024 100644 --- a/src/adapters/apple-music.ts +++ b/src/adapters/apple-music.ts @@ -21,12 +21,9 @@ const APPLE_MUSIC_SEARCH_TYPES = { export async function getAppleMusicLink(query: string, metadata: SearchMetadata) { const searchType = APPLE_MUSIC_SEARCH_TYPES[metadata.type]; + if (!searchType) return null; - if (!searchType) { - return null; - } - - // apple music does not support x-www-form-urlencoded encoding + // x-www-form-urlencoded encoding required for browser url const params = `term=${encodeURIComponent(query)}`; const url = new URL(`${ENV.adapters.appleMusic.apiUrl}/search?${params}`); diff --git a/src/adapters/deezer.ts b/src/adapters/deezer.ts index e6b84ed..9bdb421 100644 --- a/src/adapters/deezer.ts +++ b/src/adapters/deezer.ts @@ -30,10 +30,7 @@ const DEEZER_SEARCH_TYPES = { export async function getDeezerLink(query: string, metadata: SearchMetadata) { const searchType = DEEZER_SEARCH_TYPES[metadata.type]; - - if (!searchType) { - return null; - } + if (!searchType) return null; const params = new URLSearchParams({ q: query, diff --git a/src/adapters/spotify.ts b/src/adapters/spotify.ts index 43b83e8..dbefbbc 100644 --- a/src/adapters/spotify.ts +++ b/src/adapters/spotify.ts @@ -44,10 +44,7 @@ const SPOTIFY_SEARCH_TYPES = { export async function getSpotifyLink(query: string, metadata: SearchMetadata) { const searchType = SPOTIFY_SEARCH_TYPES[metadata.type]; - - if (!searchType) { - return null; - } + if (!searchType) return null; const params = new URLSearchParams({ q: query, @@ -72,7 +69,6 @@ export async function getSpotifyLink(query: string, metadata: SearchMetadata) { }); const [[, data]] = Object.entries(response); - if (data.total === 0) { throw new Error(`No results found: ${JSON.stringify(response)}`); } diff --git a/src/adapters/tidal.ts b/src/adapters/tidal.ts index c73cf1e..e249a0d 100644 --- a/src/adapters/tidal.ts +++ b/src/adapters/tidal.ts @@ -27,12 +27,13 @@ interface TidalSearchResponse { }>; included: Array<{ attributes: { - title: string; + title?: string; + name?: string; }; }>; } -const TIDAL_SEARCH_TYPES = { +export const TIDAL_SEARCH_TYPES = { [MetadataType.Song]: 'tracks', [MetadataType.Album]: 'albums', [MetadataType.Playlist]: 'playlists', @@ -43,10 +44,7 @@ const TIDAL_SEARCH_TYPES = { export async function getTidalLink(query: string, metadata: SearchMetadata) { const searchType = TIDAL_SEARCH_TYPES[metadata.type]; - - if (!searchType) { - return null; - } + if (!searchType) return null; const params = new URLSearchParams({ countryCode: 'US', @@ -58,6 +56,7 @@ export async function getTidalLink(query: string, metadata: SearchMetadata) { ); url.search = params.toString(); + // console.log('tidal', url.toString(), await getOrUpdateTidalAccessToken()); const cache = await getCachedSearchResultLink(url); if (cache) { logger.info(`[Tidal] (${url}) cache hit`); @@ -72,7 +71,6 @@ export async function getTidalLink(query: string, metadata: SearchMetadata) { }); const { data, included } = response; - if (!data || data.length === 0) { throw new Error(`No results found: ${JSON.stringify(response)}`); } @@ -81,7 +79,7 @@ export async function getTidalLink(query: string, metadata: SearchMetadata) { let highestScore = 0; for (const item of included) { - const title = item.attributes.title; + const title = item.attributes.title ?? item.attributes.name ?? ''; const score = compareTwoStrings(title.toLowerCase(), query.toLowerCase()); if (score > highestScore) { diff --git a/src/adapters/youtube.ts b/src/adapters/youtube.ts index 69252bd..27697a8 100644 --- a/src/adapters/youtube.ts +++ b/src/adapters/youtube.ts @@ -4,25 +4,60 @@ import { cacheSearchResultLink, getCachedSearchResultLink } from '~/services/cac import { SearchMetadata, SearchResultLink } from '~/services/search'; import HttpClient from '~/utils/http-client'; import { logger } from '~/utils/logger'; -import { getLinkWithPuppeteer } from '~/utils/scraper'; -const YOUTUBE_SEARCH_TYPES = { - [MetadataType.Song]: 'song', - [MetadataType.Album]: 'album', - [MetadataType.Playlist]: '', +interface YoutubeSearchResponse { + kind: string; + etag: string; + nextPageToken?: string; + regionCode: string; + pageInfo: { + totalResults: number; + resultsPerPage: number; + }; + items: Array<{ + kind: string; + etag: string; + id: { + kind: string; + videoId?: string; + playlistId?: string; + channelId?: string; + }; + }>; +} + +export const YOUTUBE_SEARCH_TYPES = { + [MetadataType.Song]: 'video', + [MetadataType.Album]: 'playlist', + [MetadataType.Playlist]: 'playlist', [MetadataType.Artist]: 'channel', - [MetadataType.Podcast]: '', - [MetadataType.Show]: '', + [MetadataType.Podcast]: 'video', + [MetadataType.Show]: undefined, }; +const YOUTUBE_SEARCH_LINK_TYPE = (item: YoutubeSearchResponse['items'][number]) => ({ + [MetadataType.Song]: `watch?v=${item.id.videoId}`, + [MetadataType.Album]: `playlist?list=${item.id.playlistId}`, + [MetadataType.Playlist]: `playlist?list=${item.id.playlistId}`, + [MetadataType.Artist]: `channel/${item.id.channelId}`, + [MetadataType.Podcast]: `podcast/${item.id.videoId}`, + [MetadataType.Show]: undefined, +}); + export async function getYouTubeLink(query: string, metadata: SearchMetadata) { - return null; // TEMPFIX: youtube blocked the server ip + const searchType = YOUTUBE_SEARCH_TYPES[metadata.type]; + if (!searchType) return null; const params = new URLSearchParams({ - q: `${query} ${YOUTUBE_SEARCH_TYPES[metadata.type]}`, + type: searchType, + regionCode: 'US', + q: query, + part: 'id', + safeSearch: 'none', + key: ENV.adapters.youTube.apiKey, }); - const url = new URL(ENV.adapters.youTube.musicUrl); + const url = new URL(ENV.adapters.youTube.apiUrl); url.search = params.toString(); const cache = await getCachedSearchResultLink(url); @@ -32,36 +67,19 @@ export async function getYouTubeLink(query: string, metadata: SearchMetadata) { } try { - const youtubeCookie = { - domain: '.youtube.com', - path: '/', - expires: Date.now() + 365 * 24 * 60 * 60 * 1000, - secure: true, - }; - - const cookies = ENV.adapters.youTube.cookies.split(';').map(cookie => { - const [name, value] = cookie.split('='); - return { - ...youtubeCookie, - name, - value, - }; - }); - - const link = await getLinkWithPuppeteer( - url.toString(), - 'ytmusic-card-shelf-renderer a', - cookies - ); + const response = await HttpClient.get(url.toString()); - if (!link) { - return null; + const { items } = response; + if (!items || !items[0]) { + throw new Error(`No results found: ${JSON.stringify(response)}`); } + const link = `${ENV.adapters.youTube.musicBaseUrl}/${YOUTUBE_SEARCH_LINK_TYPE(items[0])[metadata.type]}`; + const searchResultLink = { type: Adapter.YouTube, url: link, - isVerified: true, + isVerified: false, } as SearchResultLink; await cacheSearchResultLink(url, searchResultLink); @@ -72,15 +90,3 @@ export async function getYouTubeLink(query: string, metadata: SearchMetadata) { return null; } } - -export async function fetchYoutubeMetadata(youtubeLink: string) { - logger.info(`[${fetchYoutubeMetadata.name}] parse metadata (desktop): ${youtubeLink}`); - - const html = await HttpClient.get(youtubeLink, { - headers: { - Cookie: ENV.adapters.youTube.cookies, - }, - }); - - return html; -} diff --git a/src/config/env.ts b/src/config/env.ts index 5d96316..41dccf6 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -17,8 +17,9 @@ export const ENV = { clientSecret: Bun.env.TIDAL_CLIENT_SECRET!, }, youTube: { - musicUrl: Bun.env.YOUTUBE_MUSIC_URL!, - cookies: Bun.env.YOUTUBE_COOKIES!, + apiUrl: Bun.env.YOUTUBE_API_URL!, + apiKey: Bun.env.YOUTUBE_API_KEY!, + musicBaseUrl: Bun.env.YOUTUBE_MUSIC_BASE_URL!, }, deezer: { apiUrl: Bun.env.DEEZER_API_URL!, @@ -42,6 +43,6 @@ export const ENV = { }, cache: { databasePath: Bun.env.DATABASE_PATH ?? ':memory:', - expTime: 60 * 60 * 24 * 7, // 1 week in seconds + expTime: 60 * 60 * 24 * 7 * 4, // 4 weeks in seconds }, }; diff --git a/src/index.ts b/src/index.ts index 1b09174..235fdaa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,20 +1,13 @@ -import { Elysia } from 'elysia'; import { html } from '@elysiajs/html'; import { staticPlugin } from '@elysiajs/static'; -import compression from 'elysia-compress'; - -import { logger } from './utils/logger'; +import { Elysia } from 'elysia'; import { apiRouter } from './routes/api'; import { pageRouter } from './routes/page'; +import { logger } from './utils/logger'; export const app = new Elysia() .use(html()) - .use( - compression({ - as: 'scoped', - }) - ) .use( staticPlugin({ prefix: '', diff --git a/src/parsers/apple-music.ts b/src/parsers/apple-music.ts index 0ad4341..67a0bdc 100644 --- a/src/parsers/apple-music.ts +++ b/src/parsers/apple-music.ts @@ -27,7 +27,7 @@ export const getAppleMusicMetadata = async (id: string, link: string) => { } try { - const html = await fetchMetadata(link, {}); + const html = await fetchMetadata(link); const doc = getCheerioDoc(html); diff --git a/src/parsers/deezer.ts b/src/parsers/deezer.ts index 17bec7a..34aa668 100644 --- a/src/parsers/deezer.ts +++ b/src/parsers/deezer.ts @@ -27,7 +27,7 @@ export const getDeezerMetadata = async (id: string, link: string) => { } try { - const html = await fetchMetadata(link, {}); + const html = await fetchMetadata(link); const doc = getCheerioDoc(html); diff --git a/src/parsers/link.ts b/src/parsers/link.ts index 42414e2..2829d45 100644 --- a/src/parsers/link.ts +++ b/src/parsers/link.ts @@ -1,4 +1,4 @@ -import { ParseError } from 'elysia'; +import { InternalServerError } from 'elysia'; import { APPLE_MUSIC_LINK_REGEX, @@ -31,10 +31,7 @@ export const getSearchParser = (link?: string, searchId?: string) => { } if (!source) { - const error = new ParseError(); - error.message = 'Source not found'; - - throw error; + throw new InternalServerError('Source not found'); } let id, type; @@ -80,10 +77,7 @@ export const getSearchParser = (link?: string, searchId?: string) => { } if (!id || !type) { - const error = new ParseError(); - error.message = 'Service id could not be extracted from source.'; - - throw error; + throw new InternalServerError('Service id could not be extracted from source.'); } const searchParser = { diff --git a/src/parsers/sound-cloud.ts b/src/parsers/sound-cloud.ts index 745a961..0fb8cc7 100644 --- a/src/parsers/sound-cloud.ts +++ b/src/parsers/sound-cloud.ts @@ -25,7 +25,7 @@ export const getSoundCloudMetadata = async (id: string, link: string) => { } try { - const html = await fetchMetadata(link, {}); + const html = await fetchMetadata(link); const doc = getCheerioDoc(html); diff --git a/src/parsers/spotify.ts b/src/parsers/spotify.ts index 22b2e64..28bb400 100644 --- a/src/parsers/spotify.ts +++ b/src/parsers/spotify.ts @@ -1,10 +1,13 @@ +import { InternalServerError } from 'elysia'; + import { SPOTIFY_LINK_DESKTOP_REGEX, SPOTIFY_LINK_MOBILE_REGEX, } from '~/config/constants'; import { MetadataType, Parser } from '~/config/enum'; +import { ENV } from '~/config/env'; import { cacheSearchMetadata, getCachedSearchMetadata } from '~/services/cache'; -import { defaultHeaders, fetchMetadata } from '~/services/metadata'; +import { fetchMetadata } from '~/services/metadata'; import { SearchMetadata } from '~/services/search'; import HttpClient from '~/utils/http-client'; import { logger } from '~/utils/logger'; @@ -28,6 +31,10 @@ const SPOTIFY_METADATA_TO_METADATA_TYPE = { [SpotifyMetadataType.Show]: MetadataType.Show, }; +const spotifyClientHeaders = { + 'User-Agent': `${ENV.adapters.spotify.clientVersion} (Macintosh; Apple Silicon)`, +}; + export const getSpotifyMetadata = async (id: string, link: string) => { const cached = await getCachedSearchMetadata(id, Parser.Spotify); if (cached) { @@ -36,7 +43,7 @@ export const getSpotifyMetadata = async (id: string, link: string) => { } try { - let html = await fetchMetadata(link); + let html = await fetchMetadata(link, spotifyClientHeaders); if (SPOTIFY_LINK_MOBILE_REGEX.test(link)) { link = html.match(SPOTIFY_LINK_DESKTOP_REGEX)?.[0] ?? ''; @@ -51,7 +58,7 @@ export const getSpotifyMetadata = async (id: string, link: string) => { logger.info(`[${getSpotifyMetadata.name}] parse metadata (desktop): ${link}`); html = await HttpClient.get(link, { - headers: defaultHeaders, + headers: spotifyClientHeaders, retries: 2, }); } @@ -84,7 +91,7 @@ export const getSpotifyMetadata = async (id: string, link: string) => { return metadata; } catch (err) { - throw new Error(`[${getSpotifyMetadata.name}] (${link}) ${err}`); + throw new InternalServerError(`[${getSpotifyMetadata.name}] (${link}) ${err}`); } }; diff --git a/src/parsers/tidal-universal-link.ts b/src/parsers/tidal-universal-link.ts new file mode 100644 index 0000000..f4b66e6 --- /dev/null +++ b/src/parsers/tidal-universal-link.ts @@ -0,0 +1,51 @@ +import { Adapter } from '~/config/enum'; +import { + cacheTidalUniversalLinkResponse, + getCachedTidalUniversalLinkResponse, +} from '~/services/cache'; +import { fetchMetadata } from '~/services/metadata'; +import { SearchResultLink } from '~/services/search'; +import { logger } from '~/utils/logger'; +import { getCheerioDoc } from '~/utils/scraper'; + +export const getUniversalMetadataFromTidal = async ( + link: string, + isVerified: boolean +): Promise | undefined> => { + const cached = await getCachedTidalUniversalLinkResponse(link); + if (cached) { + logger.info(`[Tidal] (${link}) universalLink metadata cache hit`); + return cached; + } + + try { + const html = await fetchMetadata(link); + const doc = getCheerioDoc(html); + + const extractLink = (selector: string, type: Adapter): SearchResultLink | null => { + const url = doc(selector).attr('href'); + return url + ? { + type, + url, + isVerified, + } + : null; + }; + + const adapterLinks: Record = { + [Adapter.Spotify]: extractLink('a[href*="spotify.com"]', Adapter.Spotify), + [Adapter.YouTube]: extractLink('a[href*="music.youtube.com"]', Adapter.YouTube), + [Adapter.AppleMusic]: extractLink('a[href*="music.apple.com"]', Adapter.AppleMusic), + [Adapter.Deezer]: null, + [Adapter.SoundCloud]: null, + [Adapter.Tidal]: null, + }; + + await cacheTidalUniversalLinkResponse(link, adapterLinks); + + return adapterLinks; + } catch (err) { + logger.error(`[${getUniversalMetadataFromTidal.name}] (${link}) ${err}`); + } +}; diff --git a/src/parsers/tidal.ts b/src/parsers/tidal.ts index b257035..be96134 100644 --- a/src/parsers/tidal.ts +++ b/src/parsers/tidal.ts @@ -1,14 +1,7 @@ -import { CheerioAPI } from 'cheerio'; - -import { Adapter, MetadataType, Parser } from '~/config/enum'; -import { - cacheSearchMetadata, - cacheTidalUniversalLinkResponse, - getCachedSearchMetadata, - getCachedTidalUniversalLinkResponse, -} from '~/services/cache'; +import { MetadataType, Parser } from '~/config/enum'; +import { cacheSearchMetadata, getCachedSearchMetadata } from '~/services/cache'; import { fetchMetadata } from '~/services/metadata'; -import { SearchMetadata, SearchResultLink } from '~/services/search'; +import { SearchMetadata } from '~/services/search'; import { logger } from '~/utils/logger'; import { getCheerioDoc, metaTagContent } from '~/utils/scraper'; @@ -90,57 +83,3 @@ export const getTidalQueryFromMetadata = (metadata: SearchMetadata) => { return query; }; - -export const getUniversalMetadataFromTidal = async ( - link: string, - isVerified: boolean -): Promise | undefined> => { - const cached = await getCachedTidalUniversalLinkResponse(link); - if (cached) { - logger.info(`[Tidal] (${link}) universalLink metadata cache hit`); - return cached; - } - - const extractLink = ( - doc: CheerioAPI, - selector: string, - type: Adapter - ): SearchResultLink | null => { - const url = doc(selector).attr('href'); - return url - ? { - type, - url, - isVerified, - } - : null; - }; - - try { - const html = await fetchMetadata(link); - const doc = getCheerioDoc(html); - - const adapterLinks: Record = { - [Adapter.Spotify]: extractLink(doc, 'a[href*="spotify.com"]', Adapter.Spotify), - [Adapter.YouTube]: extractLink( - doc, - 'a[href*="music.youtube.com"]', - Adapter.YouTube - ), - [Adapter.AppleMusic]: extractLink( - doc, - 'a[href*="music.apple.com"]', - Adapter.AppleMusic - ), - [Adapter.Deezer]: null, - [Adapter.SoundCloud]: null, - [Adapter.Tidal]: null, - }; - - await cacheTidalUniversalLinkResponse(link, adapterLinks); - - return adapterLinks; - } catch (err) { - logger.error(`[${getUniversalMetadataFromTidal.name}] (${link}) ${err}`); - } -}; diff --git a/src/parsers/youtube.ts b/src/parsers/youtube.ts index 44bbaff..84657d2 100644 --- a/src/parsers/youtube.ts +++ b/src/parsers/youtube.ts @@ -30,7 +30,8 @@ export const getYouTubeMetadata = async (id: string, link: string) => { } try { - const html = await fetchMetadata(link, {}); + const rawLink = link.replace('music.youtube', 'www.youtube'); + const html = await fetchMetadata(rawLink); const doc = getCheerioDoc(html); diff --git a/src/routes/api.ts b/src/routes/api.ts index f2f400f..a53db65 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -1,17 +1,14 @@ import { Elysia } from 'elysia'; -import { logger } from '~/utils/logger'; import { Adapter } from '~/config/enum'; - -import { apiVersionValidator, searchPayloadValidator } from '~/validations/search'; - import { search } from '~/services/search'; +import { logger } from '~/utils/logger'; +import { apiVersionValidator, searchPayloadValidator } from '~/validations/search'; export const apiRouter = new Elysia().group('/api', app => app .onError(({ code, error }) => { logger.error(error); - return { code, message: error.message, @@ -21,7 +18,6 @@ export const apiRouter = new Elysia().group('/api', app => '/search', async ({ body: { link, adapters } }) => { const searchResult = await search({ link, adapters: adapters as Adapter[] }); - return searchResult; }, { diff --git a/src/routes/page.tsx b/src/routes/page.tsx index 951bbb2..622d499 100644 --- a/src/routes/page.tsx +++ b/src/routes/page.tsx @@ -1,32 +1,25 @@ -import { Elysia } from 'elysia'; import { Html } from '@elysiajs/html'; +import { Elysia, InternalServerError } from 'elysia'; +import { search } from '~/services/search'; import { logger } from '~/utils/logger'; - import { searchPayloadValidator, searchQueryValidator } from '~/validations/search'; - -import { search } from '~/services/search'; - +import ErrorMessage from '~/views/components/error-message'; +import SearchCard from '~/views/components/search-card'; import MainLayout from '~/views/layouts/main'; import Home from '~/views/pages/home'; -import SearchCard from '~/views/components/search-card'; -import ErrorMessage from '~/views/components/error-message'; - export const pageRouter = new Elysia() .onError(({ error, code, set }) => { - logger.error(`[pageRouter]: ${error}`); + logger.error(`[pageRouter]: ${code}:${error}`); if (code === 'NOT_FOUND') { set.headers = { 'HX-Location': '/', }; - return; } - set.status = 200; - if (code === 'VALIDATION' || code === 'PARSE') { return ; } @@ -50,12 +43,21 @@ export const pageRouter = new Elysia() ); - } catch (err) { - logger.error(err); + } catch (error) { + logger.error(`[indexRoute]: ${error}`); - set.status = 404; + if (error instanceof InternalServerError) { + set.status = 404; + return redirect('/'); + } - return redirect('/'); + return ( + + + + + + ); } }, { @@ -66,7 +68,6 @@ export const pageRouter = new Elysia() '/search', async ({ body: { link } }) => { const searchResult = await search({ link }); - return ; }, { diff --git a/src/services/metadata.ts b/src/services/metadata.ts index ab674b4..c61d44c 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -1,18 +1,14 @@ -import { ENV } from '~/config/env'; import HttpClient from '~/utils/http-client'; import { logger } from '~/utils/logger'; -export const defaultHeaders = { - 'User-Agent': `${ENV.adapters.spotify.clientVersion} (Macintosh; Apple Silicon)`, -}; +export const defaultHeaders = {}; -export async function fetchMetadata( - link: string, - headers: Record = defaultHeaders -) { +export async function fetchMetadata(link: string, headers: Record = {}) { const url = link; - - const html = await HttpClient.get(url, { headers }); + const html = await HttpClient.get(url, { + ...defaultHeaders, + headers, + }); logger.info(`[${fetchMetadata.name}] parse metadata: ${url}`); diff --git a/src/services/search.ts b/src/services/search.ts index 3484ba9..7b1a1b8 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -1,3 +1,5 @@ +import { InternalServerError } from 'elysia'; + import { getAppleMusicLink } from '~/adapters/apple-music'; import { getDeezerLink } from '~/adapters/deezer'; import { getSoundCloudLink } from '~/adapters/sound-cloud'; @@ -17,11 +19,8 @@ import { getSoundCloudQueryFromMetadata, } from '~/parsers/sound-cloud'; import { getSpotifyMetadata, getSpotifyQueryFromMetadata } from '~/parsers/spotify'; -import { - getTidalMetadata, - getTidalQueryFromMetadata, - getUniversalMetadataFromTidal, -} from '~/parsers/tidal'; +import { getTidalMetadata, getTidalQueryFromMetadata } from '~/parsers/tidal'; +import { getUniversalMetadataFromTidal } from '~/parsers/tidal-universal-link'; import { getYouTubeMetadata, getYouTubeQueryFromMetadata } from '~/parsers/youtube'; import { generateId } from '~/utils/encoding'; import { logger } from '~/utils/logger'; @@ -73,7 +72,7 @@ export const search = async ({ const searchParser = getSearchParser(link, searchId); - const metadataFetchers = { + const metadataFetchersMap = { [Parser.Spotify]: getSpotifyMetadata, [Parser.YouTube]: getYouTubeMetadata, [Parser.AppleMusic]: getAppleMusicMetadata, @@ -82,7 +81,7 @@ export const search = async ({ [Parser.Tidal]: getTidalMetadata, }; - const queryExtractors = { + const queryExtractorsMap = { [Parser.Spotify]: getSpotifyQueryFromMetadata, [Parser.YouTube]: getYouTubeQueryFromMetadata, [Parser.AppleMusic]: getAppleMusicQueryFromMetadata, @@ -91,7 +90,7 @@ export const search = async ({ [Parser.Tidal]: getTidalQueryFromMetadata, }; - const linkGetters = { + const linkGettersMap = { [Adapter.Spotify]: getSpotifyLink, [Adapter.YouTube]: getYouTubeLink, [Adapter.AppleMusic]: getAppleMusicLink, @@ -100,15 +99,15 @@ export const search = async ({ [Adapter.Tidal]: getTidalLink, }; - const fetchMetadata = metadataFetchers[searchParser.type]; - const extractQuery = queryExtractors[searchParser.type]; + const metadataFetcher = metadataFetchersMap[searchParser.type]; + const queryExtractor = queryExtractorsMap[searchParser.type]; - if (!fetchMetadata || !extractQuery) { - throw new Error('Parser not implemented yet'); + if (!metadataFetcher || !queryExtractor) { + throw new InternalServerError('Parser not implemented yet'); } - let metadata = await fetchMetadata(searchParser.id, searchParser.source); - const query = extractQuery(metadata); + let metadata = await metadataFetcher(searchParser.id, searchParser.source); + const query = queryExtractor(metadata); const parserType = searchParser.type as StreamingServiceType; logger.info( @@ -146,7 +145,6 @@ export const search = async ({ const links: SearchResultLink[] = []; const existingAdapters = new Set(links.map(link => link.type)); - // Fetch from Tidal first let tidalLink: SearchResultLink | null = linkSearchResult; if (parserType !== Adapter.Tidal) { tidalLink = await getTidalLink(query, metadata); @@ -189,10 +187,10 @@ export const search = async ({ await Promise.all( remainingAdapters .map(adapter => { - const getLink = linkGetters[adapter]; - if (!getLink) return null; + const linkGetter = linkGettersMap[adapter]; + if (!linkGetter) return null; - return getLink(query, metadata).then(link => { + return linkGetter(query, metadata).then(link => { if (link) { links.push({ type: adapter, url: link.url, isVerified: true }); existingAdapters.add(adapter); diff --git a/src/utils/http-client.ts b/src/utils/http-client.ts index 10c5b0c..133b66a 100644 --- a/src/utils/http-client.ts +++ b/src/utils/http-client.ts @@ -2,7 +2,6 @@ import axios, { AxiosError } from 'axios'; import axiosRetry from 'axios-retry'; import { DEFAULT_TIMEOUT } from '~/config/constants'; - import { logger } from '~/utils/logger'; type HttpClientOptions = { @@ -20,33 +19,21 @@ function getRandomUserAgent() { ]; const chromeVersions = [ - '86.0.4240.198', - '87.0.4280.88', - '88.0.4324.96', - '89.0.4389.72', - '90.0.4430.85', + '91.0.4472.124', + '92.0.4515.107', + '93.0.4577.63', + '94.0.4606.71', + '95.0.4638.69', ]; - const firefoxVersions = ['84.0', '85.0', '86.0', '87.0', '88.0']; - - const isChrome = Math.random() > 0.5; - const os = osOptions[Math.floor(Math.random() * osOptions.length)]; + const chromeVersion = chromeVersions[Math.floor(Math.random() * chromeVersions.length)]; - if (isChrome) { - const chromeVersion = - chromeVersions[Math.floor(Math.random() * chromeVersions.length)]; - - return `Mozilla/5.0 (${os}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion} Safari/537.36`; - } - const firefoxVersion = - firefoxVersions[Math.floor(Math.random() * firefoxVersions.length)]; - - return `Mozilla/5.0 (${os}; rv:${firefoxVersion}) Gecko/20100101 Firefox/${firefoxVersion}`; + return `Mozilla/5.0 (${os}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion} Safari/537.36 Edg/91.0.864.67`; } axiosRetry(axios, { - retries: 2, + retries: 1, retryCondition: error => { // Retry on network errors or 5xx status codes return axiosRetry.isNetworkError(error) || axiosRetry.isRetryableError(error); @@ -99,8 +86,9 @@ export default class HttpClient { return data as T; } catch (err) { + const axiosError = err as AxiosError; logger.error( - `[${HttpClient.request.name}] Request failed ${(err as Error).message}` + `[${HttpClient.request.name}] Request failed ${axiosError.message} ${axiosError.response ? axiosError.response.data : ''}` ); logger.debug(err); throw err; diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 6e4465b..73b33e8 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -7,7 +7,7 @@ export const stream = pretty({ export const logger = pino( { - level: process.env.NODE_ENV === 'test' ? 'fatal' : 'debug', + level: process.env.LOG_LEVEL ?? 'debug', }, stream ); diff --git a/src/utils/scraper.ts b/src/utils/scraper.ts index 2e5b69f..66a474a 100644 --- a/src/utils/scraper.ts +++ b/src/utils/scraper.ts @@ -1,68 +1,4 @@ import * as cheerio from 'cheerio'; -import puppeteer, { Browser, CookieParam, Page } from 'puppeteer'; - -let browser: Browser | null = null; - -async function launchBrowser(): Promise { - if (!browser) { - browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] }); - } - return browser; -} - -async function withTimeout(promise: Promise, timeout: number): Promise { - return Promise.race([ - promise, - new Promise((_, reject) => - setTimeout(() => reject(new Error('Operation timed out')), timeout) - ), - ]); -} - -export async function getLinkWithPuppeteer( - url: string, - linkQuerySelector: string, - cookies: CookieParam[] = [], - timeout: number = 4000 // Default timeout of 4 seconds -) { - const browser = await launchBrowser(); - const page: Page = await browser.newPage(); - - try { - await page.setUserAgent( - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0' - ); - - await page.setCookie(...cookies); - await page.setRequestInterception(true); - - page.on('request', request => { - if (request.resourceType() === 'image') { - request.abort(); - } else { - request.continue(); - } - }); - - await page.setViewport({ width: 768, height: 600 }); - - await withTimeout(page.goto(url, { waitUntil: 'networkidle0' }), timeout); - - const href = await withTimeout( - page.evaluate( - // eslint-disable-next-line - // @ts-ignore - selector => document.querySelector(selector)?.href, - linkQuerySelector - ), - timeout - ); - - return href; - } finally { - page.close(); - } -} export function getCheerioDoc(html: string) { return cheerio.load(html); diff --git a/src/validations/search.ts b/src/validations/search.ts index 5cf093c..e719371 100644 --- a/src/validations/search.ts +++ b/src/validations/search.ts @@ -1,4 +1,5 @@ import { t } from 'elysia'; + import { ALLOWED_LINKS_REGEX } from '~/config/constants'; import { Adapter } from '~/config/enum'; diff --git a/src/views/controllers/search_card_controller.js b/src/views/controllers/search_card_controller.js index f3c9983..3db6040 100644 --- a/src/views/controllers/search_card_controller.js +++ b/src/views/controllers/search_card_controller.js @@ -1,5 +1,6 @@ import { Controller } from '@hotwired/stimulus'; import { Howl } from 'howler'; + import { copyToClipboard } from './helpers'; export default class extends Controller { diff --git a/src/views/controllers/search_controller.js b/src/views/controllers/search_controller.js index 1943bed..803815c 100644 --- a/src/views/controllers/search_controller.js +++ b/src/views/controllers/search_controller.js @@ -1,9 +1,9 @@ import { Controller } from '@hotwired/stimulus'; -import { toast } from './helpers'; - import { SPOTIFY_LINK_REGEX, YOUTUBE_LINK_REGEX } from '~/config/constants'; +import { toast } from './helpers'; + export default class extends Controller { static targets = ['form', 'link']; diff --git a/src/views/layouts/main.tsx b/src/views/layouts/main.tsx index 527afc1..ad86cbf 100644 --- a/src/views/layouts/main.tsx +++ b/src/views/layouts/main.tsx @@ -13,6 +13,8 @@ export default function MainLayout({ image?: string; children: JSX.Element; }) { + const isProduction = process.env.NODE_ENV === 'production'; + return ( @@ -59,18 +61,20 @@ export default function MainLayout({ href="https://cdn.jsdelivr.net/npm/@tabler/icons-webfont@latest/tabler-icons.min.css" /> - + {children} -