Merge pull request #40 from sjdonado/fix/youtube-replace-puppeteer-wi…

…th-api

Fix/youtube replace puppeteer with api

.env.example

@@ -1,22 +1,27 @@
-SPOTIFY_AUTH_URL=
+LOG_LEVEL=debug
+
+APP_URL=http://localhost:3000
+DATABASE_PATH=./sqlite/db.sqlite
+
+SPOTIFY_API_URL=https://api.spotify.com/v1/search
+SPOTIFY_AUTH_URL=https://accounts.spotify.com/api/token
 SPOTIFY_CLIENT_ID=
 SPOTIFY_CLIENT_SECRET=
 SPOTIFY_CLIENT_VERSION=
 
-SPOTIFY_API_URL=https://api.spotify.com/v1/search
+TIDAL_BASE_URL=https://tidal.com/browse
+TIDAL_API_URL=https://openapi.tidal.com/v2/searchresults
+TIDAL_AUTH_URL=https://auth.tidal.com/v1/oauth2/token
+TIDAL_CLIENT_ID=
+TIDAL_CLIENT_SECRET=
 
-YOUTUBE_MUSIC_URL=https://music.youtube.com/search
-YOUTUBE_COOKIES=
+YOUTUBE_API_URL=https://content-youtube.googleapis.com/youtube/v3/search
+YOUTUBE_MUSIC_BASE_URL=https://music.youtube.com
+YOUTUBE_API_KEY=
 
 DEEZER_API_URL=https://api.deezer.com/search
-
 APPLE_MUSIC_API_URL=https://music.apple.com/us
-
 SOUNDCLOUD_BASE_URL=https://soundcloud.com
-TIDAL_BASE_URL=https://listen.tidal.com
-
-DATABASE_PATH=./sqlite/db.sqlite
-APP_URL=http://bit:3000
 
 URL_SHORTENER_API_URL=http://localhost:4000/api/links
-URL_SHORTENER_API_KEY=
+URL_SHORTENER_API_KEY=secure_12345

.env.test

@@ -0,0 +1,27 @@
+LOG_LEVEL=fatal
+
+APP_URL=http://localhost:3000
+DATABASE_PATH=./sqlite/db.sqlite
+
+SPOTIFY_API_URL=https://api.spotify.com/v1/search
+SPOTIFY_AUTH_URL=https://accounts.spotify.com/api/token
+SPOTIFY_CLIENT_ID=spotify_client_id
+SPOTIFY_CLIENT_SECRET=spotify_client_secret
+SPOTIFY_CLIENT_VERSION=spotify_client_version
+
+TIDAL_BASE_URL=https://tidal.com/browse
+TIDAL_API_URL=https://openapi.tidal.com/v2/searchresults
+TIDAL_AUTH_URL=https://auth.tidal.com/v1/oauth2/token
+TIDAL_CLIENT_ID=tidal_client_id
+TIDAL_CLIENT_SECRET=tidal_client_secret
+
+YOUTUBE_API_URL=https://content-youtube.googleapis.com/youtube/v3/search
+YOUTUBE_MUSIC_BASE_URL=https://music.youtube.com
+YOUTUBE_API_KEY=youtube_api_key
+
+DEEZER_API_URL=https://api.deezer.com/search
+APPLE_MUSIC_API_URL=https://music.apple.com/ca
+SOUNDCLOUD_BASE_URL=https://soundcloud.com
+
+URL_SHORTENER_API_URL=http://localhost:4000/api/links
+URL_SHORTENER_API_KEY=url_shortener_api_key

.github/workflows/tests.yml

@@ -4,26 +4,26 @@ on:
   pull_request:
     branches: [master]
 
-env:
-  SPOTIFY_AUTH_URL: https://accounts.spotify.com/api/token
-  SPOTIFY_CLIENT_ID: ${{ secrets.SPOTIFY_CLIENT_ID }}
-  SPOTIFY_CLIENT_SECRET: ${{ secrets.YOUTUBE_COOKIES }}
-  SPOTIFY_CLIENT_VERSION: ${{ secrets.SPOTIFY_CLIENT_VERSION }}
-  SPOTIFY_API_URL: https://api.spotify.com/v1/search
-  YOUTUBE_MUSIC_URL: https://music.youtube.com/search
-  YOUTUBE_COOKIES: ${{ secrets.YOUTUBE_COOKIES }}
-  DEEZER_API_URL: https://api.deezer.com/search
-  APPLE_MUSIC_API_URL: https://music.apple.com/ca
-  SOUNDCLOUD_BASE_URL: https://soundcloud.com
-  TIDAL_BASE_URL: https://listen.tidal.com
-
 jobs:
   tests:
     runs-on: ubuntu-latest
+    services:
+      bit:
+        image: sjdonado/bit
+        env_file: .env.test
+        ports:
+          - 4000:4000
+        options: >-
+          --volume sqlite_data:/app/sqlite
+
     steps:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v1
       - name: Install dependencies
         run: bun install
       - name: Run integration tests
         run: bun run test
+        env_file: .env.test
+
+volumes:
+  sqlite_data:

Dockerfile

@@ -4,11 +4,7 @@ EXPOSE 3000/tcp
 
 WORKDIR /usr/src/app
 
-RUN apk update && apk add --no-cache chromium nodejs python3
-
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
-ENV CHROME_PATH=/usr/bin/chromium
-ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+RUN apk update && apk add --no-cache nodejs python3
 
 COPY package.json bun.lockb .
 RUN bun install --frozen-lockfile

README.md

@@ -20,7 +20,7 @@ Adapters represent the streaming services supported by the Web App and the Rayca
 ![Uptime Badge](https://uptime.sjdonado.com/api/badge/2/uptime/24?labelPrefix=Web%20Page%20&labelSuffix=h) ![Uptime Badge](https://uptime.sjdonado.com/api/badge/2/ping/24?labelPrefix=Web%20Page%20)
 
 <div align="center">
-  <img width=1200 src="https://firebasestorage.googleapis.com/v0/b/rule-of-thumb-1c13c.appspot.com/o/idonthavespotify.webp?alt=media"/>
+  <img width="1200" alt="image" src="https://github.com/user-attachments/assets/ae6250f5-d1ed-41f2-ae21-8a2b2599a450" />
 </div>
 
 ## Raycast Extension

docker-compose.yml

@@ -6,15 +6,15 @@ services:
     volumes:
       - ./sqlite:/usr/src/app/sqlite
     ports:
-      - 3001:3000
+      - 3000:3000
     depends_on:
       - bit
   bit:
     image: sjdonado/bit
     environment:
       APP_URL: http://localhost:4001
       ADMIN_NAME: 'Admin'
-      ADMIN_API_KEY: 'E7gWaEu8JOIGKR/jTmOOgbmBCup2h48jmux2YvIzpxk='
+      ADMIN_API_KEY: 'secure_12345'
     volumes:
       - sqlite_data:/app/sqlite
     ports:

package.json

@@ -18,13 +18,11 @@
     "cache-manager": "^5.7.3",
     "cache-manager-bun-sqlite3": "^0.2.0",
     "cheerio": "^1.0.0",
-    "elysia": "^1.1.24",
-    "elysia-compress": "^1.2.1",
+    "elysia": "^1.1.26",
     "howler": "^2.2.4",
     "notyf": "^3.10.0",
     "pino": "^8.20.0",
     "pino-pretty": "^11.0.0",
-    "puppeteer": "^22.7.0",
     "string-similarity": "^4.0.4"
   },
   "devDependencies": {

src/adapters/apple-music.ts

@@ -21,12 +21,9 @@ const APPLE_MUSIC_SEARCH_TYPES = {
 
 export async function getAppleMusicLink(query: string, metadata: SearchMetadata) {
   const searchType = APPLE_MUSIC_SEARCH_TYPES[metadata.type];
+  if (!searchType) return null;
 
-  if (!searchType) {
-    return null;
-  }
-
-  // apple music does not support x-www-form-urlencoded encoding
+  // x-www-form-urlencoded encoding required for browser url
   const params = `term=${encodeURIComponent(query)}`;
 
   const url = new URL(`${ENV.adapters.appleMusic.apiUrl}/search?${params}`);

src/adapters/deezer.ts

@@ -30,10 +30,7 @@ const DEEZER_SEARCH_TYPES = {
 
 export async function getDeezerLink(query: string, metadata: SearchMetadata) {
   const searchType = DEEZER_SEARCH_TYPES[metadata.type];
-
-  if (!searchType) {
-    return null;
-  }
+  if (!searchType) return null;
 
   const params = new URLSearchParams({
     q: query,

src/adapters/spotify.ts

@@ -44,10 +44,7 @@ const SPOTIFY_SEARCH_TYPES = {
 
 export async function getSpotifyLink(query: string, metadata: SearchMetadata) {
   const searchType = SPOTIFY_SEARCH_TYPES[metadata.type];
-
-  if (!searchType) {
-    return null;
-  }
+  if (!searchType) return null;
 
   const params = new URLSearchParams({
     q: query,
@@ -72,7 +69,6 @@ export async function getSpotifyLink(query: string, metadata: SearchMetadata) {
     });
 
     const [[, data]] = Object.entries(response);
-
     if (data.total === 0) {
       throw new Error(`No results found: ${JSON.stringify(response)}`);
     }

src/adapters/tidal.ts

@@ -27,12 +27,13 @@ interface TidalSearchResponse {
   }>;
   included: Array<{
     attributes: {
-      title: string;
+      title?: string;
+      name?: string;
     };
   }>;
 }
 
-const TIDAL_SEARCH_TYPES = {
+export const TIDAL_SEARCH_TYPES = {
   [MetadataType.Song]: 'tracks',
   [MetadataType.Album]: 'albums',
   [MetadataType.Playlist]: 'playlists',
@@ -43,10 +44,7 @@ const TIDAL_SEARCH_TYPES = {
 
 export async function getTidalLink(query: string, metadata: SearchMetadata) {
   const searchType = TIDAL_SEARCH_TYPES[metadata.type];
-
-  if (!searchType) {
-    return null;
-  }
+  if (!searchType) return null;
 
   const params = new URLSearchParams({
     countryCode: 'US',
@@ -58,6 +56,7 @@ export async function getTidalLink(query: string, metadata: SearchMetadata) {
   );
   url.search = params.toString();
 
+  // console.log('tidal', url.toString(), await getOrUpdateTidalAccessToken());
   const cache = await getCachedSearchResultLink(url);
   if (cache) {
     logger.info(`[Tidal] (${url}) cache hit`);
@@ -72,7 +71,6 @@ export async function getTidalLink(query: string, metadata: SearchMetadata) {
     });
 
     const { data, included } = response;
-
     if (!data || data.length === 0) {
       throw new Error(`No results found: ${JSON.stringify(response)}`);
     }
@@ -81,7 +79,7 @@ export async function getTidalLink(query: string, metadata: SearchMetadata) {
     let highestScore = 0;
 
     for (const item of included) {
-      const title = item.attributes.title;
+      const title = item.attributes.title ?? item.attributes.name ?? '';
       const score = compareTwoStrings(title.toLowerCase(), query.toLowerCase());
 
       if (score > highestScore) {

src/adapters/youtube.ts

@@ -4,25 +4,60 @@ import { cacheSearchResultLink, getCachedSearchResultLink } from '~/services/cac
 import { SearchMetadata, SearchResultLink } from '~/services/search';
 import HttpClient from '~/utils/http-client';
 import { logger } from '~/utils/logger';
-import { getLinkWithPuppeteer } from '~/utils/scraper';
 
-const YOUTUBE_SEARCH_TYPES = {
-  [MetadataType.Song]: 'song',
-  [MetadataType.Album]: 'album',
-  [MetadataType.Playlist]: '',
+interface YoutubeSearchResponse {
+  kind: string;
+  etag: string;
+  nextPageToken?: string;
+  regionCode: string;
+  pageInfo: {
+    totalResults: number;
+    resultsPerPage: number;
+  };
+  items: Array<{
+    kind: string;
+    etag: string;
+    id: {
+      kind: string;
+      videoId?: string;
+      playlistId?: string;
+      channelId?: string;
+    };
+  }>;
+}
+
+export const YOUTUBE_SEARCH_TYPES = {
+  [MetadataType.Song]: 'video',
+  [MetadataType.Album]: 'playlist',
+  [MetadataType.Playlist]: 'playlist',
   [MetadataType.Artist]: 'channel',
-  [MetadataType.Podcast]: '',
-  [MetadataType.Show]: '',
+  [MetadataType.Podcast]: 'video',
+  [MetadataType.Show]: undefined,
 };
 
+const YOUTUBE_SEARCH_LINK_TYPE = (item: YoutubeSearchResponse['items'][number]) => ({
+  [MetadataType.Song]: `watch?v=${item.id.videoId}`,
+  [MetadataType.Album]: `playlist?list=${item.id.playlistId}`,
+  [MetadataType.Playlist]: `playlist?list=${item.id.playlistId}`,
+  [MetadataType.Artist]: `channel/${item.id.channelId}`,
+  [MetadataType.Podcast]: `podcast/${item.id.videoId}`,
+  [MetadataType.Show]: undefined,
+});
+
 export async function getYouTubeLink(query: string, metadata: SearchMetadata) {
-  return null; // TEMPFIX: youtube blocked the server ip
+  const searchType = YOUTUBE_SEARCH_TYPES[metadata.type];
+  if (!searchType) return null;
 
   const params = new URLSearchParams({
-    q: `${query} ${YOUTUBE_SEARCH_TYPES[metadata.type]}`,
+    type: searchType,
+    regionCode: 'US',
+    q: query,
+    part: 'id',
+    safeSearch: 'none',
+    key: ENV.adapters.youTube.apiKey,
   });
 
-  const url = new URL(ENV.adapters.youTube.musicUrl);
+  const url = new URL(ENV.adapters.youTube.apiUrl);
   url.search = params.toString();
 
   const cache = await getCachedSearchResultLink(url);
@@ -32,36 +67,19 @@ export async function getYouTubeLink(query: string, metadata: SearchMetadata) {
   }
 
   try {
-    const youtubeCookie = {
-      domain: '.youtube.com',
-      path: '/',
-      expires: Date.now() + 365 * 24 * 60 * 60 * 1000,
-      secure: true,
-    };
-
-    const cookies = ENV.adapters.youTube.cookies.split(';').map(cookie => {
-      const [name, value] = cookie.split('=');
-      return {
-        ...youtubeCookie,
-        name,
-        value,
-      };
-    });
-
-    const link = await getLinkWithPuppeteer(
-      url.toString(),
-      'ytmusic-card-shelf-renderer a',
-      cookies
-    );
+    const response = await HttpClient.get<YoutubeSearchResponse>(url.toString());
 
-    if (!link) {
-      return null;
+    const { items } = response;
+    if (!items || !items[0]) {
+      throw new Error(`No results found: ${JSON.stringify(response)}`);
     }
 
+    const link = `${ENV.adapters.youTube.musicBaseUrl}/${YOUTUBE_SEARCH_LINK_TYPE(items[0])[metadata.type]}`;
+
     const searchResultLink = {
       type: Adapter.YouTube,
       url: link,
-      isVerified: true,
+      isVerified: false,
     } as SearchResultLink;
 
     await cacheSearchResultLink(url, searchResultLink);
@@ -72,15 +90,3 @@ export async function getYouTubeLink(query: string, metadata: SearchMetadata) {
     return null;
   }
 }
-
-export async function fetchYoutubeMetadata(youtubeLink: string) {
-  logger.info(`[${fetchYoutubeMetadata.name}] parse metadata (desktop): ${youtubeLink}`);
-
-  const html = await HttpClient.get<string>(youtubeLink, {
-    headers: {
-      Cookie: ENV.adapters.youTube.cookies,
-    },
-  });
-
-  return html;
-}