From f295cb9363fdba06abb54285852e396d896a73da Mon Sep 17 00:00:00 2001 From: zugdev Date: Fri, 24 Jan 2025 04:54:46 -0300 Subject: [PATCH 1/6] feat: image cache timestamp check 15mins --- src/home/fetch-github/fetch-avatar.ts | 8 ++++---- src/home/getters/get-indexed-db.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/home/fetch-github/fetch-avatar.ts b/src/home/fetch-github/fetch-avatar.ts index 011e1b23..eb5e7e8a 100644 --- a/src/home/fetch-github/fetch-avatar.ts +++ b/src/home/fetch-github/fetch-avatar.ts @@ -26,10 +26,10 @@ export async function fetchAvatar(orgName: string): Promise { // It will try to fetch from IndexedDB first, then from GitHub organizations, and finally from GitHub users, returning in the first successful step const fetchPromise = (async () => { // Step 1: Try to get the avatar from IndexedDB - const avatarBlob = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` }); - if (avatarBlob) { - organizationImageCache.set(orgName, avatarBlob); // Cache it in memory - return avatarBlob; + const avatar = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` }); + if (avatar && Number(avatar.timestamp) + 60 * 1000 * 15 <= Date.now()) { // fail if the image is older than 15 minutes + organizationImageCache.set(orgName, avatar.image); // Cache it in memory + return avatar.image; } const octokit = new Octokit({ auth: await getGitHubAccessToken() }); diff --git a/src/home/getters/get-indexed-db.ts b/src/home/getters/get-indexed-db.ts index 77c06d45..6a6b6efd 100644 --- a/src/home/getters/get-indexed-db.ts +++ b/src/home/getters/get-indexed-db.ts @@ -44,7 +44,7 @@ export async function saveImageToCache({ }); } -export function getImageFromCache({ dbName, storeName, orgName }: { dbName: string; storeName: string; orgName: string }): Promise { +export function getImageFromCache({ dbName, storeName, orgName }: { dbName: string; storeName: string; orgName: string }): Promise<{image: Blob; timestamp: string} | null> { return new Promise((resolve, reject) => { const open = indexedDB.open(dbName, 2); // Increase version number to ensure onupgradeneeded is called open.onupgradeneeded = function () { @@ -59,7 +59,11 @@ export function getImageFromCache({ dbName, storeName, orgName }: { dbName: stri const store = transaction.objectStore(storeName); const getImage = store.get(`avatarUrl-${orgName}`); getImage.onsuccess = function () { - resolve(getImage.result?.image || null); + if (getImage.result) { + resolve({ image: getImage.result.image, timestamp: getImage.result.created }); + } else { + resolve(null); + } }; transaction.oncomplete = function () { db.close(); From 4196b7be7ff6939d606399ab1863f26b126fb2d9 Mon Sep 17 00:00:00 2001 From: zugdev Date: Sat, 25 Jan 2025 04:15:15 -0300 Subject: [PATCH 2/6] chore: format --- src/home/fetch-github/fetch-avatar.ts | 3 ++- src/home/getters/get-indexed-db.ts | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/home/fetch-github/fetch-avatar.ts b/src/home/fetch-github/fetch-avatar.ts index eb5e7e8a..44d674cb 100644 --- a/src/home/fetch-github/fetch-avatar.ts +++ b/src/home/fetch-github/fetch-avatar.ts @@ -27,7 +27,8 @@ export async function fetchAvatar(orgName: string): Promise { const fetchPromise = (async () => { // Step 1: Try to get the avatar from IndexedDB const avatar = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` }); - if (avatar && Number(avatar.timestamp) + 60 * 1000 * 15 <= Date.now()) { // fail if the image is older than 15 minutes + if (avatar && Number(avatar.timestamp) + 60 * 1000 * 15 <= Date.now()) { + // fail if the image is older than 15 minutes organizationImageCache.set(orgName, avatar.image); // Cache it in memory return avatar.image; } diff --git a/src/home/getters/get-indexed-db.ts b/src/home/getters/get-indexed-db.ts index 6a6b6efd..eefad51e 100644 --- a/src/home/getters/get-indexed-db.ts +++ b/src/home/getters/get-indexed-db.ts @@ -44,7 +44,15 @@ export async function saveImageToCache({ }); } -export function getImageFromCache({ dbName, storeName, orgName }: { dbName: string; storeName: string; orgName: string }): Promise<{image: Blob; timestamp: string} | null> { +export function getImageFromCache({ + dbName, + storeName, + orgName, +}: { + dbName: string; + storeName: string; + orgName: string; +}): Promise<{ image: Blob; timestamp: string } | null> { return new Promise((resolve, reject) => { const open = indexedDB.open(dbName, 2); // Increase version number to ensure onupgradeneeded is called open.onupgradeneeded = function () { From 9e1568ec222bc85ccd38d0ca51573b27d4a059c1 Mon Sep 17 00:00:00 2001 From: zugdev Date: Sat, 25 Jan 2025 04:39:13 -0300 Subject: [PATCH 3/6] feat: rollback --- src/home/fetch-github/fetch-avatar.ts | 15 ++++++++++----- src/home/getters/get-indexed-db.ts | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/home/fetch-github/fetch-avatar.ts b/src/home/fetch-github/fetch-avatar.ts index 44d674cb..38bbdf57 100644 --- a/src/home/fetch-github/fetch-avatar.ts +++ b/src/home/fetch-github/fetch-avatar.ts @@ -9,6 +9,12 @@ import { taskManager } from "../home"; // Map to track ongoing avatar fetches const pendingFetches: Map> = new Map(); +export async function fetchPartnerAvatars(): Promise { + const response = await fetch("https://raw.githubusercontent.com/ubiquity/devpool-directory/__STORAGE__/devpool-partner-avatars.json"); + const jsonData = await response.json(); + return jsonData; +} + // Fetches the avatar for a given organization from GitHub either from cache, indexedDB or GitHub API export async function fetchAvatar(orgName: string): Promise { // Check if the avatar is already cached in memory @@ -26,11 +32,10 @@ export async function fetchAvatar(orgName: string): Promise { // It will try to fetch from IndexedDB first, then from GitHub organizations, and finally from GitHub users, returning in the first successful step const fetchPromise = (async () => { // Step 1: Try to get the avatar from IndexedDB - const avatar = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` }); - if (avatar && Number(avatar.timestamp) + 60 * 1000 * 15 <= Date.now()) { - // fail if the image is older than 15 minutes - organizationImageCache.set(orgName, avatar.image); // Cache it in memory - return avatar.image; + const avatarBlob = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` }); + if (avatarBlob) { + organizationImageCache.set(orgName, avatarBlob); // Cache it in memory + return avatarBlob; } const octokit = new Octokit({ auth: await getGitHubAccessToken() }); diff --git a/src/home/getters/get-indexed-db.ts b/src/home/getters/get-indexed-db.ts index eefad51e..8d557a2b 100644 --- a/src/home/getters/get-indexed-db.ts +++ b/src/home/getters/get-indexed-db.ts @@ -52,7 +52,7 @@ export function getImageFromCache({ dbName: string; storeName: string; orgName: string; -}): Promise<{ image: Blob; timestamp: string } | null> { +}): Promise { return new Promise((resolve, reject) => { const open = indexedDB.open(dbName, 2); // Increase version number to ensure onupgradeneeded is called open.onupgradeneeded = function () { @@ -68,7 +68,7 @@ export function getImageFromCache({ const getImage = store.get(`avatarUrl-${orgName}`); getImage.onsuccess = function () { if (getImage.result) { - resolve({ image: getImage.result.image, timestamp: getImage.result.created }); + resolve(getImage.result.image); } else { resolve(null); } From ec5414649ef03eb17d245171ec00b7910f706f1a Mon Sep 17 00:00:00 2001 From: zugdev Date: Sat, 25 Jan 2025 04:39:22 -0300 Subject: [PATCH 4/6] chore: format --- src/home/getters/get-indexed-db.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/home/getters/get-indexed-db.ts b/src/home/getters/get-indexed-db.ts index 8d557a2b..caf3be1f 100644 --- a/src/home/getters/get-indexed-db.ts +++ b/src/home/getters/get-indexed-db.ts @@ -44,15 +44,7 @@ export async function saveImageToCache({ }); } -export function getImageFromCache({ - dbName, - storeName, - orgName, -}: { - dbName: string; - storeName: string; - orgName: string; -}): Promise { +export function getImageFromCache({ dbName, storeName, orgName }: { dbName: string; storeName: string; orgName: string }): Promise { return new Promise((resolve, reject) => { const open = indexedDB.open(dbName, 2); // Increase version number to ensure onupgradeneeded is called open.onupgradeneeded = function () { From db222ef15338533646ffb270fcc5cd24371e7d22 Mon Sep 17 00:00:00 2001 From: zugdev Date: Sat, 25 Jan 2025 15:16:07 -0300 Subject: [PATCH 5/6] feat: avatars from URL mapping --- src/home/fetch-github/fetch-avatar.ts | 126 ++++----------------- src/home/fetch-github/fetch-issues-full.ts | 1 - src/home/getters/get-indexed-db.ts | 76 ------------- src/home/rendering/render-github-issues.ts | 6 +- src/home/rendering/render-org-header.ts | 25 ++-- 5 files changed, 33 insertions(+), 201 deletions(-) diff --git a/src/home/fetch-github/fetch-avatar.ts b/src/home/fetch-github/fetch-avatar.ts index 38bbdf57..de7d7a0a 100644 --- a/src/home/fetch-github/fetch-avatar.ts +++ b/src/home/fetch-github/fetch-avatar.ts @@ -1,122 +1,36 @@ -import { Octokit } from "@octokit/rest"; -import { getGitHubAccessToken } from "../getters/get-github-access-token"; -import { getImageFromCache, saveImageToCache } from "../getters/get-indexed-db"; -import { renderErrorInModal } from "../rendering/display-popup-modal"; -import { organizationImageCache } from "./fetch-issues-full"; -import { GitHubIssue } from "../github-types"; -import { taskManager } from "../home"; +export const ubiquityAvatarUrl = "https://avatars.githubusercontent.com/u/76412717?v=4"; -// Map to track ongoing avatar fetches -const pendingFetches: Map> = new Map(); +export type OrgNameAndAvatarUrl = { + ownerName: string; + avatar_url?: string; +} -export async function fetchPartnerAvatars(): Promise { +export async function fetchPartnerAvatars(): Promise { const response = await fetch("https://raw.githubusercontent.com/ubiquity/devpool-directory/__STORAGE__/devpool-partner-avatars.json"); const jsonData = await response.json(); return jsonData; } -// Fetches the avatar for a given organization from GitHub either from cache, indexedDB or GitHub API -export async function fetchAvatar(orgName: string): Promise { - // Check if the avatar is already cached in memory - const cachedAvatar = organizationImageCache.get(orgName); - if (cachedAvatar) { - return cachedAvatar; - } - - // If there's a pending fetch for this organization, wait for it to complete - if (pendingFetches.has(orgName)) { - return pendingFetches.get(orgName); - } - - // Start the fetch process and store the promise in the pending fetches map - // It will try to fetch from IndexedDB first, then from GitHub organizations, and finally from GitHub users, returning in the first successful step - const fetchPromise = (async () => { - // Step 1: Try to get the avatar from IndexedDB - const avatarBlob = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` }); - if (avatarBlob) { - organizationImageCache.set(orgName, avatarBlob); // Cache it in memory - return avatarBlob; - } - - const octokit = new Octokit({ auth: await getGitHubAccessToken() }); +// A global map of partner {ownerName => avatarUrl} +const partnerAvatarMap = new Map(); - // Step 2: No avatar in IndexedDB, fetch from GitHub - try { - const { - data: { avatar_url: avatarUrl }, - } = await octokit.rest.orgs.get({ org: orgName }); - - if (avatarUrl) { - const response = await fetch(avatarUrl); - const blob = await response.blob(); - - // Cache the fetched avatar in both memory and IndexedDB - await saveImageToCache({ - dbName: "GitHubAvatars", - storeName: "ImageStore", - keyName: "name", - orgName: `avatarUrl-${orgName}`, - avatarBlob: blob, - }); - - organizationImageCache.set(orgName, blob); - return blob; - } - } catch (orgError) { - console.warn(`Failed to fetch avatar from organization ${orgName}: ${orgError}`); - } +export function fetchAvatar(orgName: string): string | undefined { + return partnerAvatarMap.get(orgName.toLowerCase()); +} - // Step 3: Try fetching from GitHub users if the organization lookup failed - try { - const { - data: { avatar_url: avatarUrl }, - } = await octokit.rest.users.getByUsername({ username: orgName }); +export async function fetchAvatars() { + try { + const partnerData = await fetchPartnerAvatars(); + console.log("partnerData", partnerData); + partnerData.forEach(({ ownerName, avatar_url: avatarUrl }) => { if (avatarUrl) { - const response = await fetch(avatarUrl); - const blob = await response.blob(); - - // Cache the fetched avatar in both memory and IndexedDB - await saveImageToCache({ - dbName: "GitHubAvatars", - storeName: "ImageStore", - keyName: "name", - orgName: `avatarUrl-${orgName}`, - avatarBlob: blob, - }); - - organizationImageCache.set(orgName, blob); - return blob; + partnerAvatarMap.set(ownerName.toLowerCase(), avatarUrl); } - } catch (innerError) { - renderErrorInModal(innerError as Error, `All tries failed to fetch avatar for ${orgName}: ${innerError}`); - } - })(); - - pendingFetches.set(orgName, fetchPromise); - - // Wait for the fetch to complete - try { - const result = await fetchPromise; - return result; - } finally { - // Remove the pending fetch once it completes - pendingFetches.delete(orgName); + }); + } catch (error) { + console.error("Failed to load partner avatars:", error); } } -// fetches avatars for all tasks (issues) cached. it will fetch only once per organization, remaining are returned from cache -export async function fetchAvatars() { - const cachedTasks = taskManager.getTasks(); - // fetches avatar for each organization for each task, but fetchAvatar() will only fetch once per organization, remaining are returned from cache - const avatarPromises = cachedTasks.map(async (task: GitHubIssue) => { - const [orgName] = task.repository_url.split("/").slice(-2); - if (orgName) { - return fetchAvatar(orgName); - } - return Promise.resolve(); - }); - - await Promise.allSettled(avatarPromises); -} diff --git a/src/home/fetch-github/fetch-issues-full.ts b/src/home/fetch-github/fetch-issues-full.ts index 2c60ff67..c70dcce3 100644 --- a/src/home/fetch-github/fetch-issues-full.ts +++ b/src/home/fetch-github/fetch-issues-full.ts @@ -2,7 +2,6 @@ import { saveIssuesToCache } from "../getters/get-indexed-db"; import { GitHubIssue } from "../github-types"; import { taskManager } from "../home"; import { displayGitHubIssues } from "./fetch-and-display-previews"; -export const organizationImageCache = new Map(); // this should be declared in image related script // Fetches the issues from `devpool-issues.json` file in the `__STORAGE__` branch of the `devpool-directory` repo // https://github.com/ubiquity/devpool-directory/blob/__STORAGE__/devpool-issues.json diff --git a/src/home/getters/get-indexed-db.ts b/src/home/getters/get-indexed-db.ts index caf3be1f..1fe604f4 100644 --- a/src/home/getters/get-indexed-db.ts +++ b/src/home/getters/get-indexed-db.ts @@ -1,81 +1,5 @@ import { GitHubIssue } from "../github-types"; -// this file contains functions to save and retrieve issues/images from IndexedDB which is client-side in-browser storage -export async function saveImageToCache({ - dbName, - storeName, - keyName, - orgName, - avatarBlob, -}: { - dbName: string; - storeName: string; - keyName: string; - orgName: string; - avatarBlob: Blob; -}): Promise { - return new Promise((resolve, reject) => { - const open = indexedDB.open(dbName, 2); // Increase version number to ensure onupgradeneeded is called - open.onupgradeneeded = function () { - const db = open.result; - if (!db.objectStoreNames.contains(storeName)) { - db.createObjectStore(storeName, { keyPath: keyName }); - } - }; - open.onsuccess = function () { - const db = open.result; - const transaction = db.transaction(storeName, "readwrite"); - const store = transaction.objectStore(storeName); - const item = { - name: `avatarUrl-${orgName}`, - image: avatarBlob, - created: new Date().getTime(), - }; - store.put(item); - transaction.oncomplete = function () { - db.close(); - resolve(); - }; - transaction.onerror = function (event) { - const errorEventTarget = event.target as IDBRequest; - reject("Error saving image to DB: " + errorEventTarget.error?.message); - }; - }; - }); -} - -export function getImageFromCache({ dbName, storeName, orgName }: { dbName: string; storeName: string; orgName: string }): Promise { - return new Promise((resolve, reject) => { - const open = indexedDB.open(dbName, 2); // Increase version number to ensure onupgradeneeded is called - open.onupgradeneeded = function () { - const db = open.result; - if (!db.objectStoreNames.contains(storeName)) { - db.createObjectStore(storeName, { keyPath: "name" }); - } - }; - open.onsuccess = function () { - const db = open.result; - const transaction = db.transaction(storeName, "readonly"); - const store = transaction.objectStore(storeName); - const getImage = store.get(`avatarUrl-${orgName}`); - getImage.onsuccess = function () { - if (getImage.result) { - resolve(getImage.result.image); - } else { - resolve(null); - } - }; - transaction.oncomplete = function () { - db.close(); - }; - transaction.onerror = function (event) { - const errorEventTarget = event.target as IDBRequest; - reject("Error retrieving image from DB: " + errorEventTarget.error?.message); - }; - }; - }); -} - async function openIssuesDB(): Promise { return new Promise((resolve, reject) => { const request = indexedDB.open("IssuesDB", 2); diff --git a/src/home/rendering/render-github-issues.ts b/src/home/rendering/render-github-issues.ts index 514aabe3..95805ad0 100644 --- a/src/home/rendering/render-github-issues.ts +++ b/src/home/rendering/render-github-issues.ts @@ -1,12 +1,12 @@ import { marked } from "marked"; import markedFootnote from "marked-footnote"; -import { organizationImageCache } from "../fetch-github/fetch-issues-full"; import { GitHubIssue } from "../github-types"; import { taskManager } from "../home"; import { renderErrorInModal } from "./display-popup-modal"; import { closeModal, modal, modalBodyInner, bottomBar, titleAnchor, titleHeader, bottomBarClearLabels } from "./render-preview-modal"; import { setupKeyboardNavigation } from "./setup-keyboard-navigation"; import { waitForElement } from "./utils"; +import { fetchAvatar, ubiquityAvatarUrl } from "../fetch-github/fetch-avatar"; export function renderGitHubIssues(tasks: GitHubIssue[], skipAnimation: boolean) { const container = taskManager.getContainer(); @@ -237,11 +237,11 @@ export function applyAvatarsToIssues() { issueElements.forEach((issueElement) => { const orgName = issueElement.querySelector(".organization-name")?.textContent; if (orgName) { - const avatarUrl = organizationImageCache.get(orgName); + const avatarUrl = fetchAvatar(orgName) ?? ubiquityAvatarUrl; if (avatarUrl) { const avatarImg = issueElement.querySelector("img"); if (avatarImg) { - avatarImg.src = URL.createObjectURL(avatarUrl); + avatarImg.src = avatarUrl; } } } diff --git a/src/home/rendering/render-org-header.ts b/src/home/rendering/render-org-header.ts index b4f18e0e..dddb2880 100644 --- a/src/home/rendering/render-org-header.ts +++ b/src/home/rendering/render-org-header.ts @@ -1,26 +1,21 @@ -import { organizationImageCache } from "../fetch-github/fetch-issues-full"; +import { fetchAvatar, ubiquityAvatarUrl } from "../fetch-github/fetch-avatar"; export function renderOrgHeaderLabel(orgName: string): void { const brandingDiv = document.getElementById("branding"); if (!brandingDiv) return; // Fetch the organization logo from the cache - const logoBlob = organizationImageCache.get(orgName); + const logoUrl = fetchAvatar(orgName) ?? ubiquityAvatarUrl; - if (logoBlob) { - // Convert Blob to a URL - const logoUrl = URL.createObjectURL(logoBlob); + const img = document.createElement("img"); + img.src = logoUrl; + img.alt = `${orgName} Logo`; + console.log("oi"); + img.id = "logo"; - const img = document.createElement("img"); - img.src = logoUrl; - img.alt = `${orgName} Logo`; - console.log("oi"); - img.id = "logo"; - - // Replace the existing SVG with the new image - const svgLogo = brandingDiv.querySelector("svg#logo"); - if (svgLogo) brandingDiv.replaceChild(img, svgLogo); - } + // Replace the existing SVG with the new image + const svgLogo = brandingDiv.querySelector("svg#logo"); + if (svgLogo) brandingDiv.replaceChild(img, svgLogo); // Update the organization name inside the span with class 'full' const orgNameSpan = brandingDiv.querySelector("span.full"); From 2688bcadb8fd5309227f22d694d3b94ffe3822d2 Mon Sep 17 00:00:00 2001 From: zugdev Date: Sat, 25 Jan 2025 15:16:33 -0300 Subject: [PATCH 6/6] feat: avatars from URL mapping --- src/home/fetch-github/fetch-avatar.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/home/fetch-github/fetch-avatar.ts b/src/home/fetch-github/fetch-avatar.ts index de7d7a0a..28f7b18b 100644 --- a/src/home/fetch-github/fetch-avatar.ts +++ b/src/home/fetch-github/fetch-avatar.ts @@ -3,7 +3,7 @@ export const ubiquityAvatarUrl = "https://avatars.githubusercontent.com/u/764127 export type OrgNameAndAvatarUrl = { ownerName: string; avatar_url?: string; -} +}; export async function fetchPartnerAvatars(): Promise { const response = await fetch("https://raw.githubusercontent.com/ubiquity/devpool-directory/__STORAGE__/devpool-partner-avatars.json"); @@ -20,9 +20,8 @@ export function fetchAvatar(orgName: string): string | undefined { export async function fetchAvatars() { try { - const partnerData = await fetchPartnerAvatars(); - console.log("partnerData", partnerData); - + const partnerData = await fetchPartnerAvatars(); + partnerData.forEach(({ ownerName, avatar_url: avatarUrl }) => { if (avatarUrl) { partnerAvatarMap.set(ownerName.toLowerCase(), avatarUrl); @@ -32,5 +31,3 @@ export async function fetchAvatars() { console.error("Failed to load partner avatars:", error); } } - -