From c19186232b308f57408933c510876a435b965496 Mon Sep 17 00:00:00 2001 From: Eliezer Steinbock <3090527+elie222@users.noreply.github.com> Date: Wed, 1 Jan 2025 15:51:03 +0200 Subject: [PATCH] Clean up logging in more files --- apps/web/.eslintrc.json | 13 +++++++- .../user/stats/tinybird/load/load-emails.ts | 32 ++++++++++--------- apps/web/utils/actions/admin.ts | 5 ++- apps/web/utils/actions/api-key.ts | 7 ++-- apps/web/utils/actions/group.ts | 15 ++++++--- .../ai-categorize-single-sender.ts | 9 ++++-- apps/web/utils/ai/choose-rule/run-rules.ts | 12 ++++--- apps/web/utils/ai/group/create-group.ts | 19 +++++++++-- apps/web/utils/logger.ts | 1 + 9 files changed, 79 insertions(+), 34 deletions(-) diff --git a/apps/web/.eslintrc.json b/apps/web/.eslintrc.json index e69121199..f2ec392c8 100644 --- a/apps/web/.eslintrc.json +++ b/apps/web/.eslintrc.json @@ -4,5 +4,16 @@ "parser": "@typescript-eslint/parser", "parserOptions": { "project": true - } + }, + "rules": { + "no-console": "warn" + }, + "overrides": [ + { + "files": ["**/*.test.ts", "**/*.test.tsx", "**/__tests__/**/*", "**/*.tsx"], + "rules": { + "no-console": "off" + } + } + ] } diff --git a/apps/web/app/api/user/stats/tinybird/load/load-emails.ts b/apps/web/app/api/user/stats/tinybird/load/load-emails.ts index 26d5dc45f..d0601099c 100644 --- a/apps/web/app/api/user/stats/tinybird/load/load-emails.ts +++ b/apps/web/app/api/user/stats/tinybird/load/load-emails.ts @@ -14,11 +14,14 @@ import { SENT_LABEL_ID, UNREAD_LABEL_ID, } from "@/utils/gmail/label"; +import { createScopedLogger } from "@/utils/logger"; const PAGE_SIZE = 20; // avoid setting too high because it will hit the rate limit const PAUSE_AFTER_RATE_LIMIT = 10_000; const MAX_PAGES = 50; +const logger = createScopedLogger("Tinybird Load Emails"); + export async function loadTinybirdEmails( options: { ownerEmail: string; @@ -40,10 +43,10 @@ export async function loadTinybirdEmails( ]); const after = newestEmailSaved.data?.[0]?.timestamp; - console.log("Loading emails after:", after); + logger.info("Loading emails after", { after }); while (pages < MAX_PAGES) { - console.log("After Page", pages); + logger.info("After Page", { pages }); let res: Awaited>; try { res = await saveBatch({ @@ -56,7 +59,7 @@ export async function loadTinybirdEmails( }); } catch (error) { // TODO save batch won't throw a rate limit error anymore. Just logs: `Error fetching message 429 Resource has been exhausted (e.g. check quota).` - console.log(`Rate limited. Waiting ${PAUSE_AFTER_RATE_LIMIT} seconds...`); + logger.info(`Rate limited. Waiting ${PAUSE_AFTER_RATE_LIMIT} seconds...`); await sleep(PAUSE_AFTER_RATE_LIMIT); res = await saveBatch({ ownerEmail, @@ -74,15 +77,15 @@ export async function loadTinybirdEmails( pages++; } - console.log("Completed emails after:", after); + logger.info("Completed emails after", { after }); if (!body.loadBefore) return { pages }; const before = oldestEmailSaved.data?.[0]?.timestamp; - console.log("Loading emails before:", before); + logger.info("Loading emails before", { before }); while (pages < MAX_PAGES) { - console.log("Before Page", pages); + logger.info("Before Page", { pages }); let res: Awaited>; try { res = await saveBatch({ @@ -94,7 +97,7 @@ export async function loadTinybirdEmails( after: undefined, }); } catch (error) { - console.log("Rate limited. Waiting 10 seconds..."); + logger.info("Rate limited. Waiting 10 seconds..."); await sleep(PAUSE_AFTER_RATE_LIMIT); res = await saveBatch({ ownerEmail, @@ -112,7 +115,7 @@ export async function loadTinybirdEmails( pages++; } - console.log("Completed emails before:", before); + logger.info("Completed emails before", { before }); return { pages }; } @@ -181,12 +184,11 @@ async function saveBatch( }; if (!tinybirdEmail.timestamp) { - console.error( - "No timestamp for email", - tinybirdEmail.ownerEmail, - tinybirdEmail.gmailMessageId, - m.headers.date, - ); + logger.error("No timestamp for email", { + ownerEmail: tinybirdEmail.ownerEmail, + gmailMessageId: tinybirdEmail.gmailMessageId, + date: m.headers.date, + }); return; } @@ -194,7 +196,7 @@ async function saveBatch( }) .filter(isDefined); - console.log("Publishing", emailsToPublish.length, "emails"); + logger.info("Publishing", { count: emailsToPublish.length }); await publishEmail(emailsToPublish); diff --git a/apps/web/utils/actions/admin.ts b/apps/web/utils/actions/admin.ts index 690c72214..fe2b58bf5 100644 --- a/apps/web/utils/actions/admin.ts +++ b/apps/web/utils/actions/admin.ts @@ -4,6 +4,9 @@ import { auth } from "@/app/api/auth/[...nextauth]/auth"; import { processHistoryForUser } from "@/app/api/google/webhook/process-history"; import { isAdmin } from "@/utils/admin"; import { withActionInstrumentation } from "@/utils/actions/middleware"; +import { createScopedLogger } from "@/utils/logger"; + +const logger = createScopedLogger("Admin Action"); export const adminProcessHistoryAction = withActionInstrumentation( "adminProcessHistory", @@ -21,7 +24,7 @@ export const adminProcessHistoryAction = withActionInstrumentation( if (!userId) return { error: "Not logged in" }; if (!isAdmin(session.user.email)) return { error: "Not admin" }; - console.log(`Processing history for ${emailAddress}`); + logger.info("Admin processing history", { emailAddress }); await processHistoryForUser( { diff --git a/apps/web/utils/actions/api-key.ts b/apps/web/utils/actions/api-key.ts index da4809a58..d0a493e30 100644 --- a/apps/web/utils/actions/api-key.ts +++ b/apps/web/utils/actions/api-key.ts @@ -13,6 +13,9 @@ import type { import prisma from "@/utils/prisma"; import { generateSecureApiKey, hashApiKey } from "@/utils/api-key"; import { withActionInstrumentation } from "@/utils/actions/middleware"; +import { createScopedLogger } from "@/utils/logger"; + +const logger = createScopedLogger("Api Key Action"); export const createApiKeyAction = withActionInstrumentation( "createApiKey", @@ -24,7 +27,7 @@ export const createApiKeyAction = withActionInstrumentation( const { data, success, error } = createApiKeyBody.safeParse(unsafeData); if (!success) return { error: error.message }; - console.log(`Creating API key for ${userId}`); + logger.info("Creating API key", { userId }); const secretKey = generateSecureApiKey(); const hashedKey = hashApiKey(secretKey); @@ -54,7 +57,7 @@ export const deactivateApiKeyAction = withActionInstrumentation( const { data, success, error } = deactivateApiKeyBody.safeParse(unsafeData); if (!success) return { error: error.message }; - console.log(`Deactivating API key for ${userId}`); + logger.info("Deactivating API key", { userId }); await prisma.apiKey.update({ where: { id: data.id, userId }, diff --git a/apps/web/utils/actions/group.ts b/apps/web/utils/actions/group.ts index d2a287d57..0300263e8 100644 --- a/apps/web/utils/actions/group.ts +++ b/apps/web/utils/actions/group.ts @@ -26,6 +26,9 @@ import { GroupName } from "@/utils/config"; import { aiGenerateGroupItems } from "@/utils/ai/group/create-group"; import type { UserAIFields } from "@/utils/llms/types"; import { withActionInstrumentation } from "@/utils/actions/middleware"; +import { createScopedLogger } from "@/utils/logger"; + +const logger = createScopedLogger("Group Action"); export const createGroupAction = withActionInstrumentation( "createGroup", @@ -72,7 +75,7 @@ export const createGroupAction = withActionInstrumentation( if (isDuplicateError(error, "name")) return { error: "Group with this name already exists" }; - console.error("Error creating group", error); + logger.error("Error creating group", { error, name, prompt }); captureException(error, { extra: { name, prompt } }, session?.user.email); return { error: "Error creating group" }; } @@ -87,16 +90,18 @@ async function generateGroupItemsFromPrompt( name: string, prompt: string, ) { - console.log(`generateGroupItemsFromPrompt: ${name} - ${prompt}`); + logger.info("generateGroupItemsFromPrompt", { name, prompt }); const result = await aiGenerateGroupItems(user, gmail, token, { name, prompt, }); - console.log( - `generateGroupItemsFromPrompt result. Senders: ${result.senders.length}, Subjects: ${result.subjects.length}`, - ); + logger.info("generateGroupItemsFromPrompt result", { + name, + senders: result.senders.length, + subjects: result.subjects.length, + }); await prisma.$transaction([ ...result.senders.map((sender) => diff --git a/apps/web/utils/ai/categorize-sender/ai-categorize-single-sender.ts b/apps/web/utils/ai/categorize-sender/ai-categorize-single-sender.ts index b4be7bb11..cf8d8b821 100644 --- a/apps/web/utils/ai/categorize-sender/ai-categorize-single-sender.ts +++ b/apps/web/utils/ai/categorize-sender/ai-categorize-single-sender.ts @@ -3,6 +3,9 @@ import { chatCompletionObject } from "@/utils/llms"; import type { UserEmailWithAI } from "@/utils/llms/types"; import type { Category } from "@prisma/client"; import { formatCategoriesForPrompt } from "@/utils/ai/categorize-sender/format-categories"; +import { createScopedLogger } from "@/utils/logger"; + +const logger = createScopedLogger("aiCategorizeSender"); const categorizeSenderSchema = z.object({ rationale: z.string().describe("Keep it short. 1-2 sentences max."), @@ -23,8 +26,6 @@ export async function aiCategorizeSender({ previousEmails: string[]; categories: Pick[]; }) { - console.log("aiCategorizeSender"); - const system = `You are an AI assistant specializing in email management and organization. Your task is to categorize an email accounts based on their name, email address, and content from previous emails. Provide an accurate categorization to help users efficiently manage their inbox.`; @@ -49,6 +50,8 @@ Instructions: 4. If you're not certain, respond with "Unknown". 5. If multiple categories are possible, respond with "Unknown".`; + logger.trace("aiCategorizeSender", { system, prompt }); + const aiResponse = await chatCompletionObject({ userAi: user, system, @@ -61,5 +64,7 @@ Instructions: if (!categories.find((c) => c.name === aiResponse.object.category)) return null; + logger.trace("aiCategorizeSender result", { aiResponse: aiResponse.object }); + return aiResponse.object; } diff --git a/apps/web/utils/ai/choose-rule/run-rules.ts b/apps/web/utils/ai/choose-rule/run-rules.ts index a486d74cd..3aeb017a2 100644 --- a/apps/web/utils/ai/choose-rule/run-rules.ts +++ b/apps/web/utils/ai/choose-rule/run-rules.ts @@ -16,9 +16,9 @@ import { getActionItemsWithAiArgs } from "@/utils/ai/choose-rule/ai-choose-args" import { executeAct } from "@/utils/ai/choose-rule/execute"; import { getEmailFromMessage } from "@/utils/ai/choose-rule/get-email-from-message"; import prisma from "@/utils/prisma"; -// import { createScopedLogger } from "@/utils/logger"; +import { createScopedLogger } from "@/utils/logger"; -// const logger = createScopedLogger("ai-run-rules"); +const logger = createScopedLogger("ai-run-rules"); export type TestResult = { rule?: Rule | null; @@ -236,9 +236,11 @@ async function upsertExecutedRule({ ) { // Unique constraint violation, ignore the error // May be due to a race condition? - console.log( - `Ignored duplicate entry for ExecutedRule: ${userId} ${threadId} ${messageId}`, - ); + logger.info("Ignored duplicate entry for ExecutedRule", { + userId, + threadId, + messageId, + }); return await prisma.executedRule.findUnique({ where: { unique_user_thread_message: { diff --git a/apps/web/utils/ai/group/create-group.ts b/apps/web/utils/ai/group/create-group.ts index 532320f99..f151d48bd 100644 --- a/apps/web/utils/ai/group/create-group.ts +++ b/apps/web/utils/ai/group/create-group.ts @@ -4,6 +4,9 @@ import { chatCompletionTools } from "@/utils/llms"; import type { Group, User } from "@prisma/client"; import { queryBatchMessages } from "@/utils/gmail/message"; import type { UserAIFields } from "@/utils/llms/types"; +import { createScopedLogger } from "@/utils/logger"; + +const logger = createScopedLogger("aiCreateGroup"); const GENERATE_GROUP_ITEMS = "generateGroupItems"; const VERIFY_GROUP_ITEMS = "verifyGroupItems"; @@ -54,7 +57,10 @@ export async function aiGenerateGroupItems( accessToken: string, group: Pick, ): Promise> { - console.log(`aiGenerateGroupItems: ${group.name} - ${group.prompt}`); + logger.info("aiGenerateGroupItems", { + name: group.name, + prompt: group.prompt, + }); const system = `You are an AI assistant specializing in email management and organization. Your task is to create highly specific email groups based on user prompts and their actual email history. @@ -79,6 +85,8 @@ Key guidelines: 8. It's better to suggest fewer, more reliable criteria than to risk overgeneralization. 9. If the user explicitly excludes certain types of emails, ensure your suggestions do not include them.`; + logger.trace("aiGenerateGroupItems", { system, prompt }); + const aiResponse = await chatCompletionTools({ userAi: user, system, @@ -99,6 +107,8 @@ Key guidelines: ({ toolName }) => toolName === GENERATE_GROUP_ITEMS, ); + logger.trace("aiGenerateGroupItems result", { generateGroupItemsToolCalls }); + const combinedArgs = generateGroupItemsToolCalls.reduce< z.infer >( @@ -122,7 +132,10 @@ async function verifyGroupItems( group: Pick, initialItems: z.infer, ): Promise> { - console.log(`verifyGroupItems: ${group.name} - ${group.prompt}`); + logger.info("verifyGroupItems", { + name: group.name, + prompt: group.prompt, + }); const system = `You are an AI assistant specializing in email management and organization. Your task is to identify and remove any incorrect or overly broad criteria from the generated email group. @@ -167,7 +180,7 @@ Guidelines: ); if (verifyGroupItemsToolCalls.length === 0) { - console.warn("No verification results found. Returning initial items."); + logger.warn("No verification results found. Returning initial items."); return initialItems; } diff --git a/apps/web/utils/logger.ts b/apps/web/utils/logger.ts index ed0052e6a..20ec72f14 100644 --- a/apps/web/utils/logger.ts +++ b/apps/web/utils/logger.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ import { log } from "next-axiom"; import { env } from "@/env";