From 6fb2a0ec43a42f1aa89846776dff3ec866961009 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 3 Dec 2024 11:05:19 +0100 Subject: [PATCH 01/12] feat: working version Signed-off-by: Jan --- apps/easypid/app.config.js | 3 + apps/easypid/metro.config.js | 1 + apps/easypid/package.json | 1 + apps/easypid/src/app/_layout.tsx | 2 + .../src/features/menu/FunkeSettingsScreen.tsx | 55 +++--- .../menu/components/LocalAiContainer.tsx | 65 +++++++ .../menu/components/LocalAiDownloader.tsx | 17 ++ ...keOpenIdPresentationNotificationScreen.tsx | 27 ++- .../FunkePresentationNotificationScreen.tsx | 8 +- .../components/RequestPurposeSection.tsx | 24 ++- .../components/VerificationAnalysisIcon.tsx | 21 -- .../share/slides/ShareCredentialsSlide.tsx | 8 +- apps/easypid/src/hooks/index.ts | 1 + apps/easypid/src/hooks/useOverAskingAi.tsx | 137 +++++++++++++ apps/easypid/src/llm/RnExecutorchModule.ts | 28 +++ apps/easypid/src/llm/constants.ts | 5 + apps/easypid/src/llm/types.ts | 15 ++ apps/easypid/src/llm/useLLM.tsx | 180 ++++++++++++++++++ .../src/use-cases/ValidateVerification.ts | 6 +- .../app/src/components/ConfirmationSheet.tsx | 6 +- packages/ui/src/base/Switch.tsx | 55 ++++++ packages/ui/src/base/index.ts | 1 + packages/ui/src/content/Icon.tsx | 2 + packages/ui/src/index.ts | 10 +- pnpm-lock.yaml | 14 ++ 25 files changed, 610 insertions(+), 82 deletions(-) create mode 100644 apps/easypid/src/features/menu/components/LocalAiContainer.tsx create mode 100644 apps/easypid/src/features/menu/components/LocalAiDownloader.tsx delete mode 100644 apps/easypid/src/features/share/components/VerificationAnalysisIcon.tsx create mode 100644 apps/easypid/src/hooks/useOverAskingAi.tsx create mode 100644 apps/easypid/src/llm/RnExecutorchModule.ts create mode 100644 apps/easypid/src/llm/constants.ts create mode 100644 apps/easypid/src/llm/types.ts create mode 100644 apps/easypid/src/llm/useLLM.tsx create mode 100644 packages/ui/src/base/Switch.tsx diff --git a/apps/easypid/app.config.js b/apps/easypid/app.config.js index 056588e4..f1f10dc2 100644 --- a/apps/easypid/app.config.js +++ b/apps/easypid/app.config.js @@ -117,6 +117,9 @@ const config = { ], }, associatedDomains: associatedDomains.map((host) => `applinks:${host}`), + entitlements: { + 'com.apple.developer.kernel.increased-memory-limit': true, + }, }, android: { adaptiveIcon: { diff --git a/apps/easypid/metro.config.js b/apps/easypid/metro.config.js index d456cb8a..5cbcdb86 100644 --- a/apps/easypid/metro.config.js +++ b/apps/easypid/metro.config.js @@ -15,6 +15,7 @@ config.resolver.nodeModulesPaths = [ path.resolve(workspaceRoot, 'node_modules'), ] config.resolver.sourceExts = [...config.resolver.sourceExts, 'js', 'json', 'ts', 'tsx', 'cjs', 'mjs'] +config.resolver.assetExts = [...config.resolver.assetExts, 'bin'] config.resolver.extraNodeModules = { // Needed for cosmjs trying to import node crypto crypto: require.resolve('./src/polyfills/crypto.ts'), diff --git a/apps/easypid/package.json b/apps/easypid/package.json index 7b80551b..576d7d47 100644 --- a/apps/easypid/package.json +++ b/apps/easypid/package.json @@ -55,6 +55,7 @@ "react": "catalog:", "react-native": "catalog:", "react-native-argon2": "^2.0.1", + "react-native-executorch": "^0.1.2", "react-native-fs": "^2.20.0", "react-native-gesture-handler": "~2.16.2", "react-native-get-random-values": "~1.11.0", diff --git a/apps/easypid/src/app/_layout.tsx b/apps/easypid/src/app/_layout.tsx index 0fcb2642..9480c6d2 100644 --- a/apps/easypid/src/app/_layout.tsx +++ b/apps/easypid/src/app/_layout.tsx @@ -6,6 +6,7 @@ import { DefaultTheme, ThemeProvider } from '@react-navigation/native' import { Slot } from 'expo-router' import * as SplashScreen from 'expo-splash-screen' +import { useCheckIncompleteDownload } from '@easypid/llm/useLLM' import tamaguiConfig from '../../tamagui.config' void SplashScreen.preventAutoHideAsync() @@ -17,6 +18,7 @@ export const unstable_settings = { export default function RootLayout() { useTransparentNavigationBar() + useCheckIncompleteDownload() return ( diff --git a/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx b/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx index 720a5a9a..9665b710 100644 --- a/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx +++ b/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx @@ -1,35 +1,46 @@ -import { Button, FlexPage, Heading, HeroIcons, Paragraph, ScrollView, Stack, YStack } from '@package/ui' +import { Button, FlexPage, Heading, HeroIcons, ScrollView, Stack, YStack } from '@package/ui' import React from 'react' import { useRouter } from 'solito/router' import { useScrollViewPosition } from '@package/app/src/hooks' +import { LocalAiContainer } from './components/LocalAiContainer' export function FunkeSettingsScreen() { const { handleScroll, isScrolledByOffset, scrollEventThrottle } = useScrollViewPosition() const router = useRouter() return ( - - - - - - Settings - + <> + + + + + + Settings + + - - - - This page is under construction. - router.back()}> - Back - - - - + + + + + + + router.back()}> + Back + + + + + ) } diff --git a/apps/easypid/src/features/menu/components/LocalAiContainer.tsx b/apps/easypid/src/features/menu/components/LocalAiContainer.tsx new file mode 100644 index 00000000..d7da0fe1 --- /dev/null +++ b/apps/easypid/src/features/menu/components/LocalAiContainer.tsx @@ -0,0 +1,65 @@ +import { HeroIcons } from '@package/ui/src/content/Icon' + +import { Switch } from '@package/ui/src/base/Switch' + +import { useLLM } from '@easypid/llm/useLLM' +import { ConfirmationSheet } from '@package/app/src/components/ConfirmationSheet' +import { useToastController } from 'packages/ui/src' +import React, { useState } from 'react' + +export function LocalAiContainer() { + const toast = useToastController() + + const [isAiModelConfirmationOpen, setIsAiModelConfirmationOpen] = useState(false) + const { loadModel, isModelReady, downloadProgress, removeModel, isModelActivated, isModelDownloading } = useLLM() + + const handleAiModelChange = (value: boolean) => { + if (isModelDownloading) { + toast.show('Model download in progress', { + message: 'Force close the app to cancel the download', + customData: { + preset: 'warning', + }, + }) + return + } + + if (value) { + setIsAiModelConfirmationOpen(true) + } else { + removeModel() + } + } + + return ( + <> + } + value={isModelActivated} + description={ + isModelReady + ? 'Model active and ready to use' + : downloadProgress + ? `Downloading model ${(downloadProgress * 100).toFixed(2)}%` + : '' + } + onChange={handleAiModelChange} + /> + { + setIsAiModelConfirmationOpen(false) + loadModel() + }} + /> + + ) +} diff --git a/apps/easypid/src/features/menu/components/LocalAiDownloader.tsx b/apps/easypid/src/features/menu/components/LocalAiDownloader.tsx new file mode 100644 index 00000000..917d696f --- /dev/null +++ b/apps/easypid/src/features/menu/components/LocalAiDownloader.tsx @@ -0,0 +1,17 @@ +import { useLocalLlm } from '@easypid/hooks/useLocalLlm' +import { MessageBox, Paragraph, YStack } from 'packages/ui/src' + +export function LocalAiDownloader() { + const { error, downloadProgress, isModelReady } = useLocalLlm() + + return ( + + {error && } + {isModelReady ? ( + Local AI model ready to use + ) : ( + Downloading model ({(downloadProgress * 100)?.toFixed(2)}%) + )} + + ) +} diff --git a/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx b/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx index e48c4c08..6fadb5d0 100644 --- a/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx +++ b/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx @@ -12,8 +12,7 @@ import React, { useEffect, useState, useCallback } from 'react' import { useAppAgent } from '@easypid/agent' import { InvalidPinError } from '@easypid/crypto/error' -import { analyzeVerification } from '@easypid/use-cases/ValidateVerification' -import type { VerificationAnalysisResponse } from '@easypid/use-cases/ValidateVerification' +import { useOverAskingAi } from '@easypid/hooks' import { usePushToWallet } from '@package/app/src/hooks/usePushToWallet' import { setWalletServiceProviderPin } from '../../crypto/WalletServiceProviderClient' import { useShouldUsePinForSubmission } from '../../hooks/useShouldUsePinForPresentation' @@ -58,26 +57,24 @@ export function FunkeOpenIdPresentationNotificationScreen() { }) }, [credentialsForRequest, params.data, params.uri, toast.show, agent, pushToWallet, toast]) - const [verificationAnalysis, setVerificationAnalysis] = useState<{ - isLoading: boolean - result: VerificationAnalysisResponse | undefined - }>({ - isLoading: false, - result: undefined, - }) + const { checkForOverAsking, isProcessingOverAsking, overAskingResponse } = useOverAskingAi() useEffect(() => { - if (!credentialsForRequest?.formattedSubmission || !credentialsForRequest?.formattedSubmission.areAllSatisfied) { + if (!credentialsForRequest?.formattedSubmission) { + return + } + + if (isProcessingOverAsking || overAskingResponse) { + // Already generating or already has result return } - setVerificationAnalysis((prev) => ({ ...prev, isLoading: true })) const submission = credentialsForRequest.formattedSubmission const requestedCards = submission.entries .filter((entry): entry is FormattedSubmissionEntrySatisfied => entry.isSatisfied) .flatMap((entry) => entry.credentials) - analyzeVerification({ + void checkForOverAsking({ verifier: { name: credentialsForRequest.verifier.name ?? 'No name provided', domain: credentialsForRequest.verifier.hostName ?? 'No domain provided', @@ -89,8 +86,8 @@ export function FunkeOpenIdPresentationNotificationScreen() { subtitle: credential.credential.display.description ?? 'Card description', requestedAttributes: getDisclosedAttributeNamesForDisplay(credential), })), - }).then((result) => setVerificationAnalysis((prev) => ({ ...prev, isLoading: false, result }))) - }, [credentialsForRequest]) + }) + }, [credentialsForRequest, checkForOverAsking, isProcessingOverAsking, overAskingResponse]) const onProofAccept = useCallback( async (pin?: string): Promise => { @@ -206,7 +203,7 @@ export function FunkeOpenIdPresentationNotificationScreen() { trustedEntities={credentialsForRequest?.verifier.trustedEntities} lastInteractionDate={lastInteractionDate} onComplete={() => pushToWallet('replace')} - verificationAnalysis={verificationAnalysis} + overAskingResponse={overAskingResponse} /> ) } diff --git a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx index c41b773e..fa72b7a9 100644 --- a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx +++ b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx @@ -1,6 +1,6 @@ import type { DisplayImage, FormattedSubmission, TrustedEntity } from '@package/agent' -import type { VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' import { type SlideStep, SlideWizard } from '@package/app' import { LoadingRequestSlide } from '../receive/slides/LoadingRequestSlide' import { VerifyPartySlide } from '../receive/slides/VerifyPartySlide' @@ -14,7 +14,7 @@ interface FunkePresentationNotificationScreenProps { verifierName?: string logo?: DisplayImage lastInteractionDate?: string - verificationAnalysis: VerificationAnalysisResult + overAskingResponse?: OverAskingResponse trustedEntities?: Array submission?: FormattedSubmission usePin: boolean @@ -35,7 +35,7 @@ export function FunkePresentationNotificationScreen({ isAccepting, submission, onComplete, - verificationAnalysis, + overAskingResponse, trustedEntities, }: FunkePresentationNotificationScreenProps) { return ( @@ -74,7 +74,7 @@ export function FunkePresentationNotificationScreen({ logo={logo} submission={submission} isAccepting={isAccepting} - verificationAnalysis={verificationAnalysis} + overAskingResponse={overAskingResponse} /> ), }, diff --git a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx index 3c06ff88..e4c92b90 100644 --- a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx +++ b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx @@ -1,4 +1,4 @@ -import type { VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse } from '@easypid/use-cases/ValidateVerification' import { AnimatedStack, Circle, @@ -7,6 +7,7 @@ import { Image, InfoSheet, MessageBox, + Spinner, Stack, XStack, YStack, @@ -16,15 +17,14 @@ import type { DisplayImage } from 'packages/agent/src' import { useState } from 'react' import React from 'react' import { FadeIn, ZoomIn } from 'react-native-reanimated' -import { VerificationAnalysisIcon } from './VerificationAnalysisIcon' interface RequestPurposeSectionProps { purpose: string logo?: DisplayImage - verificationAnalysis?: VerificationAnalysisResult + overAskingResponse?: OverAskingResponse } -export function RequestPurposeSection({ purpose, logo, verificationAnalysis }: RequestPurposeSectionProps) { +export function RequestPurposeSection({ purpose, logo, overAskingResponse }: RequestPurposeSectionProps) { const [isAnalysisModalOpen, setIsAnalysisModalOpen] = useState(false) const { handlePressIn, handlePressOut, pressStyle } = useScaleAnimation() @@ -34,7 +34,7 @@ export function RequestPurposeSection({ purpose, logo, verificationAnalysis }: R return ( <> - {verificationAnalysis?.result?.validRequest === 'no' && ( + {overAskingResponse?.validRequest === 'no' && ( PURPOSE - {verificationAnalysis && ( - - - - )} + + {!overAskingResponse ? ( + + ) : overAskingResponse.validRequest === 'yes' ? ( + + ) : overAskingResponse.validRequest === 'no' ? ( + + ) : null} + - - if (!verificationAnalysis.result || verificationAnalysis.result.validRequest === 'could_not_determine') { - // AI doesn't know or an error was thrown - return null - } - - if (verificationAnalysis.result.validRequest === 'yes') { - return - } - - return -} diff --git a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx index 4278469c..428711ea 100644 --- a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx +++ b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx @@ -1,4 +1,4 @@ -import type { VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' import type { DisplayImage, FormattedSubmission } from '@package/agent' import { DualResponseButtons, usePushToWallet, useScrollViewPosition } from '@package/app' import { useWizard } from '@package/app' @@ -17,7 +17,7 @@ interface ShareCredentialsSlideProps { isAccepting: boolean isOffline?: boolean - verificationAnalysis?: VerificationAnalysisResult + overAskingResponse?: OverAskingResponse } export const ShareCredentialsSlide = ({ @@ -27,7 +27,7 @@ export const ShareCredentialsSlide = ({ onDecline, isAccepting, isOffline, - verificationAnalysis, + overAskingResponse, }: ShareCredentialsSlideProps) => { const { onNext, onCancel } = useWizard() const [scrollViewHeight, setScrollViewHeight] = useState(0) @@ -88,7 +88,7 @@ export const ShareCredentialsSlide = ({ purpose={ submission.purpose ?? 'No information was provided on the purpose of the data request. Be cautious' } - verificationAnalysis={verificationAnalysis} + overAskingResponse={overAskingResponse} logo={logo} /> )} diff --git a/apps/easypid/src/hooks/index.ts b/apps/easypid/src/hooks/index.ts index 9d8f8014..3c02b5c6 100644 --- a/apps/easypid/src/hooks/index.ts +++ b/apps/easypid/src/hooks/index.ts @@ -1,2 +1,3 @@ export * from './usePidCredential' export * from './useWalletReset' +export * from './useOverAskingAi' diff --git a/apps/easypid/src/hooks/useOverAskingAi.tsx b/apps/easypid/src/hooks/useOverAskingAi.tsx new file mode 100644 index 00000000..03608813 --- /dev/null +++ b/apps/easypid/src/hooks/useOverAskingAi.tsx @@ -0,0 +1,137 @@ +import { useEffect, useState } from 'react' + +import { useLLM } from '@easypid/llm/useLLM' +import type { OverAskingResponse, VerificationAnalysisInput } from '@easypid/use-cases/ValidateVerification' +import { analyzeVerification as analyzeVerificationApi } from '@easypid/use-cases/ValidateVerification' + +// todos: add a timeout to both api and local calls + +const fallbackResponse: OverAskingResponse = { + validRequest: 'could_not_determine', + reason: 'Error determining if the request is valid.', +} + +export function useOverAskingAi() { + const [isProcessingOverAsking, setIsProcessingOverAsking] = useState(false) + const [overAskingResponse, setOverAskingResponse] = useState() + + const { generate, response, error, isModelReady, isModelGenerating } = useLLM() + + useEffect(() => { + console.log('response', response) + }, [response]) + + useEffect(() => { + if (error) { + console.error('Error generating using LLM:', error) + setIsProcessingOverAsking(false) + setOverAskingResponse(fallbackResponse) + return + } + + if (!response || isModelGenerating) return + + try { + const result = formatResult(response) + setOverAskingResponse(result) + } catch (e) { + console.error('Error parsing AI response:', e) + setOverAskingResponse(fallbackResponse) + setIsProcessingOverAsking(false) + } + }, [response, isModelGenerating, error]) + + const checkForOverAsking = async (input: VerificationAnalysisInput) => { + setIsProcessingOverAsking(true) + if (isModelReady) { + console.log('Model ready, using local LLM') + const prompt = formatPrompt(input) + await generate(prompt) + } else { + console.log('Local LLM not ready, using API') + await analyzeVerificationApi(input) + .then(setOverAskingResponse) + .catch((e) => { + console.error('Error analyzing verification:', e) + setOverAskingResponse(fallbackResponse) + }) + .finally(() => setIsProcessingOverAsking(false)) + } + } + + return { + isProcessingOverAsking, + checkForOverAsking, + overAskingResponse, + } +} + +const formatResult = (response: string) => { + const match = response.match(/([\s\S]*?)<\/response>/) + if (!match) return + + const responseContent = match[1] + + if (responseContent.includes('') && responseContent.includes('')) { + return { + validRequest: responseContent.split('')[1].split('')[0] as + | 'yes' + | 'no' + | 'could_not_determine', + reason: responseContent.split('')[1].split('')[0], + } + } + + return fallbackResponse +} + +const formatPrompt = (input: VerificationAnalysisInput) => { + const cards = input.cards + .map( + (credential) => + `${credential.name} - ${credential.subtitle}. Requested attributes: ${credential.requestedAttributes.join(', ')}` + ) + .join('\n') + return ` +You are an AI assistant specializing in data privacy analysis. Your task is to evaluate data verification requests and determine if they are asking for an appropriate amount of information or if they are overasking. + +Here is the information for the current request: + + +${input.verifier.name} + + + +${input.verifier.domain} + + + +${input.name} + + + +${input.purpose} + + + +${cards} + + + +Provide a small evaluation of the request, and provide your final response in the following XML structure: + + +Your concise reason for the assessment +yes + + +Example of a properly formatted response: + + +Request aligns with purpose. Information amount appropriate. Verifier seems legitimate. +yes + + +Remember: Provide a concise reason and use the correct XML structure in your response. Do not add any text outside of the specified tags. +` +} diff --git a/apps/easypid/src/llm/RnExecutorchModule.ts b/apps/easypid/src/llm/RnExecutorchModule.ts new file mode 100644 index 00000000..3cce9190 --- /dev/null +++ b/apps/easypid/src/llm/RnExecutorchModule.ts @@ -0,0 +1,28 @@ +import { NativeEventEmitter, NativeModules, Platform } from 'react-native' + +const LINKING_ERROR = `The package 'react-native-executorch' doesn't seem to be linked. Make sure: \n\n${Platform.select({ ios: "- You have run 'pod install'\n", default: '' })}- You rebuilt the app after installing the package\n- You are not using Expo Go\n` + +const RnExecutorch = NativeModules.RnExecutorch + ? NativeModules.RnExecutorch + : new Proxy( + {}, + { + get() { + throw new Error(LINKING_ERROR) + }, + } + ) + +const eventEmitter = new NativeEventEmitter(RnExecutorch) + +export const subscribeToTokenGenerated = (callback: (data?: string) => void) => { + const subscription = eventEmitter.addListener('onToken', callback) + return () => subscription.remove() +} + +export const subscribeToDownloadProgress = (callback: (data?: number) => void) => { + const subscription = eventEmitter.addListener('onDownloadProgress', callback) + return () => subscription.remove() +} + +export default RnExecutorch diff --git a/apps/easypid/src/llm/constants.ts b/apps/easypid/src/llm/constants.ts new file mode 100644 index 00000000..decd3c97 --- /dev/null +++ b/apps/easypid/src/llm/constants.ts @@ -0,0 +1,5 @@ +export const DEFAULT_SYSTEM_PROMPT = + "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text." + +export const DEFAULT_CONTEXT_WINDOW_LENGTH = 3 +export const EOT_TOKEN = '<|eot_id|>' diff --git a/apps/easypid/src/llm/types.ts b/apps/easypid/src/llm/types.ts new file mode 100644 index 00000000..ef945600 --- /dev/null +++ b/apps/easypid/src/llm/types.ts @@ -0,0 +1,15 @@ +export type ResourceSource = string | number + +export interface Model { + generate: (input: string) => Promise + response: string + downloadProgress: number + error: string | null + isModelGenerating: boolean + isModelReady: boolean + isModelActivated: boolean + isModelDownloading: boolean + interrupt: () => void + loadModel: () => Promise + removeModel: () => void +} diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx new file mode 100644 index 00000000..042be486 --- /dev/null +++ b/apps/easypid/src/llm/useLLM.tsx @@ -0,0 +1,180 @@ +import { mmkv } from '@easypid/storage/mmkv' +import { useCallback, useEffect, useRef, useState } from 'react' +import { Image } from 'react-native' +import { LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER } from 'react-native-executorch' +import { useMMKVBoolean } from 'react-native-mmkv' +import RnExecutorch, { subscribeToDownloadProgress, subscribeToTokenGenerated } from './RnExecutorchModule' +import { DEFAULT_CONTEXT_WINDOW_LENGTH, DEFAULT_SYSTEM_PROMPT, EOT_TOKEN } from './constants' +import type { Model, ResourceSource } from './types' + +const interrupt = () => { + RnExecutorch.interrupt() +} + +export function useIsModelReady() { + return useMMKVBoolean('isModelReady', mmkv) +} + +export function removeIsModelReady() { + mmkv.delete('isModelReady') +} + +export function useIsModelActivated() { + return useMMKVBoolean('isModelActivated', mmkv) +} + +export function removeIsModelActivated() { + mmkv.delete('isModelActivated') +} + +export function useIsModelDownloading() { + return useMMKVBoolean('isModelDownloading', mmkv) +} + +export function removeIsModelDownloading() { + mmkv.delete('isModelDownloading') +} + +export const useLLM = ({ + modelSource = LLAMA3_2_1B_QLORA_URL, + tokenizerSource = LLAMA3_2_1B_TOKENIZER, + systemPrompt = DEFAULT_SYSTEM_PROMPT, + contextWindowLength = DEFAULT_CONTEXT_WINDOW_LENGTH, +}: { + modelSource?: ResourceSource + tokenizerSource?: ResourceSource + systemPrompt?: string + contextWindowLength?: number +} = {}): Model => { + const [error, setError] = useState(null) + const [isModelActivated, setIsModelActivated] = useIsModelActivated() + const [isModelDownloading, setIsModelDownloading] = useIsModelDownloading() + const [isModelReady, setIsModelReady] = useIsModelReady() + const [isModelGenerating, setIsModelGenerating] = useState(false) + const [response, setResponse] = useState('') + const [downloadProgress, setDownloadProgress] = useState(0) + const initialized = useRef(false) + + useEffect(() => { + const unsubscribeDownloadProgress = subscribeToDownloadProgress((data) => { + if (data) { + setDownloadProgress(data) + } + }) + + return () => { + if (unsubscribeDownloadProgress) unsubscribeDownloadProgress() + } + }, []) + + const loadModel = useCallback(async () => { + if (initialized.current) { + return + } + + setIsModelActivated(true) + initialized.current = true + try { + try { + setIsModelDownloading(true) + await RnExecutorch.loadLLM(modelSource, tokenizerSource, systemPrompt, contextWindowLength) + await RnExecutorch + } catch (error) { + console.log('ERROR LOADING MODEL', error) + } + setIsModelDownloading(false) + setIsModelActivated(true) + setIsModelReady(true) + } catch (err) { + const message = (err as Error).message + setIsModelReady(false) + setIsModelDownloading(false) + setError(message) + initialized.current = false + } + }, [ + contextWindowLength, + modelSource, + systemPrompt, + tokenizerSource, + setIsModelReady, + setIsModelActivated, + setIsModelDownloading, + ]) + + const generate = useCallback( + async (input: string): Promise => { + if (!isModelReady || !isModelActivated) { + throw new Error('Model not active or still loading') + } + if (error) { + throw new Error(error) + } + + try { + setResponse('') // This might be causing issues - let's move it + setIsModelGenerating(true) + await RnExecutorch.runInference(input) + } catch (err) { + setIsModelGenerating(false) + throw new Error((err as Error).message) + } + }, + [isModelReady, error, isModelActivated] + ) + + // Move the token subscription to useEffect to ensure it persists + useEffect(() => { + const unsubscribeTokenGenerated = subscribeToTokenGenerated((data) => { + if (!data) return + + if (data !== EOT_TOKEN) { + setResponse((prevResponse) => prevResponse + data) + } else { + setIsModelGenerating(false) + } + }) + + return () => { + if (unsubscribeTokenGenerated) unsubscribeTokenGenerated() + } + }, []) + + // Doesn't actually remove the model, just the state + const removeModel = useCallback(() => { + removeIsModelReady() + removeIsModelActivated() + removeIsModelDownloading() + initialized.current = false + }, []) + + return { + generate, + error, + isModelActivated: isModelActivated ?? false, + isModelReady: isModelReady ?? false, + isModelDownloading: isModelDownloading ?? false, + isModelGenerating, + response, + downloadProgress, + interrupt, + loadModel, + removeModel, + } +} + +export function useCheckIncompleteDownload() { + const [isModelActivated] = useIsModelActivated() + const [isModelDownloading] = useIsModelDownloading() + const [isModelReady] = useIsModelReady() + + // biome-ignore lint/correctness/useExhaustiveDependencies: should only run once + useEffect(() => { + if (isModelActivated && isModelDownloading && !isModelReady) { + console.log('Cleaning up incomplete model download from previous session') + removeIsModelReady() + removeIsModelActivated() + removeIsModelDownloading() + } + }, []) +} diff --git a/apps/easypid/src/use-cases/ValidateVerification.ts b/apps/easypid/src/use-cases/ValidateVerification.ts index ca018295..e30834b1 100644 --- a/apps/easypid/src/use-cases/ValidateVerification.ts +++ b/apps/easypid/src/use-cases/ValidateVerification.ts @@ -16,14 +16,14 @@ export type VerificationAnalysisInput = { }> } -export type VerificationAnalysisResponse = { +export type OverAskingResponse = { validRequest: 'yes' | 'no' | 'could_not_determine' reason: string } export type VerificationAnalysisResult = { isLoading: boolean - result: VerificationAnalysisResponse | undefined + result: OverAskingResponse | undefined } export const analyzeVerification = async ({ @@ -31,7 +31,7 @@ export const analyzeVerification = async ({ name, purpose, cards, -}: VerificationAnalysisInput): Promise => { +}: VerificationAnalysisInput): Promise => { try { const cardsWithoutExcludedAttributes = cards.map((card) => ({ ...card, diff --git a/packages/app/src/components/ConfirmationSheet.tsx b/packages/app/src/components/ConfirmationSheet.tsx index bca8adf8..53843f0d 100644 --- a/packages/app/src/components/ConfirmationSheet.tsx +++ b/packages/app/src/components/ConfirmationSheet.tsx @@ -8,6 +8,7 @@ const DEFAULT_CONFIRM_TEXT = 'Yes, stop' interface ConfirmationSheetProps { type?: 'regular' | 'floating' + variant?: 'confirmation' | 'regular' title?: string description?: string confirmText?: string @@ -19,6 +20,7 @@ interface ConfirmationSheetProps { export function ConfirmationSheet({ type = 'regular', + variant = 'confirmation', title, description, confirmText, @@ -45,7 +47,7 @@ export function ConfirmationSheet({ {description || DEFAULT_DESCRIPTION} void +} + +const AnimatedSwitch = Animated.createAnimatedComponent(TamaguiSwitch) + +export function Switch({ id, label, value, disabled, onChange, icon, description }: SettingsSwitchProps) { + const { handlePressIn, handlePressOut, pressStyle } = useScaleAnimation({ scaleInValue: 0.95 }) + + return ( + + + + {icon && ( + + {cloneElement(icon, { size: 20, color: value ? '$primary-500' : '$grey-500' })} + + )} + + + + + + + {description && {description}} + + ) +} diff --git a/packages/ui/src/base/index.ts b/packages/ui/src/base/index.ts index 172f5d8c..85bd2d70 100644 --- a/packages/ui/src/base/index.ts +++ b/packages/ui/src/base/index.ts @@ -5,3 +5,4 @@ export * from './Stacks' export * from './Page' export * from './ScrollView' export * from './Separator' +export * from './Switch' diff --git a/packages/ui/src/content/Icon.tsx b/packages/ui/src/content/Icon.tsx index 1ecb2764..f451c5bf 100644 --- a/packages/ui/src/content/Icon.tsx +++ b/packages/ui/src/content/Icon.tsx @@ -64,6 +64,7 @@ import { CircleStackIcon as CircleStackFilledIcon, ClockIcon as ClockFilledIcon, Cog8ToothIcon as Cog8ToothFilledIcon, + CpuChipIcon as CpuChipFilledIcon, CreditCardIcon as CreditCardFilledIcon, ExclamationCircleIcon as ExclamationCircleFilledIcon, ExclamationTriangleIcon as ExclamationTriangleFilledIcon, @@ -175,6 +176,7 @@ export const HeroIcons = { MagnifyingGlass: wrapHeroIcon(MagnifyingGlassIcon), Link: wrapHeroIcon(LinkIcon), Cloud: wrapHeroIcon(CloudIcon), + CpuChipFilled: wrapHeroIcon(CpuChipFilledIcon), } as const export const CustomIcons = { diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index af1d2d57..0d4e7667 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -1,6 +1,14 @@ export { tokens, config, absoluteFill } from './config/tamagui.config' export * from './constants' -export { TamaguiProviderProps, TamaguiProvider, Spacer, Input, AnimatePresence, Circle, VisuallyHidden } from 'tamagui' +export { + TamaguiProviderProps, + TamaguiProvider, + Spacer, + Input, + AnimatePresence, + Circle, + VisuallyHidden, +} from 'tamagui' export { ToastProvider, useToastController, ToastViewport, useToastState } from '@tamagui/toast' export * from './panels' export * from './base' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 053b832e..c4c8d7bc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -225,6 +225,9 @@ importers: react-native-argon2: specifier: ^2.0.1 version: 2.0.1 + react-native-executorch: + specifier: ^0.1.2 + version: 0.1.2(react-native@0.74.5(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))(@types/react@18.2.79)(react@18.3.1))(react@18.3.1) react-native-fs: specifier: ^2.20.0 version: 2.20.0(react-native@0.74.5(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))(@types/react@18.2.79)(react@18.3.1)) @@ -8368,6 +8371,12 @@ packages: react-native-argon2@2.0.1: resolution: {integrity: sha512-/iOi0S+VVgS1gQGtQgL4ZxUVS4gz6Lav3bgIbtNmr9KbOunnBYzP6/yBe/XxkbpXvasHDwdQnuppOH/nuOBn7w==} + react-native-executorch@0.1.2: + resolution: {integrity: sha512-iexg2NDKO/ADVZtDYT99kap7kVJPtmTd8HbtLJRmVBT/tYhoXG9o5+vcHl5xIo9qlyZW8YqG047z4NtFAGDP1w==} + peerDependencies: + react: '*' + react-native: '*' + react-native-fit-image@1.5.5: resolution: {integrity: sha512-Wl3Vq2DQzxgsWKuW4USfck9zS7YzhvLNPpkwUUCF90bL32e1a0zOVQ3WsJILJOwzmPdHfzZmWasiiAUNBkhNkg==} @@ -20407,6 +20416,11 @@ snapshots: react-native-argon2@2.0.1: {} + react-native-executorch@0.1.2(react-native@0.74.5(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))(@types/react@18.2.79)(react@18.3.1))(react@18.3.1): + dependencies: + react: 18.3.1 + react-native: 0.74.5(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))(@types/react@18.2.79)(react@18.3.1) + react-native-fit-image@1.5.5: dependencies: prop-types: 15.8.1 From 821d4a1569e3043ef2e27b0a910ea5cf62842c26 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 3 Dec 2024 15:02:25 +0100 Subject: [PATCH 02/12] feat: warnings --- .../menu/components/LocalAiContainer.tsx | 58 +++++++++++++++---- .../menu/components/LocalAiDownloader.tsx | 17 ------ apps/easypid/src/llm/useLLM.tsx | 19 +++++- .../app/src/components/ConfirmationSheet.tsx | 20 +++++-- packages/app/src/hooks/index.ts | 1 + .../src/hooks/useHasInternetConnection.tsx | 5 ++ packages/ui/src/base/Switch.tsx | 26 +++++++-- 7 files changed, 107 insertions(+), 39 deletions(-) delete mode 100644 apps/easypid/src/features/menu/components/LocalAiDownloader.tsx diff --git a/apps/easypid/src/features/menu/components/LocalAiContainer.tsx b/apps/easypid/src/features/menu/components/LocalAiContainer.tsx index d7da0fe1..a23f01e5 100644 --- a/apps/easypid/src/features/menu/components/LocalAiContainer.tsx +++ b/apps/easypid/src/features/menu/components/LocalAiContainer.tsx @@ -2,17 +2,47 @@ import { HeroIcons } from '@package/ui/src/content/Icon' import { Switch } from '@package/ui/src/base/Switch' -import { useLLM } from '@easypid/llm/useLLM' +import { useIsDeviceCapable, useLLM } from '@easypid/llm/useLLM' import { ConfirmationSheet } from '@package/app/src/components/ConfirmationSheet' +import { useHasInternetConnection, useIsConnectedToWifi } from 'packages/app/src/hooks' import { useToastController } from 'packages/ui/src' import React, { useState } from 'react' export function LocalAiContainer() { const toast = useToastController() + const isConnectedToWifi = useIsConnectedToWifi() + const hasInternetConnection = useHasInternetConnection() + const isDeviceCapable = useIsDeviceCapable() const [isAiModelConfirmationOpen, setIsAiModelConfirmationOpen] = useState(false) const { loadModel, isModelReady, downloadProgress, removeModel, isModelActivated, isModelDownloading } = useLLM() + const onActivateModel = () => { + if (!isDeviceCapable) { + toast.show('Device not supported', { + message: 'This device is not powerful enough to run local AI models', + customData: { + preset: 'warning', + }, + }) + setIsAiModelConfirmationOpen(false) + return + } + if (!isConnectedToWifi && !hasInternetConnection) { + toast.show('WiFi connection required', { + message: 'Please connect to WiFi to activate and download the model', + customData: { + preset: 'warning', + }, + }) + setIsAiModelConfirmationOpen(false) + return + } + + setIsAiModelConfirmationOpen(false) + loadModel() + } + const handleAiModelChange = (value: boolean) => { if (isModelDownloading) { toast.show('Model download in progress', { @@ -39,26 +69,30 @@ export function LocalAiContainer() { icon={} value={isModelActivated} description={ - isModelReady - ? 'Model active and ready to use' - : downloadProgress - ? `Downloading model ${(downloadProgress * 100).toFixed(2)}%` - : '' + isModelActivated + ? isModelReady + ? 'Model active and ready to use' + : downloadProgress + ? `Downloading model ${(downloadProgress * 100).toFixed(2)}%` + : 'Getting ready...' + : '' } onChange={handleAiModelChange} + beta /> { - setIsAiModelConfirmationOpen(false) - loadModel() - }} + cancelText="Cancel" + description={[ + 'This will download a local AI model to your device which will take up to around 1.3GB of space.', + 'This is an experimental feature. Only supported on high-end devices.', + ]} + onConfirm={onActivateModel} /> ) diff --git a/apps/easypid/src/features/menu/components/LocalAiDownloader.tsx b/apps/easypid/src/features/menu/components/LocalAiDownloader.tsx deleted file mode 100644 index 917d696f..00000000 --- a/apps/easypid/src/features/menu/components/LocalAiDownloader.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import { useLocalLlm } from '@easypid/hooks/useLocalLlm' -import { MessageBox, Paragraph, YStack } from 'packages/ui/src' - -export function LocalAiDownloader() { - const { error, downloadProgress, isModelReady } = useLocalLlm() - - return ( - - {error && } - {isModelReady ? ( - Local AI model ready to use - ) : ( - Downloading model ({(downloadProgress * 100)?.toFixed(2)}%) - )} - - ) -} diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 042be486..3a8b1685 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -1,6 +1,6 @@ import { mmkv } from '@easypid/storage/mmkv' import { useCallback, useEffect, useRef, useState } from 'react' -import { Image } from 'react-native' +import { Platform } from 'react-native' import { LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER } from 'react-native-executorch' import { useMMKVBoolean } from 'react-native-mmkv' import RnExecutorch, { subscribeToDownloadProgress, subscribeToTokenGenerated } from './RnExecutorchModule' @@ -163,6 +163,8 @@ export const useLLM = ({ } } +// Check for incomplete model download from previous session +// Used when the app is opened export function useCheckIncompleteDownload() { const [isModelActivated] = useIsModelActivated() const [isModelDownloading] = useIsModelDownloading() @@ -178,3 +180,18 @@ export function useCheckIncompleteDownload() { } }, []) } + +// TODO: Add expo-device to check if the device is capable +export function useIsDeviceCapable(): boolean { + // For iOS, check if device is at least iPhone X or newer + if (Platform.OS === 'ios') { + return true + } + + // For Android, check RAM + if (Platform.OS === 'android') { + return true + } + + return false +} diff --git a/packages/app/src/components/ConfirmationSheet.tsx b/packages/app/src/components/ConfirmationSheet.tsx index 53843f0d..ce8b3abf 100644 --- a/packages/app/src/components/ConfirmationSheet.tsx +++ b/packages/app/src/components/ConfirmationSheet.tsx @@ -10,7 +10,8 @@ interface ConfirmationSheetProps { type?: 'regular' | 'floating' variant?: 'confirmation' | 'regular' title?: string - description?: string + description?: string | string[] + cancelText?: string confirmText?: string isOpen: boolean setIsOpen: (open: boolean) => void @@ -24,6 +25,7 @@ export function ConfirmationSheet({ title, description, confirmText, + cancelText, isOpen, setIsOpen, onConfirm, @@ -44,12 +46,16 @@ export function ConfirmationSheet({ - {description || DEFAULT_DESCRIPTION} + {Array.isArray(description) ? ( + description.map((desc) => {desc}) + ) : ( + {description || DEFAULT_DESCRIPTION} + )} setIsOpen(false))} /> @@ -63,14 +69,18 @@ export function ConfirmationSheet({ {title || DEFAULT_TITLE} - {description || DEFAULT_DESCRIPTION} + {Array.isArray(description) ? ( + description.map((desc) => {desc}) + ) : ( + {description || DEFAULT_DESCRIPTION} + )} setIsOpen(false))} /> diff --git a/packages/app/src/hooks/index.ts b/packages/app/src/hooks/index.ts index cac7af42..715d91ce 100644 --- a/packages/app/src/hooks/index.ts +++ b/packages/app/src/hooks/index.ts @@ -8,3 +8,4 @@ export * from './usePushToWallet' export * from './useHeaderRightAction' export * from './useHaptics' export * from './useImageScaler' +export * from './useHasInternetConnection' diff --git a/packages/app/src/hooks/useHasInternetConnection.tsx b/packages/app/src/hooks/useHasInternetConnection.tsx index b2befde9..1b71e199 100644 --- a/packages/app/src/hooks/useHasInternetConnection.tsx +++ b/packages/app/src/hooks/useHasInternetConnection.tsx @@ -7,3 +7,8 @@ export const useHasInternetConnection = () => { return (isConnected && isInternetReachable) ?? false } + +export const useIsConnectedToWifi = () => { + const { type } = useNetInfo() + return type === 'wifi' +} diff --git a/packages/ui/src/base/Switch.tsx b/packages/ui/src/base/Switch.tsx index b7cadfdf..9d678614 100644 --- a/packages/ui/src/base/Switch.tsx +++ b/packages/ui/src/base/Switch.tsx @@ -13,11 +13,12 @@ type SettingsSwitchProps = { icon?: React.ReactElement disabled?: boolean onChange: (value: boolean) => void + beta?: boolean } const AnimatedSwitch = Animated.createAnimatedComponent(TamaguiSwitch) -export function Switch({ id, label, value, disabled, onChange, icon, description }: SettingsSwitchProps) { +export function Switch({ id, label, value, disabled, onChange, icon, description, beta }: SettingsSwitchProps) { const { handlePressIn, handlePressOut, pressStyle } = useScaleAnimation({ scaleInValue: 0.95 }) return ( @@ -29,9 +30,26 @@ export function Switch({ id, label, value, disabled, onChange, icon, description {cloneElement(icon, { size: 20, color: value ? '$primary-500' : '$grey-500' })} )} - + + + {beta && ( + + + BETA + + + )} + Date: Tue, 3 Dec 2024 15:37:33 +0100 Subject: [PATCH 03/12] feat: clean up --- apps/easypid/metro.config.js | 1 - .../FunkePresentationNotificationScreen.tsx | 2 +- .../components/RequestPurposeSection.tsx | 2 +- .../share/slides/ShareCredentialsSlide.tsx | 2 +- apps/easypid/src/hooks/useOverAskingAi.tsx | 56 +++++++------------ apps/easypid/src/llm/useLLM.tsx | 28 +++++----- ...lidateVerification.ts => OverAskingApi.ts} | 11 +--- 7 files changed, 40 insertions(+), 62 deletions(-) rename apps/easypid/src/use-cases/{ValidateVerification.ts => OverAskingApi.ts} (85%) diff --git a/apps/easypid/metro.config.js b/apps/easypid/metro.config.js index 5cbcdb86..d456cb8a 100644 --- a/apps/easypid/metro.config.js +++ b/apps/easypid/metro.config.js @@ -15,7 +15,6 @@ config.resolver.nodeModulesPaths = [ path.resolve(workspaceRoot, 'node_modules'), ] config.resolver.sourceExts = [...config.resolver.sourceExts, 'js', 'json', 'ts', 'tsx', 'cjs', 'mjs'] -config.resolver.assetExts = [...config.resolver.assetExts, 'bin'] config.resolver.extraNodeModules = { // Needed for cosmjs trying to import node crypto crypto: require.resolve('./src/polyfills/crypto.ts'), diff --git a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx index fa72b7a9..2d9b22a2 100644 --- a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx +++ b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx @@ -1,6 +1,6 @@ import type { DisplayImage, FormattedSubmission, TrustedEntity } from '@package/agent' -import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/OverAskingApi' import { type SlideStep, SlideWizard } from '@package/app' import { LoadingRequestSlide } from '../receive/slides/LoadingRequestSlide' import { VerifyPartySlide } from '../receive/slides/VerifyPartySlide' diff --git a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx index e4c92b90..cb0d2005 100644 --- a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx +++ b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx @@ -1,4 +1,4 @@ -import type { OverAskingResponse } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse } from '@easypid/use-cases/OverAskingApi' import { AnimatedStack, Circle, diff --git a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx index 428711ea..3a7282a6 100644 --- a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx +++ b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx @@ -1,4 +1,4 @@ -import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/OverAskingApi' import type { DisplayImage, FormattedSubmission } from '@package/agent' import { DualResponseButtons, usePushToWallet, useScrollViewPosition } from '@package/app' import { useWizard } from '@package/app' diff --git a/apps/easypid/src/hooks/useOverAskingAi.tsx b/apps/easypid/src/hooks/useOverAskingAi.tsx index 03608813..e8feb25b 100644 --- a/apps/easypid/src/hooks/useOverAskingAi.tsx +++ b/apps/easypid/src/hooks/useOverAskingAi.tsx @@ -1,10 +1,8 @@ import { useEffect, useState } from 'react' import { useLLM } from '@easypid/llm/useLLM' -import type { OverAskingResponse, VerificationAnalysisInput } from '@easypid/use-cases/ValidateVerification' -import { analyzeVerification as analyzeVerificationApi } from '@easypid/use-cases/ValidateVerification' - -// todos: add a timeout to both api and local calls +import type { OverAskingInput, OverAskingResponse } from '@easypid/use-cases/OverAskingApi' +import { checkForOverAskingApi as analyzeVerificationApi } from '@easypid/use-cases/OverAskingApi' const fallbackResponse: OverAskingResponse = { validRequest: 'could_not_determine', @@ -17,13 +15,8 @@ export function useOverAskingAi() { const { generate, response, error, isModelReady, isModelGenerating } = useLLM() - useEffect(() => { - console.log('response', response) - }, [response]) - useEffect(() => { if (error) { - console.error('Error generating using LLM:', error) setIsProcessingOverAsking(false) setOverAskingResponse(fallbackResponse) return @@ -32,7 +25,7 @@ export function useOverAskingAi() { if (!response || isModelGenerating) return try { - const result = formatResult(response) + const result = formatLocalResult(response) setOverAskingResponse(result) } catch (e) { console.error('Error parsing AI response:', e) @@ -41,18 +34,18 @@ export function useOverAskingAi() { } }, [response, isModelGenerating, error]) - const checkForOverAsking = async (input: VerificationAnalysisInput) => { + const checkForOverAsking = async (input: OverAskingInput) => { setIsProcessingOverAsking(true) if (isModelReady) { - console.log('Model ready, using local LLM') - const prompt = formatPrompt(input) + console.debug('Local LLM ready, using local LLM') + const prompt = formatLocalPrompt(input) await generate(prompt) } else { - console.log('Local LLM not ready, using API') + console.debug('Local LLM not ready, using API') await analyzeVerificationApi(input) .then(setOverAskingResponse) .catch((e) => { - console.error('Error analyzing verification:', e) + console.error('Error analyzing verification using API:', e) setOverAskingResponse(fallbackResponse) }) .finally(() => setIsProcessingOverAsking(false)) @@ -66,9 +59,15 @@ export function useOverAskingAi() { } } -const formatResult = (response: string) => { +// AI responds in XML format, so we need to parse it +// Expected format: +// +// Your concise reason for the assessment +// yes +// +const formatLocalResult = (response: string) => { const match = response.match(/([\s\S]*?)<\/response>/) - if (!match) return + if (!match) return fallbackResponse const responseContent = match[1] @@ -85,13 +84,11 @@ const formatResult = (response: string) => { return fallbackResponse } -const formatPrompt = (input: VerificationAnalysisInput) => { +const formatLocalPrompt = (input: OverAskingInput) => { const cards = input.cards - .map( - (credential) => - `${credential.name} - ${credential.subtitle}. Requested attributes: ${credential.requestedAttributes.join(', ')}` - ) + .map((credential) => `- ${credential.name}. Requested attributes: ${credential.requestedAttributes.join(', ')}`) .join('\n') + return ` You are an AI assistant specializing in data privacy analysis. Your task is to evaluate data verification requests and determine if they are asking for an appropriate amount of information or if they are overasking. @@ -105,10 +102,6 @@ ${input.verifier.name} ${input.verifier.domain} - -${input.name} - - ${input.purpose} @@ -118,20 +111,13 @@ ${cards} -Provide a small evaluation of the request, and provide your final response in the following XML structure: +Provide a short reason for your assessment of the request. Use the following XML structure: Your concise reason for the assessment yes -Example of a properly formatted response: - - -Request aligns with purpose. Information amount appropriate. Verifier seems legitimate. -yes - - -Remember: Provide a concise reason and use the correct XML structure in your response. Do not add any text outside of the specified tags. +Remember: DO NOT add any text outside of the specified tags. DO NOT bother responding with anything other than the XML structure. ` } diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 3a8b1685..65e2fbe3 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -4,7 +4,7 @@ import { Platform } from 'react-native' import { LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER } from 'react-native-executorch' import { useMMKVBoolean } from 'react-native-mmkv' import RnExecutorch, { subscribeToDownloadProgress, subscribeToTokenGenerated } from './RnExecutorchModule' -import { DEFAULT_CONTEXT_WINDOW_LENGTH, DEFAULT_SYSTEM_PROMPT, EOT_TOKEN } from './constants' +import { DEFAULT_CONTEXT_WINDOW_LENGTH, EOT_TOKEN } from './constants' import type { Model, ResourceSource } from './types' const interrupt = () => { @@ -38,13 +38,9 @@ export function removeIsModelDownloading() { export const useLLM = ({ modelSource = LLAMA3_2_1B_QLORA_URL, tokenizerSource = LLAMA3_2_1B_TOKENIZER, - systemPrompt = DEFAULT_SYSTEM_PROMPT, - contextWindowLength = DEFAULT_CONTEXT_WINDOW_LENGTH, }: { modelSource?: ResourceSource tokenizerSource?: ResourceSource - systemPrompt?: string - contextWindowLength?: number } = {}): Model => { const [error, setError] = useState(null) const [isModelActivated, setIsModelActivated] = useIsModelActivated() @@ -55,6 +51,16 @@ export const useLLM = ({ const [downloadProgress, setDownloadProgress] = useState(0) const initialized = useRef(false) + useEffect(() => { + if (!response) return + console.debug('Local LLM response', response) + }, [response]) + + useEffect(() => { + if (!error) return + console.debug('Local LLM error', error) + }, [error]) + useEffect(() => { const unsubscribeDownloadProgress = subscribeToDownloadProgress((data) => { if (data) { @@ -77,7 +83,7 @@ export const useLLM = ({ try { try { setIsModelDownloading(true) - await RnExecutorch.loadLLM(modelSource, tokenizerSource, systemPrompt, contextWindowLength) + await RnExecutorch.loadLLM(modelSource, tokenizerSource, '', DEFAULT_CONTEXT_WINDOW_LENGTH) await RnExecutorch } catch (error) { console.log('ERROR LOADING MODEL', error) @@ -92,15 +98,7 @@ export const useLLM = ({ setError(message) initialized.current = false } - }, [ - contextWindowLength, - modelSource, - systemPrompt, - tokenizerSource, - setIsModelReady, - setIsModelActivated, - setIsModelDownloading, - ]) + }, [modelSource, tokenizerSource, setIsModelReady, setIsModelActivated, setIsModelDownloading]) const generate = useCallback( async (input: string): Promise => { diff --git a/apps/easypid/src/use-cases/ValidateVerification.ts b/apps/easypid/src/use-cases/OverAskingApi.ts similarity index 85% rename from apps/easypid/src/use-cases/ValidateVerification.ts rename to apps/easypid/src/use-cases/OverAskingApi.ts index e30834b1..0b1f07a7 100644 --- a/apps/easypid/src/use-cases/ValidateVerification.ts +++ b/apps/easypid/src/use-cases/OverAskingApi.ts @@ -2,7 +2,7 @@ const PLAYGROUND_URL = 'https://funke.animo.id' export const EXCLUDED_ATTRIBUTES_FOR_ANALYSIS = ['Issuing authority', 'Issuing country', 'Issued at', 'Expires at'] -export type VerificationAnalysisInput = { +export type OverAskingInput = { verifier: { name: string domain: string @@ -21,17 +21,12 @@ export type OverAskingResponse = { reason: string } -export type VerificationAnalysisResult = { - isLoading: boolean - result: OverAskingResponse | undefined -} - -export const analyzeVerification = async ({ +export const checkForOverAskingApi = async ({ verifier, name, purpose, cards, -}: VerificationAnalysisInput): Promise => { +}: OverAskingInput): Promise => { try { const cardsWithoutExcludedAttributes = cards.map((card) => ({ ...card, From 02fccbf7915c1eb7277f4feeb54c940c9dfbda78 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 3 Dec 2024 15:42:59 +0100 Subject: [PATCH 04/12] chore: refactor --- apps/easypid/src/app/_layout.tsx | 2 +- .../menu/components/LocalAiContainer.tsx | 2 +- apps/easypid/src/hooks/useOverAskingAi.tsx | 2 +- apps/easypid/src/llm/index.ts | 3 ++ apps/easypid/src/llm/state.ts | 27 +++++++++++++++ apps/easypid/src/llm/useLLM.tsx | 34 +++++-------------- 6 files changed, 41 insertions(+), 29 deletions(-) create mode 100644 apps/easypid/src/llm/index.ts create mode 100644 apps/easypid/src/llm/state.ts diff --git a/apps/easypid/src/app/_layout.tsx b/apps/easypid/src/app/_layout.tsx index 9480c6d2..da05c8dc 100644 --- a/apps/easypid/src/app/_layout.tsx +++ b/apps/easypid/src/app/_layout.tsx @@ -6,7 +6,7 @@ import { DefaultTheme, ThemeProvider } from '@react-navigation/native' import { Slot } from 'expo-router' import * as SplashScreen from 'expo-splash-screen' -import { useCheckIncompleteDownload } from '@easypid/llm/useLLM' +import { useCheckIncompleteDownload } from '@easypid/llm' import tamaguiConfig from '../../tamagui.config' void SplashScreen.preventAutoHideAsync() diff --git a/apps/easypid/src/features/menu/components/LocalAiContainer.tsx b/apps/easypid/src/features/menu/components/LocalAiContainer.tsx index a23f01e5..0eac525b 100644 --- a/apps/easypid/src/features/menu/components/LocalAiContainer.tsx +++ b/apps/easypid/src/features/menu/components/LocalAiContainer.tsx @@ -2,7 +2,7 @@ import { HeroIcons } from '@package/ui/src/content/Icon' import { Switch } from '@package/ui/src/base/Switch' -import { useIsDeviceCapable, useLLM } from '@easypid/llm/useLLM' +import { useIsDeviceCapable, useLLM } from '@easypid/llm' import { ConfirmationSheet } from '@package/app/src/components/ConfirmationSheet' import { useHasInternetConnection, useIsConnectedToWifi } from 'packages/app/src/hooks' import { useToastController } from 'packages/ui/src' diff --git a/apps/easypid/src/hooks/useOverAskingAi.tsx b/apps/easypid/src/hooks/useOverAskingAi.tsx index e8feb25b..825a3708 100644 --- a/apps/easypid/src/hooks/useOverAskingAi.tsx +++ b/apps/easypid/src/hooks/useOverAskingAi.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from 'react' -import { useLLM } from '@easypid/llm/useLLM' +import { useLLM } from '@easypid/llm' import type { OverAskingInput, OverAskingResponse } from '@easypid/use-cases/OverAskingApi' import { checkForOverAskingApi as analyzeVerificationApi } from '@easypid/use-cases/OverAskingApi' diff --git a/apps/easypid/src/llm/index.ts b/apps/easypid/src/llm/index.ts new file mode 100644 index 00000000..6684c967 --- /dev/null +++ b/apps/easypid/src/llm/index.ts @@ -0,0 +1,3 @@ +export * from './useLLM' +export * from './state' +export * from './types' diff --git a/apps/easypid/src/llm/state.ts b/apps/easypid/src/llm/state.ts new file mode 100644 index 00000000..39ef5cf2 --- /dev/null +++ b/apps/easypid/src/llm/state.ts @@ -0,0 +1,27 @@ +import { useMMKVBoolean } from 'react-native-mmkv' + +import { mmkv } from '@easypid/storage/mmkv' + +export function useIsModelReady() { + return useMMKVBoolean('isModelReady', mmkv) +} + +export function removeIsModelReady() { + mmkv.delete('isModelReady') +} + +export function useIsModelActivated() { + return useMMKVBoolean('isModelActivated', mmkv) +} + +export function removeIsModelActivated() { + mmkv.delete('isModelActivated') +} + +export function useIsModelDownloading() { + return useMMKVBoolean('isModelDownloading', mmkv) +} + +export function removeIsModelDownloading() { + mmkv.delete('isModelDownloading') +} diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 65e2fbe3..3e1312da 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -1,40 +1,22 @@ -import { mmkv } from '@easypid/storage/mmkv' import { useCallback, useEffect, useRef, useState } from 'react' import { Platform } from 'react-native' import { LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER } from 'react-native-executorch' -import { useMMKVBoolean } from 'react-native-mmkv' import RnExecutorch, { subscribeToDownloadProgress, subscribeToTokenGenerated } from './RnExecutorchModule' import { DEFAULT_CONTEXT_WINDOW_LENGTH, EOT_TOKEN } from './constants' +import { + removeIsModelActivated, + removeIsModelDownloading, + removeIsModelReady, + useIsModelActivated, + useIsModelDownloading, + useIsModelReady, +} from './state' import type { Model, ResourceSource } from './types' const interrupt = () => { RnExecutorch.interrupt() } -export function useIsModelReady() { - return useMMKVBoolean('isModelReady', mmkv) -} - -export function removeIsModelReady() { - mmkv.delete('isModelReady') -} - -export function useIsModelActivated() { - return useMMKVBoolean('isModelActivated', mmkv) -} - -export function removeIsModelActivated() { - mmkv.delete('isModelActivated') -} - -export function useIsModelDownloading() { - return useMMKVBoolean('isModelDownloading', mmkv) -} - -export function removeIsModelDownloading() { - mmkv.delete('isModelDownloading') -} - export const useLLM = ({ modelSource = LLAMA3_2_1B_QLORA_URL, tokenizerSource = LLAMA3_2_1B_TOKENIZER, From 050d209efccb5ca9055d021657aa80f398ba7025 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 3 Dec 2024 15:43:58 +0100 Subject: [PATCH 05/12] fix: types --- .../src/features/share/FunkePresentationNotificationScreen.tsx | 2 +- .../easypid/src/features/share/slides/ShareCredentialsSlide.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx index 2d9b22a2..ba3aec92 100644 --- a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx +++ b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx @@ -1,6 +1,6 @@ import type { DisplayImage, FormattedSubmission, TrustedEntity } from '@package/agent' -import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/OverAskingApi' +import type { OverAskingResponse } from '@easypid/use-cases/OverAskingApi' import { type SlideStep, SlideWizard } from '@package/app' import { LoadingRequestSlide } from '../receive/slides/LoadingRequestSlide' import { VerifyPartySlide } from '../receive/slides/VerifyPartySlide' diff --git a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx index 3a7282a6..a48c13d6 100644 --- a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx +++ b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx @@ -1,4 +1,4 @@ -import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/OverAskingApi' +import type { OverAskingResponse } from '@easypid/use-cases/OverAskingApi' import type { DisplayImage, FormattedSubmission } from '@package/agent' import { DualResponseButtons, usePushToWallet, useScrollViewPosition } from '@package/app' import { useWizard } from '@package/app' From 6f306ef6d7fadc226b905eb5828fc0c878890934 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 09:33:43 +0100 Subject: [PATCH 06/12] feat: added expo device info --- apps/easypid/package.json | 1 + .../components/RequestPurposeSection.tsx | 3 ++- apps/easypid/src/llm/useLLM.tsx | 15 ++++++++++----- pnpm-lock.yaml | 19 +++++++++++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/apps/easypid/package.json b/apps/easypid/package.json index 576d7d47..355fbefb 100644 --- a/apps/easypid/package.json +++ b/apps/easypid/package.json @@ -38,6 +38,7 @@ "expo-blur": "^13.0.2", "expo-constants": "~16.0.2", "expo-dev-client": "~4.0.16", + "expo-device": "~6.0.2", "expo-font": "~12.0.7", "expo-haptics": "~13.0.1", "expo-image": "~1.13.0", diff --git a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx index cb0d2005..4c8fce8b 100644 --- a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx +++ b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx @@ -14,6 +14,7 @@ import { useScaleAnimation, } from '@package/ui' import type { DisplayImage } from 'packages/agent/src' +import { isAndroid } from 'packages/app/src' import { useState } from 'react' import React from 'react' import { FadeIn, ZoomIn } from 'react-native-reanimated' @@ -41,7 +42,7 @@ export function RequestPurposeSection({ purpose, logo, overAskingResponse }: Req onPressIn={handlePressIn} onPressOut={handlePressOut} onPress={toggleAnalysisModal} - mt="$-2" + mt={isAndroid() ? '$0' : '$-2'} mb="$4" > = MIN_REQUIRED_RAM } return false diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c4c8d7bc..c8439302 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,6 +174,9 @@ importers: expo-dev-client: specifier: ~4.0.16 version: 4.0.28(expo@51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))) + expo-device: + specifier: ~6.0.2 + version: 6.0.2(expo@51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))) expo-font: specifier: ~12.0.7 version: 12.0.10(expo@51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))) @@ -6017,6 +6020,11 @@ packages: peerDependencies: expo: '*' + expo-device@6.0.2: + resolution: {integrity: sha512-sCt91CuTmAuMXX4SlFOn4lIos2UIr8vb0jDstDDZXys6kErcj0uynC7bQAMreU5uRUTKMAl4MAMpKt9ufCXPBw==} + peerDependencies: + expo: '*' + expo-eas-client@0.12.0: resolution: {integrity: sha512-Jkww9Cwpv0z7DdLYiRX0r4fqBEcI9cKqTn7cHx63S09JaZ2rcwEE4zYHgrXwjahO+tU2VW8zqH+AJl6RhhW4zA==} @@ -9395,6 +9403,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + ua-parser-js@0.7.39: + resolution: {integrity: sha512-IZ6acm6RhQHNibSt7+c09hhvsKy9WUr4DVbeq9U8o71qxyYtJpQeDxQnMrVqnIFMLcQjHO0I9wgfO2vIahht4w==} + hasBin: true + ua-parser-js@1.0.39: resolution: {integrity: sha512-k24RCVWlEcjkdOxYmVJgeD/0a1TiSpqLg+ZalVGV9lsnr4yqu0w7tX/x2xX6G4zpkgQnRf89lxuZ1wsbjXM8lw==} hasBin: true @@ -17605,6 +17617,11 @@ snapshots: expo-dev-menu-interface: 1.8.3(expo@51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))) semver: 7.6.3 + expo-device@6.0.2(expo@51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))): + dependencies: + expo: 51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0)) + ua-parser-js: 0.7.39 + expo-eas-client@0.12.0: {} expo-file-system@17.0.1(expo@51.0.39(@babel/core@7.26.0)(@babel/preset-env@7.26.0(@babel/core@7.26.0))): @@ -21723,6 +21740,8 @@ snapshots: typescript@5.3.3: {} + ua-parser-js@0.7.39: {} + ua-parser-js@1.0.39: {} uc.micro@1.0.6: {} From 592e784af5a2827714ad6ec809d02fecbf4cb721 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 12:24:49 +0100 Subject: [PATCH 07/12] feat: performance improvements --- apps/easypid/app.config.js | 3 + .../src/features/menu/FunkeSettingsScreen.tsx | 56 +++++------- .../FunkeCredentialNotificationScreen.tsx | 2 + ...keOpenIdPresentationNotificationScreen.tsx | 8 +- .../share/slides/ShareCredentialsSlide.tsx | 4 +- apps/easypid/src/hooks/useOverAskingAi.tsx | 69 +++++++-------- apps/easypid/src/llm/constants.ts | 4 - apps/easypid/src/llm/prompt.ts | 87 +++++++++++++++++++ apps/easypid/src/llm/useLLM.tsx | 18 ++-- 9 files changed, 161 insertions(+), 90 deletions(-) create mode 100644 apps/easypid/src/llm/prompt.ts diff --git a/apps/easypid/app.config.js b/apps/easypid/app.config.js index f1f10dc2..fea3fb7d 100644 --- a/apps/easypid/app.config.js +++ b/apps/easypid/app.config.js @@ -148,6 +148,9 @@ const config = { })) ), ], + config: { + largeHeap: true, + }, }, experiments: { tsconfigPaths: true, diff --git a/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx b/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx index 9665b710..a79caaa5 100644 --- a/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx +++ b/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx @@ -1,46 +1,38 @@ -import { Button, FlexPage, Heading, HeroIcons, ScrollView, Stack, YStack } from '@package/ui' +import { FlexPage, Heading, ScrollView, Stack, YStack } from '@package/ui' import React from 'react' -import { useRouter } from 'solito/router' import { useScrollViewPosition } from '@package/app/src/hooks' +import { TextBackButton } from 'packages/app/src' import { LocalAiContainer } from './components/LocalAiContainer' export function FunkeSettingsScreen() { const { handleScroll, isScrolledByOffset, scrollEventThrottle } = useScrollViewPosition() - const router = useRouter() return ( - <> - - - - - - Settings - - + + + + + + Settings + - - - - - + + + + + + - router.back()}> - Back - + + - - - + + + ) } diff --git a/apps/easypid/src/features/receive/FunkeCredentialNotificationScreen.tsx b/apps/easypid/src/features/receive/FunkeCredentialNotificationScreen.tsx index 7c2f8e6b..bdd4b628 100644 --- a/apps/easypid/src/features/receive/FunkeCredentialNotificationScreen.tsx +++ b/apps/easypid/src/features/receive/FunkeCredentialNotificationScreen.tsx @@ -418,6 +418,8 @@ export function FunkeCredentialNotificationScreen() { logo={credentialsForRequest.verifier.logo} submission={credentialsForRequest.formattedSubmission} isAccepting={isSharingPresentation} + // Not supported for this flow atm + overAskingResponse={{ validRequest: 'could_not_determine', reason: '' }} /> ), } diff --git a/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx b/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx index 6fadb5d0..2c40a8aa 100644 --- a/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx +++ b/apps/easypid/src/features/share/FunkeOpenIdPresentationNotificationScreen.tsx @@ -57,10 +57,10 @@ export function FunkeOpenIdPresentationNotificationScreen() { }) }, [credentialsForRequest, params.data, params.uri, toast.show, agent, pushToWallet, toast]) - const { checkForOverAsking, isProcessingOverAsking, overAskingResponse } = useOverAskingAi() + const { checkForOverAsking, isProcessingOverAsking, overAskingResponse, stopOverAsking } = useOverAskingAi() useEffect(() => { - if (!credentialsForRequest?.formattedSubmission) { + if (!credentialsForRequest?.formattedSubmission || !credentialsForRequest?.formattedSubmission.areAllSatisfied) { return } @@ -99,6 +99,7 @@ export function FunkeOpenIdPresentationNotificationScreen() { }, } + stopOverAsking() setIsSharing(true) if (shouldUsePin) { @@ -174,10 +175,11 @@ export function FunkeOpenIdPresentationNotificationScreen() { } } }, - [credentialsForRequest, agent, shouldUsePin] + [credentialsForRequest, agent, shouldUsePin, stopOverAsking] ) const onProofDecline = async () => { + stopOverAsking() if (credentialsForRequest) { await addSharedActivityForCredentialsForRequest( agent, diff --git a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx index a48c13d6..e053f9a6 100644 --- a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx +++ b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx @@ -88,7 +88,9 @@ export const ShareCredentialsSlide = ({ purpose={ submission.purpose ?? 'No information was provided on the purpose of the data request. Be cautious' } - overAskingResponse={overAskingResponse} + overAskingResponse={ + submission.areAllSatisfied ? overAskingResponse : { validRequest: 'could_not_determine', reason: '' } + } logo={logo} /> )} diff --git a/apps/easypid/src/hooks/useOverAskingAi.tsx b/apps/easypid/src/hooks/useOverAskingAi.tsx index 825a3708..bd73732c 100644 --- a/apps/easypid/src/hooks/useOverAskingAi.tsx +++ b/apps/easypid/src/hooks/useOverAskingAi.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from 'react' import { useLLM } from '@easypid/llm' import type { OverAskingInput, OverAskingResponse } from '@easypid/use-cases/OverAskingApi' -import { checkForOverAskingApi as analyzeVerificationApi } from '@easypid/use-cases/OverAskingApi' +import { EXCLUDED_ATTRIBUTES_FOR_ANALYSIS, checkForOverAskingApi } from '@easypid/use-cases/OverAskingApi' const fallbackResponse: OverAskingResponse = { validRequest: 'could_not_determine', @@ -13,7 +13,7 @@ export function useOverAskingAi() { const [isProcessingOverAsking, setIsProcessingOverAsking] = useState(false) const [overAskingResponse, setOverAskingResponse] = useState() - const { generate, response, error, isModelReady, isModelGenerating } = useLLM() + const { generate, response, error, isModelReady, isModelGenerating, interrupt } = useLLM() useEffect(() => { if (error) { @@ -42,7 +42,7 @@ export function useOverAskingAi() { await generate(prompt) } else { console.debug('Local LLM not ready, using API') - await analyzeVerificationApi(input) + await checkForOverAskingApi(input) .then(setOverAskingResponse) .catch((e) => { console.error('Error analyzing verification using API:', e) @@ -52,19 +52,20 @@ export function useOverAskingAi() { } } + const stopOverAsking = () => { + if (isModelReady) interrupt() + if (!overAskingResponse) setOverAskingResponse(fallbackResponse) + setIsProcessingOverAsking(false) + } + return { isProcessingOverAsking, checkForOverAsking, overAskingResponse, + stopOverAsking, } } -// AI responds in XML format, so we need to parse it -// Expected format: -// -// Your concise reason for the assessment -// yes -// const formatLocalResult = (response: string) => { const match = response.match(/([\s\S]*?)<\/response>/) if (!match) return fallbackResponse @@ -86,38 +87,28 @@ const formatLocalResult = (response: string) => { const formatLocalPrompt = (input: OverAskingInput) => { const cards = input.cards - .map((credential) => `- ${credential.name}. Requested attributes: ${credential.requestedAttributes.join(', ')}`) + .map( + (credential) => ` + + ${credential.name} + + ${credential.requestedAttributes + .filter((attr) => !EXCLUDED_ATTRIBUTES_FOR_ANALYSIS.includes(attr)) + .map((attr) => `${attr}`) + .join('\n ')} + + ` + ) .join('\n') return ` -You are an AI assistant specializing in data privacy analysis. Your task is to evaluate data verification requests and determine if they are asking for an appropriate amount of information or if they are overasking. - -Here is the information for the current request: - - -${input.verifier.name} - - - -${input.verifier.domain} - - - -${input.purpose} - - - -${cards} - - - -Provide a short reason for your assessment of the request. Use the following XML structure: - - -Your concise reason for the assessment -yes - - -Remember: DO NOT add any text outside of the specified tags. DO NOT bother responding with anything other than the XML structure. + + ${input.verifier.name} + ${input.verifier.domain} + ${input.purpose} + + ${cards} + + ` } diff --git a/apps/easypid/src/llm/constants.ts b/apps/easypid/src/llm/constants.ts index decd3c97..86bab8a3 100644 --- a/apps/easypid/src/llm/constants.ts +++ b/apps/easypid/src/llm/constants.ts @@ -1,5 +1 @@ -export const DEFAULT_SYSTEM_PROMPT = - "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text." - -export const DEFAULT_CONTEXT_WINDOW_LENGTH = 3 export const EOT_TOKEN = '<|eot_id|>' diff --git a/apps/easypid/src/llm/prompt.ts b/apps/easypid/src/llm/prompt.ts new file mode 100644 index 00000000..f5f585ac --- /dev/null +++ b/apps/easypid/src/llm/prompt.ts @@ -0,0 +1,87 @@ +export const OVERASKING_PROMPT = ` +You are an AI assistant specializing in data privacy analysis. Your task is to evaluate data verification requests and determine if they are asking for an appropriate amount of information or if they are overasking. + +=== INFORMATION AVAILABLE === + +You will be provided with the following information: + +- Verifier name: the name of the requesting party +- Verifier domain: the domain of the requesting party +- Request purpose: the purpose of the verification request; why is the verifier requesting this information? +- Cards and requested attributes: the specific cards and requested attributes per card that the verifier is requesting. + +Based on this information, you should determine if the request matches the verifier and the purpose, or if the verifier is overasking for information. Focus only on personal information that could be sensitive. Overasking of metadata related to the card is not a reason to reject the request. + +=== OUTPUT === + +Your output should consist of two parts: + +1. Reason: Use 10-20 words describing why the request is overasking or not. Use specifics from the request to justify your answer. +2. Valid: Your final verdict which can be 'yes', 'no' or 'could_not_determine'. + +Your response should be formatted in XML, as shown below: + + +Your concise reason for the assessment +yes + + +This output structure is VERY important and should be followed exactly. It will be parsed by the app, so make sure it's correct. DO NOT include any other text than the XML tags and content specified above. + + +=== EXAMPLES === + +=== EXAMPLE 1 === + + + HealthCare Plus + healthcareplus.med + Medical appointment scheduling and insurance verification + + + Insurance Card + + policy_number + expiration_date + + + + + + + Request aligns with medical purpose, asking only for relevant insurance information needed for appointment scheduling. + yes + + + +=== EXAMPLE 2 === + + + OnlineShop + onlineshop.com + Shipping a purchased item + + + Personalausweis + + full_name + address + date_of_birth + portrait + + + + + + + Online shop requesting a portrait photo for simple shipping is excessive and unnecessary for stated purpose. + no + + + + +====== GUIDELINES ====== + +Return ONLY the XML ... tags and content specified above. DO NOT repeat the input or any other text. + +` diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 02b95982..8c9973a3 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -3,7 +3,8 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { Platform } from 'react-native' import { LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER } from 'react-native-executorch' import RnExecutorch, { subscribeToDownloadProgress, subscribeToTokenGenerated } from './RnExecutorchModule' -import { DEFAULT_CONTEXT_WINDOW_LENGTH, EOT_TOKEN } from './constants' +import { EOT_TOKEN } from './constants' +import { OVERASKING_PROMPT } from './prompt' import { removeIsModelActivated, removeIsModelDownloading, @@ -12,19 +13,14 @@ import { useIsModelDownloading, useIsModelReady, } from './state' -import type { Model, ResourceSource } from './types' +import type { Model } from './types' const interrupt = () => { RnExecutorch.interrupt() } -export const useLLM = ({ - modelSource = LLAMA3_2_1B_QLORA_URL, - tokenizerSource = LLAMA3_2_1B_TOKENIZER, -}: { - modelSource?: ResourceSource - tokenizerSource?: ResourceSource -} = {}): Model => { +// FIXME: The model expects a system prompt on initializing, but this blocks it from being used for different tasks. +export const useLLM = (): Model => { const [error, setError] = useState(null) const [isModelActivated, setIsModelActivated] = useIsModelActivated() const [isModelDownloading, setIsModelDownloading] = useIsModelDownloading() @@ -66,7 +62,7 @@ export const useLLM = ({ try { try { setIsModelDownloading(true) - await RnExecutorch.loadLLM(modelSource, tokenizerSource, '', DEFAULT_CONTEXT_WINDOW_LENGTH) + await RnExecutorch.loadLLM(LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER, OVERASKING_PROMPT, 2) await RnExecutorch } catch (error) { console.log('ERROR LOADING MODEL', error) @@ -81,7 +77,7 @@ export const useLLM = ({ setError(message) initialized.current = false } - }, [modelSource, tokenizerSource, setIsModelReady, setIsModelActivated, setIsModelDownloading]) + }, [setIsModelReady, setIsModelActivated, setIsModelDownloading]) const generate = useCallback( async (input: string): Promise => { From 06b15ecb0996d1ab290b3d5529eb84f41b7cadde Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 14:12:56 +0100 Subject: [PATCH 08/12] chore: make error scrollable --- .../src/features/menu/FunkeSettingsScreen.tsx | 6 ++--- .../receive/slides/CredentialErrorSlide.tsx | 23 +++++++++++-------- packages/ui/src/content/Icon.tsx | 2 ++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx b/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx index 743f5ed5..ee8210dc 100644 --- a/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx +++ b/apps/easypid/src/features/menu/FunkeSettingsScreen.tsx @@ -1,4 +1,4 @@ -import { FlexPage, Heading, ScrollView, Stack, Switch, YStack } from '@package/ui' +import { FlexPage, Heading, HeroIcons, ScrollView, Stack, Switch, YStack } from '@package/ui' import React from 'react' import { TextBackButton } from 'packages/app/src' @@ -28,15 +28,15 @@ export function FunkeSettingsScreen() { > - } value={isDevelopmentModeEnabled ?? false} onChange={setIsDevelopmentModeEnabled} /> + - diff --git a/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx b/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx index ad6df95d..9cf21af9 100644 --- a/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx +++ b/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx @@ -1,4 +1,5 @@ -import { Button, Heading, HeroIcons, Paragraph, Stack, XStack, YStack } from '@package/ui' +import { Button, Heading, HeroIcons, Paragraph, ScrollView, Stack, XStack, YStack } from '@package/ui' +import { useState } from 'react' interface CredentialErrorSlideProps { reason?: string @@ -6,10 +7,12 @@ interface CredentialErrorSlideProps { } export const CredentialErrorSlide = ({ reason, onCancel }: CredentialErrorSlideProps) => { + const [scrollViewHeight, setScrollViewHeight] = useState(0) + return ( - - + setScrollViewHeight(event.nativeEvent.layout.height)}> + Something went wrong @@ -22,13 +25,15 @@ export const CredentialErrorSlide = ({ reason, onCancel }: CredentialErrorSlideP again later. - {reason && ( - - Reason: - {reason} - + {reason && scrollViewHeight !== 0 && ( + + + Reason: + {reason} + + )} - + diff --git a/packages/ui/src/content/Icon.tsx b/packages/ui/src/content/Icon.tsx index f451c5bf..f8358a1b 100644 --- a/packages/ui/src/content/Icon.tsx +++ b/packages/ui/src/content/Icon.tsx @@ -64,6 +64,7 @@ import { CircleStackIcon as CircleStackFilledIcon, ClockIcon as ClockFilledIcon, Cog8ToothIcon as Cog8ToothFilledIcon, + CommandLineIcon, CpuChipIcon as CpuChipFilledIcon, CreditCardIcon as CreditCardFilledIcon, ExclamationCircleIcon as ExclamationCircleFilledIcon, @@ -177,6 +178,7 @@ export const HeroIcons = { Link: wrapHeroIcon(LinkIcon), Cloud: wrapHeroIcon(CloudIcon), CpuChipFilled: wrapHeroIcon(CpuChipFilledIcon), + CommandLineFilled: wrapHeroIcon(CommandLineIcon), } as const export const CustomIcons = { From 87902fc0406905f7e554895616d8a75dc921cda9 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 14:13:59 +0100 Subject: [PATCH 09/12] chore: spacing --- .../src/features/receive/slides/CredentialErrorSlide.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx b/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx index 9cf21af9..6b1c88b1 100644 --- a/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx +++ b/apps/easypid/src/features/receive/slides/CredentialErrorSlide.tsx @@ -12,7 +12,7 @@ export const CredentialErrorSlide = ({ reason, onCancel }: CredentialErrorSlideP return ( setScrollViewHeight(event.nativeEvent.layout.height)}> - + Something went wrong From 54b694f081927e435cb7b07ce12c9aa4aebc1985 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 14:14:21 +0100 Subject: [PATCH 10/12] chore: comment --- apps/easypid/src/llm/useLLM.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 8c9973a3..8289585d 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -89,7 +89,7 @@ export const useLLM = (): Model => { } try { - setResponse('') // This might be causing issues - let's move it + setResponse('') setIsModelGenerating(true) await RnExecutorch.runInference(input) } catch (err) { From a5d285c040c4d6dab53640ed7f596f3aaa7f6a6a Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 14:37:38 +0100 Subject: [PATCH 11/12] fix: stricter reqs --- apps/easypid/src/llm/useLLM.tsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 8289585d..74445074 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -159,19 +159,21 @@ export function useCheckIncompleteDownload() { } export function useIsDeviceCapable(): boolean { - // For iOS, check if device is at least iPhone 12 or newer + // For iOS, check if device is at least iPhone 12 or newer and iOS 17.0 or newer if (Platform.OS === 'ios') { const modelId = Device.modelId ?? '' + const systemVersion = Number.parseFloat(Device.osVersion ?? '0') // iPhone 12 series and newer start with iPhone13 (iPhone12 = iPhone13,2) - return /iPhone1[3-9]/.test(modelId) || /iPhone[2-9][0-9]/.test(modelId) + const isSupportedModel = /iPhone1[3-9]/.test(modelId) || /iPhone[2-9][0-9]/.test(modelId) + return isSupportedModel && systemVersion >= 17.0 } - // For Android, check for minimum 4GB RAM + // For Android, check for minimum 4GB RAM and Android 13 or newer if (Platform.OS === 'android') { const totalMemory = Device.totalMemory ?? 0 - // totalMemory is in bytes, convert 4GB to bytes (4 * 1024 * 1024 * 1024) const MIN_REQUIRED_RAM = 4 * 1024 * 1024 * 1024 - return totalMemory >= MIN_REQUIRED_RAM + const systemVersion = Number.parseInt(Device.osVersion ?? '0', 10) + return totalMemory >= MIN_REQUIRED_RAM && systemVersion >= 13 } return false From 281faf5f81a9619a672774f1f23b709854540147 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 4 Dec 2024 14:39:49 +0100 Subject: [PATCH 12/12] chore: changelog entry --- apps/easypid/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/easypid/README.md b/apps/easypid/README.md index 39401d04..89d0a779 100644 --- a/apps/easypid/README.md +++ b/apps/easypid/README.md @@ -388,6 +388,9 @@ The following standards and specifications were implemented. - Fixed an issue where the PIN screen would get stuck in a loading state when an incorrect PIN was entered [commit](https://github.com/animo/paradym-wallet/commit/0f65ef98f5f26c3afc0968e4f848bf538a86cfd7) - Fixed an issue with redirect based auth flow if the authorization flow left the in-app browser (e.g. when requiring authentication using the native AusweisApp with the eID card) [commit](https://github.com/animo/paradym-wallet/commit/eb333b81fe5662cc2f010e1ee9bbdc83a7e19aa3) - Fixed an issue where the PID setup would get stuck if you skipped it during onboarding [commit](https://github.com/animo/openid4vc-playground-funke/commit/65178e776bc421b9ca413542ea0e86db4ad1ead4) +- Added support for on-device local AI model for oversharing detection on higher-end devices (can be enabled in the settings) [commit](https://github.com/animo/paradym-wallet/commit) + + #### 28-11-2024