diff --git a/.gitignore b/.gitignore index 3efc9d887..40f731fe8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,5 @@ /docs/Manifest.toml /docs/build/ -/.DS_Store # macOS folder metadata -/.vscode \ No newline at end of file +**/.DS_Store +**/.vscode \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..9f2cfffe9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Added +- Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). \ No newline at end of file diff --git a/Project.toml b/Project.toml index 0454efa99..18294e9af 100644 --- a/Project.toml +++ b/Project.toml @@ -7,11 +7,15 @@ version = "0.2.0-DEV" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" +PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" [compat] +Aqua = "0.7" HTTP = "1" JSON3 = "1" OpenAI = "0.8.7" +PrecompileTools = "1" +Test = "<0.0.1, 1" julia = "1.9,1.10" [extras] diff --git a/README.md b/README.md index 9ada7e181..62342a48c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ ai"What is the capital of \$(country)?" # AIMessage("The capital of Spain is Madrid.") ``` -Pro tip: Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4`. Great for those extra hard questions! +Pro tip: Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4` (use `gpt4t` for the new GPT-4 Turbo model). Great for those extra hard questions! For more complex prompt templates, you can use handlebars-style templating and provide variables as keyword arguments: @@ -50,6 +50,8 @@ Pro tip: Use `asyncmap` to run multiple AI-powered tasks concurrently. Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` +For more practical examples, see the `examples/` folder and the [Advanced Examples](#advanced-examples) section below. + ## Table of Contents - [PromptingTools.jl: "Your Daily Dose of AI Efficiency."](#promptingtoolsjl-your-daily-dose-of-ai-efficiency) @@ -57,7 +59,9 @@ Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_ - [Table of Contents](#table-of-contents) - [Why PromptingTools.jl](#why-promptingtoolsjl) - [Advanced Examples](#advanced-examples) + - [Seamless Integration Into Your Workflow](#seamless-integration-into-your-workflow) - [Advanced Prompts / Conversations](#advanced-prompts--conversations) + - [Templated Prompts](#templated-prompts) - [Asynchronous Execution](#asynchronous-execution) - [Model Aliases](#model-aliases) - [Embeddings](#embeddings) @@ -90,11 +94,32 @@ Some features: ## Advanced Examples -TODO: +TODOs: + +- [ ] Add more practical examples (with DataFrames!) +- [ ] Add mini tasks with structured extraction +- [ ] Add an example of how to build a RAG app in 50 lines + +### Seamless Integration Into Your Workflow +Google search is great, but it's a context switch. You often have to open a few pages and read through the discussion to find the answer you need. Same with the ChatGPT website. + +Imagine you are in VSCode, editing your `.gitignore` file. How do I ignore a file in all subfolders again? -[ ] Add more practical examples (DataFrames!) -[ ] Add mini tasks with structured extraction -[ ] Add an example of how to build a RAG app in 50 lines +All you need to do is to type: +`aai"What to write in .gitignore to ignore file XYZ in any folder or subfolder?"` + +With `aai""` (as opposed to `ai""`), we make a non-blocking call to the LLM to not prevent you from continuing your work. When the answer is ready, we log it from the background: + +> [ Info: Tokens: 102 @ Cost: $0.0002 in 2.7 seconds +> ┌ Info: AIMessage> To ignore a file called "XYZ" in any folder or subfolder, you can add the following line to your .gitignore file: +> │ +> │ ``` +> │ **/XYZ +> │ ``` +> │ +> └ This pattern uses the double asterisk (`**`) to match any folder or subfolder, and then specifies the name of the file you want to ignore. + +You probably saved 3-5 minutes on this task and probably another 5-10 minutes, because of the context switch/distraction you avoided. It's a small win, but it adds up quickly. ### Advanced Prompts / Conversations @@ -126,6 +151,59 @@ aigenerate(new_conversation; object = "old iPhone") ``` > AIMessage("Hmm, possess an old iPhone, I do not. But experience with attachments, I have. Detachment, I learned. True power and freedom, it brings...") +### Templated Prompts + +With LLMs, the quality / robustness of your results depends on the quality of your prompts. But writing prompts is hard! That's why we offer a templating system to save you time and effort. + +To use a specific template (eg, `` to ask a Julia language): +```julia +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +``` + +The above is equivalent to a more verbose version that explicitly uses the dispatch on `AITemplate`: +```julia +msg = aigenerate(AITemplate(:JuliaExpertAsk); ask = "How do I add packages?") +``` + +Find available templates with `aitemplates`: +```julia +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). + +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay +``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! + +You can inspect any template by "rendering" it (this is what the LLM will see): +```julia +julia> AITemplate(:JudgeIsItTrue) |> PromptingTools.render +``` + +See more examples in the [examples/](examples/) folder. + ### Asynchronous Execution You can leverage `asyncmap` to run multiple AI-powered tasks concurrently, improving performance for batch operations. @@ -183,14 +261,16 @@ aiclassify("Is two plus two four?") System prompts and higher-quality models can be used for more complex tasks, including knowing when to defer to a human: ```julia -aiclassify(:IsStatementTrue; statement = "Is two plus three a vegetable on Mars?", model = "gpt4") +aiclassify(:JudgeIsItTrue; it = "Is two plus three a vegetable on Mars?", model = "gpt4t") # unknown ``` -In the above example, we used a prompt template `:IsStatementTrue`, which automatically expands into the following system prompt (and a separate user prompt): +In the above example, we used a prompt template `:JudgeIsItTrue`, which automatically expands into the following system prompt (and a separate user prompt): > "You are an impartial AI judge evaluating whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide." +For more information on templates, see the [Templated Prompts](#templated-prompts) section. + ### Data Extraction !!! Experimental diff --git a/examples/working_with_aitemplates.jl b/examples/working_with_aitemplates.jl new file mode 100644 index 000000000..520f3533b --- /dev/null +++ b/examples/working_with_aitemplates.jl @@ -0,0 +1,73 @@ +# This file contains examples of how to work with AITemplate(s). + +using PromptingTools +const PT = PromptingTools + +# LLM responses are only as good as the prompts you give them. However, great prompts take long time to write -- AITemplate are a way to re-use great prompts! +# +# AITemplates are just a collection of templated prompts (ie, set of "messages" that have placeholders like {{question}}) +# +# They are saved as JSON files in the `templates` directory. +# They are automatically loaded on package import, but you can always force a re-load with `PT.load_templates!()` +PT.load_templates!(); + +# You can (create them) and use them for any ai* function instead of a prompt: +# Let's use a template called :JuliaExpertAsk +# alternatively, you can use `AITemplate(:JuliaExpertAsk)` for cleaner dispatch +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +# ... some response from GPT3.5 +# +# You can see that it had a placeholder for the actual question (`ask`) that we provided as a keyword argument. +# We did not have to write any system prompt for personas, tone, etc. -- it was all provided by the template! +# +# How to know which templates are available? You can search for them with `aitemplates()`: +# You can search by Symbol (only for partial name match), String (partial match on name or description), or Regex (more fields) +tmps = aitemplates("JuliaExpertAsk") +# Outputs a list of available templates that match the search -- there is just one in this case: +# +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +# +# You see not just the description, but also a preview of the actual prompts, placeholders available, and the length (to gauge how much it would cost). +# +# If you use VSCode, you can display them in a nice scrollable table with `vscodedisplay`: +using DataFrames +DataFrame(tmp) |> vscodedisplay +# +# +# You can also just `render` the template to see the underlying mesages: +msgs = PT.render(AITemplate(:JuliaExpertAsk)) +# +# 2-element Vector{PromptingTools.AbstractChatMessage}: +# PromptingTools.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") +# PromptingTools.UserMessage("# Question\n\n{{ask}}") +# +# Now, you know exactly what's in the template! +# +# If you want to modify it, simply change it and save it as a new file with `save_template` (see the docs `?save_template` for more details). +# +# Let's adjust the previous template to be more specific to a data analysis question: +tpl = [PT.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PT.UserMessage("# Question\n\n{{ask}}")] +# Templates are saved in the `templates` directory of the package. Name of the file will become the template name (eg, call `:JuliaDataExpertAsk`) +filename = joinpath(pkgdir(PromptingTools), + "templates", + "persona-task", + "JuliaDataExpertAsk.json") +PT.save_template(filename, + tpl; + description = "For asking data analysis questions in Julia language. Placeholders: `ask`") +rm(filename) # cleanup if we don't like it +# +# When you create a new template, remember to re-load the templates with `load_templates!()` so that it's available for use. +PT.load_templates!(); +# +# !!! If you have some good templates (or suggestions for the existing ones), please consider sharing them with the community by opening a PR to the `templates` directory! diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 42feb29a8..8e7697005 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -2,7 +2,9 @@ module PromptingTools using OpenAI using JSON3 +using JSON3: StructTypes using HTTP +using PrecompileTools # GLOBALS const MODEL_CHAT = "gpt-3.5-turbo" @@ -20,7 +22,7 @@ const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", "gpt4t" => "gpt-4-1106-preview", # 4t is for "4 turbo" "gpt3t" => "gpt-3.5-turbo-1106", # 3t is for "3 turbo" "ada" => "text-embedding-ada-002") -# below is defined in llm_interace.jl ! +# the below default is defined in llm_interace.jl ! # const PROMPT_SCHEMA = OpenAISchema() include("utils.jl") @@ -34,6 +36,12 @@ export AIMessage # export UserMessage, SystemMessage, DataMessage # for debugging only include("messages.jl") +export aitemplates, AITemplate +include("templates.jl") + +const TEMPLATE_STORE = Dict{Symbol, Any}() +const TEMPLATE_METADATA = Vector{AITemplateMetadata}() + ## Individual interfaces include("llm_openai.jl") @@ -41,4 +49,12 @@ include("llm_openai.jl") export @ai_str, @aai_str include("macros.jl") +function __init__() + # Load templates + load_templates!() end + +# Enable precompilation to reduce start time +@compile_workload include("precompilation.jl") + +end # module PromptingTools diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 526abec38..31576bc56 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -40,7 +40,9 @@ end abstract type AbstractChatMLSchema <: AbstractPromptSchema end """ -ChatMLSchema is used by many open-source chatbots, by OpenAI models under the hood and by several models and inferfaces (eg, Ollama, vLLM) +ChatMLSchema is used by many open-source chatbots, by OpenAI models (under the hood) and by several models and inferfaces (eg, Ollama, vLLM) + +You can explore it on [tiktokenizer](https://tiktokenizer.vercel.app/) It uses the following conversation structure: ``` @@ -54,11 +56,23 @@ It uses the following conversation structure: """ struct ChatMLSchema <: AbstractChatMLSchema end -## Dispatch into defaults +abstract type AbstractManagedSchema <: AbstractPromptSchema end + +""" +Ollama by default manages different models and their associated prompt schemas when you pass `system_prompt` and `prompt` fields to the API. + +Warning: It works only for 1 system message and 1 user message, so anything more than that has to be rejected. + +If you need to pass more messagese / longer conversational history, you can use define the model-specific schema directly and pass your Ollama requests with `raw=true`, + which disables and templating and schema management by Ollama. +""" +struct OllamaManagedSchema <: AbstractManagedSchema end + +## Dispatch into default schema const PROMPT_SCHEMA = OpenAISchema() aigenerate(prompt; kwargs...) = aigenerate(PROMPT_SCHEMA, prompt; kwargs...) function aiembed(doc_or_docs, args...; kwargs...) aiembed(PROMPT_SCHEMA, doc_or_docs, args...; kwargs...) end -aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) +aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 83d42258a..a5041e25c 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -1,8 +1,8 @@ ## Rendering of converation history for the OpenAI API "Builds a history of the conversation to provide the prompt to the API. All kwargs are passed as replacements such that `{{key}}=>value` in the template.}}" function render(schema::AbstractOpenAISchema, - messages::Vector{<:AbstractMessage}; - kwargs...) + messages::Vector{<:AbstractMessage}; + kwargs...) ## conversation = Dict{String, String}[] # TODO: concat system messages together @@ -39,7 +39,7 @@ end ## User-Facing API """ - aigenerate([prompt_schema::AbstractOpenAISchema,] prompt; verbose::Bool = true, + aigenerate([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, model::String = MODEL_CHAT, http_kwargs::NamedTuple = (; retry_non_idempotent = true, @@ -97,13 +97,14 @@ msg=aigenerate(conversation) # AIMessage("Ah, strong feelings you have for your iPhone. A Jedi's path, this is not... ") ``` """ -function aigenerate(prompt_schema::AbstractOpenAISchema, prompt; verbose::Bool = true, - api_key::String = API_KEY, - model::String = MODEL_CHAT, - http_kwargs::NamedTuple = (retry_non_idempotent = true, - retries = 5, - readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), - kwargs...) +function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) ## global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided @@ -126,15 +127,15 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt; verbose::Bool = end # Extend OpenAI create_chat to allow for testing/debugging function OpenAI.create_chat(schema::AbstractOpenAISchema, - api_key::AbstractString, - model::AbstractString, - conversation; - kwargs...) + api_key::AbstractString, + model::AbstractString, + conversation; + kwargs...) OpenAI.create_chat(api_key, model, conversation; kwargs...) end function OpenAI.create_chat(schema::TestEchoOpenAISchema, api_key::AbstractString, - model::AbstractString, - conversation; kwargs...) + model::AbstractString, + conversation; kwargs...) schema.model_id = model schema.inputs = conversation return schema @@ -194,14 +195,14 @@ msg.content' * msg.content[:, 1] # [1.0, 0.787] """ function aiembed(prompt_schema::AbstractOpenAISchema, - doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, - postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, - model::String = MODEL_EMBEDDING, - http_kwargs::NamedTuple = (retry_non_idempotent = true, - retries = 5, - readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), - kwargs...) where {F <: Function} + doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, + postprocess::F = identity; verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_EMBEDDING, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) where {F <: Function} ## global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided @@ -224,31 +225,31 @@ function aiembed(prompt_schema::AbstractOpenAISchema, end # Extend OpenAI create_embeddings to allow for testing function OpenAI.create_embeddings(schema::AbstractOpenAISchema, - api_key::AbstractString, - docs, - model::AbstractString; - kwargs...) + api_key::AbstractString, + docs, + model::AbstractString; + kwargs...) OpenAI.create_embeddings(api_key, docs, model; kwargs...) end function OpenAI.create_embeddings(schema::TestEchoOpenAISchema, api_key::AbstractString, - docs, - model::AbstractString; kwargs...) + docs, + model::AbstractString; kwargs...) schema.model_id = model schema.inputs = docs return schema end """ - aiclassify(prompt_schema::AbstractOpenAISchema, prompt; + aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), max_tokens = 1, temperature = 0), kwargs...) Classifies the given prompt/statement as true/false/unknown. -Note: this is a very simple classifier, it is not meant to be used in production. Credit goes to: https://twitter.com/AAAzzam/status/1669753721574633473 +Note: this is a very simple classifier, it is not meant to be used in production. Credit goes to [AAAzzam](https://twitter.com/AAAzzam/status/1669753721574633473). -It uses Logit bias trick to force the model to output only true/false/unknown. +It uses Logit bias trick and limits the output to 1 token to force the model to output only true/false/unknown. Output tokens used (via `api_kwargs`): - 837: ' true' @@ -272,24 +273,24 @@ tryparse(Bool, aiclassify("Is two plus two four?")) isa Bool # true Output of type `Nothing` marks that the model couldn't classify the statement as true/false. Ideally, we would like to re-use some helpful system prompt to get more accurate responses. -For this reason we have templates, eg, `:IsStatementTrue`. By specifying the template, we can provide our statement as the expected variable (`statement` in this case) +For this reason we have templates, eg, `:JudgeIsItTrue`. By specifying the template, we can provide our statement as the expected variable (`it` in this case) See that the model now correctly classifies the statement as "unknown". ```julia -aiclassify(:IsStatementTrue; statement = "Is two plus three a vegetable on Mars?") # unknown +aiclassify(:JudgeIsItTrue; it = "Is two plus three a vegetable on Mars?") # unknown ``` For better results, use higher quality models like gpt4, eg, ```julia -aiclassify(:IsStatementTrue; - statement = "If I had two apples and I got three more, I have five apples now.", +aiclassify(:JudgeIsItTrue; + it = "If I had two apples and I got three more, I have five apples now.", model = "gpt4") # true ``` """ -function aiclassify(prompt_schema::AbstractOpenAISchema, prompt; - api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), - max_tokens = 1, temperature = 0), - kwargs...) +function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), + max_tokens = 1, temperature = 0), + kwargs...) ## msg = aigenerate(prompt_schema, prompt; @@ -297,11 +298,3 @@ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt; kwargs...) return msg end -# Dispatch for templates -function aiclassify(prompt_schema::AbstractOpenAISchema, - template_sym::Symbol; - kwargs...) - # render template into prompt - prompt = render(prompt_schema, Val(template_sym)) - return aiclassify(prompt_schema, prompt; kwargs...) -end diff --git a/src/messages.jl b/src/messages.jl index b8f254d5c..bc460f7c7 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -5,25 +5,45 @@ abstract type AbstractMessage end abstract type AbstractChatMessage <: AbstractMessage end # with text-based content abstract type AbstractDataMessage <: AbstractMessage end # with data-based content, eg, embeddings -Base.@kwdef mutable struct SystemMessage{T <: AbstractString} <: AbstractChatMessage +## Allowed inputs for ai* functions, AITemplate is resolved one level higher +const ALLOWED_PROMPT_TYPE = Union{ + AbstractString, + AbstractMessage, + Vector{<:AbstractMessage}, +} + +# Workaround to be able to add metadata to serialized conversations, templates, etc. +# Ignored by `render` directives +Base.@kwdef struct MetadataMessage{T <: AbstractString} <: AbstractChatMessage + content::T + description::String = "" + version::String = "1" + source::String = "" + _type::Symbol = :metadatamessage +end +Base.@kwdef struct SystemMessage{T <: AbstractString} <: AbstractChatMessage content::T variables::Vector{Symbol} = _extract_handlebar_variables(content) + _type::Symbol = :systemmessage end -Base.@kwdef mutable struct UserMessage{T <: AbstractString} <: AbstractChatMessage +Base.@kwdef struct UserMessage{T <: AbstractString} <: AbstractChatMessage content::T variables::Vector{Symbol} = _extract_handlebar_variables(content) + _type::Symbol = :usermessage end Base.@kwdef struct AIMessage{T <: Union{AbstractString, Nothing}} <: AbstractChatMessage content::T = nothing status::Union{Int, Nothing} = nothing tokens::Tuple{Int, Int} = (-1, -1) elapsed::Float64 = -1.0 + _type::Symbol = :aimessage end -Base.@kwdef mutable struct DataMessage{T <: Any} <: AbstractDataMessage +Base.@kwdef struct DataMessage{T <: Any} <: AbstractDataMessage content::T status::Union{Int, Nothing} = nothing tokens::Tuple{Int, Int} = (-1, -1) elapsed::Float64 = -1.0 + _type::Symbol = :datamessage end # content-only constructor @@ -45,6 +65,8 @@ function Base.show(io::IO, ::MIME"text/plain", m::AbstractChatMessage) printstyled(io, type_; color = :light_green) elseif m isa UserMessage printstyled(io, type_; color = :light_red) + elseif m isa MetadataMessage + printstyled(io, type_; color = :light_blue) else print(io, type_) end @@ -59,8 +81,8 @@ end ## Dispatch for render function render(schema::AbstractPromptSchema, - messages::Vector{<:AbstractMessage}; - kwargs...) + messages::Vector{<:AbstractMessage}; + kwargs...) render(schema, messages; kwargs...) end function render(schema::AbstractPromptSchema, msg::AbstractMessage; kwargs...) @@ -70,13 +92,16 @@ function render(schema::AbstractPromptSchema, msg::AbstractString; kwargs...) render(schema, [UserMessage(; content = msg)]; kwargs...) end -## Prompt Templates -# ie, a way to re-use similar prompting patterns (eg, aiclassifier) -# flow: template -> messages |+ kwargs variables -> chat history -# Defined through Val() to allow for dispatch -function render(prompt_schema::AbstractOpenAISchema, template::Val{:IsStatementTrue}) - [ - SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), - UserMessage("##Statement\n\n{{statement}}"), - ] -end +## Serialization via JSON3 +StructTypes.StructType(::Type{AbstractChatMessage}) = StructTypes.AbstractType() +StructTypes.StructType(::Type{MetadataMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{SystemMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{UserMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{AIMessage}) = StructTypes.Struct() +StructTypes.subtypekey(::Type{AbstractChatMessage}) = :_type +function StructTypes.subtypes(::Type{AbstractChatMessage}) + (usermessage = UserMessage, + aimessage = AIMessage, + systemmessage = SystemMessage, + metadatamessage = MetadataMessage) +end \ No newline at end of file diff --git a/src/precompilation.jl b/src/precompilation.jl new file mode 100644 index 000000000..61b0c0aa8 --- /dev/null +++ b/src/precompilation.jl @@ -0,0 +1,15 @@ +# Load templates +load_templates!(); + +# API Calls +mock_response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) +schema = TestEchoOpenAISchema(; response = mock_response, status = 200) +# API calls +msg = aigenerate(schema, "I want to ask {{it}}"; it = "Is this correct?") +msg = aiclassify(schema, "I want to ask {{it}}"; it = "Is this correct?") + +# Use of Templates +template_name = :JudgeIsItTrue +msg = aigenerate(schema, template_name; it = "Is this correct?") +msg = aiclassify(schema, template_name; it = "Is this correct?"); \ No newline at end of file diff --git a/src/templates.jl b/src/templates.jl new file mode 100644 index 000000000..772aa21ec --- /dev/null +++ b/src/templates.jl @@ -0,0 +1,318 @@ +# This file contains the templating system which translates a symbol (=template name) into a set of messages under the specified schema. +# Templates are stored as JSON files in the `templates` folder. +# Once loaded, they are stored in global variable `TEMPLATE_STORE` +# +# Flow: template -> messages |+ kwargs variables -> "conversation" to pass to the model + +## Types +""" + AITemplate + +AITemplate is a template for a conversation prompt. + This type is merely a container for the template name, which is resolved into a set of messages (=prompt) by `render`. + +# Naming Convention +- Template names should be in CamelCase +- Follow the format `......` where possible, eg, `JudgeIsItTrue`, `` + - Starting with the Persona (=System prompt), eg, `Judge` = persona is meant to `judge` some provided information + - Variable to be filled in with context, eg, `It` = placeholder `it` + - Ending with the variable name is helpful, eg, `JuliaExpertTask` for a persona to be an expert in Julia language and `task` is the placeholder name +- Ideally, the template name should be self-explanatory, eg, `JudgeIsItTrue` = persona is meant to `judge` some provided information where it is true or false + +# Examples + +Save time by re-using pre-made templates, just fill in the placeholders with the keyword arguments: +```julia +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +``` + +The above is equivalent to a more verbose version that explicitly uses the dispatch on `AITemplate`: +```julia +msg = aigenerate(AITemplate(:JuliaExpertAsk); ask = "How do I add packages?") +``` + +Find available templates with `aitemplates`: +```julia +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). + +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay +``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! + +You can inspect any template by "rendering" it (this is what the LLM will see): +```julia +julia> AITemplate(:JudgeIsItTrue) |> PromptingTools.render +``` + +See also: `save_template`, `load_template`, `load_templates!` for more advanced use cases (and the corresponding script in `examples/` folder) +""" +struct AITemplate + name::Symbol +end + +"Helper for easy searching and reviewing of templates. Defined on loading of each template." +Base.@kwdef struct AITemplateMetadata + name::Symbol + description::String = "" + version::String = "-" + wordcount::Int + variables::Vector{Symbol} = Symbol[] + system_preview::String = "" + user_preview::String = "" + source::String = "" +end +function Base.show(io::IO, ::MIME"text/plain", t::AITemplateMetadata) + # just dumping seems to give ok output + dump(IOContext(io, :limit => true), t, maxdepth = 1) +end +# overload also the vector printing for nicer search results with `aitemplates` +function Base.show(io::IO, m::MIME"text/plain", v::Vector{<:AITemplateMetadata}) + printstyled(io, "$(length(v))-element Vector{AITemplateMetadata}:"; color = :light_blue) + println(io) + [(show(io, m, v[i]); println(io)) for i in eachindex(v)] + nothing +end + +## Rendering messages from templates +function render(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + global TEMPLATE_STORE + haskey(TEMPLATE_STORE, template.name) || + error("Template $(template.name) not found in TEMPLATE_STORE") + # get template + return TEMPLATE_STORE[template.name] +end + +# dispatch on default schema +"Renders provided messaging template (`template`) under the default schema (`PROMPT_SCHEMA`)." +function render(template::AITemplate; kwargs...) + global PROMPT_SCHEMA + render(PROMPT_SCHEMA, template; kwargs...) +end + +## Loading / Saving +"Saves provided messaging template (`messages`) to `io_or_file`. Automatically adds metadata based on provided keyword arguments." +function save_template(io_or_file::Union{IO, AbstractString}, + messages::AbstractVector{<:AbstractChatMessage}; + content::AbstractString = "Template Metadata", + description::AbstractString = "", + version::AbstractString = "1", + source::AbstractString = "") + + # create metadata + metadata_msg = MetadataMessage(; content, description, version, source) + + # save template to IO or file + JSON3.write(io_or_file, [metadata_msg, messages...]) +end +"Loads messaging template from `io_or_file` and returns tuple of template messages and metadata." +function load_template(io_or_file::Union{IO, AbstractString}) + messages = JSON3.read(io_or_file, Vector{AbstractChatMessage}) + template, metadata = AbstractChatMessage[], MetadataMessage[] + for i in eachindex(messages) + msg = messages[i] + if msg isa MetadataMessage + push!(metadata, msg) + else + push!(template, msg) + end + end + return template, metadata +end + +""" + remove_templates!() + +Removes all templates from `TEMPLATE_STORE` and `TEMPLATE_METADATA`. +""" +remove_templates!(; store = TEMPLATE_STORE, metadata_store = TEMPLATE_METADATA) = (empty!(store); empty!(metadata_store); nothing) + +""" + load_templates!(; remove_templates::Bool=true) + +Loads templates from folder `templates/` in the package root and stores them in `TEMPLATE_STORE` and `TEMPLATE_METADATA`. + +Note: Automatically removes any existing templates and metadata from `TEMPLATE_STORE` and `TEMPLATE_METADATA` if `remove_templates=true`. +""" +function load_templates!(dir_templates::String = joinpath(@__DIR__, "..", "templates"); + remove_templates::Bool = true, + store::Dict{Symbol, <:Any} = TEMPLATE_STORE, + metadata_store::Vector{<:AITemplateMetadata} = TEMPLATE_METADATA,) + # first remove any old templates and their metadata + remove_templates && remove_templates!(; store, metadata_store) + # recursively load all templates from the `templates` folder + for (root, dirs, files) in walkdir(dir_templates) + for file in files + if endswith(file, ".json") + template_name = Symbol(split(basename(file), ".")[begin]) + template, metadata_msgs = load_template(joinpath(root, file)) + # add to store + if haskey(store, template_name) + @warn("Template $(template_name) already exists, overwriting! Metadata will be duplicated.") + end + store[template_name] = template + + # prepare the metadata + wordcount = 0 + system_preview = "" + user_preview = "" + variables = Symbol[] + for i in eachindex(template) + msg = template[i] + wordcount += length(msg.content) + if hasproperty(msg, :variables) + append!(variables, msg.variables) + end + # truncate previews to 100 characters + if msg isa SystemMessage && length(system_preview) < 100 + system_preview *= first(msg.content, 100) + elseif msg isa UserMessage && length(user_preview) < 100 + user_preview *= first(msg.content, 100) + end + end + if !isempty(metadata_msgs) + # use the first metadata message found if available + meta = first(metadata_msgs) + metadata = AITemplateMetadata(; name = template_name, + meta.description, meta.version, meta.source, + wordcount, + system_preview = first(system_preview, 100), + user_preview = first(user_preview, 100), + variables = unique(variables)) + else + metadata = AITemplateMetadata(; name = template_name, + wordcount, + system_preview = first(system_preview, 100), + user_preview = first(user_preview, 100), + variables = unique(variables)) + end + # add metadata to store + push!(metadata_store, metadata) + end + end + end + return nothing +end + +## Searching for templates +""" + aitemplates + +Find easily the most suitable templates for your use case. + +You can search by: +- `query::Symbol` which looks look only for partial matches in the template `name` +- `query::AbstractString` which looks for partial matches in the template `name` or `description` +- `query::Regex` which looks for matches in the template `name`, `description` or any of the message previews + +# Keyword Arguments +- `limit::Int` limits the number of returned templates (Defaults to 10) + +# Examples + +Find available templates with `aitemplates`: +```julia +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). + +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay +``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! +""" +function aitemplates end + +"Find the top-`limit` templates whose `name::Symbol` partially matches the `query_name::Symbol` in `TEMPLATE_METADATA`." +function aitemplates(query_name::Symbol; + limit::Int = 10, + metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) + query_str = lowercase(string(query_name)) + found_templates = filter(x -> occursin(query_str, + lowercase(string(x.name))), metadata_store) + return first(found_templates, limit) +end +"Find the top-`limit` templates whose `name` or `description` fields partially match the `query_key::String` in `TEMPLATE_METADATA`." +function aitemplates(query_key::AbstractString; + limit::Int = 10, + metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) + query_str = lowercase(query_key) + found_templates = filter(x -> occursin(query_str, lowercase(string(x.name))) || + occursin(query_str, lowercase(string(x.description))), + metadata_store) + return first(found_templates, limit) +end +"Find the top-`limit` templates where provided `query_key::Regex` matches either of `name`, `description` or previews or User or System messages in `TEMPLATE_METADATA`." +function aitemplates(query_key::Regex; + limit::Int = 10, + metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) + found_templates = filter(x -> occursin(query_key, + string(x.name)) || + occursin(query_key, + x.description) || + occursin(query_key, + x.system_preview) || + occursin(query_key, x.user_preview), + metadata_store) + return first(found_templates, limit) +end + +## Dispatch for AI templates (unpacks the messages) +function aigenerate(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aigenerate(schema, render(schema, template); kwargs...) +end +function aiclassify(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aiclassify(schema, render(schema, template); kwargs...) +end + +# Shortcut for symbols +function aigenerate(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aigenerate(schema, AITemplate(template); kwargs...) +end +function aiclassify(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aiclassify(schema, AITemplate(template); kwargs...) +end \ No newline at end of file diff --git a/templates/classification/JudgeIsItTrue.json b/templates/classification/JudgeIsItTrue.json new file mode 100644 index 000000000..9ca7c0e02 --- /dev/null +++ b/templates/classification/JudgeIsItTrue.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "LLM-based classification whether provided statement is true/false/unknown. Statement is provided via `it` placeholder.", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Statement\n\n{{it}}", + "variables": [ + "it" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/AssistantAsk.json b/templates/persona-task/AssistantAsk.json new file mode 100644 index 000000000..d3d97601e --- /dev/null +++ b/templates/persona-task/AssistantAsk.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "Helpful assistant for asking generic questions. Placeholders: `ask`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Question\n\n{{ask}}", + "variables": [ + "ask" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/DetailOrientedTask.json b/templates/persona-task/DetailOrientedTask.json new file mode 100644 index 000000000..dd62cf896 --- /dev/null +++ b/templates/persona-task/DetailOrientedTask.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholders: `task`, `data`.", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class AI assistant. You are detail oriented, diligent, and have a great memory. Your communication is brief and concise.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}", + "variables": [ + "task", + "data" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertAsk.json b/templates/persona-task/JuliaExpertAsk.json new file mode 100644 index 000000000..428cbe533 --- /dev/null +++ b/templates/persona-task/JuliaExpertAsk.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "For asking questions about Julia language. Placeholders: `ask`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Question\n\n{{ask}}", + "variables": [ + "ask" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertCoTTask.json b/templates/persona-task/JuliaExpertCoTTask.json new file mode 100644 index 000000000..a45885bc4 --- /dev/null +++ b/templates/persona-task/JuliaExpertCoTTask.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. When no data is provided, create some examples. First, think through your approach step by step. Then implement the solution.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}", + "variables": [ + "task", + "data" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/test/messages.jl b/test/messages.jl index 0068712c3..ec1aa0a31 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -1,9 +1,9 @@ -using PromptingTools: AIMessage, SystemMessage, UserMessage, DataMessage +using PromptingTools: AIMessage, SystemMessage, MetadataMessage, UserMessage, DataMessage @testset "Message constructors" begin # Creates an instance of MSG with the given content string. content = "Hello, world!" - for T in [AIMessage, SystemMessage, UserMessage] + for T in [AIMessage, SystemMessage, UserMessage, MetadataMessage] # args msg = T(content) @test typeof(msg) <: T diff --git a/test/runtests.jl b/test/runtests.jl index a9e3a46ea..b7f138d24 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using PromptingTools +using JSON3 using Test using Aqua const PT = PromptingTools @@ -10,4 +11,5 @@ end include("utils.jl") include("messages.jl") include("llm_openai.jl") + include("templates.jl") end diff --git a/test/templates.jl b/test/templates.jl new file mode 100644 index 000000000..51f027461 --- /dev/null +++ b/test/templates.jl @@ -0,0 +1,74 @@ +using PromptingTools: AbstractChatMessage, SystemMessage, UserMessage, MetadataMessage +using PromptingTools: render +using PromptingTools: save_template, load_template, load_templates!, aitemplates +using PromptingTools: TestEchoOpenAISchema + +@testset "Templates - save/load" begin + description = "Some description" + version = "1.1" + msgs = [ + SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}"), + ] + tmp, _ = mktemp() + save_template(tmp, + msgs; + description, version) + template, metadata = load_template(tmp) + @test template == msgs + @test metadata[1].description == description + @test metadata[1].version == version + @test metadata[1].content == "Template Metadata" + @test metadata[1].source == "" +end + +@testset "Template rendering" begin + template = AITemplate(:JudgeIsItTrue) + expected_output = AbstractChatMessage[SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}")] + @test expected_output == render(PT.PROMPT_SCHEMA, template) + @test expected_output == render(template) +end + +@testset "Templates - search" begin + # search all + tmps = aitemplates("") + @test tmps == PT.TEMPLATE_METADATA + # Exact search for JudgeIsItTrue + tmps = aitemplates(:JudgeIsItTrue) + @test length(tmps) == 1 + @test tmps[1].name == :JudgeIsItTrue + # Search for multiple with :Task in name + tmps1 = aitemplates(:Task) + @test length(tmps1) >= 1 + tmps2 = aitemplates("Task") + @test length(tmps2) == length(tmps1) + # Search via regex + tmps = aitemplates(r"IMPARTIAL AI JUDGE"i) + @test length(tmps) >= 1 +end + +@testset "Templates - Echo aigenerate call" begin + # E2E test for aigenerate with rendering template and filling the placeholders + template_name = :JudgeIsItTrue + expected_template_rendered = render(AITemplate(template_name)) |> + x -> render(PT.PROMPT_SCHEMA, x; it = "Is this correct?") + # corresponds to OpenAI API v1 + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + + # AIGeneration API - use AITemplate(:) + schema1 = TestEchoOpenAISchema(; response, status = 200) + msg = aigenerate(schema1, AITemplate(template_name); it = "Is this correct?") + @test schema1.inputs == expected_template_rendered + + # AIGeneration API - use template name as symbol + schema2 = TestEchoOpenAISchema(; response, status = 200) + msg = aigenerate(schema2, template_name; it = "Is this correct?") + @test schema2.inputs == expected_template_rendered + + # AIClassify API - use symbol dispatch + schema3 = TestEchoOpenAISchema(; response, status = 200) + msg = aiclassify(schema3, template_name; it = "Is this correct?") + @test schema3.inputs == expected_template_rendered +end \ No newline at end of file