From 0a90895e163b6c1d988e0967b04249a5457ab1f4 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Thu, 25 Jul 2024 23:24:36 +0200 Subject: [PATCH 01/16] basic implementation of curl --- CMakeLists.txt | 5 +- docs/API.md | 49 ++++++++++++++ src/webview_candidate_window.cpp | 113 +++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 docs/API.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 82c28c2..0d4b319 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,14 +16,15 @@ endif() find_package(PkgConfig REQUIRED) pkg_check_modules(NlohmannJson REQUIRED IMPORTED_TARGET "nlohmann_json") +find_package(CURL REQUIRED) if(LINUX) pkg_check_modules(webkit2gtk REQUIRED IMPORTED_TARGET "webkit2gtk-4.1") - set(LIBS PkgConfig::NlohmannJson PkgConfig::webkit2gtk) + set(LIBS PkgConfig::NlohmannJson PkgConfig::webkit2gtk CURL::libcurl) elseif(APPLE) find_library(COCOA_LIB Cocoa REQUIRED) find_library(WEBKIT_LIB WebKit REQUIRED) - set(LIBS PkgConfig::NlohmannJson ${COCOA_LIB} ${WEBKIT_LIB}) + set(LIBS PkgConfig::NlohmannJson ${COCOA_LIB} ${WEBKIT_LIB} CURL::libcurl) endif() include_directories(pugixml/src) diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..a99533a --- /dev/null +++ b/docs/API.md @@ -0,0 +1,49 @@ +# API + +## `curl` (async) + +With `curl`, you can interact with network resources directly. + +```ts +async function curl(url: string, args: CurlArgs) => CurlResponse + +type CurlArgs = { + method?: "GET" | "POST" | "DELETE" | "HEAD", + headers?: object, + data?: string, // ignored if `json` exists + json?: JSON, +} + +type CurlResponse = { + status: number, + data: string, +} +``` + +**Example** POST w/ JSON: + +```js +# Basic usage +curl("https://httpbin.org/post", { + json: {arr: [1,2,3]} +}).then(r => { + console.log(r.status, r.status==200) + return JSON.parse(r.data) +}).then(j => console.log(j.data)) + +# OpenAI +curl("https://api.openai.com/v1/chat/completions", { + headers: { "Authorization": "Bearer $OPENAI_API_KEY" }, + json: { + "model": "gpt-4o-mini", + "messages": [{ + "role": "system", + "content": "You are a poetic assistant, skilled in explaining complex programming concepts with creative flair." + }, { + "role": "user", + "content": "Compose a poem that explains the concept of recursion in programming." + }] + } +}).then(r => JSON.parse(r.data)) + .then(j => console.log(j)) +``` diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 2d86b28..1a6c430 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -2,9 +2,13 @@ #include "html_template.hpp" #include "utility.hpp" #include +#include #include #include +static size_t _curl_cb(const char *data, size_t size, size_t nmemb, + std::string *outbuf); + namespace candidate_window { void to_json(nlohmann::json &j, const CandidateAction &a) { @@ -64,6 +68,109 @@ WebviewCandidateWindow::WebviewCandidateWindow() bind("_copyHTML", [this](std::string html) { write_clipboard(html); }); + w_->bind( + "curl", + [this](std::string id, std::string req, void *) { + // NOTE: resolve status 0=fulfilled, otherwise rejected + + // TODO: for now, we use a one-thread-per-connection way. + // this is inscalable. need to be changed later. + auto j = nlohmann::json::parse(req); + std::string url; + try { + url = j[0].get(); + } catch (const std::exception &e) { + std::cerr << "[JS] Insufficient number of arguments to 'curl', " + "needed 1 or 2, got 0\n"; + return; + } + auto args = j[1]; + + std::thread([this, id = std::move(id), url = std::move(url), + args = std::move(args)] { + CURL *curl = curl_easy_init(); + if (!curl) { + w_->resolve(id, 1, "\"Failed to initialize curl\""); + return; + } + + std::string recvBuf, postData; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _curl_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &recvBuf); + + CURLcode res; + nlohmann::json j; + std::unordered_map headers; + struct curl_slist *hlist = NULL; + + // method + if (args.contains("method") && args["method"].is_string()) { + std::string method = args["method"]; + if (method == "GET") { + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); + } else if (method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1); + } else if (method == "DELETE") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + } else if (method == "HEAD") { + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + } else { + w_->resolve(id, 1, + nlohmann::json("Unknown HTTP method")); + goto done; + } + } + + // json, data + if (args.contains("json")) { + postData = args["json"].dump(); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, + postData.c_str()); + headers["Content-Type"] = "application/json"; + } else if (args.contains("data") && args["data"].is_string()) { + postData = args["data"].get(); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, + postData.c_str()); + } + + // headers + if (args.contains("headers") && args["headers"].is_object()) { + for (const auto &el : args["headers"].items()) { + try { + headers[el.key()] = el.value(); + } catch (...) { + std::cerr << "[JS] Cannot get the value of header '" + << el.key() << "', value is " + << el.value() << "\n"; + } + } + } + for (const auto &[key, value] : headers) { + std::string s = key + ": " + value; + hlist = curl_slist_append(hlist, s.c_str()); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::string errmsg = "Error getting data from remote: "; + errmsg += curl_easy_strerror(res); + w_->resolve(id, 1, nlohmann::json(errmsg).dump().c_str()); + } else { + int status = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + j["status"] = status; + j["data"] = recvBuf; + w_->resolve(id, 0, j.dump()); + } + + done: + curl_easy_cleanup(curl); + }).detach(); + }, + nullptr); + std::string html_template(reinterpret_cast(HTML_TEMPLATE), HTML_TEMPLATE_len); w_->set_html(html_template.c_str()); @@ -208,3 +315,9 @@ void WebviewCandidateWindow::update_input_panel( void WebviewCandidateWindow::copy_html() { invoke_js("copyHTML"); } } // namespace candidate_window + +static size_t _curl_cb(const char *data, size_t size, size_t nmemb, + std::string *outbuf) { + outbuf->append(data, size * nmemb); + return size * nmemb; +} From 6a3933801d33e93d668dfb803ab99a86078df2ab Mon Sep 17 00:00:00 2001 From: ksqsf Date: Thu, 25 Jul 2024 23:24:53 +0200 Subject: [PATCH 02/16] bring back preview.app --- preview/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/preview/CMakeLists.txt b/preview/CMakeLists.txt index 1150c51..cf6d96f 100644 --- a/preview/CMakeLists.txt +++ b/preview/CMakeLists.txt @@ -1,9 +1,11 @@ -add_executable(preview preview.cpp) -target_link_libraries(preview WebviewCandidateWindow) -target_include_directories(preview PRIVATE "${PROJECT_SOURCE_DIR}/include") - if(APPLE) + add_executable(preview MACOSX_BUNDLE preview.cpp) target_compile_options(preview PRIVATE "-Wno-auto-var-id" "-ObjC++") +else() + add_executable(preview preview.cpp) endif() +target_link_libraries(preview WebviewCandidateWindow) +target_include_directories(preview PRIVATE "${PROJECT_SOURCE_DIR}/include") + add_dependencies(preview GenerateHTML) From fb9bb615c1b7a0658168d9a600ce757c78cf210a Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 01:00:01 +0200 Subject: [PATCH 03/16] use curl_multi --- include/curl.hpp | 25 ++++ include/webview_candidate_window.hpp | 4 + src/CMakeLists.txt | 1 + src/curl.cpp | 107 ++++++++++++++ src/webview_candidate_window.cpp | 202 +++++++++++++-------------- 5 files changed, 231 insertions(+), 108 deletions(-) create mode 100644 include/curl.hpp create mode 100644 src/curl.cpp diff --git a/include/curl.hpp b/include/curl.hpp new file mode 100644 index 0000000..a72e37f --- /dev/null +++ b/include/curl.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +class CurlMultiManager { + public: + using Callback = std::function; + using HandleData = std::pair; + + static CurlMultiManager &shared(); + CurlMultiManager(); + ~CurlMultiManager(); + void add(CURL *easy, CurlMultiManager::Callback cb); + + private: + CURLM *multi; + std::thread worker_thread; + int controlfd[2]; + std::shared_mutex m; + std::unordered_map buf; + std::unordered_map cb; + + void run(); +}; diff --git a/include/webview_candidate_window.hpp b/include/webview_candidate_window.hpp index 07c2d27..f8107f0 100644 --- a/include/webview_candidate_window.hpp +++ b/include/webview_candidate_window.hpp @@ -58,6 +58,10 @@ class WebviewCandidateWindow : public CandidateWindow { void *platform_data = nullptr; void platform_init(); + private: + /* API */ + void api_curl(std::string id, std::string req); + private: /* Invoke a JavaScript function. */ template diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd3d193..be8d99a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(WebviewCandidateWindow "${PROJECT_SOURCE_DIR}/pugixml/src/pugixml.cpp" webview_candidate_window.cpp platform.cpp + curl.cpp ) target_include_directories(WebviewCandidateWindow PUBLIC "${PROJECT_SOURCE_DIR}/include" "${PROJECT_SOURCE_DIR}/webview") target_link_libraries(WebviewCandidateWindow ${LIBS}) diff --git a/src/curl.cpp b/src/curl.cpp new file mode 100644 index 0000000..057d9cc --- /dev/null +++ b/src/curl.cpp @@ -0,0 +1,107 @@ +#include "curl.hpp" +#include +#include +#include +#include +#include + +static size_t _on_data_cb(char *data, size_t size, size_t nmemb, + std::string *outbuf); +std::mutex m; +std::atomic running; + +CurlMultiManager &CurlMultiManager::shared() { + static CurlMultiManager instance; + return instance; +} + +CurlMultiManager::CurlMultiManager() { + bool expected = false; + if (!running.compare_exchange_strong(expected, true)) { + throw std::runtime_error("should only run one curl manager"); + } + if (pipe(controlfd) < 0) { + throw std::runtime_error("failed to create curl control pipe"); + } + curl_global_init(CURL_GLOBAL_ALL); + multi = curl_multi_init(); + worker_thread = std::thread(&CurlMultiManager::run, this); +} + +CurlMultiManager::~CurlMultiManager() { + write(controlfd[1], "q", 1); + if (worker_thread.joinable()) { + worker_thread.join(); + } + curl_multi_cleanup(multi); + curl_global_cleanup(); + close(controlfd[0]); + close(controlfd[1]); + running.store(false); +} + +void CurlMultiManager::add(CURL *easy, CurlMultiManager::Callback callback) { + { + std::unique_lock g(m); + buf[easy] = ""; + cb[easy] = callback; + } + curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, _on_data_cb); + curl_easy_setopt(easy, CURLOPT_WRITEDATA, &buf[easy]); + curl_multi_add_handle(multi, easy); + std::atomic_thread_fence(std::memory_order_seq_cst); + write(controlfd[1], "a", 1); +} + +void CurlMultiManager::run() { + while (true) { + int still_running = 0; + int ret; + ret = curl_multi_perform(multi, &still_running); + int numfds; + struct curl_waitfd wfd; + wfd.fd = controlfd[0]; + wfd.events = CURL_WAIT_POLLIN; + wfd.revents = 0; + curl_multi_poll(multi, &wfd, 1, 1000, &numfds); + std::atomic_thread_fence(std::memory_order_seq_cst); + if (wfd.revents) { + char cmd; + read(controlfd[0], &cmd, 1); + switch (cmd) { + case 'q': // quit + return; + case 'a': // added a new handle + break; + default: + assert(false && "unreachable"); + } + } + CURLMsg *msg; + int msgs_left; + while ((msg = curl_multi_info_read(multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE) { + CURL *easy = msg->easy_handle; + CURLcode res = msg->data.result; + { + // do this because the cb might be slow + std::shared_lock g(m); + cb[easy](res, easy, buf[easy]); + } + curl_multi_remove_handle(multi, easy); + curl_easy_cleanup(easy); + { + std::unique_lock g(m); + cb.erase(easy); + buf.erase(easy); + } + } + } + } +} + +size_t _on_data_cb(char *data, size_t size, size_t nmemb, std::string *outbuf) { + size_t realsize = size * nmemb; + outbuf->append(data, realsize); + return realsize; +} diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 1a6c430..7416568 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -1,14 +1,11 @@ #include "webview_candidate_window.hpp" +#include "curl.hpp" #include "html_template.hpp" #include "utility.hpp" #include -#include #include #include -static size_t _curl_cb(const char *data, size_t size, size_t nmemb, - std::string *outbuf); - namespace candidate_window { void to_json(nlohmann::json &j, const CandidateAction &a) { @@ -70,105 +67,7 @@ WebviewCandidateWindow::WebviewCandidateWindow() w_->bind( "curl", - [this](std::string id, std::string req, void *) { - // NOTE: resolve status 0=fulfilled, otherwise rejected - - // TODO: for now, we use a one-thread-per-connection way. - // this is inscalable. need to be changed later. - auto j = nlohmann::json::parse(req); - std::string url; - try { - url = j[0].get(); - } catch (const std::exception &e) { - std::cerr << "[JS] Insufficient number of arguments to 'curl', " - "needed 1 or 2, got 0\n"; - return; - } - auto args = j[1]; - - std::thread([this, id = std::move(id), url = std::move(url), - args = std::move(args)] { - CURL *curl = curl_easy_init(); - if (!curl) { - w_->resolve(id, 1, "\"Failed to initialize curl\""); - return; - } - - std::string recvBuf, postData; - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _curl_cb); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &recvBuf); - - CURLcode res; - nlohmann::json j; - std::unordered_map headers; - struct curl_slist *hlist = NULL; - - // method - if (args.contains("method") && args["method"].is_string()) { - std::string method = args["method"]; - if (method == "GET") { - curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); - } else if (method == "POST") { - curl_easy_setopt(curl, CURLOPT_POST, 1); - } else if (method == "DELETE") { - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - } else if (method == "HEAD") { - curl_easy_setopt(curl, CURLOPT_NOBODY, 1); - } else { - w_->resolve(id, 1, - nlohmann::json("Unknown HTTP method")); - goto done; - } - } - - // json, data - if (args.contains("json")) { - postData = args["json"].dump(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, - postData.c_str()); - headers["Content-Type"] = "application/json"; - } else if (args.contains("data") && args["data"].is_string()) { - postData = args["data"].get(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, - postData.c_str()); - } - - // headers - if (args.contains("headers") && args["headers"].is_object()) { - for (const auto &el : args["headers"].items()) { - try { - headers[el.key()] = el.value(); - } catch (...) { - std::cerr << "[JS] Cannot get the value of header '" - << el.key() << "', value is " - << el.value() << "\n"; - } - } - } - for (const auto &[key, value] : headers) { - std::string s = key + ": " + value; - hlist = curl_slist_append(hlist, s.c_str()); - } - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist); - - res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::string errmsg = "Error getting data from remote: "; - errmsg += curl_easy_strerror(res); - w_->resolve(id, 1, nlohmann::json(errmsg).dump().c_str()); - } else { - int status = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); - j["status"] = status; - j["data"] = recvBuf; - w_->resolve(id, 0, j.dump()); - } - - done: - curl_easy_cleanup(curl); - }).detach(); - }, + [this](std::string id, std::string req, void *) { api_curl(id, req); }, nullptr); std::string html_template(reinterpret_cast(HTML_TEMPLATE), @@ -314,10 +213,97 @@ void WebviewCandidateWindow::update_input_panel( void WebviewCandidateWindow::copy_html() { invoke_js("copyHTML"); } -} // namespace candidate_window +enum PromiseResolution { + kFulfilled, + kRejected, +}; + +void WebviewCandidateWindow::api_curl(std::string id, std::string req) { + auto j = nlohmann::json::parse(req); + std::string url; + try { + url = j[0].get(); + } catch (const std::exception &e) { + std::cerr << "[JS] Insufficient number of arguments to 'curl', " + "needed 1 or 2, got 0\n"; + w_->resolve(id, kRejected, "\"Bad call to 'curl'\""); + return; + } + auto args = j[1]; + + CURL *curl = curl_easy_init(); + if (!curl) { + w_->resolve(id, kRejected, "\"Failed to initialize curl\""); + return; + } + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + CURLcode res; + std::unordered_map headers; + struct curl_slist *hlist = NULL; + + // method + if (args.contains("method") && args["method"].is_string()) { + std::string method = args["method"]; + if (method == "GET") { + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); + } else if (method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1); + } else if (method == "DELETE") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + } else if (method == "HEAD") { + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + } else { + w_->resolve(id, kRejected, nlohmann::json("Unknown HTTP method")); + curl_easy_cleanup(curl); + return; + } + } -static size_t _curl_cb(const char *data, size_t size, size_t nmemb, - std::string *outbuf) { - outbuf->append(data, size * nmemb); - return size * nmemb; + // json, data + if (args.contains("json")) { + curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, + args["json"].dump().c_str()); + headers["Content-Type"] = "application/json"; + } else if (args.contains("data") && args["data"].is_string()) { + curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, + args["data"].get().c_str()); + } + + // headers + if (args.contains("headers") && args["headers"].is_object()) { + for (const auto &el : args["headers"].items()) { + try { + headers[el.key()] = el.value(); + } catch (...) { + std::cerr << "[JS] Cannot get the value of header '" << el.key() + << "', value is " << el.value() << "\n"; + } + } + } + for (const auto &[key, value] : headers) { + std::string s = key + ": " + value; + hlist = curl_slist_append(hlist, s.c_str()); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist); + + CurlMultiManager::shared().add(curl, [this, id](CURLcode res, CURL *curl, + const std::string &data) { + if (res != CURLE_OK) { + std::string errmsg = "CURL error: "; + errmsg += curl_easy_strerror(res); + w_->resolve(id, kRejected, nlohmann::json(errmsg).dump().c_str()); + } else { + int status = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + nlohmann::json j{ + {"status", status}, + {"data", data}, + }; + w_->resolve(id, kFulfilled, j.dump()); + } + }); } + +} // namespace candidate_window From 8c2bc5a0e528f10c9a85b52b37f7847e0f098ea8 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 01:39:25 +0200 Subject: [PATCH 04/16] avoid cross-thread exception --- include/curl.hpp | 2 ++ src/curl.cpp | 6 +++++- src/webview_candidate_window.cpp | 32 ++++++++++++++++++++------------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/include/curl.hpp b/include/curl.hpp index a72e37f..38ac7b7 100644 --- a/include/curl.hpp +++ b/include/curl.hpp @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include class CurlMultiManager { public: diff --git a/src/curl.cpp b/src/curl.cpp index 057d9cc..f1bb77e 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -86,7 +86,11 @@ void CurlMultiManager::run() { { // do this because the cb might be slow std::shared_lock g(m); - cb[easy](res, easy, buf[easy]); + try { + cb[easy](res, easy, buf[easy]); + } catch (...) { + assert(false && "curl callback must not throw!"); + } } curl_multi_remove_handle(multi, easy); curl_easy_cleanup(easy); diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 7416568..9af04c8 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -290,18 +290,26 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { CurlMultiManager::shared().add(curl, [this, id](CURLcode res, CURL *curl, const std::string &data) { - if (res != CURLE_OK) { - std::string errmsg = "CURL error: "; - errmsg += curl_easy_strerror(res); - w_->resolve(id, kRejected, nlohmann::json(errmsg).dump().c_str()); - } else { - int status = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); - nlohmann::json j{ - {"status", status}, - {"data", data}, - }; - w_->resolve(id, kFulfilled, j.dump()); + try { + if (res != CURLE_OK) { + std::string errmsg = "CURL error: "; + errmsg += curl_easy_strerror(res); + w_->resolve(id, kRejected, + nlohmann::json(errmsg).dump().c_str()); + } else { + int status = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + nlohmann::json j{ + {"status", status}, + {"data", data}, + }; + w_->resolve(id, kFulfilled, j.dump()); + } + } catch (const std::exception &e) { + std::cerr << "[JS] curl callback throws " << e.what() << "\n"; + } catch (...) { + std::cerr << "[JS] FATAL! Unhandled exception in curl callback\n"; + std::terminate(); } }); } From 77e53805c23d2e55348a31a995f1b8330cee8566 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 01:48:34 +0200 Subject: [PATCH 05/16] set_api --- include/webview_candidate_window.hpp | 5 +++++ src/webview_candidate_window.cpp | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/webview_candidate_window.hpp b/include/webview_candidate_window.hpp index f8107f0..cd9de55 100644 --- a/include/webview_candidate_window.hpp +++ b/include/webview_candidate_window.hpp @@ -9,6 +9,9 @@ #include namespace candidate_window { + +enum CustomAPI : uint64_t { kCurl = 1 }; + class WebviewCandidateWindow : public CandidateWindow { public: WebviewCandidateWindow(); @@ -33,6 +36,8 @@ class WebviewCandidateWindow : public CandidateWindow { void set_accent_color(); void copy_html(); + void set_api(uint64_t apis); + private: std::shared_ptr w_; double cursor_x_ = 0; diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 9af04c8..aec52b8 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -65,14 +65,11 @@ WebviewCandidateWindow::WebviewCandidateWindow() bind("_copyHTML", [this](std::string html) { write_clipboard(html); }); - w_->bind( - "curl", - [this](std::string id, std::string req, void *) { api_curl(id, req); }, - nullptr); - std::string html_template(reinterpret_cast(HTML_TEMPLATE), HTML_TEMPLATE_len); w_->set_html(html_template.c_str()); + + set_api(kCurl); // FIXME: remove before PR } void WebviewCandidateWindow::set_accent_color() { @@ -213,6 +210,19 @@ void WebviewCandidateWindow::update_input_panel( void WebviewCandidateWindow::copy_html() { invoke_js("copyHTML"); } +void WebviewCandidateWindow::set_api(uint64_t apis) { + if (apis & kCurl) { + w_->bind( + "curl", + [this](std::string id, std::string req, void *) { + api_curl(id, req); + }, + nullptr); + } else { + w_->eval("curl = undefined;"); + } +} + enum PromiseResolution { kFulfilled, kRejected, From 50592224f11196951c02f57f5bb46dbb872cd468 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 02:38:08 +0200 Subject: [PATCH 06/16] binary base64 --- docs/API.md | 3 +++ include/utility.hpp | 1 + src/utility.cpp | 22 ++++++++++++++++++++++ src/webview_candidate_window.cpp | 13 +++++++++---- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/API.md b/docs/API.md index a99533a..bbc6607 100644 --- a/docs/API.md +++ b/docs/API.md @@ -12,6 +12,7 @@ type CurlArgs = { headers?: object, data?: string, // ignored if `json` exists json?: JSON, + binary?: bool, } type CurlResponse = { @@ -20,6 +21,8 @@ type CurlResponse = { } ``` +- If `args.binary` is `true`, then `response.data` will be a base64-encoded representation of the original data. + **Example** POST w/ JSON: ```js diff --git a/include/utility.hpp b/include/utility.hpp index 891ee83..e13ec9e 100644 --- a/include/utility.hpp +++ b/include/utility.hpp @@ -50,3 +50,4 @@ template function_traits make_function_traits(const T &) { } std::string escape_html(const std::string &content); +std::string base64(const std::string &s); diff --git a/src/utility.cpp b/src/utility.cpp index 80f4e66..b0d74ac 100644 --- a/src/utility.cpp +++ b/src/utility.cpp @@ -9,3 +9,25 @@ std::string escape_html(const std::string &content) { doc.print(ss, "", pugi::format_raw); return ss.str(); } + +std::string base64(const std::string &s) { + static const char *chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string ret; + ret.reserve((s.size() + 2) / 3 * 4); + unsigned int w = 0; + int b = -6; + for (unsigned char c : s) { + w = (w << 8) + c; + b += 8; + while (b >= 0) { + ret += chars[(w >> b) & 0x3F]; + b -= 6; + } + } + if (b > -6) + ret += chars[((w << 8) >> (b + 8)) & 0x3F]; + while (ret.size() % 4) + ret += '='; + return ret; +} diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index aec52b8..b57e0ea 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -249,7 +249,7 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - CURLcode res; + bool binary = false; std::unordered_map headers; struct curl_slist *hlist = NULL; @@ -280,6 +280,9 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, args["data"].get().c_str()); } + if (args.contains("binary") && args["binary"].is_boolean()) { + binary = args["binary"]; + } // headers if (args.contains("headers") && args["headers"].is_object()) { @@ -298,8 +301,9 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist); - CurlMultiManager::shared().add(curl, [this, id](CURLcode res, CURL *curl, - const std::string &data) { + CurlMultiManager::shared().add(curl, [this, id, + binary](CURLcode res, CURL *curl, + const std::string &data) { try { if (res != CURLE_OK) { std::string errmsg = "CURL error: "; @@ -311,12 +315,13 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); nlohmann::json j{ {"status", status}, - {"data", data}, + {"data", !binary ? data : base64(data)}, }; w_->resolve(id, kFulfilled, j.dump()); } } catch (const std::exception &e) { std::cerr << "[JS] curl callback throws " << e.what() << "\n"; + w_->resolve(id, kRejected, nlohmann::json(e.what()).dump()); } catch (...) { std::cerr << "[JS] FATAL! Unhandled exception in curl callback\n"; std::terminate(); From 968512f42a0f465c4bb03a49fb70d7a050838872 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 14:56:42 +0200 Subject: [PATCH 07/16] relax memory barrier --- src/curl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/curl.cpp b/src/curl.cpp index f1bb77e..dacca41 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -49,7 +49,7 @@ void CurlMultiManager::add(CURL *easy, CurlMultiManager::Callback callback) { curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, _on_data_cb); curl_easy_setopt(easy, CURLOPT_WRITEDATA, &buf[easy]); curl_multi_add_handle(multi, easy); - std::atomic_thread_fence(std::memory_order_seq_cst); + std::atomic_thread_fence(std::memory_order_release); write(controlfd[1], "a", 1); } @@ -64,7 +64,7 @@ void CurlMultiManager::run() { wfd.events = CURL_WAIT_POLLIN; wfd.revents = 0; curl_multi_poll(multi, &wfd, 1, 1000, &numfds); - std::atomic_thread_fence(std::memory_order_seq_cst); + std::atomic_thread_fence(std::memory_order_acquire); if (wfd.revents) { char cmd; read(controlfd[0], &cmd, 1); From 94bf1f1038c4ff665ffb34daa57579c12c62d41d Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 15:30:02 +0200 Subject: [PATCH 08/16] done --- src/webview_candidate_window.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index b57e0ea..6e18ca7 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -68,8 +68,6 @@ WebviewCandidateWindow::WebviewCandidateWindow() std::string html_template(reinterpret_cast(HTML_TEMPLATE), HTML_TEMPLATE_len); w_->set_html(html_template.c_str()); - - set_api(kCurl); // FIXME: remove before PR } void WebviewCandidateWindow::set_accent_color() { From be963ece87a545613bd9b8f188a8b2fbcacb3ca7 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 15:41:58 +0200 Subject: [PATCH 09/16] install libcurl-dev on ubuntu --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e118cf3..b482908 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,8 @@ jobs: run: | sudo apt install -y ninja-build \ nlohmann-json3-dev \ - libwebkit2gtk-4.1-dev + libwebkit2gtk-4.1-dev \ + libcurl4-openssl-dev # npx playwright install-deps - name: Configure (macOS) From c4e0399701b5280be3ff09553faf9eb0bdd900a4 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 16:29:21 +0200 Subject: [PATCH 10/16] fix ubuntu build --- src/curl.cpp | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/curl.cpp b/src/curl.cpp index dacca41..8235ff6 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -1,12 +1,16 @@ #include "curl.hpp" #include #include +#include +#include #include #include #include static size_t _on_data_cb(char *data, size_t size, size_t nmemb, std::string *outbuf); +static bool write_char_strong(int fd, char c); +static bool read_char_strong(int fd, char *c); std::mutex m; std::atomic running; @@ -29,7 +33,7 @@ CurlMultiManager::CurlMultiManager() { } CurlMultiManager::~CurlMultiManager() { - write(controlfd[1], "q", 1); + write_char_strong(controlfd[1], 'q'); if (worker_thread.joinable()) { worker_thread.join(); } @@ -50,7 +54,7 @@ void CurlMultiManager::add(CURL *easy, CurlMultiManager::Callback callback) { curl_easy_setopt(easy, CURLOPT_WRITEDATA, &buf[easy]); curl_multi_add_handle(multi, easy); std::atomic_thread_fence(std::memory_order_release); - write(controlfd[1], "a", 1); + write_char_strong(controlfd[1], 'a'); } void CurlMultiManager::run() { @@ -67,7 +71,7 @@ void CurlMultiManager::run() { std::atomic_thread_fence(std::memory_order_acquire); if (wfd.revents) { char cmd; - read(controlfd[0], &cmd, 1); + read_char_strong(controlfd[0], &cmd); switch (cmd) { case 'q': // quit return; @@ -109,3 +113,19 @@ size_t _on_data_cb(char *data, size_t size, size_t nmemb, std::string *outbuf) { outbuf->append(data, realsize); return realsize; } + +static bool write_char_strong(int fd, char c) { + ssize_t ret; + do { + ret = write(fd, &c, 1); + } while (ret == -1 && errno == EINTR); + return ret == 1; +} + +static bool read_char_strong(int fd, char *c) { + ssize_t ret; + do { + ret = read(fd, c, 1); + } while (ret == -1 && errno == EINTR); + return ret == 1; +} From dcf22ad2df16d12b6699011fee3c2a58246887e9 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 18:05:01 +0200 Subject: [PATCH 11/16] add more methods --- docs/API.md | 2 +- src/webview_candidate_window.cpp | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/API.md b/docs/API.md index bbc6607..0b071d1 100644 --- a/docs/API.md +++ b/docs/API.md @@ -8,7 +8,7 @@ With `curl`, you can interact with network resources directly. async function curl(url: string, args: CurlArgs) => CurlResponse type CurlArgs = { - method?: "GET" | "POST" | "DELETE" | "HEAD", + method?: "GET" | "POST" | "DELETE" | "HEAD" | "OPTIONS" | "PUT" | "PATCH", headers?: object, data?: string, // ignored if `json` exists json?: JSON, diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 6e18ca7..72cafc0 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -258,10 +258,13 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); } else if (method == "POST") { curl_easy_setopt(curl, CURLOPT_POST, 1); - } else if (method == "DELETE") { - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + } else if (method == "DELETE" || method == "PUT" || method == "PATCH") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str()); } else if (method == "HEAD") { curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + } else if (method == "OPTIONS") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str()); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); } else { w_->resolve(id, kRejected, nlohmann::json("Unknown HTTP method")); curl_easy_cleanup(curl); From 984b8796ba513f33d14519094b8c401c619a99ed Mon Sep 17 00:00:00 2001 From: Qijia Liu Date: Sat, 27 Jul 2024 13:01:24 -0400 Subject: [PATCH 12/16] fix enable after disable --- src/webview_candidate_window.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 72cafc0..f9b65d5 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -217,7 +217,7 @@ void WebviewCandidateWindow::set_api(uint64_t apis) { }, nullptr); } else { - w_->eval("curl = undefined;"); + w_->unbind("curl"); } } From 418707c5ab746567ab2850be38671587bc7f3427 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 19:02:58 +0200 Subject: [PATCH 13/16] add timeout --- docs/API.md | 3 ++- src/webview_candidate_window.cpp | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/API.md b/docs/API.md index 0b071d1..5014a1d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -10,9 +10,10 @@ async function curl(url: string, args: CurlArgs) => CurlResponse type CurlArgs = { method?: "GET" | "POST" | "DELETE" | "HEAD" | "OPTIONS" | "PUT" | "PATCH", headers?: object, - data?: string, // ignored if `json` exists + data?: string, // ignored if `json` exists json?: JSON, binary?: bool, + timeout?: uint64 // milliseconds } type CurlResponse = { diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index f9b65d5..28236ea 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -302,16 +302,17 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hlist); + // timeout + if (args.contains("timeout") && args["timeout"].is_number_integer()) { + uint64_t timeout = args["timeout"]; + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout); + } + CurlMultiManager::shared().add(curl, [this, id, binary](CURLcode res, CURL *curl, const std::string &data) { try { - if (res != CURLE_OK) { - std::string errmsg = "CURL error: "; - errmsg += curl_easy_strerror(res); - w_->resolve(id, kRejected, - nlohmann::json(errmsg).dump().c_str()); - } else { + if (res == CURLE_OK) { int status = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); nlohmann::json j{ @@ -319,6 +320,11 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { {"data", !binary ? data : base64(data)}, }; w_->resolve(id, kFulfilled, j.dump()); + } else { + std::string errmsg = "CURL error: "; + errmsg += curl_easy_strerror(res); + w_->resolve(id, kRejected, + nlohmann::json(errmsg).dump().c_str()); } } catch (const std::exception &e) { std::cerr << "[JS] curl callback throws " << e.what() << "\n"; From 7a73f3645e35879b58e4843e1ba52cd5982da445 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 19:32:47 +0200 Subject: [PATCH 14/16] reduce poll timeout --- docs/API.md | 1 + src/curl.cpp | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/API.md b/docs/API.md index 5014a1d..a4fe9ff 100644 --- a/docs/API.md +++ b/docs/API.md @@ -23,6 +23,7 @@ type CurlResponse = { ``` - If `args.binary` is `true`, then `response.data` will be a base64-encoded representation of the original data. +- The precision of `args.timeout` is 50ms. **Example** POST w/ JSON: diff --git a/src/curl.cpp b/src/curl.cpp index 8235ff6..3e421e7 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -67,7 +67,10 @@ void CurlMultiManager::run() { wfd.fd = controlfd[0]; wfd.events = CURL_WAIT_POLLIN; wfd.revents = 0; - curl_multi_poll(multi, &wfd, 1, 1000, &numfds); + // NOTE: poll does not return when any of the easy handle's + // timeout expires. By setting poll's timeout to 50ms, we + // effectively set the timeout precision to 50ms. + curl_multi_poll(multi, &wfd, 1, 50, &numfds); std::atomic_thread_fence(std::memory_order_acquire); if (wfd.revents) { char cmd; From 52678e59d81525d796ef94c9d3abc60f5d291159 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 20:03:05 +0200 Subject: [PATCH 15/16] revert back if-else --- src/webview_candidate_window.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 28236ea..5ffd64b 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -312,7 +312,12 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { binary](CURLcode res, CURL *curl, const std::string &data) { try { - if (res == CURLE_OK) { + if (res != CURLE_OK) { + std::string errmsg = "CURL error: "; + errmsg += curl_easy_strerror(res); + w_->resolve(id, kRejected, + nlohmann::json(errmsg).dump().c_str()); + } else { int status = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); nlohmann::json j{ @@ -320,11 +325,6 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { {"data", !binary ? data : base64(data)}, }; w_->resolve(id, kFulfilled, j.dump()); - } else { - std::string errmsg = "CURL error: "; - errmsg += curl_easy_strerror(res); - w_->resolve(id, kRejected, - nlohmann::json(errmsg).dump().c_str()); } } catch (const std::exception &e) { std::cerr << "[JS] curl callback throws " << e.what() << "\n"; From 4bb33f7aa92bd16745bda8759e136e2c1f223ce2 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 27 Jul 2024 20:30:54 +0200 Subject: [PATCH 16/16] fix curl_easy_getinfo --- src/webview_candidate_window.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/webview_candidate_window.cpp b/src/webview_candidate_window.cpp index 5ffd64b..5f8119a 100644 --- a/src/webview_candidate_window.cpp +++ b/src/webview_candidate_window.cpp @@ -312,19 +312,19 @@ void WebviewCandidateWindow::api_curl(std::string id, std::string req) { binary](CURLcode res, CURL *curl, const std::string &data) { try { - if (res != CURLE_OK) { - std::string errmsg = "CURL error: "; - errmsg += curl_easy_strerror(res); - w_->resolve(id, kRejected, - nlohmann::json(errmsg).dump().c_str()); - } else { - int status = 0; + if (res == CURLE_OK) { + long status = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); nlohmann::json j{ {"status", status}, {"data", !binary ? data : base64(data)}, }; w_->resolve(id, kFulfilled, j.dump()); + } else { + std::string errmsg = "CURL error: "; + errmsg += curl_easy_strerror(res); + w_->resolve(id, kRejected, + nlohmann::json(errmsg).dump().c_str()); } } catch (const std::exception &e) { std::cerr << "[JS] curl callback throws " << e.what() << "\n";