From 463cebf0c4d0fa4a6b8de2b8f0f150e2014c8b23 Mon Sep 17 00:00:00 2001 From: HJfod <60038575+HJfod@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:26:34 +0200 Subject: [PATCH] finish new web requests api --- loader/include/Geode/utils/web.hpp | 13 +- loader/include/Geode/utils/web2.hpp | 84 ++++++++ loader/src/utils/web2.cpp | 313 ++++++++++++++++++++++++++++ 3 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 loader/include/Geode/utils/web2.hpp create mode 100644 loader/src/utils/web2.cpp diff --git a/loader/include/Geode/utils/web.hpp b/loader/include/Geode/utils/web.hpp index d77fd31b..01d27f3a 100644 --- a/loader/include/Geode/utils/web.hpp +++ b/loader/include/Geode/utils/web.hpp @@ -4,6 +4,7 @@ #include "MiniFunction.hpp" #include #include "Result.hpp" +#include "Promise.hpp" #include "general.hpp" #include @@ -19,6 +20,7 @@ namespace geode::utils::web { * @param url URL to fetch * @returns Returned data as bytes, or error on error */ + [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] GEODE_DLL Result fetchBytes(std::string const& url); /** @@ -26,6 +28,7 @@ namespace geode::utils::web { * @param url URL to fetch * @returns Returned data as string, or error on error */ + [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] GEODE_DLL Result fetch(std::string const& url); /** @@ -38,6 +41,7 @@ namespace geode::utils::web { * automatically remove the file that was being downloaded * @returns Returned data as JSON, or error on error */ + [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] GEODE_DLL Result<> fetchFile( std::string const& url, ghc::filesystem::path const& into, FileProgressCallback prog = nullptr ); @@ -47,6 +51,7 @@ namespace geode::utils::web { * @param url URL to fetch * @returns Returned data as JSON, or error on error */ + [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] GEODE_DLL Result fetchJSON(std::string const& url); class SentAsyncWebRequest; @@ -65,7 +70,7 @@ namespace geode::utils::web { * A handle to an in-progress sent asynchronous web request. Use this to * cancel the request / query information about it */ - class GEODE_DLL SentAsyncWebRequest { + class GEODE_DLL [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] SentAsyncWebRequest { private: class Impl; std::shared_ptr m_impl; @@ -114,7 +119,7 @@ namespace geode::utils::web { * internet without slowing the main thread. All callbacks are run in the * GD thread, so interacting with the Cocos2d UI is perfectly safe */ - class GEODE_DLL AsyncWebRequest { + class GEODE_DLL [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] AsyncWebRequest { private: class Impl; std::unique_ptr m_impl; @@ -262,7 +267,7 @@ namespace geode::utils::web { }; template - class AsyncWebResult { + class [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] AsyncWebResult { private: AsyncWebRequest& m_request; DataConverter m_converter; @@ -291,7 +296,7 @@ namespace geode::utils::web { AsyncWebRequest& then(utils::MiniFunction handle); }; - class GEODE_DLL AsyncWebResponse { + class GEODE_DLL [[deprecated("Use the WebRequest class from the web2.hpp header instead")]] AsyncWebResponse { private: AsyncWebRequest& m_request; diff --git a/loader/include/Geode/utils/web2.hpp b/loader/include/Geode/utils/web2.hpp new file mode 100644 index 00000000..fe62ffba --- /dev/null +++ b/loader/include/Geode/utils/web2.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include +#include "Result.hpp" +#include "Promise.hpp" +#include + +namespace geode::utils::web { + class WebRequest; + + class GEODE_DLL WebResponse final { + private: + class Impl; + + std::shared_ptr m_impl; + + friend class WebRequest; + + public: + // Must be default-constructible for use in Promise + WebResponse(); + + int code() const; + + Result string() const; + Result json() const; + ByteVector data() const; + + std::vector headers() const; + std::optional header(std::string_view name) const; + }; + + using WebError = WebResponse; + + class GEODE_DLL WebProgress final { + private: + class Impl; + + std::shared_ptr m_impl; + + friend class WebRequest; + + public: + // Must be default-constructible for use in Promise + WebProgress(); + + size_t downloaded() const; + size_t downloadTotal() const; + std::optional downloadProgress() const; + + size_t uploaded() const; + size_t uploadTotal() const; + std::optional uploadProgress() const; + }; + + using WebPromise = Promise; + + class GEODE_DLL WebRequest final { + private: + class Impl; + + std::shared_ptr m_impl; + + public: + WebRequest(); + ~WebRequest(); + + WebPromise send(std::string_view method, std::string_view url); + WebPromise post(std::string_view url); + WebPromise get(std::string_view url); + WebPromise put(std::string_view url); + WebPromise patch(std::string_view url); + + WebRequest& header(std::string_view name, std::string_view value); + WebRequest& param(std::string_view name, std::string_view value); + WebRequest& userAgent(std::string_view name); + + WebRequest& timeout(std::chrono::seconds time); + + WebRequest& body(ByteVector raw); + WebRequest& bodyString(std::string_view str); + WebRequest& bodyJSON(matjson::Value const& json); + }; +} diff --git a/loader/src/utils/web2.cpp b/loader/src/utils/web2.cpp new file mode 100644 index 00000000..6e49f2c4 --- /dev/null +++ b/loader/src/utils/web2.cpp @@ -0,0 +1,313 @@ +#include +#include +#include + +using namespace geode::prelude; +using namespace geode::utils::web; + +class WebResponse::Impl { +public: + int m_code; + ByteVector m_data; + std::unordered_map m_headers; +}; + +WebResponse::WebResponse() : m_impl(std::make_shared()) {} + +int WebResponse::code() const { + return m_impl->m_code; +} + +Result WebResponse::string() const { + return Ok(std::string(m_impl->m_data.begin(), m_impl->m_data.end())); +} +Result WebResponse::json() const { + GEODE_UNWRAP_INTO(auto value, this->string()); + std::string error; + auto res = matjson::parse(value, error); + if (error.size() > 0) { + return Err("Error parsing JSON: " + error); + } + return Ok(res.value()); +} +ByteVector WebResponse::data() const { + return m_impl->m_data; +} + +std::vector WebResponse::headers() const { + return map::keys(m_impl->m_headers); +} + +std::optional WebResponse::header(std::string_view name) const { + auto str = std::string(name); + if (m_impl->m_headers.contains(str)) { + return m_impl->m_headers.at(str); + } + return std::nullopt; +} + +class WebProgress::Impl { +public: + size_t m_downloadCurrent; + size_t m_downloadTotal; + size_t m_uploadCurrent; + size_t m_uploadTotal; +}; + +WebProgress::WebProgress() : m_impl(std::make_shared()) {} + +size_t WebProgress::downloaded() const { + return m_impl->m_downloadCurrent; +} +size_t WebProgress::downloadTotal() const { + return m_impl->m_downloadTotal; +} +std::optional WebProgress::downloadProgress() const { + return downloadTotal() > 0 ? std::optional(downloaded() * 100.f / downloadTotal()) : std::nullopt; +} + +size_t WebProgress::uploaded() const { + return m_impl->m_uploadCurrent; +} +size_t WebProgress::uploadTotal() const { + return m_impl->m_uploadTotal; +} +std::optional WebProgress::uploadProgress() const { + return uploadTotal() > 0 ? std::optional(uploaded() * 100.f / uploadTotal()) : std::nullopt; +} + +class WebRequest::Impl { +public: + std::string m_method; + std::string m_url; + std::unordered_map m_headers; + std::unordered_map m_urlParameters; + std::optional m_userAgent; + std::optional m_body; + std::optional m_timeout; + + WebResponse makeError(int code, std::string const& msg) { + auto res = WebResponse(); + res.m_impl->m_code = code; + res.m_impl->m_data = toByteArray(msg); + return res; + } +}; + +WebRequest::WebRequest() : m_impl(std::make_shared()) {} +WebRequest::~WebRequest() {} + +WebPromise WebRequest::send(std::string_view method, std::string_view url) { + m_impl->m_method = method; + m_impl->m_url = url; + return WebPromise([impl = m_impl](auto resolve, auto reject, auto progress, auto cancelled) { + // Init Curl + auto curl = curl_easy_init(); + if (!curl) { + reject(impl->makeError(-1, "Curl not initialized")); + return; + } + + // todo: in the future, we might want to support downloading directly into + // files / in-memory streams like the old AsyncWebRequest class + + // Struct that holds values for the curl callbacks + struct ResponseData { + WebResponse response; + Impl* impl; + WebPromise::Progress progress; + PromiseCancellationToken cancelled; + } responseData = { + .response = WebResponse(), + .impl = impl.get(), + .progress = progress, + .cancelled = cancelled, + }; + + // Store downloaded response data into a byte vector + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseData); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](char* data, size_t size, size_t nmemb, void* ptr) { + auto target = static_cast(ptr)->response.m_impl->m_data; + target.insert(target.end(), data, data + size * nmemb); + return size * nmemb; + }); + + // Set headers + curl_slist* headers = nullptr; + for (auto& [name, value] : impl->m_headers) { + // Sanitize header name + auto header = name; + header.erase(std::remove_if(header.begin(), header.end(), [](char c) { + return c == '\r' || c == '\n'; + }), header.end()); + // Append value + header += ": " + value; + headers = curl_slist_append(headers, header.c_str()); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + // Add parameters to the URL and pass it to curl + auto url = impl->m_url; + for (auto param : impl->m_urlParameters) { + url += "&" + param.first + "=" + param.second; + } + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + // Set request method + if (impl->m_method != "GET") { + if (impl->m_method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + } + else { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, impl->m_method.c_str()); + } + } + + // Set body if provided + if (impl->m_body) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, impl->m_body->data()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, impl->m_body->size()); + } + + // No need to verify SSL, we trust our domains :-) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + + // Set user agent if provided + if (impl->m_userAgent) { + curl_easy_setopt(curl, CURLOPT_USERAGENT, impl->m_userAgent->c_str()); + } + + // Set timeout + if (impl->m_timeout) { + curl_easy_setopt(curl, CURLOPT_TIMEOUT, impl->m_timeout->count()); + } + + // Track progress + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + + // Follow redirects + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + + // Do not fail if response code is 4XX or 5XX + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0L); + + // Get headers from the response + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &responseData); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (+[](char* buffer, size_t size, size_t nitems, void* ptr) { + auto& headers = static_cast(ptr)->response.m_impl->m_headers; + std::string line; + std::stringstream ss(std::string(buffer, size * nitems)); + while (std::getline(ss, line)) { + auto colon = line.find(':'); + if (colon == std::string::npos) continue; + auto key = line.substr(0, colon); + auto value = line.substr(colon + 2); + if (value.ends_with('\r')) { + value = value.substr(0, value.size() - 1); + } + headers.insert_or_assign(key, value); + } + return size * nitems; + })); + + // Track & post progress on the Promise + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &responseData); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, +[](void* ptr, double dtotal, double dnow, double utotal, double unow) -> int { + auto data = static_cast(ptr); + + // Check for cancellation and abort if so + if (data->cancelled) { + return 1; + } + + // Post progress to Promise listener + auto progress = WebProgress(); + progress.m_impl->m_downloadTotal = dtotal; + progress.m_impl->m_downloadCurrent = dnow; + progress.m_impl->m_uploadTotal = utotal; + progress.m_impl->m_uploadCurrent = unow; + data->progress(std::move(progress)); + + // Continue as normal + return 0; + }); + + // Make the actual web request + auto curlResponse = curl_easy_perform(curl); + + // Get the response code; note that this will be invalid if the + // curlResponse is not CURLE_OK + long code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + responseData.response.m_impl->m_code = static_cast(code); + + // Free up curl memory + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + // Check if the request failed on curl's side or because of cancellation + if (curlResponse != CURLE_OK) { + if (cancelled) { + reject(impl->makeError(-1, "Request cancelled")); + } + else { + reject(impl->makeError(-1, "Curl failed: " + std::string(curl_easy_strerror(curlResponse)))); + } + return; + } + + // Check if the response was an error code + if (code >= 400 && code <= 600) { + reject(std::move(responseData.response)); + return; + } + + // Otherwise resolve with success :-) + resolve(std::move(responseData.response)); + }); +} +WebPromise WebRequest::post(std::string_view url) { + return this->send("POST", url); +} +WebPromise WebRequest::get(std::string_view url) { + return this->send("GET", url); +} +WebPromise WebRequest::put(std::string_view url) { + return this->send("PUT", url); +} +WebPromise WebRequest::patch(std::string_view url) { + return this->send("PATCH", url); +} + +WebRequest& WebRequest::header(std::string_view name, std::string_view value) { + m_impl->m_headers.insert_or_assign(std::string(name), std::string(value)); + return *this; +} +WebRequest& WebRequest::param(std::string_view name, std::string_view value) { + m_impl->m_urlParameters.insert_or_assign(std::string(name), std::string(value)); + return *this; +} +WebRequest& WebRequest::userAgent(std::string_view name) { + m_impl->m_userAgent = name; + return *this; +} + +WebRequest& WebRequest::timeout(std::chrono::seconds time) { + m_impl->m_timeout = time; + return *this; +} + +WebRequest& WebRequest::body(ByteVector raw) { + m_impl->m_body = raw; + return *this; +} +WebRequest& WebRequest::bodyString(std::string_view str) { + m_impl->m_body = toByteArray(str); + return *this; +} +WebRequest& WebRequest::bodyJSON(matjson::Value const& json) { + this->header("Content-Type", "application/json"); + m_impl->m_body = toByteArray(json); + return *this; +}