More web options (HTTP version, accept encoding, transfer body, follow request, range) (#943)

loader/include/Geode/utils/web.hpp

@@ -25,6 +25,18 @@ namespace geode::utils::web {
         constexpr static long AWS_SIGV4 = 0x0400;
     }
 
+    // https://curl.se/libcurl/c/CURLOPT_HTTP_VERSION.html
+    enum class HttpVersion {
+        DEFAULT,
+        VERSION_1_0,
+        VERSION_1_1,
+        VERSION_2_0,
+        VERSION_2TLS,
+        VERSION_2_PRIOR_KNOWLEDGE,
+        VERSION_3 = 30,
+        VERSION_3ONLY = 31
+    };
+
     // https://curl.se/libcurl/c/CURLOPT_PROXYTYPE.html
     enum class ProxyType {
         HTTP, // HTTP
@@ -68,7 +80,7 @@ namespace geode::utils::web {
         Result<matjson::Value> json() const;
         ByteVector data() const;
         Result<> into(std::filesystem::path const& path) const;
-        
+
         std::vector<std::string> headers() const;
         std::optional<std::string> header(std::string_view name) const;
     };
@@ -120,16 +132,124 @@ namespace geode::utils::web {
             return this->param(name, std::to_string(value));
         }
 
+        /**
+         * Sets the request's user agent.
+         * Defaults to not sending the User-Agent: header.
+         *
+         * @param name
+         * @return WebRequest&
+         */
         WebRequest& userAgent(std::string_view name);
 
+        /**
+         * Sets the response's encoding. Valid values include: br, gzip, deflate, ...
+         * You can set multiple encoding types by calling this method with a comma separated list
+         * of the encodings of your choosing.
+         * Defaults to not sending an Accept-Encoding: header, and in turn, does not decompress received contents automatically.
+         *
+         * @example
+         * auto req = web::WebRequest()
+         *  .acceptEncoding("gzip, deflate")
+         *  .get(url);
+         *
+         * @param encodingType Target response encoding type. An empty string ("") will use all built-in supported encodings.
+         * @return WebRequest&
+         */
+        WebRequest& acceptEncoding(std::string_view encodingType);
+
+        /**
+         * Sets the maximum amount of seconds to allow the entire transfer operation to take.
+         * The default timeout is 0, which means the request never times out during transfer.
+         *
+         * @param time
+         * @return WebRequest&
+         */
         WebRequest& timeout(std::chrono::seconds time);
 
+        /**
+         * Sets the target byte range to request.
+         * Defaults to receiving the full request.
+         *
+         * @param byteRange a pair of ints, first value is what byte to start from, second value is the last byte to get (both inclusive)
+         * @return WebRequest&
+         */
+        WebRequest& downloadRange(std::pair<std::uint64_t, std::uint64_t> byteRange);
+
+        /**
+         * Enable or disables peer verification in SSL handshake.
+         * The default is true.
+         *
+         * @param enabled
+         * @return WebRequest&
+         */
         WebRequest& certVerification(bool enabled);
+
+        /**
+         * Enables or disabled getting the body of a request. For HTTP(S), this does a HEAD request.
+         * For most other protocols it means just not asking to transfer the body data. 
+         * The default is true.
+         *
+         * @param enabled
+         * @return WebRequest&
+         */
+        WebRequest& transferBody(bool enabled);
+
+        /**
+         * Follow HTTP 3xx redirects.
+         * The default is true.
+         *
+         * @param enabled
+         * @return WebRequest&
+         */
+        WebRequest& followRedirects(bool enabled);
+
+        /**
+         * Sets the Certificate Authority (CA) bundle content.
+         * Defaults to not sending a CA bundle.
+         *
+         * @param content
+         * @return WebRequest&
+         */
         WebRequest& CABundleContent(std::string_view content);
+
+        /**
+         * Sets the request's proxy.
+         * Defaults to not using a proxy.
+         *
+         * @param proxyOpts
+         * @return WebRequest&
+         */
         WebRequest& proxyOpts(ProxyOpts const& proxyOpts);
 
+        /**
+         * Sets the request's HTTP version.
+         * The default is HttpVersion::VERSION_2TLS.
+         *
+         * @param httpVersion
+         * @return WebRequest&
+         */
+        WebRequest& version(HttpVersion httpVersion);
+
+        /**
+         * Sets the body of the request to a byte vector.
+         *
+         * @param raw The raw bytes to set as the body.
+         * @return WebRequest&
+         */
         WebRequest& body(ByteVector raw);
+        /**
+         * Sets the body of the request to a string.
+         *
+         * @param str The string to set as the body.
+         * @return WebRequest&
+         */
         WebRequest& bodyString(std::string_view str);
+        /**
+         * Sets the body of the request to a json object.
+         *
+         * @param json
+         * @return WebRequest&
+         */
         WebRequest& bodyJSON(matjson::Value const& json);
     };
 }

loader/src/utils/web.cpp

@@ -65,6 +65,33 @@ static long unwrapHttpAuth(long auth) {
     return unwrapped;
 }
 
+static long unwrapHttpVersion(HttpVersion version)
+{
+    switch (version) {
+        using enum HttpVersion;
+
+        case DEFAULT:
+            return CURL_HTTP_VERSION_NONE;
+        case VERSION_1_0:
+            return CURL_HTTP_VERSION_1_0;
+        case VERSION_1_1:
+            return CURL_HTTP_VERSION_1_1;
+        case VERSION_2_0:
+            return CURL_HTTP_VERSION_2_0;
+        case VERSION_2TLS:
+            return CURL_HTTP_VERSION_2TLS;
+        case VERSION_2_PRIOR_KNOWLEDGE:
+            return CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE;
+        case VERSION_3:
+            return CURL_HTTP_VERSION_3;
+        case VERSION_3ONLY:
+            return CURL_HTTP_VERSION_3ONLY;
+    }
+
+    // Shouldn't happen.
+    unreachable("Unexpected HTTP Version!");
+}
+
 class WebResponse::Impl {
 public:
     int m_code;
@@ -166,11 +193,16 @@ public:
     std::unordered_map<std::string, std::string> m_headers;
     std::unordered_map<std::string, std::string> m_urlParameters;
     std::optional<std::string> m_userAgent;
+    std::optional<std::string> m_acceptEncodingType;
     std::optional<ByteVector> m_body;
     std::optional<std::chrono::seconds> m_timeout;
+    std::optional<std::pair<std::uint64_t, std::uint64_t>> m_range;
     bool m_certVerification = true;
+    bool m_transferBody = true;
+    bool m_followRedirects = true;
     std::string m_CABundleContent;
     ProxyOpts m_proxyOpts = {};
+    HttpVersion m_httpVersion = HttpVersion::DEFAULT;
 
     WebResponse makeError(int code, std::string const& msg) {
         auto res = WebResponse();
@@ -254,6 +286,9 @@ WebTask WebRequest::send(std::string_view method, std::string_view url) {
         }
         curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 
+        // Set HTTP version
+        curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, unwrapHttpVersion(impl->m_httpVersion));
+
         // Set request method
         if (impl->m_method != "GET") {
             if (impl->m_method == "POST") {
@@ -273,7 +308,7 @@ WebTask WebRequest::send(std::string_view method, std::string_view url) {
             // why? god knows
             curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");
         }
-        
+
         // Cert verification
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, impl->m_certVerification ? 1 : 0);
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2);
@@ -289,18 +324,31 @@ WebTask WebRequest::send(std::string_view method, std::string_view url) {
             caBundleBlob.len = impl->m_CABundleContent.size();
             caBundleBlob.flags = CURL_BLOB_COPY;
             curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &caBundleBlob);
-        }  
+        }
+
+        // Transfer body
+        curl_easy_setopt(curl, CURLOPT_NOBODY, impl->m_transferBody ? 0L : 1L);
 
         // Set user agent if provided
         if (impl->m_userAgent) {
             curl_easy_setopt(curl, CURLOPT_USERAGENT, impl->m_userAgent->c_str());
         }
 
+        // Set encoding
+        if (impl->m_acceptEncodingType) {
+            curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, impl->m_acceptEncodingType->c_str());
+        }
+
         // Set timeout
         if (impl->m_timeout) {
             curl_easy_setopt(curl, CURLOPT_TIMEOUT, impl->m_timeout->count());
         }
 
+        // Set range
+        if (impl->m_range) {
+            curl_easy_setopt(curl, CURLOPT_RANGE, fmt::format("{}-{}", impl->m_range->first, impl->m_range->second).c_str());
+        }
+
         // Set proxy options
         auto const& proxyOpts = impl->m_proxyOpts;
         if (!proxyOpts.address.empty()) {
@@ -323,11 +371,12 @@ WebTask WebRequest::send(std::string_view method, std::string_view url) {
             curl_easy_setopt(curl, CURLOPT_PROXY_SSL_VERIFYHOST, 2);
         }
 
+        // Follow request through 3xx responses
+        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, impl->m_followRedirects ? 1L : 0L);
+
         // Track progress
         curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
 
-        // Follow redirects
-        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
         // don't change the method from POST to GET when following a redirect
         curl_easy_setopt(curl, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
 
@@ -438,10 +487,24 @@ WebRequest& WebRequest::timeout(std::chrono::seconds time) {
     return *this;
 }
 
+WebRequest& WebRequest::downloadRange(std::pair<std::uint64_t, std::uint64_t> byteRange) {
+    m_impl->m_range = byteRange;
+    return *this;
+}
+
 WebRequest& WebRequest::certVerification(bool enabled) {
     m_impl->m_certVerification = enabled;
     return *this;
 }
+WebRequest& WebRequest::transferBody(bool enabled) {
+    m_impl->m_transferBody = enabled;
+    return *this;
+}
+
+WebRequest& WebRequest::followRedirects(bool enabled) {
+    m_impl->m_followRedirects = enabled;
+    return *this;
+}
 
 WebRequest& WebRequest::CABundleContent(std::string_view content) {
     m_impl->m_CABundleContent = content;
@@ -453,6 +516,16 @@ WebRequest& WebRequest::proxyOpts(ProxyOpts const& proxyOpts) {
     return *this;
 }
 
+WebRequest& WebRequest::version(HttpVersion httpVersion) {
+    m_impl->m_httpVersion = httpVersion;
+    return *this;
+}
+
+WebRequest& WebRequest::acceptEncoding(std::string_view str) {
+    m_impl->m_acceptEncodingType = str;
+    return *this;
+}
+
 WebRequest& WebRequest::body(ByteVector raw) {
     m_impl->m_body = raw;
     return *this;