From d43da2300c9e345e25016888f4418b00c142e6ec Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Fri, 16 Aug 2024 19:52:52 +0000 Subject: [PATCH 1/7] Add a plugin mechanism to resource_retriever Signed-off-by: Michael Carroll Signed-off-by: William Woodall --- resource_retriever/CMakeLists.txt | 8 +- .../include/resource_retriever/exception.hpp | 49 ++++++ .../resource_retriever/memory_resource.hpp | 56 +++++++ .../plugins/curl_retriever.hpp | 68 ++++++++ .../plugins/filesystem_retriever.hpp | 57 +++++++ .../plugins/retriever_plugin.hpp | 55 ++++++ .../include/resource_retriever/retriever.hpp | 36 +--- .../src/plugins/curl_retriever.cpp | 157 ++++++++++++++++++ .../src/plugins/filesystem_retriever.cpp | 91 ++++++++++ .../src/plugins/retriever_plugin.cpp | 91 ++++++++++ resource_retriever/src/retriever.cpp | 151 +++-------------- resource_retriever/test/test.cpp | 80 +++++---- 12 files changed, 711 insertions(+), 188 deletions(-) create mode 100644 resource_retriever/include/resource_retriever/exception.hpp create mode 100644 resource_retriever/include/resource_retriever/memory_resource.hpp create mode 100644 resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp create mode 100644 resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp create mode 100644 resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp create mode 100644 resource_retriever/src/plugins/curl_retriever.cpp create mode 100644 resource_retriever/src/plugins/filesystem_retriever.cpp create mode 100644 resource_retriever/src/plugins/retriever_plugin.cpp diff --git a/resource_retriever/CMakeLists.txt b/resource_retriever/CMakeLists.txt index 9e05996..f5e8ee3 100644 --- a/resource_retriever/CMakeLists.txt +++ b/resource_retriever/CMakeLists.txt @@ -17,8 +17,12 @@ find_package(libcurl_vendor REQUIRED) find_package(CURL REQUIRED) # TODO(wjwwood): split cpp and python apis into separate packages - -add_library(${PROJECT_NAME} src/retriever.cpp) +add_library(${PROJECT_NAME} + src/retriever.cpp + src/plugins/retriever_plugin.cpp + src/plugins/filesystem_retriever.cpp + src/plugins/curl_retriever.cpp +) target_include_directories(${PROJECT_NAME} PUBLIC $ diff --git a/resource_retriever/include/resource_retriever/exception.hpp b/resource_retriever/include/resource_retriever/exception.hpp new file mode 100644 index 0000000..1a075f6 --- /dev/null +++ b/resource_retriever/include/resource_retriever/exception.hpp @@ -0,0 +1,49 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#ifndef RESOURCE_RETRIEVER__EXCEPTION_HPP_ +#define RESOURCE_RETRIEVER__EXCEPTION_HPP_ + +#include +#include + +#include "resource_retriever/visibility_control.hpp" + +namespace resource_retriever +{ +class RESOURCE_RETRIEVER_PUBLIC Exception : public std::runtime_error +{ +public: + Exception(const std::string & file, const std::string & error_msg) + : std::runtime_error("Error retrieving file [" + file + "]: " + error_msg) + { + } +}; +} // namespace resource_retriever + +#endif // RESOURCE_RETRIEVER__EXCEPTION_HPP_ diff --git a/resource_retriever/include/resource_retriever/memory_resource.hpp b/resource_retriever/include/resource_retriever/memory_resource.hpp new file mode 100644 index 0000000..c5ddabe --- /dev/null +++ b/resource_retriever/include/resource_retriever/memory_resource.hpp @@ -0,0 +1,56 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#ifndef RESOURCE_RETRIEVER__MEMORY_RESOURCE_HPP_ +#define RESOURCE_RETRIEVER__MEMORY_RESOURCE_HPP_ + +#include +#include +#include + +#include "resource_retriever/visibility_control.hpp" + +namespace resource_retriever +{ +/** + * \brief A combination of a pointer to data in memory along with the data's size. + */ +struct RESOURCE_RETRIEVER_PUBLIC MemoryResource +{ + explicit MemoryResource(std::vector data): + data(std::move(data)) + { + } + const std::vector data; +}; + +using MemoryResourcePtr = std::shared_ptr; + +} // namespace resource_retriever + +#endif // RESOURCE_RETRIEVER__MEMORY_RESOURCE_HPP_ diff --git a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp new file mode 100644 index 0000000..3bf0a60 --- /dev/null +++ b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp @@ -0,0 +1,68 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#ifndef RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ +#define RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ + +#include +#include +#include +#include +#include + +#include "resource_retriever/plugins/retriever_plugin.hpp" +#include "resource_retriever/visibility_control.hpp" + +using CURL = void; + +namespace resource_retriever::plugins +{ + +class RESOURCE_RETRIEVER_PUBLIC CurlRetriever: public RetrieverPlugin +{ +public: + CurlRetriever(); + ~CurlRetriever() override; + + CurlRetriever(const CurlRetriever & ret) = delete; + CurlRetriever & operator=(const CurlRetriever & other) = delete; + + CurlRetriever(CurlRetriever && other) noexcept; + CurlRetriever & operator=(CurlRetriever && other) noexcept; + + bool can_handle(const std::string & url) override; + std::string name() override; + MemoryResourcePtr get(const std::string & url) override; + +private: + CURL * curl_handle_ {nullptr}; +}; + +} // namespace resource_retriever::plugins + +#endif // RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ diff --git a/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp new file mode 100644 index 0000000..bc90622 --- /dev/null +++ b/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp @@ -0,0 +1,57 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#ifndef RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ +#define RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ + +#include +#include +#include +#include +#include + +#include "resource_retriever/plugins/retriever_plugin.hpp" +#include "resource_retriever/visibility_control.hpp" + +namespace resource_retriever::plugins +{ + +class RESOURCE_RETRIEVER_PUBLIC FilesystemRetriever: public RetrieverPlugin +{ +public: + FilesystemRetriever(); + ~FilesystemRetriever() override; + + bool can_handle(const std::string & url) override; + std::string name() override; + MemoryResourcePtr get(const std::string & url) override; +}; + +} // namespace resource_retriever::plugins + +#endif // RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ diff --git a/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp b/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp new file mode 100644 index 0000000..91f1d54 --- /dev/null +++ b/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp @@ -0,0 +1,55 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#ifndef RESOURCE_RETRIEVER__PLUGINS__RETRIEVER_PLUGIN_HPP_ +#define RESOURCE_RETRIEVER__PLUGINS__RETRIEVER_PLUGIN_HPP_ + +#include + +#include +#include "resource_retriever/visibility_control.hpp" + +namespace resource_retriever::plugins +{ +std::string expand_package_url(const std::string & url); +std::string escape_spaces(const std::string & url); + +class RESOURCE_RETRIEVER_PUBLIC RetrieverPlugin +{ +public: + RetrieverPlugin(); + virtual ~RetrieverPlugin() = 0; + + virtual std::string name() = 0; + virtual bool can_handle(const std::string & url) = 0; + virtual MemoryResourcePtr get(const std::string & url) = 0; +}; + +} // namespace resource_retriever::plugins + +#endif // RESOURCE_RETRIEVER__PLUGINS__RETRIEVER_PLUGIN_HPP_ diff --git a/resource_retriever/include/resource_retriever/retriever.hpp b/resource_retriever/include/resource_retriever/retriever.hpp index 2a1dcdc..cdeeb3d 100644 --- a/resource_retriever/include/resource_retriever/retriever.hpp +++ b/resource_retriever/include/resource_retriever/retriever.hpp @@ -33,30 +33,18 @@ #include #include #include +#include +#include "resource_retriever/plugins/retriever_plugin.hpp" #include "resource_retriever/visibility_control.hpp" -using CURL = void; - namespace resource_retriever { -class Exception : public std::runtime_error -{ -public: - Exception(const std::string & file, const std::string & error_msg) - : std::runtime_error("Error retrieving file [" + file + "]: " + error_msg) - { - } -}; -/** - * \brief A combination of a pointer to data in memory along with the data's size. - */ -struct MemoryResource -{ - std::shared_ptr data; - size_t size {0}; -}; +using RetrieverPluginPtr = std::shared_ptr; +using RetrieverVec = std::vector; + +RetrieverVec RESOURCE_RETRIEVER_PUBLIC default_plugins(); /** * \brief Retrieves files from from a url. Caches a CURL handle so multiple accesses to a single url @@ -65,26 +53,20 @@ struct MemoryResource class RESOURCE_RETRIEVER_PUBLIC Retriever { public: - Retriever(); + explicit Retriever(RetrieverVec plugins = default_plugins()); ~Retriever(); - Retriever(const Retriever & ret) = delete; - Retriever & operator=(const Retriever & other) = delete; - - Retriever(Retriever && other) noexcept; - Retriever & operator=(Retriever && other) noexcept; - /** * \brief Get a file and store it in memory * \param url The url to retrieve. package://package/file will be turned into the correct file:// invocation * \return The file, loaded into memory * \throws resource_retriever::Exception if anything goes wrong. */ - MemoryResource get(const std::string & url); + MemoryResourcePtr get(const std::string & url); private: - CURL * curl_handle_ {nullptr}; + RetrieverVec plugins; }; } // namespace resource_retriever diff --git a/resource_retriever/src/plugins/curl_retriever.cpp b/resource_retriever/src/plugins/curl_retriever.cpp new file mode 100644 index 0000000..09f96bb --- /dev/null +++ b/resource_retriever/src/plugins/curl_retriever.cpp @@ -0,0 +1,157 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "resource_retriever/plugins/curl_retriever.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +#include "resource_retriever/exception.hpp" + +namespace +{ + +class CURLStaticInit +{ +public: + CURLStaticInit() + { + CURLcode ret = curl_global_init(CURL_GLOBAL_ALL); + if (ret != 0) { + fprintf(stderr, "Error initializing libcurl! retcode = %d", ret); + } else { + initialized_ = true; + } + } + + ~CURLStaticInit() + { + if (initialized_) { + curl_global_cleanup(); + } + } + +private: + bool initialized_ {false}; +}; +CURLStaticInit g_curl_init; +} // namespace + +namespace resource_retriever::plugins +{ + +struct MemoryBuffer +{ + std::vector v; +}; + +size_t curlWriteFunc(void * buffer, size_t size, size_t nmemb, void * userp) +{ + auto * membuf = reinterpret_cast(userp); + size_t prev_size = membuf->v.size(); + membuf->v.resize(prev_size + size * nmemb); + memcpy(&membuf->v[prev_size], buffer, size * nmemb); + return size * nmemb; +} + +CurlRetriever::CurlRetriever(): + curl_handle_(curl_easy_init()) +{ + +} +CurlRetriever::~CurlRetriever() +{ + if (curl_handle_ != nullptr) { + curl_easy_cleanup(curl_handle_); + } +} + +CurlRetriever::CurlRetriever(CurlRetriever && other) noexcept +: curl_handle_(std::exchange(other.curl_handle_, nullptr)) +{ +} + +CurlRetriever & CurlRetriever::operator=(CurlRetriever && other) noexcept +{ + std::swap(curl_handle_, other.curl_handle_); + return *this; +} + + + +std::string CurlRetriever::name() { + return "resource_retriever::plugins::CurlRetriever"; +} + +bool CurlRetriever::can_handle(const std::string & url) +{ + return (url.find("package://") == 0 || + url.find("file://") == 0 || + url.find("http://") == 0 || + url.find("https://") == 0); +} + +MemoryResourcePtr CurlRetriever::get(const std::string & url) +{ + // Expand package:// url into file:// + auto mod_url = url; + try { + mod_url = expand_package_url(mod_url); + } catch (const resource_retriever::Exception &e) { + return nullptr; + } + + // newer versions of curl do not accept spaces in URLs + mod_url = escape_spaces(mod_url); + + curl_easy_setopt(curl_handle_, CURLOPT_URL, mod_url.c_str()); + curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION, curlWriteFunc); + + char error_buffer[CURL_ERROR_SIZE]; + curl_easy_setopt(curl_handle_, CURLOPT_ERRORBUFFER, error_buffer); + + MemoryResourcePtr res {nullptr}; + MemoryBuffer buf; + curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, &buf); + + CURLcode ret = curl_easy_perform(curl_handle_); + + if (ret == 0 && !buf.v.empty()) { + res = std::make_shared(buf.v); + } + + return res; +} + +} // namespace resource_retriever::plugins diff --git a/resource_retriever/src/plugins/filesystem_retriever.cpp b/resource_retriever/src/plugins/filesystem_retriever.cpp new file mode 100644 index 0000000..dd4bc91 --- /dev/null +++ b/resource_retriever/src/plugins/filesystem_retriever.cpp @@ -0,0 +1,91 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "resource_retriever/plugins/filesystem_retriever.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "resource_retriever/exception.hpp" + +namespace resource_retriever::plugins +{ + +FilesystemRetriever::FilesystemRetriever() = default; + +FilesystemRetriever::~FilesystemRetriever() = default; + +std::string FilesystemRetriever::name() { + return "resource_retriever::plugins::FilesystemRetriever"; +} + +bool FilesystemRetriever::can_handle(const std::string & url) +{ + return url.find("package://") == 0 || url.find("file://") == 0; +} + +MemoryResourcePtr FilesystemRetriever::get(const std::string & url) +{ + // Expand package:// url into file:// + auto mod_url = url; + try { + mod_url = expand_package_url(mod_url); + } catch (const resource_retriever::Exception &e) { + return nullptr; + } + + if (mod_url.find("file://") == 0) + { + mod_url = mod_url.substr(7); + } + + std::ifstream file(mod_url, std::ios::binary); + MemoryResourcePtr res {nullptr}; + + if (file.is_open()) { + // Get the file size + file.seekg(0, std::ios::end); + std::streampos fileSize = file.tellg(); + file.seekg(0, std::ios::beg); + + // Create the vector and read the file + std::vector data(fileSize); + file.read(reinterpret_cast(data.data()), fileSize); + file.close(); + res = std::make_shared(data); + } + + return res; +} + +} // namespace resource_retriever::plugins diff --git a/resource_retriever/src/plugins/retriever_plugin.cpp b/resource_retriever/src/plugins/retriever_plugin.cpp new file mode 100644 index 0000000..908704c --- /dev/null +++ b/resource_retriever/src/plugins/retriever_plugin.cpp @@ -0,0 +1,91 @@ +// Copyright 2009, Willow Garage, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the Willow Garage, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "resource_retriever/plugins/retriever_plugin.hpp" + +#include + +#include "resource_retriever/exception.hpp" + +#include "ament_index_cpp/get_package_prefix.hpp" +#include "ament_index_cpp/get_package_share_directory.hpp" + +namespace resource_retriever::plugins +{ + +std::string escape_spaces(const std::string & url) +{ + std::string new_mod_url; + new_mod_url.reserve(url.length()); + + std::string::size_type last_pos = 0; + std::string::size_type find_pos; + + while (std::string::npos != (find_pos = url.find(' ', last_pos))) { + new_mod_url.append(url, last_pos, find_pos - last_pos); + new_mod_url += "%20"; + last_pos = find_pos + std::string(" ").length(); + } + + // Take care for the rest after last occurrence + new_mod_url.append(url, last_pos, url.length() - last_pos); + return new_mod_url; +} + +std::string expand_package_url(const std::string & url) +{ + std::string mod_url = url; + if (url.find("package://") == 0) { + mod_url.erase(0, strlen("package://")); + size_t pos = mod_url.find('/'); + if (pos == std::string::npos) { + throw Exception(url, "Could not parse package:// format into file:// format"); + } + + std::string package = mod_url.substr(0, pos); + if (package.empty()) { + throw Exception(url, "Package name must not be empty"); + } + mod_url.erase(0, pos); + std::string package_path; + try { + package_path = ament_index_cpp::get_package_share_directory(package); + } catch (const ament_index_cpp::PackageNotFoundError &) { + throw Exception(url, "Package [" + package + "] does not exist"); + } + + mod_url = "file://" + package_path + mod_url; + } + return mod_url; +}; + +RetrieverPlugin::RetrieverPlugin() = default; + +RetrieverPlugin::~RetrieverPlugin() = default; + +} // namespace resource_retriever::plugins diff --git a/resource_retriever/src/retriever.cpp b/resource_retriever/src/retriever.cpp index 3ca5371..aae79bd 100644 --- a/resource_retriever/src/retriever.cpp +++ b/resource_retriever/src/retriever.cpp @@ -28,159 +28,48 @@ #include "resource_retriever/retriever.hpp" -#include - #include #include #include #include #include -#include "ament_index_cpp/get_package_prefix.hpp" -#include "ament_index_cpp/get_package_share_directory.hpp" - - -namespace -{ -class CURLStaticInit -{ -public: - CURLStaticInit() - { - CURLcode ret = curl_global_init(CURL_GLOBAL_ALL); - if (ret != 0) { - fprintf(stderr, "Error initializing libcurl! retcode = %d", ret); - } else { - initialized_ = true; - } - } - - ~CURLStaticInit() - { - if (initialized_) { - curl_global_cleanup(); - } - } - -private: - bool initialized_ {false}; -}; -CURLStaticInit g_curl_init; -} // namespace +#include "resource_retriever/exception.hpp" +#include "resource_retriever/plugins/curl_retriever.hpp" +#include "resource_retriever/plugins/filesystem_retriever.hpp" namespace resource_retriever { -Retriever::Retriever() -: curl_handle_(curl_easy_init()) -{ -} - -Retriever::~Retriever() -{ - if (curl_handle_ != nullptr) { - curl_easy_cleanup(curl_handle_); - } -} - -Retriever::Retriever(Retriever && other) noexcept -: curl_handle_(std::exchange(other.curl_handle_, nullptr)) -{ -} - -Retriever & Retriever::operator=(Retriever && other) noexcept +RetrieverVec default_plugins() { - std::swap(curl_handle_, other.curl_handle_); - return *this; + return { + std::make_shared(), + std::make_shared(), + }; } -struct MemoryBuffer +Retriever::Retriever(RetrieverVec plugins): + plugins(std::move(plugins)) { - std::vector v; -}; - -size_t curlWriteFunc(void * buffer, size_t size, size_t nmemb, void * userp) -{ - MemoryBuffer * membuf = reinterpret_cast(userp); - - size_t prev_size = membuf->v.size(); - membuf->v.resize(prev_size + size * nmemb); - memcpy(&membuf->v[prev_size], buffer, size * nmemb); - - return size * nmemb; } -static std::string escape_spaces(const std::string & url) -{ - std::string new_mod_url; - new_mod_url.reserve(url.length()); - - std::string::size_type last_pos = 0; - std::string::size_type find_pos; +Retriever::~Retriever() = default; - while (std::string::npos != (find_pos = url.find(" ", last_pos))) { - new_mod_url.append(url, last_pos, find_pos - last_pos); - new_mod_url += "%20"; - last_pos = find_pos + std::string(" ").length(); - } - - // Take care for the rest after last occurrence - new_mod_url.append(url, last_pos, url.length() - last_pos); - return new_mod_url; -} - -MemoryResource Retriever::get(const std::string & url) +MemoryResourcePtr Retriever::get(const std::string & url) { - std::string mod_url = url; - if (url.find("package://") == 0) { - mod_url.erase(0, strlen("package://")); - size_t pos = mod_url.find('/'); - if (pos == std::string::npos) { - throw Exception(url, "Could not parse package:// format into file:// format"); - } + for (auto & plugin : plugins) + { + if (plugin->can_handle(url)) + { + auto res = plugin->get(url); - std::string package = mod_url.substr(0, pos); - if (package.empty()) { - throw Exception(url, "Package name must not be empty"); + if (res != nullptr) + return res; } - mod_url.erase(0, pos); - std::string package_path; - try { - package_path = ament_index_cpp::get_package_share_directory(package); - } catch (const ament_index_cpp::PackageNotFoundError &) { - throw Exception(url, "Package [" + package + "] does not exist"); - } - - mod_url = "file://" + package_path + mod_url; } - - // newer versions of curl do not accept spaces in URLs - mod_url = escape_spaces(mod_url); - - curl_easy_setopt(curl_handle_, CURLOPT_URL, mod_url.c_str()); - curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION, curlWriteFunc); - - char error_buffer[CURL_ERROR_SIZE]; - curl_easy_setopt(curl_handle_, CURLOPT_ERRORBUFFER, error_buffer); - - MemoryResource res; - MemoryBuffer buf; - curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, &buf); - - CURLcode ret = curl_easy_perform(curl_handle_); - if (ret != 0) { - throw Exception(mod_url, error_buffer); - } - - if (!buf.v.empty()) { - res.size = buf.v.size(); - // Converted from boost::shared_array, see: https://stackoverflow.com/a/8624884 - res.data.reset(new uint8_t[res.size], std::default_delete()); - memcpy(res.data.get(), &buf.v[0], res.size); - } - - return res; + return nullptr; } } // namespace resource_retriever diff --git a/resource_retriever/test/test.cpp b/resource_retriever/test/test.cpp index 4f14aa2..b8d8b33 100644 --- a/resource_retriever/test/test.cpp +++ b/resource_retriever/test/test.cpp @@ -31,16 +31,26 @@ #include "gtest/gtest.h" +#include "resource_retriever/memory_resource.hpp" +#include "resource_retriever/plugins/retriever_plugin.hpp" #include "resource_retriever/retriever.hpp" +TEST(Retriever, defaultPlugins) +{ + auto plugins = resource_retriever::default_plugins(); + + EXPECT_EQ(plugins.size(), 2u); +} + TEST(Retriever, getByPackage) { try { resource_retriever::Retriever r; - resource_retriever::MemoryResource res = r.get("package://resource_retriever/test/test.txt"); + auto res = r.get("package://resource_retriever/test/test.txt"); - ASSERT_EQ(res.size, 1u); - ASSERT_EQ(*res.data, 'A'); + ASSERT_NE(nullptr, res); + ASSERT_EQ(res->data.size(), 1u); + ASSERT_EQ(*res->data.data(), 'A'); } catch (const std::exception & e) { FAIL() << "Exception caught: " << e.what() << '\n'; } @@ -50,9 +60,10 @@ TEST(Retriever, http) { try { resource_retriever::Retriever r; - resource_retriever::MemoryResource res = r.get("http://packages.ros.org/ros.key"); + auto res = r.get("http://packages.ros.org/ros.key"); - ASSERT_GT(res.size, 0u); + ASSERT_NE(nullptr, res); + ASSERT_GT(res->data.size(), 0u); } catch (const std::exception & e) { FAIL() << "Exception caught: " << e.what() << '\n'; } @@ -62,34 +73,47 @@ TEST(Retriever, invalidFiles) { resource_retriever::Retriever r; - try { - r.get("file://fail"); - FAIL(); - } catch (const std::exception & e) { - (void)e; - } + EXPECT_EQ(nullptr, r.get("file://fail")); + EXPECT_EQ(nullptr, r.get("package://roscpp")); + EXPECT_EQ(nullptr, r.get("package://invalid_package_blah/test.xml")); + EXPECT_EQ(nullptr, r.get("package:///test.xml")); +} - try { - r.get("package://roscpp"); - FAIL(); - } catch (const std::exception & e) { - (void)e; - } +class TestRetrieverPlugin: public resource_retriever::plugins::RetrieverPlugin +{ +public: + TestRetrieverPlugin() = default; + ~TestRetrieverPlugin() override = default; - try { - r.get("package://invalid_package_blah/test.xml"); - FAIL(); - } catch (const std::exception & e) { - (void)e; + std::string name() override { + return "TestRetrieverPlugin"; + } + + bool can_handle(const std::string & url) override { + return url.find("test://") == 0; } - // Empty package name - try { - r.get("package:///test.xml"); - FAIL(); - } catch (const std::exception & e) { - (void)e; + resource_retriever::MemoryResourcePtr get(const std::string & url) override { + (void) url; + return std::make_shared(std::vector{0, 1, 2, 3, 4, 5}); } +}; + +TEST(Retriever, customPlugin) +{ + resource_retriever::RetrieverVec plugins { + std::make_shared() + }; + + resource_retriever::Retriever r(plugins); + EXPECT_EQ(nullptr, r.get("package://resource_retriever/test/text.txt")); + EXPECT_EQ(nullptr, r.get("file://foo/bar/text.txt")); + EXPECT_NE(nullptr, r.get("test://foo")); + + auto res = r.get("test://foo"); + EXPECT_EQ(res->data.size(), 6u); + EXPECT_EQ(res->data[0], 0u); + EXPECT_EQ(res->data[1], 1u); } int main(int argc, char ** argv) From ceee468efdf2e7c65b0fc3b85992573deb1b40c2 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Wed, 21 Aug 2024 19:10:39 +0000 Subject: [PATCH 2/7] Update memory resource Signed-off-by: Michael Carroll Signed-off-by: William Woodall --- .../include/resource_retriever/memory_resource.hpp | 7 ++++++- resource_retriever/src/plugins/curl_retriever.cpp | 2 +- resource_retriever/src/plugins/filesystem_retriever.cpp | 2 +- resource_retriever/test/test.cpp | 3 +-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/resource_retriever/include/resource_retriever/memory_resource.hpp b/resource_retriever/include/resource_retriever/memory_resource.hpp index c5ddabe..b9475e8 100644 --- a/resource_retriever/include/resource_retriever/memory_resource.hpp +++ b/resource_retriever/include/resource_retriever/memory_resource.hpp @@ -30,6 +30,7 @@ #define RESOURCE_RETRIEVER__MEMORY_RESOURCE_HPP_ #include +#include #include #include @@ -42,10 +43,14 @@ namespace resource_retriever */ struct RESOURCE_RETRIEVER_PUBLIC MemoryResource { - explicit MemoryResource(std::vector data): + explicit MemoryResource(std::string url, std::string expanded_url, std::vector data): + url(std::move(url)), + expanded_url(std::move(expanded_url)), data(std::move(data)) { } + const std::string url; + const std::string expanded_url; const std::vector data; }; diff --git a/resource_retriever/src/plugins/curl_retriever.cpp b/resource_retriever/src/plugins/curl_retriever.cpp index 09f96bb..803a3f8 100644 --- a/resource_retriever/src/plugins/curl_retriever.cpp +++ b/resource_retriever/src/plugins/curl_retriever.cpp @@ -148,7 +148,7 @@ MemoryResourcePtr CurlRetriever::get(const std::string & url) CURLcode ret = curl_easy_perform(curl_handle_); if (ret == 0 && !buf.v.empty()) { - res = std::make_shared(buf.v); + res = std::make_shared(url, mod_url, buf.v); } return res; diff --git a/resource_retriever/src/plugins/filesystem_retriever.cpp b/resource_retriever/src/plugins/filesystem_retriever.cpp index dd4bc91..98349ac 100644 --- a/resource_retriever/src/plugins/filesystem_retriever.cpp +++ b/resource_retriever/src/plugins/filesystem_retriever.cpp @@ -82,7 +82,7 @@ MemoryResourcePtr FilesystemRetriever::get(const std::string & url) std::vector data(fileSize); file.read(reinterpret_cast(data.data()), fileSize); file.close(); - res = std::make_shared(data); + res = std::make_shared(url, mod_url, data); } return res; diff --git a/resource_retriever/test/test.cpp b/resource_retriever/test/test.cpp index b8d8b33..b4b6233 100644 --- a/resource_retriever/test/test.cpp +++ b/resource_retriever/test/test.cpp @@ -94,8 +94,7 @@ class TestRetrieverPlugin: public resource_retriever::plugins::RetrieverPlugin } resource_retriever::MemoryResourcePtr get(const std::string & url) override { - (void) url; - return std::make_shared(std::vector{0, 1, 2, 3, 4, 5}); + return std::make_shared(url, url, std::vector{0, 1, 2, 3, 4, 5}); } }; From cbffde2dbda35f8af25e8f03d99f7504a615dfb0 Mon Sep 17 00:00:00 2001 From: Eloy Briceno <51831786+Voldivh@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:29:30 -0500 Subject: [PATCH 3/7] Address comments (#104) Signed-off-by: Voldivh Signed-off-by: William Woodall --- .../include/resource_retriever/plugins/curl_retriever.hpp | 6 ------ .../resource_retriever/plugins/filesystem_retriever.hpp | 6 ------ resource_retriever/src/plugins/curl_retriever.cpp | 2 -- 3 files changed, 14 deletions(-) diff --git a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp index 3bf0a60..5da7b23 100644 --- a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp +++ b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp @@ -29,12 +29,6 @@ #ifndef RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ #define RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ -#include -#include -#include -#include -#include - #include "resource_retriever/plugins/retriever_plugin.hpp" #include "resource_retriever/visibility_control.hpp" diff --git a/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp index bc90622..d1248d5 100644 --- a/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp +++ b/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp @@ -29,12 +29,6 @@ #ifndef RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ #define RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ -#include -#include -#include -#include -#include - #include "resource_retriever/plugins/retriever_plugin.hpp" #include "resource_retriever/visibility_control.hpp" diff --git a/resource_retriever/src/plugins/curl_retriever.cpp b/resource_retriever/src/plugins/curl_retriever.cpp index 803a3f8..431651f 100644 --- a/resource_retriever/src/plugins/curl_retriever.cpp +++ b/resource_retriever/src/plugins/curl_retriever.cpp @@ -108,8 +108,6 @@ CurlRetriever & CurlRetriever::operator=(CurlRetriever && other) noexcept return *this; } - - std::string CurlRetriever::name() { return "resource_retriever::plugins::CurlRetriever"; } From ec6214650673ebb308db2ccdf0bc8cc820219b0e Mon Sep 17 00:00:00 2001 From: William Woodall Date: Tue, 19 Nov 2024 09:32:55 -0800 Subject: [PATCH 4/7] avoid variable shadowing Signed-off-by: William Woodall --- .../include/resource_retriever/memory_resource.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resource_retriever/include/resource_retriever/memory_resource.hpp b/resource_retriever/include/resource_retriever/memory_resource.hpp index b9475e8..4aa271e 100644 --- a/resource_retriever/include/resource_retriever/memory_resource.hpp +++ b/resource_retriever/include/resource_retriever/memory_resource.hpp @@ -43,10 +43,10 @@ namespace resource_retriever */ struct RESOURCE_RETRIEVER_PUBLIC MemoryResource { - explicit MemoryResource(std::string url, std::string expanded_url, std::vector data): - url(std::move(url)), - expanded_url(std::move(expanded_url)), - data(std::move(data)) + explicit MemoryResource(std::string url_in, std::string expanded_url_in, std::vector data_in): + url(std::move(url_in)), + expanded_url(std::move(expanded_url_in)), + data(std::move(data_in)) { } const std::string url; From 8e6d4ee5b8e5fb2a2bbf80c86a7aac6286333667 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Tue, 3 Dec 2024 16:19:32 +0000 Subject: [PATCH 5/7] Lint Signed-off-by: Michael Carroll Signed-off-by: William Woodall --- .../resource_retriever/memory_resource.hpp | 7 ++++-- .../plugins/curl_retriever.hpp | 4 +++- .../plugins/filesystem_retriever.hpp | 4 +++- .../plugins/retriever_plugin.hpp | 2 +- .../src/plugins/curl_retriever.cpp | 22 ++++++++++--------- .../src/plugins/filesystem_retriever.cpp | 10 ++++----- .../src/plugins/retriever_plugin.cpp | 2 +- resource_retriever/src/retriever.cpp | 13 +++++------ resource_retriever/test/test.cpp | 22 +++++++++++-------- 9 files changed, 49 insertions(+), 37 deletions(-) diff --git a/resource_retriever/include/resource_retriever/memory_resource.hpp b/resource_retriever/include/resource_retriever/memory_resource.hpp index 4aa271e..f02a2e3 100644 --- a/resource_retriever/include/resource_retriever/memory_resource.hpp +++ b/resource_retriever/include/resource_retriever/memory_resource.hpp @@ -43,8 +43,11 @@ namespace resource_retriever */ struct RESOURCE_RETRIEVER_PUBLIC MemoryResource { - explicit MemoryResource(std::string url_in, std::string expanded_url_in, std::vector data_in): - url(std::move(url_in)), + explicit MemoryResource( + std::string url_in, + std::string expanded_url_in, + std::vector data_in) + :url(std::move(url_in)), expanded_url(std::move(expanded_url_in)), data(std::move(data_in)) { diff --git a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp index 5da7b23..9be1a79 100644 --- a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp +++ b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp @@ -29,6 +29,8 @@ #ifndef RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ #define RESOURCE_RETRIEVER__PLUGINS__CURL_RETRIEVER_HPP_ +#include + #include "resource_retriever/plugins/retriever_plugin.hpp" #include "resource_retriever/visibility_control.hpp" @@ -37,7 +39,7 @@ using CURL = void; namespace resource_retriever::plugins { -class RESOURCE_RETRIEVER_PUBLIC CurlRetriever: public RetrieverPlugin +class RESOURCE_RETRIEVER_PUBLIC CurlRetriever : public RetrieverPlugin { public: CurlRetriever(); diff --git a/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp index d1248d5..ec424f9 100644 --- a/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp +++ b/resource_retriever/include/resource_retriever/plugins/filesystem_retriever.hpp @@ -29,13 +29,15 @@ #ifndef RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ #define RESOURCE_RETRIEVER__PLUGINS__FILESYSTEM_RETRIEVER_HPP_ +#include + #include "resource_retriever/plugins/retriever_plugin.hpp" #include "resource_retriever/visibility_control.hpp" namespace resource_retriever::plugins { -class RESOURCE_RETRIEVER_PUBLIC FilesystemRetriever: public RetrieverPlugin +class RESOURCE_RETRIEVER_PUBLIC FilesystemRetriever : public RetrieverPlugin { public: FilesystemRetriever(); diff --git a/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp b/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp index 91f1d54..e2b6967 100644 --- a/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp +++ b/resource_retriever/include/resource_retriever/plugins/retriever_plugin.hpp @@ -36,7 +36,7 @@ namespace resource_retriever::plugins { -std::string expand_package_url(const std::string & url); +std::string expand_package_url(const std::string & url); std::string escape_spaces(const std::string & url); class RESOURCE_RETRIEVER_PUBLIC RetrieverPlugin diff --git a/resource_retriever/src/plugins/curl_retriever.cpp b/resource_retriever/src/plugins/curl_retriever.cpp index 431651f..47a6cc0 100644 --- a/resource_retriever/src/plugins/curl_retriever.cpp +++ b/resource_retriever/src/plugins/curl_retriever.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -49,7 +50,7 @@ class CURLStaticInit { CURLcode ret = curl_global_init(CURL_GLOBAL_ALL); if (ret != 0) { - fprintf(stderr, "Error initializing libcurl! retcode = %d", ret); + std::cerr << "Error initializing libcurl! retcode = " << ret; } else { initialized_ = true; } @@ -85,11 +86,11 @@ size_t curlWriteFunc(void * buffer, size_t size, size_t nmemb, void * userp) return size * nmemb; } -CurlRetriever::CurlRetriever(): - curl_handle_(curl_easy_init()) +CurlRetriever::CurlRetriever() +:curl_handle_(curl_easy_init()) { - } + CurlRetriever::~CurlRetriever() { if (curl_handle_ != nullptr) { @@ -108,16 +109,17 @@ CurlRetriever & CurlRetriever::operator=(CurlRetriever && other) noexcept return *this; } -std::string CurlRetriever::name() { +std::string CurlRetriever::name() +{ return "resource_retriever::plugins::CurlRetriever"; } bool CurlRetriever::can_handle(const std::string & url) { - return (url.find("package://") == 0 || - url.find("file://") == 0 || - url.find("http://") == 0 || - url.find("https://") == 0); + return url.find("package://") == 0 || + url.find("file://") == 0 || + url.find("http://") == 0 || + url.find("https://") == 0; } MemoryResourcePtr CurlRetriever::get(const std::string & url) @@ -126,7 +128,7 @@ MemoryResourcePtr CurlRetriever::get(const std::string & url) auto mod_url = url; try { mod_url = expand_package_url(mod_url); - } catch (const resource_retriever::Exception &e) { + } catch (const resource_retriever::Exception & e) { return nullptr; } diff --git a/resource_retriever/src/plugins/filesystem_retriever.cpp b/resource_retriever/src/plugins/filesystem_retriever.cpp index 98349ac..5d90d9e 100644 --- a/resource_retriever/src/plugins/filesystem_retriever.cpp +++ b/resource_retriever/src/plugins/filesystem_retriever.cpp @@ -45,7 +45,8 @@ FilesystemRetriever::FilesystemRetriever() = default; FilesystemRetriever::~FilesystemRetriever() = default; -std::string FilesystemRetriever::name() { +std::string FilesystemRetriever::name() +{ return "resource_retriever::plugins::FilesystemRetriever"; } @@ -60,12 +61,11 @@ MemoryResourcePtr FilesystemRetriever::get(const std::string & url) auto mod_url = url; try { mod_url = expand_package_url(mod_url); - } catch (const resource_retriever::Exception &e) { + } catch (const resource_retriever::Exception & e) { return nullptr; } - if (mod_url.find("file://") == 0) - { + if (mod_url.find("file://") == 0) { mod_url = mod_url.substr(7); } @@ -80,7 +80,7 @@ MemoryResourcePtr FilesystemRetriever::get(const std::string & url) // Create the vector and read the file std::vector data(fileSize); - file.read(reinterpret_cast(data.data()), fileSize); + file.read(reinterpret_cast(data.data()), fileSize); file.close(); res = std::make_shared(url, mod_url, data); } diff --git a/resource_retriever/src/plugins/retriever_plugin.cpp b/resource_retriever/src/plugins/retriever_plugin.cpp index 908704c..d4260ad 100644 --- a/resource_retriever/src/plugins/retriever_plugin.cpp +++ b/resource_retriever/src/plugins/retriever_plugin.cpp @@ -82,7 +82,7 @@ std::string expand_package_url(const std::string & url) mod_url = "file://" + package_path + mod_url; } return mod_url; -}; +} RetrieverPlugin::RetrieverPlugin() = default; diff --git a/resource_retriever/src/retriever.cpp b/resource_retriever/src/retriever.cpp index aae79bd..f521d6a 100644 --- a/resource_retriever/src/retriever.cpp +++ b/resource_retriever/src/retriever.cpp @@ -50,8 +50,8 @@ RetrieverVec default_plugins() }; } -Retriever::Retriever(RetrieverVec plugins): - plugins(std::move(plugins)) +Retriever::Retriever(RetrieverVec plugins) +:plugins(std::move(plugins)) { } @@ -59,14 +59,13 @@ Retriever::~Retriever() = default; MemoryResourcePtr Retriever::get(const std::string & url) { - for (auto & plugin : plugins) - { - if (plugin->can_handle(url)) - { + for (auto & plugin : plugins) { + if (plugin->can_handle(url)) { auto res = plugin->get(url); - if (res != nullptr) + if (res != nullptr) { return res; + } } } return nullptr; diff --git a/resource_retriever/test/test.cpp b/resource_retriever/test/test.cpp index b4b6233..4593b95 100644 --- a/resource_retriever/test/test.cpp +++ b/resource_retriever/test/test.cpp @@ -79,22 +79,26 @@ TEST(Retriever, invalidFiles) EXPECT_EQ(nullptr, r.get("package:///test.xml")); } -class TestRetrieverPlugin: public resource_retriever::plugins::RetrieverPlugin +class TestRetrieverPlugin : public resource_retriever::plugins::RetrieverPlugin { public: - TestRetrieverPlugin() = default; - ~TestRetrieverPlugin() override = default; + TestRetrieverPlugin() = default; + ~TestRetrieverPlugin() override = default; - std::string name() override { - return "TestRetrieverPlugin"; - } + std::string name() override + { + return "TestRetrieverPlugin"; + } - bool can_handle(const std::string & url) override { + bool can_handle(const std::string & url) override + { return url.find("test://") == 0; } - resource_retriever::MemoryResourcePtr get(const std::string & url) override { - return std::make_shared(url, url, std::vector{0, 1, 2, 3, 4, 5}); + resource_retriever::MemoryResourcePtr get(const std::string & url) override + { + return std::make_shared( + url, url, std::vector{0, 1, 2, 3, 4, 5}); } }; From 8e0fae727583a280fa70c5dff9b1c88bf869bbb4 Mon Sep 17 00:00:00 2001 From: William Woodall Date: Fri, 13 Dec 2024 16:23:33 -0800 Subject: [PATCH 6/7] address review comments and change error handling for libcurl Signed-off-by: William Woodall --- .../plugins/curl_retriever.hpp | 4 ++ .../src/plugins/curl_retriever.cpp | 38 +++++++++++++------ .../src/plugins/filesystem_retriever.cpp | 3 -- .../src/plugins/retriever_plugin.cpp | 12 ++++-- resource_retriever/src/retriever.cpp | 2 - resource_retriever/test/test.cpp | 4 +- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp index 9be1a79..5322c40 100644 --- a/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp +++ b/resource_retriever/include/resource_retriever/plugins/curl_retriever.hpp @@ -42,6 +42,10 @@ namespace resource_retriever::plugins class RESOURCE_RETRIEVER_PUBLIC CurlRetriever : public RetrieverPlugin { public: + /// Construct a CurlRetriever plugin and initialize libcurl. + /** + * \throws std::runtime_error if libcurl fails to initialize + */ CurlRetriever(); ~CurlRetriever() override; diff --git a/resource_retriever/src/plugins/curl_retriever.cpp b/resource_retriever/src/plugins/curl_retriever.cpp index 47a6cc0..a6c9f06 100644 --- a/resource_retriever/src/plugins/curl_retriever.cpp +++ b/resource_retriever/src/plugins/curl_retriever.cpp @@ -30,10 +30,9 @@ #include -#include #include -#include #include +#include #include #include #include @@ -42,16 +41,13 @@ namespace { - class CURLStaticInit { public: CURLStaticInit() { - CURLcode ret = curl_global_init(CURL_GLOBAL_ALL); - if (ret != 0) { - std::cerr << "Error initializing libcurl! retcode = " << ret; - } else { + ret_ = curl_global_init(CURL_GLOBAL_ALL); + if (ret_ == CURLE_OK) { initialized_ = true; } } @@ -63,8 +59,19 @@ class CURLStaticInit } } + /// Check that libcurl is globally initialized, otherwise throw. + void check_if_initialized() const + { + if (!this->initialized_) { + throw std::runtime_error( + "curl_global_init(CURL_GLOBAL_ALL) failed (" + std::to_string(ret_) + "): " + + curl_easy_strerror(ret_)); + } + } + private: bool initialized_ {false}; + CURLcode ret_ = CURLE_OK; }; CURLStaticInit g_curl_init; } // namespace @@ -86,8 +93,14 @@ size_t curlWriteFunc(void * buffer, size_t size, size_t nmemb, void * userp) return size * nmemb; } +CURL * do_curl_easy_init() +{ + ::g_curl_init.check_if_initialized(); + return curl_easy_init(); +} + CurlRetriever::CurlRetriever() -:curl_handle_(curl_easy_init()) +: curl_handle_(do_curl_easy_init()) { } @@ -116,10 +129,11 @@ std::string CurlRetriever::name() bool CurlRetriever::can_handle(const std::string & url) { - return url.find("package://") == 0 || - url.find("file://") == 0 || - url.find("http://") == 0 || - url.find("https://") == 0; + return + url.find("package://") == 0 || + url.find("file://") == 0 || + url.find("http://") == 0 || + url.find("https://") == 0; } MemoryResourcePtr CurlRetriever::get(const std::string & url) diff --git a/resource_retriever/src/plugins/filesystem_retriever.cpp b/resource_retriever/src/plugins/filesystem_retriever.cpp index 5d90d9e..07b09c4 100644 --- a/resource_retriever/src/plugins/filesystem_retriever.cpp +++ b/resource_retriever/src/plugins/filesystem_retriever.cpp @@ -28,12 +28,9 @@ #include "resource_retriever/plugins/filesystem_retriever.hpp" -#include -#include #include #include #include -#include #include #include "resource_retriever/exception.hpp" diff --git a/resource_retriever/src/plugins/retriever_plugin.cpp b/resource_retriever/src/plugins/retriever_plugin.cpp index d4260ad..fc31463 100644 --- a/resource_retriever/src/plugins/retriever_plugin.cpp +++ b/resource_retriever/src/plugins/retriever_plugin.cpp @@ -28,7 +28,8 @@ #include "resource_retriever/plugins/retriever_plugin.hpp" -#include +#include +#include #include "resource_retriever/exception.hpp" @@ -59,12 +60,15 @@ std::string escape_spaces(const std::string & url) std::string expand_package_url(const std::string & url) { + constexpr std::string_view package_url_prefix = "package://"; std::string mod_url = url; - if (url.find("package://") == 0) { - mod_url.erase(0, strlen("package://")); + if (url.find(package_url_prefix) == 0) { + mod_url.erase(0, package_url_prefix.length()); size_t pos = mod_url.find('/'); if (pos == std::string::npos) { - throw Exception(url, "Could not parse package:// format into file:// format"); + throw Exception( + url, + "Could not parse " + std::string(package_url_prefix) + " format into file:// format"); } std::string package = mod_url.substr(0, pos); diff --git a/resource_retriever/src/retriever.cpp b/resource_retriever/src/retriever.cpp index f521d6a..8e3d0f4 100644 --- a/resource_retriever/src/retriever.cpp +++ b/resource_retriever/src/retriever.cpp @@ -28,11 +28,9 @@ #include "resource_retriever/retriever.hpp" -#include #include #include #include -#include #include "resource_retriever/exception.hpp" #include "resource_retriever/plugins/curl_retriever.hpp" diff --git a/resource_retriever/test/test.cpp b/resource_retriever/test/test.cpp index 4593b95..3044dd8 100644 --- a/resource_retriever/test/test.cpp +++ b/resource_retriever/test/test.cpp @@ -26,8 +26,10 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include #include +#include +#include +#include #include "gtest/gtest.h" From 00b410f5b5d150217862c4b8543f2534ce8f3b3b Mon Sep 17 00:00:00 2001 From: William Woodall Date: Tue, 17 Dec 2024 07:58:40 -0500 Subject: [PATCH 7/7] fixup Signed-off-by: William Woodall --- .../include/resource_retriever/memory_resource.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource_retriever/include/resource_retriever/memory_resource.hpp b/resource_retriever/include/resource_retriever/memory_resource.hpp index f02a2e3..dc38446 100644 --- a/resource_retriever/include/resource_retriever/memory_resource.hpp +++ b/resource_retriever/include/resource_retriever/memory_resource.hpp @@ -47,7 +47,7 @@ struct RESOURCE_RETRIEVER_PUBLIC MemoryResource std::string url_in, std::string expanded_url_in, std::vector data_in) - :url(std::move(url_in)), + : url(std::move(url_in)), expanded_url(std::move(expanded_url_in)), data(std::move(data_in)) {