(.*)<\\/tr>"};
+ static const std::string_view filename_regex_str{"| .*<\\/a>"};
+ static const std::string_view size_regex_str{" | (.*)<\\/td> | "};
+
+ static const std::string_view begin_file_element = ")";
+
+ static const std::string_view begin_size_element = " | ";
+ static const std::string_view end_size_element = " | ";
+
+ const auto &text = response.text;
+ std::regex word_regex(row_regex_str.data());
+ auto words_begin = std::sregex_iterator(text.begin(), text.end(), word_regex);
+ auto words_end = std::sregex_iterator();
+
+ GodotFileObjects godot_file_objects;
+
+ for (std::sregex_iterator i = words_begin; i != words_end; ++i) {
+ const std::smatch &match = *i;
+ std::string match_str = match[1].str();
+
+ auto begin_pos = match_str.find(begin_file_element) + begin_file_element.size();
+ auto end_pos = match_str.find(end_file_element, begin_pos);
+ if (end_pos == std::string::npos || begin_pos == std::string::npos) {
+ continue;
+ }
+
+ const std::string file_element = match_str.substr(begin_pos, end_pos - begin_pos);
+
+ if (isInvalidFileOrFolder(file_element)) {
+ continue;
+ }
+
+ const std::string &url = file_element;
+ const std::string full_url = response.url.str() + url;
+
+ if (cb::string::ends_with(file_element, "/")) {
+ godot_file_objects.directories.emplace_back(GodotDirectory{file_element, full_url, url});
+ } else {
+ begin_pos = match_str.find(begin_size_element) + begin_size_element.size();
+ end_pos = match_str.find(end_size_element, begin_pos);
+ std::string file_size = "0M";
+ if (end_pos == std::string::npos || begin_pos == std::string::npos) {
+ std::cout << "couldn't parse file size for " << file_element << std::endl;
+ } else {
+ file_size = match_str.substr(begin_pos, end_pos - begin_pos);
+ }
+
+ const GodotFileObject &godot_filename = GodotFileObject{
+ file_element,
+ full_url,
+ file_size,
+ extract_version_from_url(full_url),
+ };
+ godot_file_objects.files.emplace_back(godot_filename);
+ }
+ }
+
+ return godot_file_objects;
+}
+
+bool GodotVersionScraper::isInvalidFileOrFolder(const std::string &file_element) {
+// using namespace cb::string;
+ return cb::string::starts_with(file_element, "media")
+ or cb::string::starts_with(file_element, "patreon")
+ or cb::string::starts_with(file_element, "testing")
+ or cb::string::starts_with(file_element, "toolchains")
+ or cb::string::starts_with(file_element, "s=")
+ or cb::string::starts_with(file_element, "..")
+ or cb::string::ends_with(file_element, ".txt")
+ or cb::string::ends_with(file_element, ".sha256");
+}
+
+GodotFileObjects GodotVersionScraper::request_godot_versions() {
+ const std::string url = "https://downloads.tuxfamily.org/godotengine/";
+
+ GodotFileObjects file_objects = request_godot_file_objects(url);
+ std::cout << "got " << file_objects.files.size() << " files" << std::endl;
+ std::cout << "got " << file_objects.directories.size() << " directories" << std::endl;
+
+ return file_objects;
+}
+
+std::string GodotVersionScraper::extract_version_from_url(const std::string &url) {
+ std::size_t start_pos = url.rfind("/Godot");
+ if (start_pos == std::string::npos) {
+ start_pos = url.rfind("/godot");
+ if (start_pos == std::string::npos) {
+ return "";
+ }
+ }
+ // + 1 because of Godot_ or Godot- -> remove the -/_
+ start_pos += 6 + 1;
+
+ std::size_t endPos = url.rfind('.');
+ if (endPos == std::string::npos) {
+ return "";
+ }
+
+ return url.substr(start_pos, endPos - start_pos);
+}
diff --git a/src/godot_version_scraper/godot_version_scraper.h b/src/godot_version_scraper/godot_version_scraper.h
new file mode 100644
index 0000000..f80f678
--- /dev/null
+++ b/src/godot_version_scraper/godot_version_scraper.h
@@ -0,0 +1,30 @@
+
+#ifndef GODOTHUB_GODOTVERSIONSCRAPER_H
+#define GODOTHUB_GODOTVERSIONSCRAPER_H
+
+
+#include "godot_structs.h"
+
+#include
+#include
+
+
+class GodotVersionScraper {
+public:
+ [[nodiscard]]
+ static GodotFileObjects request_godot_versions();
+
+private:
+ [[nodiscard]]
+ static GodotFileObjects request_godot_file_objects(const std::string &base_url);
+
+ [[nodiscard]]
+ static GodotFileObjects fetch_page_content(const cpr::Response &response);
+
+ static bool isInvalidFileOrFolder(const std::string &file_element);
+
+ static std::string extract_version_from_url(const std::string &url);
+};
+
+
+#endif //GODOTHUB_GODOTVERSIONSCRAPER_H
diff --git a/src/main.cpp b/src/main.cpp
new file mode 100644
index 0000000..502a97b
--- /dev/null
+++ b/src/main.cpp
@@ -0,0 +1,32 @@
+#include "godot_version_scraper/godot_structs.h"
+#include "apps/fetch_godot.h"
+
+#define USE_CBLIB_DEFINES
+
+#include
+
+#include
+
+
+int main(int argc, char **argv) {
+ using namespace cb::argparser;
+ ArgumentParser argument_parser{};
+
+ argument_parser.add_argument(
+ "-o", "--output", Type::STRING_TYPE,
+ "config.json", "Output file path for the config",
+ Mode::REQUIRED_MODE);
+
+ ArgumentMap arguments = argument_parser.parse(argc, argv);
+
+ if (arguments["-h"]) {
+ std::cout << argument_parser.get_help() << std::endl;
+ return 0;
+ }
+
+ auto output_file_path = arguments["-o"].as_str();
+
+ FetchGodot godot_fetcher{};
+ godot_fetcher.run();
+ godot_fetcher.save(output_file_path);
+}