From 9aac6ff6597e8135811b9de2bb023e9f8c8d6d96 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Sun, 18 Oct 2026 00:37:48 +0100 Subject: [PATCH 1/7] Add automatic debuginfod use to Linux crash catcher. The implementation isn't as simple as I'd have liked because the experience on Ubuntu was truly awful, with crash logs taking over half an hour to generate. There's therefore extra protection in case of servers that are too slow or go down, and deduplication so the per-server problem cost is minimised. --- .gitlab-ci.yml | 3 +- CI/before_script.linux.sh | 1 + components/CMakeLists.txt | 3 + components/crashcatcher/crashcatcher.cpp | 109 ++++++++++++++++++++++- 4 files changed, 110 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af0bca8986..60f727d8b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -171,8 +171,7 @@ Almalinux_Generic: echo "Extracting '$package'..." tar -xf "$package" -C install done - - echo "set debuginfod enabled on" >> ~/.gdbinit - - export DEBUGINFOD_URLS="https://openmw-sym.rgw.ctrl-c.liu.se $DEBUGINFOD_URLS https://debuginfod.elfutils.org" + - export OPENMW_DEBUGINFOD=ON - CI/run_integration_tests.sh after_script: - | diff --git a/CI/before_script.linux.sh b/CI/before_script.linux.sh index fbf061b99c..5a66b7b7f2 100755 --- a/CI/before_script.linux.sh +++ b/CI/before_script.linux.sh @@ -95,6 +95,7 @@ if [[ $CI_OPENMW_USE_VCPKG ]]; then -DVCPKG_HOST_TRIPLET="$VCPKG_TARGET_TRIPLET" -DVCPKG_TARGET_TRIPLET="$VCPKG_TARGET_TRIPLET" -DCMAKE_TOOLCHAIN_FILE="$DEPENDENCIES_ROOT_PATH/scripts/buildsystems/vcpkg.cmake" + -DEXTRA_DEBUGINFOD_URLS="https://openmw-sym.rgw.ctrl-c.liu.se" ) else CMAKE_CONF_OPTS+=( diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index e4e3e66aa5..cec93eb6b9 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -402,6 +402,9 @@ elseif(NOT ANDROID) add_component_dir (crashcatcher crashcatcher ) + if (EXTRA_DEBUGINFOD_URLS) + set_source_files_properties(crashcatcher/crashcatcher.cpp PROPERTIES COMPILE_DEFINITIONS "EXTRA_DEBUGINFOD_URLS=\"${EXTRA_DEBUGINFOD_URLS}\"") + endif () endif() add_component_dir(detournavigator diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index e460f04dcc..445c37f1e0 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -178,6 +179,101 @@ namespace } std::unique_ptr scopedFile(file); + bool forceDebuginfod = false; +#ifdef __linux__ + char* openmwDebuginfod = getenv("OPENMW_DEBUGINFOD"); + if (openmwDebuginfod) + { + std::string_view asStringView(openmwDebuginfod); + forceDebuginfod = !(asStringView == "0" || asStringView == "OFF" || asStringView == "NO"); + } + else + { + const auto buttons = std::to_array({ + { SDL_MESSAGEBOX_BUTTON_RETURNKEY_DEFAULT, 1, "Yes" }, + { SDL_MESSAGEBOX_BUTTON_ESCAPEKEY_DEFAULT, 0, "No" } + }); + const SDL_MessageBoxData messageBoxData = { + SDL_MESSAGEBOX_ERROR, + nullptr, + "Fatal Error", + "OpenMW has encountered a fatal error.\n" + "Allow debugger to connect to debuginfod server to generate a more detailed crash log?\n" + "It may take longer to generate the crash log.", + buttons.size(), + buttons.data(), + nullptr + }; + int buttonId; + if (!SDL_ShowMessageBox(&messageBoxData, &buttonId)) + forceDebuginfod = buttonId != 0; + else + printf( + "Failed to show SDL message box. Debuginfod server will not be force-enabled. " + "Set an environment variable called OPENMW_DEBUGINFOD to override this.\n\n"); + } + + if (forceDebuginfod) + { + std::vector debuginfodUrls; + auto add = [&](std::string_view url) { + if (url.empty()) + return; + + // duplicates are removed automatically if the debuginfod client includes a commit from July 2021, + // so we might not need this check + if (std::find(debuginfodUrls.begin(), debuginfodUrls.end(), url) == debuginfodUrls.end()) + { + std::stringstream command; + // filter out excessively slow servers, like the Ubuntu one that's been down for months + // when https://sourceware.org/bugzilla/show_bug.cgi?id=34421 is resolved and propagated to enough + // user machines, we may be able to remove this + command << "curl --max-time 3 --silent '" << url << "' -o /dev/null"; + auto ret = system(command.str().c_str()); + if (ret == 0) + debuginfodUrls.emplace_back(url); + else + printf("Skipping %s as it took too long to respond.\n", std::string(url).c_str()); + } + else + printf("Skipping %s as it is a duplicate.\n", std::string(url).c_str()); + }; +#ifdef EXTRA_DEBUGINFOD_URLS + std::istringstream extraDebuginfodUrls(EXTRA_DEBUGINFOD_URLS); + while (extraDebuginfodUrls) + { + std::string url; + extraDebuginfodUrls >> url; + add(url); + } +#endif + char* existingDebuginfodUrls = getenv("DEBUGINFOD_URLS"); + if (existingDebuginfodUrls) + { + std::istringstream existingDebuginfodUrlsStream(existingDebuginfodUrls); + while (existingDebuginfodUrlsStream) + { + std::string url; + existingDebuginfodUrlsStream >> url; + add(url); + } + } + + add("https://debuginfod.elfutils.org"); + + std::stringstream debuginfodUrlsStream; + for (const auto& url : debuginfodUrls) + debuginfodUrlsStream << url << " "; + setenv("DEBUGINFOD_URLS", debuginfodUrlsStream.str().c_str(), 1); + + // The default timeout is ninety seconds per shared library per server, so if anything goes down, + // this takes far too long + // As of October 2026, the Ubuntu debuginfod server has been down for months, + // so we can't ignore this as a unlikely hypothetical + setenv("DEBUGINFOD_TIMEOUT", "3", 1); + } +#endif + if (fprintf(file, "%s", T::sScript) < 0) { printf("Failed to write debugger script to file \"%s\": %s\n", scriptPath, @@ -188,8 +284,9 @@ namespace scopedFile = nullptr; scopedFd = nullptr; - char command[128]; - snprintf(command, sizeof(command), T::sCommandTemplate, pid, scriptPath); + char command[192]; + snprintf(command, sizeof(command), T::sCommandTemplate, pid, scriptPath, + forceDebuginfod ? T::sDebuginfodEnableArgument : ""); printf("Executing: %s\n", command); fflush(stdout); @@ -218,7 +315,10 @@ namespace struct Gdb { static constexpr char sName[] = "gdb"; + static constexpr char sDebuginfodEnableArgument[] = "--init-eval-command=\"set debuginfod enabled on\""; static constexpr char sScript[] = R"(shell echo "" +show debuginfod enabled +show debuginfod urls shell echo "* Loaded Libraries" info sharedlibrary shell echo "" @@ -236,12 +336,13 @@ thread apply all backtrace full 1000 detach quit )"; - static constexpr char sCommandTemplate[] = "gdb --pid %d --quiet --batch --command %s"; + static constexpr char sCommandTemplate[] = "gdb --pid %d --quiet --batch --command %s %s"; }; struct Lldb { static constexpr char sName[] = "lldb"; + static constexpr char sDebuginfodEnableArgument[] = ""; // debuginfod is always enabled if compiled in static constexpr char sScript[] = R"(script print("\n* Loaded Libraries") image list script print('\n* Threads') @@ -253,7 +354,7 @@ script print(''.join(f'{t}\n' + ''.join(''.join([f' {f}\n', ''.join(f' {s}\n detach quit )"; - static constexpr char sCommandTemplate[] = "lldb --attach-pid %d --batch --source %s"; + static constexpr char sCommandTemplate[] = "lldb --attach-pid %d --batch --source %s %s"; }; void printProcessInfo(pid_t pid) From fe56ace9aced4cd9b117e892a7c9f5d6f176c25d Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Fri, 23 Oct 2026 00:45:41 +0100 Subject: [PATCH 2/7] Add wget fallback --- components/crashcatcher/crashcatcher.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index 445c37f1e0..fe1d8cbff4 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -220,8 +220,6 @@ namespace if (url.empty()) return; - // duplicates are removed automatically if the debuginfod client includes a commit from July 2021, - // so we might not need this check if (std::find(debuginfodUrls.begin(), debuginfodUrls.end(), url) == debuginfodUrls.end()) { std::stringstream command; @@ -230,13 +228,19 @@ namespace // user machines, we may be able to remove this command << "curl --max-time 3 --silent '" << url << "' -o /dev/null"; auto ret = system(command.str().c_str()); - if (ret == 0) + if (WIFEXITED(ret) && WEXITSTATUS(ret) == 127) + { + command.clear(); + command << "wget --timeout=3 --tries=1 --quiet -O /dev/null '" << url << "'"; + ret = system(command.str().c_str()); + if (WIFEXITED(ret) && WEXITSTATUS(ret) == 127) + ret = 0; + } + if (WEXITSTATUS(ret) == 0) debuginfodUrls.emplace_back(url); else printf("Skipping %s as it took too long to respond.\n", std::string(url).c_str()); } - else - printf("Skipping %s as it is a duplicate.\n", std::string(url).c_str()); }; #ifdef EXTRA_DEBUGINFOD_URLS std::istringstream extraDebuginfodUrls(EXTRA_DEBUGINFOD_URLS); From ab6be282cd078ff0be6c8cd652a2a67bab2970bd Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Fri, 23 Oct 2026 00:46:00 +0100 Subject: [PATCH 3/7] Emphasise time --- components/crashcatcher/crashcatcher.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index fe1d8cbff4..0749664107 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -199,7 +199,7 @@ namespace "Fatal Error", "OpenMW has encountered a fatal error.\n" "Allow debugger to connect to debuginfod server to generate a more detailed crash log?\n" - "It may take longer to generate the crash log.", + "It may take several minutes longer to generate the crash log.", buttons.size(), buttons.data(), nullptr From acf26e4c75feaaebd08dd3ecfcaf33234a5b81bf Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Fri, 23 Oct 2026 00:52:02 +0100 Subject: [PATCH 4/7] Some whitespace --- components/crashcatcher/crashcatcher.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index 0749664107..c0fd13584e 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -262,7 +262,7 @@ namespace add(url); } } - + add("https://debuginfod.elfutils.org"); std::stringstream debuginfodUrlsStream; From 5f12751c040e9e539874ae8405ba0a0c71add3a4 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Fri, 23 Oct 2026 16:01:32 +0100 Subject: [PATCH 5/7] Appease clang-format --- components/crashcatcher/crashcatcher.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index c0fd13584e..db31bcf503 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -191,7 +191,7 @@ namespace { const auto buttons = std::to_array({ { SDL_MESSAGEBOX_BUTTON_RETURNKEY_DEFAULT, 1, "Yes" }, - { SDL_MESSAGEBOX_BUTTON_ESCAPEKEY_DEFAULT, 0, "No" } + { SDL_MESSAGEBOX_BUTTON_ESCAPEKEY_DEFAULT, 0, "No" }, }); const SDL_MessageBoxData messageBoxData = { SDL_MESSAGEBOX_ERROR, @@ -202,7 +202,7 @@ namespace "It may take several minutes longer to generate the crash log.", buttons.size(), buttons.data(), - nullptr + nullptr, }; int buttonId; if (!SDL_ShowMessageBox(&messageBoxData, &buttonId)) @@ -476,6 +476,9 @@ static void crash_catcher(int signum, siginfo_t* siginfo, void* /*context*/) exit(1); } + if (crash_info.pid > 0) + kill(crash_info.pid, SIGSTOP); + const char* sigdesc = findSignalDescription(signals, crash_info.signum); if (crash_info.siginfo.has_value()) From 7dab7f7b5fbbbb2a3d28ef5d06f79ded59edef3b Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 28 Oct 2026 23:28:56 +0000 Subject: [PATCH 6/7] Reset stringstream --- components/crashcatcher/crashcatcher.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index db31bcf503..e31991a3af 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -230,7 +230,7 @@ namespace auto ret = system(command.str().c_str()); if (WIFEXITED(ret) && WEXITSTATUS(ret) == 127) { - command.clear(); + command = {}; command << "wget --timeout=3 --tries=1 --quiet -O /dev/null '" << url << "'"; ret = system(command.str().c_str()); if (WIFEXITED(ret) && WEXITSTATUS(ret) == 127) From c04f5057f2f24f9c00d82567569f89ea68dcaf18 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 28 Oct 2026 23:38:07 +0000 Subject: [PATCH 7/7] Always check WIFEXITED --- components/crashcatcher/crashcatcher.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/crashcatcher/crashcatcher.cpp b/components/crashcatcher/crashcatcher.cpp index e31991a3af..d725f493c0 100644 --- a/components/crashcatcher/crashcatcher.cpp +++ b/components/crashcatcher/crashcatcher.cpp @@ -236,7 +236,7 @@ namespace if (WIFEXITED(ret) && WEXITSTATUS(ret) == 127) ret = 0; } - if (WEXITSTATUS(ret) == 0) + if (!WIFEXITED(ret) || WEXITSTATUS(ret) == 0) debuginfodUrls.emplace_back(url); else printf("Skipping %s as it took too long to respond.\n", std::string(url).c_str());