kde-plasma/discover: Fix "too many open files" w/ flatpak backend

KDE-bug: https://bugs.kde.org/show_bug.cgi?id=474231

Signed-off-by: Andreas Sturmlechner <asturm@gentoo.org>

2 files changed, 286 insertions, 0 deletions

diff --git a/kde-plasma/discover/discover-5.27.8-r1.ebuild b/kde-plasma/discover/discover-5.27.8-r1.ebuild
new file mode 100644
index 000000000000..2fabb5fd60ad
--- /dev/null
+++ b/kde-plasma/discover/discover-5.27.8-r1.ebuild
@@ -0,0 +1,99 @@
+# Copyright 1999-2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+ECM_TEST="true"
+KFMIN=5.106.0
+QTMIN=5.15.9
+inherit ecm plasma.kde.org
+
+DESCRIPTION="KDE Plasma resources management GUI"
+HOMEPAGE="https://userbase.kde.org/Discover"
+
+LICENSE="GPL-2" # TODO: CHECK
+SLOT="5"
+KEYWORDS="~amd64 ~arm ~arm64 ~loong ~ppc64 ~riscv ~x86"
+IUSE="+firmware flatpak snap telemetry webengine"
+
+# libmarkdown (app-text/discount) only used in PackageKitBackend
+DEPEND="
+	>=dev-libs/appstream-0.15.3:=
+	>=dev-qt/qtconcurrent-${QTMIN}:5
+	>=dev-qt/qtdbus-${QTMIN}:5
+	>=dev-qt/qtdeclarative-${QTMIN}:5
+	>=dev-qt/qtgui-${QTMIN}:5
+	>=dev-qt/qtnetwork-${QTMIN}:5
+	>=dev-qt/qtwidgets-${QTMIN}:5
+	>=kde-frameworks/attica-${KFMIN}:5
+	>=kde-frameworks/kcmutils-${KFMIN}:5
+	>=kde-frameworks/kconfig-${KFMIN}:5
+	>=kde-frameworks/kconfigwidgets-${KFMIN}:5
+	>=kde-frameworks/kcoreaddons-${KFMIN}:5
+	>=kde-frameworks/kcrash-${KFMIN}:5
+	>=kde-frameworks/kdbusaddons-${KFMIN}:5
+	>=kde-frameworks/kdeclarative-${KFMIN}:5
+	>=kde-frameworks/ki18n-${KFMIN}:5
+	>=kde-frameworks/kidletime-${KFMIN}:5
+	>=kde-frameworks/kio-${KFMIN}:5
+	>=kde-frameworks/kirigami-${KFMIN}:5
+	>=kde-frameworks/knewstuff-${KFMIN}:5
+	>=kde-frameworks/knotifications-${KFMIN}:5
+	>=kde-frameworks/kwidgetsaddons-${KFMIN}:5
+	>=kde-frameworks/kxmlgui-${KFMIN}:5
+	>=kde-frameworks/purpose-${KFMIN}:5
+	firmware? ( >=sys-apps/fwupd-1.5.0 )
+	flatpak? ( sys-apps/flatpak )
+	snap? ( sys-libs/snapd-glib:=[qt5] )
+	telemetry? ( dev-libs/kuserfeedback:5 )
+	webengine? ( >=dev-qt/qtwebview-${QTMIN}:5 )
+"
+RDEPEND="${DEPEND}
+	>=dev-qt/qtquickcontrols2-${QTMIN}:5
+	snap? ( app-containers/snapd )
+"
+BDEPEND=">=kde-frameworks/kcmutils-${KFMIN}:5"
+
+PATCHES=(
+	"${FILESDIR}/${PN}-5.25.90-tests-optional.patch"
+	"${FILESDIR}/${P}-flatpak-qrunnable-thread.patch" # KDE-bug 474231
+)
+
+src_prepare() {
+	ecm_src_prepare
+	# we don't need it with PackageKitBackend off
+	ecm_punt_kf_module Archive
+	# we don't do anything with this
+	sed -e "s/^pkg_check_modules.*RpmOstree/#&/" \
+		-e "s/^pkg_check_modules.*Ostree/#&/" \
+		-i CMakeLists.txt || die
+}
+
+src_configure() {
+	local mycmakeargs=(
+		# TODO: Port PackageKit's portage back-end to python3
+		-DCMAKE_DISABLE_FIND_PACKAGE_packagekitqt5=ON
+		# Automated updates will not work for us
+		# https://invent.kde.org/plasma/discover/-/merge_requests/142
+		-DWITH_KCM=OFF
+		-DBUILD_DummyBackend=OFF
+		-DBUILD_FlatpakBackend=$(usex flatpak)
+		-DBUILD_FwupdBackend=$(usex firmware)
+		-DBUILD_RpmOstreeBackend=OFF
+		-DBUILD_SnapBackend=$(usex snap)
+		-DBUILD_SteamOSBackend=OFF
+		$(cmake_use_find_package telemetry KUserFeedback)
+		$(cmake_use_find_package webengine Qt5WebView)
+	)
+
+	ecm_src_configure
+}
+
+src_test() {
+	# bug 686392: needs network connection
+	local myctestargs=(
+		-E "(knsbackendtest|flatpaktest)"
+	)
+
+	ecm_src_test
+}
diff --git a/kde-plasma/discover/files/discover-5.27.8-flatpak-qrunnable-thread.patch b/kde-plasma/discover/files/discover-5.27.8-flatpak-qrunnable-thread.patch
new file mode 100644
index 000000000000..20525969b988
--- /dev/null
+++ b/kde-plasma/discover/files/discover-5.27.8-flatpak-qrunnable-thread.patch
@@ -0,0 +1,187 @@
+From 46d14515c3105e4e318d28db41057d9f1df3ce4d Mon Sep 17 00:00:00 2001
+From: Harald Sitter <sitter@kde.org>
+Date: Mon, 11 Sep 2023 07:05:52 +0200
+Subject: [PATCH] flatpak: make FlatpakTransactionThread a qrunnable instead
+
+we can't just have an unlimited number of threads for flatpak
+transactions. it'd eventually cause excessive load on both CPU and
+network to the point where things will start misbehaving. we also run
+risk of exhausting other software limited resources such as file
+descriptors.
+
+to resolve this problem we now treat the transactionthread as runnable
+and put it in a limited threadpool for concurrent execution. the new
+runnable has a finished signal that is emitted on every return from
+run() to match the QThread API.
+
+concurrency is limited to no more than 4 runnables at a time. that
+should still be plenty concurrent while generally unexpected to exhaust
+the default 1024 file descriptor limit - an install transaction appears
+to weigh between 60 and 100 fds
+
+other backends don't necessarily have this problem since they have
+daemons that do the work for us so we have way fewer open fds for them.
+
+BUG: 474231
+
+(cherry picked from commit db0ebc855517f189f64c1602a5d27e185cf02833)
+---
+ .../FlatpakBackend/FlatpakJobTransaction.cpp  | 34 ++++++++++++++++---
+ .../FlatpakTransactionThread.cpp              |  9 +++--
+ .../FlatpakBackend/FlatpakTransactionThread.h |  5 ++-
+ 3 files changed, 40 insertions(+), 8 deletions(-)
+
+diff --git a/libdiscover/backends/FlatpakBackend/FlatpakJobTransaction.cpp b/libdiscover/backends/FlatpakBackend/FlatpakJobTransaction.cpp
+index 613563755..b2c1fcc20 100644
+--- a/libdiscover/backends/FlatpakBackend/FlatpakJobTransaction.cpp
++++ b/libdiscover/backends/FlatpakBackend/FlatpakJobTransaction.cpp
+@@ -1,6 +1,7 @@
+ /*
+  *   SPDX-FileCopyrightText: 2013 Aleix Pol Gonzalez <aleixpol@blue-systems.com>
+  *   SPDX-FileCopyrightText: 2017 Jan Grulich <jgrulich@redhat.com>
++ *   SPDX-FileCopyrightText: 2023 Harald Sitter <sitter@kde.org>
+  *
+  *   SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
+  */
+@@ -10,9 +11,30 @@
+ #include "FlatpakResource.h"
+ #include "FlatpakTransactionThread.h"
+ 
++#include <thread>
++
+ #include <QDebug>
+ #include <QTimer>
+ 
++namespace
++{
++class ThreadPool : public QThreadPool
++{
++public:
++    ThreadPool()
++    {
++        // Cap the amount of concurrency to prevent too many in-flight transactions. This in particular
++        // prevents running out of file descriptors or other limited resources.
++        // https://bugs.kde.org/show_bug.cgi?id=474231
++        constexpr auto arbitraryMaxConcurrency = 4U;
++        const auto concurrency = std::min(std::thread::hardware_concurrency(), arbitraryMaxConcurrency);
++        setMaxThreadCount(std::make_signed_t<decltype(concurrency)>(concurrency));
++    }
++};
++} // namespace
++
++Q_GLOBAL_STATIC(ThreadPool, s_pool);
++
+ FlatpakJobTransaction::FlatpakJobTransaction(FlatpakResource *app, Role role, bool delayStart)
+     : Transaction(app->backend(), app, role, {})
+     , m_app(app)
+@@ -27,11 +49,12 @@ FlatpakJobTransaction::FlatpakJobTransaction(FlatpakResource *app, Role role, bo
+ 
+ FlatpakJobTransaction::~FlatpakJobTransaction()
+ {
+-    if (m_appJob->isRunning()) {
+-        cancel();
+-        m_appJob->wait();
++    cancel();
++    if (s_pool->tryTake(m_appJob)) { // immediately delete if the runnable hasn't started yet
++        delete m_appJob;
++    } else { // otherwise defer cleanup to the pool
++        m_appJob->setAutoDelete(true);
+     }
+-    delete m_appJob;
+ }
+ 
+ void FlatpakJobTransaction::cancel()
+@@ -45,6 +68,7 @@ void FlatpakJobTransaction::start()
+ 
+     // App job will be added every time
+     m_appJob = new FlatpakTransactionThread(m_app, role());
++    m_appJob->setAutoDelete(false);
+     connect(m_appJob, &FlatpakTransactionThread::finished, this, &FlatpakJobTransaction::finishTransaction);
+     connect(m_appJob, &FlatpakTransactionThread::progressChanged, this, &FlatpakJobTransaction::setProgress);
+     connect(m_appJob, &FlatpakTransactionThread::speedChanged, this, &FlatpakJobTransaction::setDownloadSpeed);
+@@ -52,7 +76,7 @@ void FlatpakJobTransaction::start()
+     connect(m_appJob, &FlatpakTransactionThread::webflowStarted, this, &FlatpakJobTransaction::webflowStarted);
+     connect(m_appJob, &FlatpakTransactionThread::webflowDone, this, &FlatpakJobTransaction::webflowDone);
+ 
+-    m_appJob->start();
++    s_pool->start(m_appJob);
+ }
+ 
+ void FlatpakJobTransaction::finishTransaction()
+diff --git a/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.cpp b/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.cpp
+index 7072460ad..738a97a8c 100644
+--- a/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.cpp
++++ b/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.cpp
+@@ -1,5 +1,6 @@
+ /*
+  *   SPDX-FileCopyrightText: 2017 Jan Grulich <jgrulich@redhat.com>
++ *   SPDX-FileCopyrightText: 2023 Harald Sitter <sitter@kde.org>
+  *
+  *   SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
+  */
+@@ -10,6 +11,7 @@
+ #include <KLocalizedString>
+ #include <QDebug>
+ #include <QDesktopServices>
++#include <QScopeGuard>
+ 
+ static int FLATPAK_CLI_UPDATE_FREQUENCY = 150;
+ 
+@@ -92,8 +94,7 @@ void FlatpakTransactionThread::webflowDoneCallback(FlatpakTransaction *transacti
+ }
+ 
+ FlatpakTransactionThread::FlatpakTransactionThread(FlatpakResource *app, Transaction::Role role)
+-    : QThread()
+-    , m_result(false)
++    : m_result(false)
+     , m_app(app)
+     , m_role(role)
+ {
+@@ -131,6 +132,10 @@ void FlatpakTransactionThread::cancel()
+ 
+ void FlatpakTransactionThread::run()
+ {
++    auto finish = qScopeGuard([this] {
++        Q_EMIT finished();
++    });
++
+     if (!m_transaction)
+         return;
+     g_autoptr(GError) localError = nullptr;
+diff --git a/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.h b/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.h
+index 277c21902..8e3d0e2e2 100644
+--- a/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.h
++++ b/libdiscover/backends/FlatpakBackend/FlatpakTransactionThread.h
+@@ -1,5 +1,6 @@
+ /*
+  *   SPDX-FileCopyrightText: 2017 Jan Grulich <jgrulich@redhat.com>
++ *   SPDX-FileCopyrightText: 2023 Harald Sitter <sitter@kde.org>
+  *
+  *   SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
+  */
+@@ -11,12 +12,13 @@
+ #include <glib.h>
+ 
+ #include <QMap>
++#include <QRunnable>
+ #include <QStringList>
+ #include <QThread>
+ #include <Transaction/Transaction.h>
+ 
+ class FlatpakResource;
+-class FlatpakTransactionThread : public QThread
++class FlatpakTransactionThread : public QObject, public QRunnable
+ {
+     Q_OBJECT
+ public:
+@@ -49,6 +51,7 @@ Q_SIGNALS:
+     void passiveMessage(const QString &msg);
+     void webflowStarted(const QUrl &url, int id);
+     void webflowDone(int id);
++    void finished();
+ 
+ private:
+     static gboolean
+-- 
+GitLab
+